diff --git a/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.py b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.py new file mode 100644 index 0000000000..6b01434f43 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.py @@ -0,0 +1,25 @@ +"""The loader-dependencies example from the loaders docs, as written. + +raw_accounts is an intermediate loader without a write_strategy that returns rows, so it writes +them to __loader__raw_accounts as a table. +""" + +from sqlbuild.executor.load.models import LoaderContext +from sqlbuild.loaders import loader + + +@loader +def raw_accounts(ctx: LoaderContext) -> list[dict[str, object]]: + """Load customer accounts.""" + return [ + {"account_id": 1, "account_name": "Pawnee Parks"}, + {"account_id": 2, "account_name": "Eagleton"}, + ] + + +@loader(depends_on=[raw_accounts]) +def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]: + """Account metrics.""" + accounts = ctx.loader(raw_accounts) + rows = ctx.query(f"SELECT account_id FROM {accounts.destination}") + return [{"account_id": row[0], "metric": "active"} for row in rows.fetchall()] diff --git a/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.yml b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.yml new file mode 100644 index 0000000000..90e7a2605a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.yml @@ -0,0 +1,5 @@ +sources: + - name: raw_account_metrics + description: Account metrics read from the intermediate raw_accounts loader. + managed: true + write_strategy: table diff --git a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml new file mode 100644 index 0000000000..78acf58022 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml @@ -0,0 +1,15 @@ +# Roles over a loader chain whose middle loader no source declares: running the job needs a grant +# on the intermediate loader's node as well as the model and the source it feeds. +[roles.shipments_runner] +description = "Run the shipments build, including its staging loader" + +[[roles.shipments_runner.grants]] +nodes = ["orders", "source:raw_shipments", "loader:stage_shipments"] +permissions = ["job.run"] + +[roles.orders_runner] +description = "Run the orders build without the staging loader" + +[[roles.orders_runner.grants]] +nodes = ["orders", "source:raw_shipments"] +permissions = ["job.run"] diff --git a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py index 9512d788c5..70db157dde 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py +++ b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py @@ -1,9 +1,12 @@ -"""A source loader that depends on an intermediate loader no source declares.""" +"""A source loader that depends on an intermediate loader no source declares. + +A job that selects the model must claim the intermediate loader's relation as well as the source. +""" from sqlbuild.loaders import loader -@loader +@loader(write_strategy="table") def stage_shipments(ctx): """Stage synthetic shipments in the intermediate loader's own relation.""" return [{"order_id": 9}] diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py new file mode 100644 index 0000000000..d153e434ff --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py @@ -0,0 +1,10 @@ +"""Jobs whose selections are Python nodes, which `--select job:` must select as published.""" + +from sqlbuild.orchestration import job + +customer_counts = job( + "customer_counts", + description="Count customers.", + select="task:count_customers asset:customers_extract", +) +order_counts = job("order_counts", description="Count orders.", select="+task:count_orders") diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py new file mode 100644 index 0000000000..221447d1c8 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py @@ -0,0 +1,10 @@ +"""A task whose Python upstream another job builds, so a job group must order the two jobs.""" + +from python.orders.partner_export.node import partner_orders_file + +from sqlbuild.tasks import task + + +@task(depends_on=partner_orders_file, tags=["orders"], description="Book collection.") +def request_carrier_collection(ctx): + return None diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py new file mode 100644 index 0000000000..2f8b9b8329 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py @@ -0,0 +1,16 @@ +"""A Python check on the orders model that a job's tag selector matches. + +A job matching the check and the model it checks publishes and runs the check with the model. A job +matching the check alone fails publication, naming the check and the model it leaves out. +""" + +from sqlbuild.checks import check +from sqlbuild.refs import model + + +@check(depends_on=model("orders"), tags=["order_checks"]) +def orders_present(ctx): + """Orders are present.""" + orders = ctx.relation(model("orders")) + count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0] + return ctx.pass_() if count else ctx.fail(message="no orders") diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py new file mode 100644 index 0000000000..dfe6194d20 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py @@ -0,0 +1,9 @@ +"""An asset one job builds that a task in another job of the same group depends on.""" + +from sqlbuild.assets import asset +from sqlbuild.refs import model + + +@asset(depends_on=model("orders"), tags=["orders"], description="Partner order file.") +def partner_orders_file(ctx): + return None diff --git a/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py b/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py new file mode 100644 index 0000000000..0723fa403a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py @@ -0,0 +1,15 @@ +"""A job that reaches a managed source through `+`, so its run selects the source's loader chain. + +`+orders` selects the model, its managed source raw_shipments, the raw_shipments loader and the +intermediate stage_shipments loader it depends on. The run records each loader as the source it +writes, which is the node its load reports under. +""" + +from sqlbuild.orchestration import job, schedule + +annual_orders = job( + "annual_orders", + description="Build orders and everything upstream of it, loaders included.", + select="+orders", + schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), +) diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs b/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs index 329bbbc646..f1c9480ace 100644 --- a/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs +++ b/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs @@ -8,17 +8,23 @@ use crate::graph::errors::SelectorError; use crate::graph::models::{JobSelection, ProjectGraph}; /// Resolve every job on the graph without jobs attached, as publication does, then attach them. -pub(crate) fn attach( +pub(crate) fn attach(graph: ProjectGraph, catalog: BTreeMap) -> ProjectGraph { + attach_with(graph, catalog, |graph, select, exclude| { + resolve(graph, select, exclude).map(|keys| keys.into_iter().collect()) + }) +} + +/// [`attach`] with each job's selectors resolved by `resolve_job` on the graph without jobs. +pub(crate) fn attach_with( mut graph: ProjectGraph, catalog: BTreeMap, + resolve_job: impl Fn(&ProjectGraph, &[String], &[String]) -> Result, ) -> ProjectGraph { graph.jobs = None; let mut resolved: BTreeMap> = BTreeMap::new(); for (name, selection) in &catalog { let keys: Result = match selection { - JobSelection::Job { select, exclude } => { - resolve(&graph, select, exclude).map(|keys| keys.into_iter().collect()) - } + JobSelection::Job { select, exclude } => resolve_job(&graph, select, exclude), JobSelection::Resolved { keys } => Ok(keys.iter().cloned().collect()), JobSelection::Group { .. } => continue, }; diff --git a/crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs b/crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs new file mode 100644 index 0000000000..091081bc50 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs @@ -0,0 +1,18 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::models::{JobSelection, ProjectGraph}; + +/// [`super::attach_jobs::attach_jobs`] with each job's selectors resolved by `resolve_job`. +pub fn attach_jobs_with( + graph: ProjectGraph, + catalog: BTreeMap, + resolve_job: impl Fn( + &ProjectGraph, + &[String], + &[String], + ) -> Result, SelectorError>, +) -> ProjectGraph { + crate::graph::_helpers::jobs::attach_with(graph, catalog, resolve_job) +} diff --git a/crates/sqlbuild-analysis/src/graph/main/mod.rs b/crates/sqlbuild-analysis/src/graph/main/mod.rs index ccd21dc172..05d0272f4d 100644 --- a/crates/sqlbuild-analysis/src/graph/main/mod.rs +++ b/crates/sqlbuild-analysis/src/graph/main/mod.rs @@ -1,4 +1,5 @@ pub mod attach_jobs; +pub mod attach_jobs_with; pub mod build_project_graph; pub mod build_resources; pub mod downstream_edges; diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs b/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs index 4d2f33a411..e80d101d53 100644 --- a/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs +++ b/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs @@ -5,7 +5,7 @@ use crate::planner_scope::constants::{MODEL, SOURCE}; use crate::planner_scope::models::LoaderGraph; use std::collections::{BTreeMap, BTreeSet}; -/// Managed sources each model's execution deps read and each managed source's declared loader chain, as `expand_loaders` keys them. +/// Managed sources each model's execution deps read and each managed source's loader chain, intermediate loaders included, as `expand_loaders` keys them. pub fn node_loads( graph: &LoaderGraph, upstream: &[(ObjectKey, Vec)], @@ -24,7 +24,6 @@ pub fn node_loads( graph .upstream(chain) .iter() - .filter(|name| !graph.intermediate(name)) .map(|name| graph.key(name)) .collect() } else { diff --git a/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs index b90277e273..8fd13b6768 100644 --- a/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs +++ b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs @@ -147,8 +147,8 @@ fn given_loader_chains_and_unmanaged_reads_when_listing_node_loads_then_only_dec expected_loads: vec![("model:orders", vec!["source:raw_returns"])], }, NodeLoadsTestCase { - // stage_orders has no declaring source, so a run that loads it still fails in the worker. - description: "a chain keeps declared loaders behind an intermediate one and drops the intermediate", + // stage_orders has no declaring source; the snapshot publishes it as source:stage_orders. + description: "a chain keeps an intermediate loader and the declared loaders behind it", loaders: vec![ ("raw_orders", vec!["stage_orders"]), ("stage_orders", vec!["raw_customers"]), @@ -162,7 +162,10 @@ fn given_loader_chains_and_unmanaged_reads_when_listing_node_loads_then_only_dec ("source:raw_orders", vec![]), ("source:raw_customers", vec![]), ], - expected_loads: vec![("source:raw_orders", vec!["source:raw_customers"])], + expected_loads: vec![( + "source:raw_orders", + vec!["source:stage_orders", "source:raw_customers"], + )], }, ]; let key = |text: &str| { diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs index 9a416f0c44..944eb5294f 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs @@ -2,6 +2,9 @@ use std::collections::HashMap; +use crate::assembly::project::types::ObjectKey; +use crate::definition_changes::constants::PYTHON_NODE_KINDS; + use crate::python_selection::constants::{CURRENT_DIRECTORY, ROOT_DIRECTORY}; use crate::python_selection::models::{InventoryIndexes, InventoryNode}; @@ -83,3 +86,17 @@ pub(crate) fn indexes(nodes: &[InventoryNode]) -> InventoryIndexes { paths: paths.rows, } } + +/// A published node's dependency keys split into its Python upstream names and its SQL keys. +pub(crate) fn split_published_deps(deps: &[ObjectKey]) -> (Vec, Vec) { + let mut python: Vec = Vec::new(); + let mut sql: Vec = Vec::new(); + for (kind, name) in deps { + if PYTHON_NODE_KINDS.contains(&kind.as_str()) { + python.push(name.clone()); + } else { + sql.push((kind.clone(), name.clone())); + } + } + (python, sql) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs index e2390a571b..7fc0e9cd4a 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs @@ -17,7 +17,9 @@ use crate::python_selection::constants::{ UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, UNMAPPED_KIND_CODE, }; -use crate::python_selection::models::{NodeIndex, PythonSelectionGraph, PythonSelectionNode}; +use crate::python_selection::models::{ + NodeIndex, PythonSelectionGraph, PythonSelectionNode, SelectionContext, SqlSelectionGraph, +}; use sqlbuild_core::constants::PYTHON_NODE_ROOT as PYTHON_NODE_PATH_ROOT; pub(crate) type Names = BTreeSet; @@ -59,6 +61,19 @@ fn keyed(rows: &[(String, Vec)]) -> HashMap<&str, &[String]> { .collect() } +impl<'graph> SelectionContext<'graph> { + pub(crate) fn new( + sql: SqlSelectionGraph<'graph>, + python: &'graph PythonSelectionGraph, + ) -> Self { + Self { + sql, + nodes: NodeIndex::new(python), + sql_names: sql_names(sql.graph), + } + } +} + impl<'graph> NodeIndex<'graph> { pub(crate) fn new(graph: &'graph PythonSelectionGraph) -> Self { Self { diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index b98e48dca6..56bf52afce 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -7,6 +7,7 @@ use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; use sqlbuild_core::text::main::glob_match::glob_match; use crate::assembly::project::types::ObjectKey; +use crate::definition_changes::constants::PYTHON_NODE_KINDS; use crate::graph::errors::SelectorError; use crate::graph::main::build_resources::build_resources; use crate::graph::main::edge_keys::edge_keys; @@ -88,29 +89,93 @@ pub(crate) fn run_selection( Atom::Sql(_) => None, }) .collect(); - if !select.is_empty() && !checks.is_empty() { - return Err(error( - PLANNER_DEFAULT_CODE, - format!( - "Python checks are not selectable here: {}. Use sqb check instead.", - checks.join(", ") - ), - )); + if !select.is_empty() + && let Some(message) = unselected_checked_nodes(context, &selected, &checks) + { + return Err(error(PLANNER_DEFAULT_CODE, message)); } let runnable: Atoms = selected .into_iter() .filter(|atom| !is_check(atom)) .collect(); validate_selected_dependencies(context, &runnable)?; + Ok(selection(context, sql_keys(&runnable), &runnable)) +} + +/// The first matched check a build cannot run after the nodes it checks, as an error message. +fn unselected_checked_nodes( + context: &SelectionContext<'_>, + selected: &Atoms, + checks: &[&str], +) -> Option { + let loaded: HashSet<&str> = source_loaders(context.sql) + .into_iter() + .map(|(source, _)| source) + .collect(); + for check in checks { + if let Some(message) = unrunnable_check(context, selected, &loaded, check) { + return Some(message); + } + } + None +} + +/// Why a build cannot run `check`: it checks a source no loader writes, or an unselected node. +fn unrunnable_check( + context: &SelectionContext<'_>, + selected: &Atoms, + loaded: &HashSet<&str>, + check: &str, +) -> Option { + let sql: &[ObjectKey] = context + .nodes + .node(check) + .map_or(&[][..], |node| &node.sql_deps); + let unloaded: Vec = sql + .iter() + .filter(|(kind, name)| kind == SOURCE_RESOURCE && !loaded.contains(name.as_str())) + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + if !unloaded.is_empty() { + return Some(format!( + "Python check '{check}' checks {}, which a build does not load, so the check \ + would not run; run it with sqb check, or narrow the selector so it no longer \ + matches the check.", + unloaded.join(", ") + )); + } + let missing: Vec = context + .nodes + .edges(check, EdgeDirection::Upstream) + .iter() + .filter(|name| !selected.contains(&Atom::Python((*name).clone()))) + .filter_map(|name| Some(format!("{}:{name}", context.nodes.node(name)?.kind))) + .chain( + sql.iter() + .filter(|key| !selected.contains(&Atom::Sql((*key).clone()))) + .map(|(kind, name)| format!("{kind}:{name}")), + ) + .collect(); + (!missing.is_empty()).then(|| { + format!( + "Python check '{check}' checks {}, which the selection does not include; \ + select it too, or narrow the selector so it no longer matches the check.", + missing.join(", ") + ) + }) +} + +/// The SQL keys, with the source each selected intermediate loader writes, and the selected Python node names. +fn selection(context: &SelectionContext<'_>, sql: Vec, atoms: &Atoms) -> RunSelection { let terminal: HashSet<&str> = context .sql .sources .iter() .filter_map(|(_, loader)| loader.as_deref()) .collect(); - let mut keys: BTreeSet = sql_keys(&runnable).into_iter().collect(); + let mut keys: BTreeSet = sql.into_iter().collect(); let mut names: Vec = Vec::new(); - for atom in &runnable { + for atom in atoms { let Atom::Python(name) = atom else { continue; }; @@ -119,10 +184,29 @@ pub(crate) fn run_selection( } names.push(name.clone()); } - Ok(RunSelection { + RunSelection { sql_keys: keys.into_iter().collect(), python_node_names: names, - }) + } +} + +/// The SQL keys with their build functions and the Python node names, checks included, that selectors name; nothing is refused. +pub(crate) fn node_selection( + context: &SelectionContext<'_>, + select: &[String], + exclude: &[String], +) -> Result { + validate_boundaries(context)?; + let selected: Atoms = groups(context, select)? + .difference(&groups(context, exclude)?) + .cloned() + .collect(); + let sql: Vec = build_resources( + context.sql.graph, + &sql_keys(&selected), + BuildResources::SELECTION, + ); + Ok(selection(context, sql, &selected)) } fn groups(context: &SelectionContext<'_>, raw_groups: &[String]) -> Resolved { @@ -258,7 +342,8 @@ fn single_side(context: &SelectionContext<'_>, raw: &str) -> Resolved { let atoms: Atoms = match kind.as_str() { TEST_KIND => return Err(unit_test_error(raw)), PATH_KIND => resolve_path(context, raw, &value)?, - SEED_KIND | SOURCE_KIND | JOB_KIND => resolve_sql(context, raw)?, + SEED_KIND | SOURCE_KIND => resolve_sql(context, raw)?, + JOB_KIND => resolve_job(context, raw)?, TASK_KIND | ASSET_KIND | LOADER_KIND | CHECK_KIND => { return resolve_typed(context, raw, (&kind, &value)); } @@ -290,6 +375,48 @@ fn resolve_sql(context: &SelectionContext<'_>, raw: &str) -> Resolved { )?)) } +/// A job's SQL keys and Python nodes, with `+` also expanding them over the Python node graph. +fn resolve_job(context: &SelectionContext<'_>, raw: &str) -> Resolved { + let ParsedSelector::Kind { + upstream, + downstream, + .. + } = parse_selector(raw)? + else { + return resolve_sql(context, raw); + }; + let mut atoms: Atoms = Atoms::new(); + let mut members: Vec = Vec::new(); + for atom in resolve_sql(context, raw)? { + match atom { + Atom::Sql((kind, name)) + if PYTHON_NODE_KINDS.contains(&kind.as_str()) + && context.nodes.node(&name).is_some() => + { + members.push(name); + } + atom => { + atoms.insert(atom); + } + } + } + let mut names: Names = members.iter().cloned().collect(); + for (expand, direction) in [ + (upstream, EdgeDirection::Upstream), + (downstream, EdgeDirection::Downstream), + ] { + if expand { + names.extend( + context + .nodes + .closure(members.iter().map(String::as_str), direction), + ); + } + } + atoms.extend(python_atoms(names)); + Ok(atoms) +} + fn resolve_python(context: &SelectionContext<'_>, raw: &str) -> Resolved { Ok(python_atoms(select_nodes( &context.nodes, @@ -383,7 +510,7 @@ fn resolve_name(context: &SelectionContext<'_>, raw: &str, value: &str) -> Resol (true, true) => Err(both_error(value)), (true, false) => resolve_sql(context, raw), (false, true) => resolve_python(context, raw), - (false, false) if is_job(context, value) => resolve_sql(context, raw), + (false, false) if is_job(context, value) => resolve_job(context, raw), (false, false) => Err(unknown_name(value)), }; } diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs index ddacd27837..9bed1e9624 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs @@ -157,7 +157,7 @@ fn validate_loader_upstreams(context: &SelectionContext<'_>) -> Result<(), Selec Ok(()) } -/// Python's `_validate_selected_dependencies`: selected sources and nodes have their loaders. +/// Selected managed sources have their loaders selected; a Python node may run without its Python upstream, whose latest persisted result it reads. pub(crate) fn validate_selected_dependencies( context: &SelectionContext<'_>, atoms: &BTreeSet, @@ -188,18 +188,5 @@ pub(crate) fn validate_selected_dependencies( select it directly or use upstream expansion" ))); } - for atom in atoms { - let Atom::Python(name) = atom else { - continue; - }; - for upstream in context.nodes.edges(name, EdgeDirection::Upstream) { - if !selected.contains(upstream.as_str()) { - return Err(failure(format!( - "Python node '{name}' depends on unselected Python node '{upstream}'; \ - select it directly or use upstream expansion" - ))); - } - } - } Ok(()) } diff --git a/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs new file mode 100644 index 0000000000..21edb1f7eb --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs @@ -0,0 +1,38 @@ +use std::collections::BTreeMap; + +use crate::graph::main::attach_jobs_with::attach_jobs_with; +use crate::graph::models::{JobSelection, ProjectGraph}; +use crate::python_selection::_helpers::unified::run_selection; +use crate::python_selection::models::{ + PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, +}; + +/// Attach jobs as publication resolves them, keeping each job's Python nodes as `(kind, name)` keys. +pub fn attach_python_jobs( + sql: SqlSelectionGraph<'_>, + python: &PythonSelectionGraph, + catalog: BTreeMap, +) -> ProjectGraph { + let kinds: BTreeMap<&str, &str> = python + .nodes + .iter() + .map(|node| (node.name.as_str(), node.kind.as_str())) + .collect(); + attach_jobs_with(sql.graph.clone(), catalog, |graph, select, exclude| { + let context: SelectionContext<'_> = SelectionContext::new( + SqlSelectionGraph { + graph, + sources: sql.sources, + }, + python, + ); + let RunSelection { + sql_keys, + python_node_names, + } = run_selection(&context, select, exclude)?; + let python_keys = python_node_names + .into_iter() + .filter_map(|name| Some((kinds.get(name.as_str())?.to_string(), name))); + Ok(sql_keys.into_iter().chain(python_keys).collect()) + }) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs index 884bfa0b85..4630bc0f8c 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs @@ -1,4 +1,8 @@ +pub mod attach_python_jobs; +pub mod published_python_graph; pub mod python_node_graph_indexes; +pub mod python_run_key; pub mod resolve_python_node_excludes; pub mod resolve_python_node_selectors; +pub mod resolve_python_sql_nodes; pub mod resolve_python_sql_run_selection; diff --git a/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs b/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs new file mode 100644 index 0000000000..695361c4e6 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs @@ -0,0 +1,40 @@ +use crate::python_selection::_helpers::inventory::{indexes, split_published_deps}; +use crate::python_selection::models::{ + InventoryIndexes, InventoryNode, PublishedPythonNode, PythonSelectionGraph, PythonSelectionNode, +}; + +/// The selection graph of Python nodes rebuilt from a published snapshot, so the server selects them as the CLI does. +pub fn published_python_graph(nodes: &[PublishedPythonNode]) -> PythonSelectionGraph { + let mut inventory: Vec = Vec::with_capacity(nodes.len()); + let mut selection_nodes: Vec = Vec::with_capacity(nodes.len()); + for node in nodes { + let (upstream, sql_deps) = split_published_deps(&node.deps); + inventory.push(InventoryNode { + name: node.name.clone(), + tags: node.tags.clone(), + relative_path: String::new(), + upstream, + }); + selection_nodes.push(PythonSelectionNode { + name: node.name.clone(), + kind: node.kind.clone(), + sql_deps, + }); + } + let InventoryIndexes { + upstream, + downstream, + tags, + .. + } = indexes(&inventory); + PythonSelectionGraph { + nodes: selection_nodes, + upstream, + downstream, + tags, + paths: nodes + .iter() + .map(|node| (node.name.clone(), node.folder.clone())) + .collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/python_selection/main/python_run_key.rs b/crates/sqlbuild-analysis/src/python_selection/main/python_run_key.rs new file mode 100644 index 0000000000..a23f1075fe --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/python_run_key.rs @@ -0,0 +1,13 @@ +use crate::python_selection::constants::LOADER_KIND; + +/// The key a selected Python node's run reports under; a loader reports as the source it writes. +pub fn python_run_key(sources: &[(String, Option)], (kind, name): (&str, &str)) -> String { + if kind != LOADER_KIND { + return format!("{kind}:{name}"); + } + let source = sources + .iter() + .find(|(_, loader)| loader.as_deref() == Some(name)) + .map_or(name, |(source, _)| source.as_str()); + format!("source:{source}") +} diff --git a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs new file mode 100644 index 0000000000..b482df85e7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs @@ -0,0 +1,15 @@ +use crate::graph::errors::SelectorError; +use crate::python_selection::_helpers::unified::node_selection; +use crate::python_selection::models::{ + PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, +}; + +/// The SQL resources and Python nodes, checks included, that selectors name, as grants and sensor conditions read them. +pub fn resolve_python_sql_nodes( + sql: SqlSelectionGraph<'_>, + python: &PythonSelectionGraph, + select: &[String], + exclude: &[String], +) -> Result { + node_selection(&SelectionContext::new(sql, python), select, exclude) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs index a820fd41d7..fef9983c7b 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs @@ -1,8 +1,7 @@ use crate::graph::errors::SelectorError; -use crate::python_selection::_helpers::nodes::sql_names; use crate::python_selection::_helpers::unified::run_selection; use crate::python_selection::models::{ - NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, + PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, }; /// Python's `resolve_python_sql_run_selectors`: run selectors over SQL resources and Python nodes. @@ -12,10 +11,5 @@ pub fn resolve_python_sql_run_selection( select: &[String], exclude: &[String], ) -> Result { - let context: SelectionContext<'_> = SelectionContext { - sql, - nodes: NodeIndex::new(python), - sql_names: sql_names(sql.graph), - }; - run_selection(&context, select, exclude) + run_selection(&SelectionContext::new(sql, python), select, exclude) } diff --git a/crates/sqlbuild-analysis/src/python_selection/models.rs b/crates/sqlbuild-analysis/src/python_selection/models.rs index b0d615e1dd..63b7aacf07 100644 --- a/crates/sqlbuild-analysis/src/python_selection/models.rs +++ b/crates/sqlbuild-analysis/src/python_selection/models.rs @@ -52,6 +52,19 @@ pub struct InventoryNode { pub upstream: Vec, } +/// One Python node as a published snapshot lists it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PublishedPythonNode { + pub name: String, + /// `task`, `asset`, `loader` or `check`. + pub kind: String, + pub tags: Vec, + /// The folder of the node's file relative to the project, in POSIX form. + pub folder: String, + /// The Python nodes and SQL resources it depends on, as published keys. + pub deps: Vec, +} + /// Python's `PythonNodeGraph` edges and indexes, each in Python's dict order. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct InventoryIndexes { diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs index 92b68329c3..2e8c3134f8 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs @@ -2,13 +2,14 @@ use crate::assembly::project::types::ObjectKey; use crate::graph::errors::SelectorError; use crate::graph::main::build_project_graph::build_project_graph; use crate::graph::models::{GraphResource, ProjectGraph}; +use crate::python_selection::main::published_python_graph::published_python_graph; use crate::python_selection::main::python_node_graph_indexes::python_node_graph_indexes; use crate::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use crate::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; use crate::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection; use crate::python_selection::models::{ - InventoryIndexes, InventoryNode, PythonSelectionGraph, PythonSelectionNode, RunSelection, - SqlSelectionGraph, + InventoryIndexes, InventoryNode, PublishedPythonNode, PythonSelectionGraph, + PythonSelectionNode, RunSelection, SqlSelectionGraph, }; use crate::python_selection::tests::test_types::{ExpectedRun, SelectorFailure}; @@ -66,6 +67,14 @@ const ORDERS_NODES: &[NodeSpec] = &[ &[], &[], ), + ( + "check_raw_customers", + "check", + "python/checks/customers.py", + &["customer_checks"], + &[], + &[("source", "raw_customers")], + ), ]; fn owned(values: &[&str]) -> Vec { @@ -127,9 +136,45 @@ fn orders_sql_graph() -> ProjectGraph { tags: Vec::new(), folder: None, }, + GraphResource { + key: key(&("source", "raw_customers")), + deps: Vec::new(), + tags: Vec::new(), + folder: None, + }, ]) } +/// The orders Python node graph as the server rebuilds it from published snapshot nodes. +pub(super) fn published_orders_graph() -> PythonSelectionGraph { + let nodes: Vec = ORDERS_NODES + .iter() + .map( + |(name, kind, path, tags, upstream, sql_deps)| PublishedPythonNode { + name: (*name).to_owned(), + kind: (*kind).to_owned(), + tags: owned(tags), + folder: path + .rsplit_once('/') + .map_or(".", |(folder, _)| folder) + .to_owned(), + deps: upstream + .iter() + .map(|upstream| { + let kind: &str = ORDERS_NODES + .iter() + .find(|node| node.0 == *upstream) + .map_or("task", |node| node.1); + key(&(kind, upstream)) + }) + .chain(sql_deps.iter().map(key)) + .collect(), + }, + ) + .collect(); + published_python_graph(&nodes) +} + /// The Python node names the selectors resolve to over the orders graph. pub(super) fn node_names(select: &[&str], exclude: &[&str]) -> Result, OwnedFailure> { resolve_python_node_selectors(&orders_python_graph(), &owned(select), &owned(exclude)) @@ -138,6 +183,15 @@ pub(super) fn node_names(select: &[&str], exclude: &[&str]) -> Result Result { + run_over(&orders_python_graph(), select, exclude) +} + +/// The run selection over the orders SQL graph and the given Python graph. +pub(super) fn run_over( + python: &PythonSelectionGraph, + select: &[&str], + exclude: &[&str], +) -> Result { let graph: ProjectGraph = orders_sql_graph(); let sources: Vec<(String, Option)> = vec![("raw_orders".to_owned(), Some("raw_orders".to_owned()))]; @@ -146,7 +200,7 @@ pub(super) fn run(select: &[&str], exclude: &[&str]) -> Result freshness_sources(publication, &select, &exclude)?, - _ => resolve(publication, &select, &exclude)?, + _ => { + let python = verb == BUILD_VERB + && settings.get(INCLUDE_PYTHON_DEST) != Some(&Json::Bool(false)); + resolve_with_tests((verb, python), publication, &select, &exclude)? + } }; Ok(CommandPlan { verb: verb.clone(), @@ -191,36 +202,161 @@ fn resolve( select: &[String], exclude: &[String], ) -> Result, String> { - let selected: BTreeSet = resolved(publication, select, exclude)?; + let selected: BTreeSet = resolved(publication, select, exclude, false)?; if selected.is_empty() { return Err("the selectors match no nodes in the current publication".into()); } Ok(selected) } +/// [`resolve`] with `test:` and unit-test names selecting `sql_test:` keys, as `sqb test` does. +fn resolve_with_tests( + (verb, python): (&str, bool), + (resources, jobs): (&[PlannerResource], &PublishedJobs), + select: &[String], + exclude: &[String], +) -> Result, String> { + let names = UnitTestSelectorNames { + tests: resources + .iter() + .filter(|resource| is_sql_test(resource)) + .map(|resource| name_of(&resource.key).to_owned()) + .collect(), + resources: resources + .iter() + .filter(|resource| !is_sql_test(resource)) + .map(|resource| name_of(&resource.key).to_owned()) + .chain(jobs.keys().cloned()) + .collect(), + }; + let split = split_unit_test_selectors( + SelectorGroups { select, exclude }, + &names, + UNIT_TEST_VERBS.contains(&verb), + ) + .map_err(|error| format!("{}: {}", error.code, error.message))?; + if !split.excluded_test_names.is_empty() { + return Err( + "an ad-hoc run cannot exclude unit tests; select the unit tests to run instead".into(), + ); + } + let mut keys: BTreeSet = if split.models_selected { + resolved((resources, jobs), &split.select, &split.exclude, python)? + } else { + BTreeSet::new() + }; + keys.extend( + split + .test_names + .iter() + .map(|name| format!("{SQL_TEST_KIND}:{name}")), + ); + if keys.is_empty() { + return Err("the selectors match no nodes in the current publication".into()); + } + Ok(keys) +} + +fn is_sql_test(resource: &PlannerResource) -> bool { + resource.key.split_once(':').map(|(kind, _)| kind) == Some(SQL_TEST_KIND) +} + +/// The published keys the selectors match; Python nodes only when the command runs them. fn resolved( (resources, jobs): (&[PlannerResource], &PublishedJobs), select: &[String], exclude: &[String], + include_python: bool, ) -> Result, String> { - let graph_resources: Vec = resources.iter().filter_map(graph_resource).collect(); + let (mut python, sql): (Vec<&PlannerResource>, Vec<&PlannerResource>) = resources + .iter() + .filter(|resource| !is_sql_test(resource)) + .partition(|resource| python_kind(&resource.key).is_some()); + if !include_python { + python.clear(); + } + let graph_resources: Vec = sql + .iter() + .filter_map(|resource| graph_resource(resource)) + .collect(); let mut graph = build_project_graph(&graph_resources); let selectors: Vec = select.iter().chain(exclude).cloned().collect(); - if selectors_need_jobs(&graph, &selectors, &[]) { - graph = attach_jobs(graph, published_jobs(jobs)); + let node_names: Vec<&str> = python + .iter() + .map(|resource| name_of(&resource.key)) + .collect(); + if selectors_need_jobs(&graph, &selectors, &node_names) { + graph = attach_jobs(graph, published_jobs(jobs, include_python)); } - Ok(resolve_selectors(&graph, select, exclude) - .map_err(|error| format!("{}: {}", error.code, error.message))? + let sources: Vec<(String, Option)> = sql + .iter() + .filter(|resource| resource.key.starts_with("source:")) + .map(|resource| (name_of(&resource.key).to_owned(), resource.loader.clone())) + .collect(); + let nodes: Vec = python + .iter() + .filter_map(|resource| published_python_node(resource)) + .collect(); + let python_graph = published_python_graph(&nodes); + let selection = resolve_python_sql_run_selection( + SqlSelectionGraph { + graph: &graph, + sources: &sources, + }, + &python_graph, + select, + exclude, + ) + .map_err(|error| format!("{}: {}", error.code, error.message))?; + let kinds: BTreeMap<&str, &str> = nodes + .iter() + .map(|node| (node.name.as_str(), node.kind.as_str())) + .collect(); + Ok(selection + .sql_keys .into_iter() .map(|(kind, name)| format!("{kind}:{name}")) + .chain( + selection.python_node_names.iter().filter_map(|name| { + Some(python_run_key(&sources, (kinds.get(name.as_str())?, name))) + }), + ) .collect()) } +/// The Python node kind of a published key, such as `task` for `task:export_orders`. +fn python_kind(key: &str) -> Option<&'static str> { + let (kind, _) = key.split_once(':')?; + PYTHON_NODE_KINDS.into_iter().find(|python| *python == kind) +} + +fn published_python_node(resource: &PlannerResource) -> Option { + Some(PublishedPythonNode { + name: name_of(&resource.key).to_owned(), + kind: python_kind(&resource.key)?.to_owned(), + tags: resource.tags.clone(), + folder: resource.folder.clone().unwrap_or_else(|| ".".to_owned()), + deps: resource + .deps + .iter() + .filter_map(|dep| object_key(dep)) + .collect(), + }) +} + +fn name_of(key: &str) -> &str { + key.split_once(':').map_or(key, |(_, name)| name) +} + /// Published jobs as already-resolved graph selections. -fn published_jobs(jobs: &PublishedJobs) -> BTreeMap { +fn published_jobs(jobs: &PublishedJobs, include_python: bool) -> BTreeMap { let mut selections: BTreeMap = BTreeMap::new(); for (name, keys) in jobs { - let keys: Vec<(String, String)> = keys.iter().filter_map(|key| object_key(key)).collect(); + let keys: Vec<(String, String)> = keys + .iter() + .filter(|key| include_python || python_kind(key).is_none()) + .filter_map(|key| object_key(key)) + .collect(); selections.insert(name.clone(), JobSelection::Resolved { keys }); } selections @@ -233,11 +369,11 @@ fn freshness_sources( exclude: &[String], ) -> Result, String> { let selected: BTreeSet = - upstream_sources(publication.0, &resolved(publication, select, &[])?); + upstream_sources(publication.0, &resolved(publication, select, &[], false)?); let excluded: BTreeSet = if exclude.is_empty() { BTreeSet::new() } else { - upstream_sources(publication.0, &resolved(publication, exclude, &[])?) + upstream_sources(publication.0, &resolved(publication, exclude, &[], false)?) }; let sources: BTreeSet = selected.difference(&excluded).cloned().collect(); if sources.is_empty() { diff --git a/crates/sqlbuild-cli/src/adhoc/constants.rs b/crates/sqlbuild-cli/src/adhoc/constants.rs index c4fc1ca671..dc93f3bcba 100644 --- a/crates/sqlbuild-cli/src/adhoc/constants.rs +++ b/crates/sqlbuild-cli/src/adhoc/constants.rs @@ -6,6 +6,12 @@ pub(crate) const ADHOC_VERBS: [&str; 5] = ["build", "test", "seed", "audit", "fr /// The verb whose settings must also form valid job settings. pub(crate) const BUILD_VERB: &str = "build"; +/// Verbs that select SQL unit tests by `test:` or by name, as on the command line. +pub(crate) const UNIT_TEST_VERBS: [&str; 2] = ["build", "test"]; + +/// The published kind of a SQL unit test. +pub(crate) const SQL_TEST_KIND: &str = "sql_test"; + /// The verb that runs only the seeds of its selection. pub(crate) const SEED_VERB: &str = "seed"; @@ -21,6 +27,9 @@ pub(crate) const CHANGED_COUNTS_DEST: &str = "changed_counts"; /// The job setting `--changed` becomes. pub(crate) const CHANGED_DEST: &str = "changed"; +/// The destination `--no-python` clears; only a build with Python nodes resolves them. +pub(crate) const INCLUDE_PYTHON_DEST: &str = "include_python"; + /// The destination `--load` and `--no-load` share; only `--no-load` is accepted. pub(crate) const LOAD_SOURCES_DEST: &str = "load_sources"; diff --git a/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs b/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs index 0725ac9380..3cd4087f5e 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs @@ -13,6 +13,7 @@ pub(super) fn resource( deps: deps.iter().map(|dep| (*dep).to_owned()).collect(), tags: tags.iter().map(|tag| (*tag).to_owned()).collect(), folder: folder.map(str::to_owned), + loader: None, } } diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index 8631f17666..df7501c9c1 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::json; +use sqlbuild_orchestration::runs::models::PlannerResource; use sqlbuild_orchestration::runs::types::PublishedJobs; use crate::adhoc::main::adhoc_planner::adhoc_planner; @@ -28,6 +29,7 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select resource("model:customers", &[], &[], Some("models/staging")), resource("source:shop.raw_orders", &[], &[], None), resource("seed:currencies", &[], &[], None), + resource("sql_test:orders_keep_amounts", &["model:orders"], &[], None), ]; let jobs: PublishedJobs = BTreeMap::from([ ( @@ -229,6 +231,40 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select command: &["test", "-s", "orders", "--case", "happy_path"], expected_plan: Ok((&["model:orders"], json!({"case": "happy_path"}))), }, + AdhocPlanTestCase { + description: "test: selects one unit test", + command: &["test", "--select", "test:orders_keep_amounts"], + expected_plan: Ok((&["sql_test:orders_keep_amounts"], json!({}))), + }, + AdhocPlanTestCase { + description: "a unit test's bare name selects it, as on the command line", + command: &["test", "--select", "orders_keep_amounts"], + expected_plan: Ok((&["sql_test:orders_keep_amounts"], json!({}))), + }, + AdhocPlanTestCase { + description: "build selects a model and a unit test together", + command: &["build", "--select", "customers", "test:orders_keep_amounts"], + expected_plan: Ok(( + &["model:customers", "sql_test:orders_keep_amounts"], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "excluding a unit test is refused, since a run freezes only what it selects", + command: &[ + "test", + "--select", + "orders", + "--exclude", + "test:orders_keep_amounts", + ], + expected_plan: Err("an ad-hoc run cannot exclude unit tests"), + }, + AdhocPlanTestCase { + description: "audit refuses unit-test selectors as the command line does", + command: &["audit", "--select", "test:orders_keep_amounts"], + expected_plan: Err("selects a unit test"), + }, AdhocPlanTestCase { description: "audit settings come from the audit parser", command: &["audit", "-s", "orders", "--concurrency", "2"], @@ -289,3 +325,204 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select ); } } + +#[test] +fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_command_line() { + let resources = [ + resource("model:stg_orders", &[], &["orders"], Some("models/staging")), + resource( + "model:orders", + &["model:stg_orders"], + &["orders"], + Some("models/marts"), + ), + resource("model:customers", &[], &[], Some("models/marts")), + resource( + "asset:partner_orders_file", + &["model:orders"], + &["orders"], + Some("python/orders/partner_export"), + ), + resource( + "task:request_carrier_collection", + &["asset:partner_orders_file"], + &["orders"], + Some("python/fulfillment/carrier_sync"), + ), + PlannerResource { + loader: Some("raw_shipments".to_owned()), + ..resource("source:raw_shipments", &[], &[], None) + }, + resource( + "model:shipments", + &["source:raw_shipments"], + &[], + Some("models/marts"), + ), + resource( + "loader:raw_shipments", + &["loader:stage_shipments"], + &[], + Some("python/loaders"), + ), + resource("loader:stage_shipments", &[], &[], Some("python/loaders")), + ]; + let jobs: PublishedJobs = BTreeMap::from([( + "carrier_sync".to_owned(), + BTreeSet::from(["task:request_carrier_collection".to_owned()]), + )]); + let test_cases = [ + AdhocPlanTestCase { + description: "a published job selects its Python nodes", + command: &["build", "--select", "job:carrier_sync"], + expected_plan: Ok((&["task:request_carrier_collection"], json!({}))), + }, + AdhocPlanTestCase { + description: "upstream of a published job crosses its Python node's dependencies", + command: &["build", "--select", "+job:carrier_sync"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "typed selectors select the Python nodes", + command: &[ + "build", + "--select", + "task:request_carrier_collection", + "asset:partner_orders_file", + ], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "a task runs without the Python node it depends on, as on the command line", + command: &["build", "--select", "task:request_carrier_collection"], + expected_plan: Ok((&["task:request_carrier_collection"], json!({}))), + }, + AdhocPlanTestCase { + description: "a python folder selects the nodes under it", + command: &["build", "--select", "python/orders"], + expected_plan: Ok((&["asset:partner_orders_file"], json!({}))), + }, + AdhocPlanTestCase { + description: "a tag shared by models and Python nodes selects both", + command: &["build", "--select", "tag:orders"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "upstream of a task crosses its Python and SQL dependencies", + command: &["build", "--select", "+task:request_carrier_collection"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "downstream of a model reaches the Python nodes that read it", + command: &["build", "--select", "orders+", "--exclude", "stg_orders"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "upstream of a model runs its source's loader chain as the sources they write", + command: &["build", "--select", "+shipments"], + expected_plan: Ok(( + &[ + "model:shipments", + "source:raw_shipments", + "source:stage_shipments", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "an intermediate loader runs as the source it writes", + command: &["build", "--select", "loader:stage_shipments"], + expected_plan: Ok((&["source:stage_shipments"], json!({}))), + }, + AdhocPlanTestCase { + description: "test selects only the SQL nodes a tag shared with Python nodes matches", + command: &["test", "--select", "tag:orders"], + expected_plan: Ok((&["model:orders", "model:stg_orders"], json!({}))), + }, + AdhocPlanTestCase { + description: "test downstream of a model stops at the Python nodes that read it", + command: &["test", "--select", "orders+", "--exclude", "stg_orders"], + expected_plan: Ok((&["model:orders"], json!({}))), + }, + AdhocPlanTestCase { + description: "a build without Python nodes selects only the SQL nodes a shared tag matches", + command: &["build", "--no-python", "--select", "tag:orders"], + expected_plan: Ok(( + &["model:orders", "model:stg_orders"], + json!({"include_python": false}), + )), + }, + AdhocPlanTestCase { + description: "test of a job with only Python nodes has nothing to test", + command: &["test", "--select", "job:carrier_sync"], + expected_plan: Err("the selectors match no nodes in the current publication"), + }, + ]; + let planner = adhoc_planner(); + for test_case in test_cases { + let command: Vec = test_case + .command + .iter() + .map(|token| (*token).to_owned()) + .collect(); + let actual = planner + .plan(&command, (&resources, &jobs)) + .map(|plan| { + ( + plan.selection.into_iter().collect::>(), + json!(plan.settings), + ) + }) + .map_err(|error| { + test_case + .expected_plan + .as_ref() + .err() + .filter(|fragment| error.contains(**fragment)) + .map_or(error.clone(), |fragment| (*fragment).to_owned()) + }); + assert_eq!( + actual, + owned_plan(&test_case.expected_plan), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index 7a7d6be746..d94c0d4073 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -31,6 +31,14 @@ use crate::store::models::{ControlTransaction, Parameter}; /// Graph kinds in the order the CLI builds its project graph: models, sources, seeds, then functions. const GRAPH_KINDS: [&str; 5] = ["model", "source", "seed", "udf", "table_fn"]; +/// Python node kinds, in the order the CLI discovers them. +const PYTHON_NODE_KINDS: [&str; 4] = ["task", "asset", "loader", "check"]; +/// SQL unit tests, which `test:` selects; each lists the models it tests. +const SQL_TEST_KIND: &str = "sql_test"; +/// Where a SQL unit test's definition lists the models it tests. +const SQL_TEST_MODELS_POINTER: &str = "/test/payload/expected_model_names"; +/// Marks the source node an intermediate loader writes, which the CLI's SQL graph does not list. +const INTERMEDIATE_LOADER_FIELD: &str = "intermediate_loader"; /// The frozen request and its verb, or why the command cannot run; `run_id` comes from the caller. pub(crate) async fn frozen_adhoc_request( @@ -142,23 +150,26 @@ pub(crate) async fn publication_plan( )) } -/// The selector graph rebuilt from published definitions, in the CLI's kind order. +/// The selector graph rebuilt from published definitions, in the CLI's kind order; sources intermediate loaders write are selected through their loader. fn planner_resources( nodes: &std::collections::BTreeMap, ) -> Vec { let mut resources: Vec = Vec::new(); - for kind in GRAPH_KINDS { + for kind in GRAPH_KINDS.iter().chain(&PYTHON_NODE_KINDS) { for (key, node) in nodes { - if node.kind != kind { + let definition = &node.definition; + if node.kind != *kind || definition.get(INTERMEDIATE_LOADER_FIELD).is_some() { continue; } - let definition = &node.definition; - let (tags, folder) = match kind { + let (tags, folder) = match *kind { "model" => ( definition.pointer("/config/tags"), Some(model_folder(definition)), ), "seed" => (definition.pointer("/schema/tags"), None), + _ if PYTHON_NODE_KINDS.contains(kind) => { + (definition.get("tags"), Some(model_folder(definition))) + } _ => (definition.get("tags"), None), }; resources.push(PlannerResource { @@ -166,9 +177,25 @@ fn planner_resources( deps: texts(definition.get("deps")), tags: texts(tags), folder, + loader: definition + .pointer("/entry/loader") + .and_then(Value::as_str) + .map(str::to_owned), }); } } + resources.extend( + nodes + .iter() + .filter(|(_, node)| node.kind == SQL_TEST_KIND) + .map(|(key, node)| PlannerResource { + key: key.clone(), + deps: tested_models(&node.definition), + tags: Vec::new(), + folder: None, + loader: None, + }), + ); resources } @@ -183,6 +210,14 @@ fn model_folder(definition: &Value) -> String { .map_or_else(|| ".".to_owned(), |(folder, _)| folder.to_owned()) } +/// The model keys a SQL unit test tests. +fn tested_models(definition: &Value) -> Vec { + texts(definition.pointer(SQL_TEST_MODELS_POINTER)) + .into_iter() + .map(|model| format!("model:{model}")) + .collect() +} + fn texts(value: Option<&Value>) -> Vec { value .and_then(Value::as_array) diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 90423bd549..169212f1bb 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -763,13 +763,14 @@ pub struct AdhocCheckRequest { pub priority: i64, } -/// One graph resource selectors resolve over: its `kind:name` key, upstream keys, tags and model folder. +/// One SQL resource or Python node selectors resolve over: its `kind:name` key, upstream keys, tags, file folder and a source's loader. #[derive(Clone, Debug, PartialEq, Eq)] pub struct PlannerResource { pub key: String, pub deps: Vec, pub tags: Vec, pub folder: Option, + pub loader: Option, } /// A parsed command: its verb, the exact node keys it selects and the flags it sets, by parser destination. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs index f5122b631c..79b2b2e30c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs @@ -61,15 +61,25 @@ pub(crate) fn graph_with_jobs<'g>( selectors: &[String], also_known: &[&str], ) -> PyResult> { - if graph.jobs.is_some() || !selectors_need_jobs(graph, selectors, also_known) { - return Ok(Cow::Borrowed(graph)); - } - Ok(match job_catalog(py)? { - Some((catalog, _)) => Cow::Owned(attach_jobs(graph.clone(), catalog)), + Ok(match needed_jobs(py, graph, selectors, also_known)? { + Some(catalog) => Cow::Owned(attach_jobs(graph.clone(), catalog)), None => Cow::Borrowed(graph), }) } +/// The command's jobs when `selectors` name one and `graph` has none attached yet. +pub(crate) fn needed_jobs( + py: Python<'_>, + graph: &ProjectGraph, + selectors: &[String], + also_known: &[&str], +) -> PyResult>> { + if graph.jobs.is_some() || !selectors_need_jobs(graph, selectors, also_known) { + return Ok(None); + } + Ok(job_catalog(py)?.map(|(catalog, _)| catalog)) +} + /// The command's job and job group names, loaded on first use; empty outside a command. pub(crate) fn job_names(py: Python<'_>) -> PyResult> { Ok(job_catalog(py)? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index c761476387..194c416314 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -6,7 +6,6 @@ use std::path::Path; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use serde_json::Value; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; @@ -20,6 +19,7 @@ use sqlbuild_orchestration::declarations::models::{ }; use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::native_host::orchestration_declarations::{ AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, declared_at, job_resources, module_jobs, run_controls, @@ -60,7 +60,7 @@ pub(crate) fn discover_declarations<'py>( pub(crate) fn compile_declarations( py: Python<'_>, mut input: SnapshotInput, - (project_dir, graph): (&Path, &ProjectGraph), + (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), declarations: Vec, ) -> PyResult { if declarations.is_empty() { @@ -113,12 +113,9 @@ pub(crate) fn compile_declarations( job.name ))); } - let selection: BTreeSet = - resolve_selectors(graph, &tokens(&job.select), &tokens(&job.exclude)) - .map_err(|error| PyValueError::new_err(error.message))? - .into_iter() - .map(|(kind, name)| format!("{kind}:{name}")) - .collect(); + let selection: BTreeSet = selected + .run_keys(graph, &tokens(&job.select), &tokens(&job.exclude)) + .map_err(|error| PyValueError::new_err(format!("job {}: {error}", job.name)))?; let declared_at: Option = declared_at(project_dir, job.declared_at)?; let controls = run_controls( &job.name, @@ -235,7 +232,7 @@ pub(crate) fn compile_declarations( .. } = compile_sensors( py, - (project_dir, &input, graph), + (project_dir, &input, (graph, selected)), authored_sensors, compile_calendars( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index f20b858789..061052b577 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -324,12 +324,15 @@ impl WorkerBuild { self.complete(py, key, (outcome, None, (None, None))) } - /// A test or audit the snapshot records has started; the node it attaches to starts with it. + /// A test or audit the snapshot records has started; a selected SQL test reports as a node. pub(crate) fn start_attached(&self, py: Python<'_>, key: &str) -> PyResult<()> { self.check_cancelled(py)?; let Some(nodes) = self.attached.get(key) else { return Ok(()); }; + if self.execution.claim.request.selection.contains(key) { + return self.start_key(py, key); + } if self .checks .lock() @@ -377,7 +380,11 @@ impl WorkerBuild { } else { outcome }; - self.finish_check(py, key, outcome)?; + if self.execution.claim.request.selection.contains(key) { + self.complete(py, key, (outcome, None, (None, None)))?; + } else { + self.finish_check(py, key, outcome)?; + } if outcome == NodeOutcome::Skipped { return Ok(()); } @@ -413,27 +420,33 @@ impl WorkerBuild { Ok(None) } - /// A failing SQL test, found by its compiled name such as `parent [case]`, fails the nodes it blocks with its error; each test that reports releases a function waiting on its tests. + /// A selected SQL test completes itself; a failing one, found by its compiled name such as `parent [case]`, fails the nodes it blocks; each test that reports releases a function waiting on its tests. fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { let py = result.py(); let outcome: String = field(result, "outcome")?.str()?.extract()?; + let passed = SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass); + let cancelled = !passed && self.worker.bind(py).get().refresh_cancellation(py)?; let name: String = field(result, "test_name")?.extract()?; let key = format!("{SQL_TEST_KEY_PREFIX}{name}"); - if SqlTestOutcome::from_value(&outcome) != Some(SqlTestOutcome::Pass) - && !self.worker.bind(py).get().refresh_cancellation(py)? - { - let error: Option = match result.getattr_opt("error_code")? { - Some(value) => value.extract()?, - None => None, - }; - let message: Option = match result.getattr_opt("error_message")? { - Some(value) => value.extract()?, - None => None, - }; - for node in self.blocked.get(&key).into_iter().flatten() { + let error: Option = match result.getattr_opt("error_code")? { + Some(value) => value.extract()?, + None => None, + }; + let message: Option = match result.getattr_opt("error_message")? { + Some(value) => value.extract()?, + None => None, + }; + let node = match (passed, cancelled) { + (true, _) => NodeOutcome::Success, + (false, true) => NodeOutcome::Cancelled, + (false, false) => NodeOutcome::Failed, + }; + self.complete(py, &key, (node, None, (error.clone(), message.clone())))?; + if !passed && !cancelled { + for model in self.blocked.get(&key).into_iter().flatten() { self.complete( py, - node, + model, (NodeOutcome::Failed, None, (error.clone(), message.clone())), )?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs index 4867f861ab..26e7bc1266 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs @@ -1,35 +1,23 @@ //! Project roles from the reviewed `roles.toml` file, their selectors resolved against this snapshot. -use std::collections::BTreeSet; use std::path::Path; use pyo3::exceptions::PyValueError; use pyo3::prelude::{PyResult, Python}; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_orchestration::authorization::main::compile_roles::compile_roles; use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::native_host::host::write_console; /// The project-root file that defines project roles. pub(crate) const ROLES_FILE: &str = "roles.toml"; -/// Node keys of one grant selector, in the job selection grammar. -fn selector_nodes(graph: &ProjectGraph, selector: &str) -> Result, String> { - let tokens: Vec = selector.split_whitespace().map(str::to_owned).collect(); - let keys = resolve_selectors(graph, &tokens, &[]).map_err(|error| error.message)?; - let mut nodes: BTreeSet = BTreeSet::new(); - for (kind, name) in keys { - nodes.insert(format!("{kind}:{name}")); - } - Ok(nodes) -} - pub(crate) fn compile_project_roles( py: Python<'_>, mut input: SnapshotInput, - (project_dir, graph): (&Path, &ProjectGraph), + (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), ) -> PyResult { let path = project_dir.join(ROLES_FILE); if !path.is_file() { @@ -42,8 +30,11 @@ pub(crate) fn compile_project_roles( } let text = std::fs::read_to_string(&path) .map_err(|error| PyValueError::new_err(format!("roles.toml: {error}")))?; - let (document, warnings) = compile_roles(&text, |selector| selector_nodes(graph, selector)) - .map_err(PyValueError::new_err)?; + let (document, warnings) = compile_roles(&text, |selector| { + let tokens: Vec = selector.split_whitespace().map(str::to_owned).collect(); + selected.node_keys(graph, &tokens, &[]) + }) + .map_err(PyValueError::new_err)?; for warning in warnings { write_console(py, "stderr", &format!("Warning: {warning}\n"))?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 1dd9d85d85..f9d1b4e03f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -1,5 +1,8 @@ //! Python node and unified SQL/Python selector resolution, and the Python node graph indexes. +use std::borrow::Cow; +use std::collections::BTreeSet; + use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -7,9 +10,13 @@ use pyo3::types::{PyDict, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::errors::SelectorError; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_analysis::python_selection::main::attach_python_jobs::attach_python_jobs; use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; +use sqlbuild_analysis::python_selection::main::python_run_key::python_run_key; use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; +use sqlbuild_analysis::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection as run_selection; use sqlbuild_analysis::python_selection::models::{ InventoryIndexes, InventoryNode, PythonSelectionGraph, PythonSelectionNode, RunSelection, @@ -17,8 +24,9 @@ use sqlbuild_analysis::python_selection::models::{ }; use crate::bindings::_helpers::boundary::panics::compiler_guard; -use crate::bindings::_helpers::compiled_project::job_catalog::graph_with_jobs; +use crate::bindings::_helpers::compiled_project::job_catalog::needed_jobs; use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_items; @@ -128,6 +136,101 @@ fn project_sources(project: &Bound<'_, PyAny>) -> PyResult)>, + pub(crate) nodes: PythonSelectionGraph, +} + +impl ProjectSelection { + pub(crate) fn new( + project: &Bound<'_, PyAny>, + discovered_inputs: &Bound<'_, PyAny>, + ) -> PyResult { + Ok(Self { + sources: project_sources(project)?, + nodes: selection_graph(&python_node_graph(discovered_inputs)?)?, + }) + } + + /// The `kind:name` keys `sqb build --select` runs for these selectors. + pub(crate) fn run_keys( + &self, + graph: &ProjectGraph, + select: &[String], + exclude: &[String], + ) -> Result, String> { + self.keys(run_selection(self.sql(graph), &self.nodes, select, exclude)) + } + + /// The `kind:name` keys these selectors name, Python checks included. + pub(crate) fn node_keys( + &self, + graph: &ProjectGraph, + select: &[String], + exclude: &[String], + ) -> Result, String> { + self.keys(resolve_python_sql_nodes( + self.sql(graph), + &self.nodes, + select, + exclude, + )) + } + + /// The published key of a Python node, such as `task:export_orders`. + pub(crate) fn node_key(&self, name: &str) -> Option { + self.nodes + .nodes + .iter() + .rev() + .find(|node| node.name == name) + .map(|node| format!("{}:{name}", node.kind)) + } + + /// The key a selected Python node's run reports under; a loader reports as the source it writes. + fn run_key(&self, name: &str) -> Option { + let node = self + .nodes + .nodes + .iter() + .rev() + .find(|node| node.name == name)?; + Some(python_run_key(&self.sources, (&node.kind, name))) + } + + /// Whether a loader writes no declared source and so runs only as part of another loader's chain. + pub(crate) fn intermediate_loader(&self, name: &str) -> bool { + !self + .sources + .iter() + .any(|(_, loader)| loader.as_deref() == Some(name)) + } + + fn sql<'graph>(&'graph self, graph: &'graph ProjectGraph) -> SqlSelectionGraph<'graph> { + SqlSelectionGraph { + graph, + sources: &self.sources, + } + } + + fn keys( + &self, + selection: Result, + ) -> Result, String> { + let selection: RunSelection = selection.map_err(|error| error.message)?; + let mut keys: BTreeSet = selection + .sql_keys + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + for name in &selection.python_node_names { + keys.extend(self.run_key(name)); + } + Ok(keys) + } +} + /// The Python node names `select` minus `exclude` resolve to; every node when unselected. pub(crate) fn python_node_names<'py>( graph: &Bound<'py, PyAny>, @@ -183,7 +286,18 @@ pub(crate) fn resolve_python_sql_run_selection( let nodes: PythonSelectionGraph = selection_graph(python_graph)?; let node_names: Vec<&str> = nodes.nodes.iter().map(|node| node.name.as_str()).collect(); let selectors: Vec = select.iter().chain(&exclude).cloned().collect(); - let graph = graph_with_jobs(py, native.get().graph(), &selectors, &node_names)?; + let graph: Cow<'_, ProjectGraph> = + match needed_jobs(py, native.get().graph(), &selectors, &node_names)? { + Some(catalog) => Cow::Owned(attach_python_jobs( + SqlSelectionGraph { + graph: native.get().graph(), + sources: &sources, + }, + &nodes, + catalog, + )), + None => Cow::Borrowed(native.get().graph()), + }; let selection: RunSelection = compiler_guard(|| { Ok(run_selection( SqlSelectionGraph { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index b9e42eaa95..fb1c70fdfc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -33,3 +33,6 @@ pub(crate) const REACH_SETTING: &str = "reach"; pub(crate) const JOB_RUN_REACH: &str = "all"; /// What a job run with `changed: True` counts when `[changes] counts` is not set: every change. pub(crate) const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; + +/// A SQL unit test's published kind; a frozen run selects it back with `test:`. +pub(crate) const SQL_TEST_NODE_KIND: &str = "sql_test"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 830e43a841..40a0682a21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -1,6 +1,6 @@ //! Location publication assembles catalog data and planner-owned local identities. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; use pyo3::exceptions::PyValueError; @@ -24,6 +24,7 @@ use crate::bindings::_helpers::compiled_project::snapshot_roles::compile_project use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::discovery::project_facts::{named_node_rows, project_facts}; use crate::bindings::_helpers::discovery::project_validation::validation_error; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; use crate::bindings::_helpers::native_host::orchestration_declarations::{ @@ -36,6 +37,9 @@ use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; use crate::bindings::_helpers::planner_scope::values::project_graph; use crate::bindings::_helpers::records::fields::field; +/// The Python node kind whose functions load sources. +const LOADER_KIND: &str = "loader"; + /// Where declarations passed to the snapshot directly are reported as declared. const DECLARATIONS_ARGUMENT: &str = "declarations argument"; @@ -111,6 +115,9 @@ pub(crate) fn orchestration_snapshot<'py>( } input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; drop(owner); + let selected = + ProjectSelection::new((&project).into_pyobject(py)?.as_any(), &discovered_inputs)?; + input.nodes = python_nodes(&selected, input.nodes)?; if let Some(providers) = providers_document(&discovered_inputs, &input.nodes)? { input.documents.insert("providers".into(), providers); } @@ -137,11 +144,15 @@ pub(crate) fn orchestration_snapshot<'py>( input = compile_declarations( py, input, - (&project_dir, graph.bind(py).get().graph()), + (&project_dir, graph.bind(py).get().graph(), &selected), authored, )?; } - input = compile_project_roles(py, input, (&project_dir, graph.bind(py).get().graph()))?; + input = compile_project_roles( + py, + input, + (&project_dir, graph.bind(py).get().graph(), &selected), + )?; let project = project.into_pyobject(py)?; if let Some(lineage) = column_lineage_document(project.as_any())? { input.documents.insert("lineage".into(), lineage); @@ -151,6 +162,77 @@ pub(crate) fn orchestration_snapshot<'py>( Ok(NativeOrchestrationSnapshot { input }) } +/// Python nodes publish their `deps` and tags, and each intermediate loader publishes the source it writes, so the server orders, selects and grants them as the CLI does. +fn python_nodes( + selected: &ProjectSelection, + mut nodes: BTreeMap, +) -> PyResult> { + let graph = &selected.nodes; + let mut tags: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + for (tag, names) in &graph.tags { + for name in names { + tags.entry(name.as_str()).or_default().push(tag.as_str()); + } + } + let upstream: BTreeMap<&str, &[String]> = graph + .upstream + .iter() + .map(|(name, names)| (name.as_str(), names.as_slice())) + .collect(); + let mut outputs: Vec<(String, SnapshotNode)> = Vec::new(); + for node in &graph.nodes { + let key = format!("{}:{}", node.kind, node.name); + let Some(published) = nodes.get_mut(&key) else { + return Err(PyValueError::new_err(format!( + "Python node {key} is missing from the snapshot" + ))); + }; + let deps: BTreeSet = upstream + .get(node.name.as_str()) + .into_iter() + .flat_map(|names| names.iter()) + .filter_map(|name| selected.node_key(name)) + .chain( + node.sql_deps + .iter() + .map(|(kind, name)| format!("{kind}:{name}")), + ) + .collect(); + published.definition["deps"] = json!(deps); + published.definition["tags"] = + json!(tags.get(node.name.as_str()).cloned().unwrap_or_default()); + if node.kind == LOADER_KIND && selected.intermediate_loader(&node.name) { + let mut definition = json!({ + "name": node.name, + "entry": {"loader": node.name}, + "intermediate_loader": true, + "deps": [key], + }); + if let Some(path) = published.definition.get("path") { + definition["path"] = path.clone(); + } + outputs.push(( + format!("source:{}", node.name), + SnapshotNode { + kind: "source".into(), + definition, + raw_sql: None, + compiled_sql: None, + python_source: None, + }, + )); + } + } + for (key, node) in outputs { + if nodes.insert(key.clone(), node).is_some() { + return Err(PyValueError::new_err(format!( + "intermediate loader {key} collides with a published node" + ))); + } + } + Ok(nodes) +} + fn identity_nodes( project: &Bound<'_, PyAny>, mut nodes: BTreeMap, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index df3da90b96..0d4d8157c5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -8,6 +8,7 @@ use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyString; use serde_json::Value; +use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; use sqlbuild_cli::argv::main::command_settings_arguments::command_settings_arguments; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; @@ -35,6 +36,7 @@ use crate::bindings::_helpers::discovery::project_config::configured_changes; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::constants::{ CHANGED_SETTING, COMMAND_IMPLEMENTATIONS, JOB_RUN_COUNTS, JOB_RUN_REACH, REACH_SETTING, + SQL_TEST_NODE_KIND, }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::recall::console::write_stderr; @@ -60,7 +62,7 @@ fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap definition + SQL_TEST_NODE_KIND => definition .pointer("/test/payload/expected_model_names") .and_then(Value::as_array) .into_iter() @@ -186,10 +188,17 @@ fn command_request<'py>( if execution.skipped_nodes.contains_key(key) { continue; } - let (_, name) = key + let (kind, name) = key .split_once(':') .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; - argv.extend([PyString::new(py, "--select"), PyString::new(py, name)]); + let selector: String = if PYTHON_NODE_KINDS.contains(&kind) { + key.clone() + } else if kind == SQL_TEST_NODE_KIND { + format!("test:{name}") + } else { + name.to_owned() + }; + argv.extend([PyString::new(py, "--select"), PyString::new(py, &selector)]); } let parsed = parse_cli_invocation(py, argv, false)?; let args = field(&parsed, "args")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index d19d662071..d0ef09dcb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -4,8 +4,6 @@ pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; pub(crate) const RUN_EVENT_CONDITION: &str = "run_event"; -pub(crate) const NODE_EVENT_CONDITION: &str = "node_event"; -pub(crate) const CHECK_KIND: &str = "check"; pub(crate) const SOURCE_NEW_DATA_CONDITION: &str = "source_new_data"; pub(crate) const SOURCE_STALE_CONDITION: &str = "source_stale"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 8779ef8608..b8923950a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -1,6 +1,6 @@ //! Authored trigger controls and bounded calendars evaluated only in the location interpreter. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; use std::path::Path; use chrono::{DateTime, TimeDelta, Timelike, Utc}; @@ -10,21 +10,20 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use serde_json::{Value, json}; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; use sqlbuild_orchestration::sensors::models::SensorDefinition; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::native_host::constants::{ - CALENDAR_WINDOW_PARAMETER, CHECK_KIND, NODE_EVENT_CONDITION, SOURCE_NEW_DATA_CONDITION, - SOURCE_STALE_CONDITION, + CALENDAR_WINDOW_PARAMETER, SOURCE_NEW_DATA_CONDITION, SOURCE_STALE_CONDITION, }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_declarations::{ @@ -253,7 +252,7 @@ pub(crate) fn compile_calendars( pub(crate) fn compile_sensors( py: Python<'_>, - (project_dir, input, graph): (&Path, &SnapshotInput, &ProjectGraph), + (project_dir, input, graph): (&Path, &SnapshotInput, (&ProjectGraph, &ProjectSelection)), authored: Vec, mut definitions: Definitions, ) -> PyResult { @@ -281,12 +280,11 @@ pub(crate) fn compile_sensors( if let Some(select) = when.select { let selectors: Vec = select.split_whitespace().map(str::to_owned).collect(); - condition["selection"] = - sensor_nodes(graph, &input.nodes, &selectors, &when.kind)?; + condition["selection"] = sensor_nodes(graph, &selectors, &when.kind)?; } if let Some(selectors) = when.executed_successfully { condition["executed_successfully"] = - sensor_nodes(graph, &input.nodes, &selectors, "executed_successfully")?; + sensor_nodes(graph, &selectors, "executed_successfully")?; } condition } @@ -314,8 +312,7 @@ pub(crate) fn compile_sensors( } fn sensor_nodes( - graph: &ProjectGraph, - available: &BTreeMap, + (graph, selected): (&ProjectGraph, &ProjectSelection), selectors: &[String], field: &str, ) -> PyResult { @@ -324,31 +321,9 @@ fn sensor_nodes( "{field} needs nonempty selectors" ))); } - let mut nodes: BTreeSet = BTreeSet::new(); - let mut sql_selectors: Vec = Vec::new(); - for selector in selectors { - if field == NODE_EVENT_CONDITION && selector.starts_with("check:") { - if !available - .get(selector) - .is_some_and(|node| node.kind == CHECK_KIND) - { - return Err(PyValueError::new_err(format!( - "unknown check node {selector}" - ))); - } - nodes.insert(selector.clone()); - } else { - sql_selectors.push(selector.clone()); - } - } - if !sql_selectors.is_empty() { - nodes.extend( - resolve_selectors(graph, &sql_selectors, &[]) - .map_err(|error| PyValueError::new_err(error.message))? - .into_iter() - .map(|(kind, name)| format!("{kind}:{name}")), - ); - } + let nodes: BTreeSet = selected + .node_keys(graph, selectors, &[]) + .map_err(|error| PyValueError::new_err(format!("{field}: {error}")))?; if nodes.is_empty() { return Err(PyValueError::new_err(format!( "{field} needs nonempty selectors" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md index 4dc3e69973..fe54d4880a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md @@ -120,7 +120,11 @@ Checks do not support `columns`, `column_lineage`, or `retry`. ## Running checks -Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. To run checks on their own, use [`sqb check`](../../cli/check.md): +Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. + +A `sqb build` selector that matches a check must also select the nodes the check depends on. Otherwise the build fails with an error that names the check and the missing node; select that node too, or narrow the selector so it no longer matches the check. A check on a source that no loader writes never runs in a build, so a selector that matches it fails too; run that check with `sqb check`. Jobs and server runs resolve selectors the same way. + +To run checks on their own, use [`sqb check`](../../cli/check.md): ```bash # Run all checks diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index 17cdf8438d..c4c2db12e9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -143,7 +143,7 @@ def export_orders(ctx): - `ctx.skip(reason, mode=...)` skips the node. `mode` accepts `"soft"` (default, skips only this node) or `"hard"` (also blocks dependents), as a string or the `SkipMode` enum from `sqlbuild.tasks`/`sqlbuild.assets`. - Assets may pass `materialized=True`/`False` to record whether an artifact was produced. -Downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. +Within one run, downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. ## Result persistence @@ -157,7 +157,7 @@ Python nodes are selected like SQL resources, by bare name or typed selector: sqb build --select export_orders # bare name sqb build --select task:export_orders # typed sqb build --select asset:orders_export -sqb build --select check:check_orders +sqb build --select export_orders check:check_orders_exported # a check with the node it checks sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` @@ -177,6 +177,18 @@ dependency fails or is skipped. A dependency that is not part of the run (not se unmanaged source) is read as it currently exists, so `sqb build --select task:export_orders` runs the task against the relations already in the warehouse. +Python dependencies follow the same rule. A task or asset selected without its Python upstream +reads that upstream's latest successful result through `ctx.result_of`. If the upstream has never +succeeded for the target, the node fails with an error that names the upstream, for example: + +```text +Upstream task 'prepare_orders' has no successful result to read. Build it first with +'sqb build --select task:prepare_orders', or build both with 'sqb build --select +asset:export_orders'. +``` + +Jobs, sensors, roles and ad-hoc runs on the server resolve selectors the same way, so a job can +select one Python node and a job group orders it after the job that builds its upstream. + The commands differ in what they include by default: | Command | SQL | Loaders / tasks / assets | Checks | Audits | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md index cf563c84a9..09e66562eb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md @@ -82,7 +82,7 @@ sqb build --select source:raw__orders ### Job -Select the nodes a job or job group declared under `orchestration/` builds. Job names are globally +Select the SQL and Python nodes a job or job group declared under `orchestration/` builds. Job names are globally unique, so a bare name works; `job:` is the explicit form and accepts glob patterns: ```bash diff --git a/src/sqlbuild/executor/load/_helpers/execution.py b/src/sqlbuild/executor/load/_helpers/execution.py index 955a267d89..673e81e4af 100644 --- a/src/sqlbuild/executor/load/_helpers/execution.py +++ b/src/sqlbuild/executor/load/_helpers/execution.py @@ -3,8 +3,9 @@ from __future__ import annotations from collections.abc import Callable +from dataclasses import replace -from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry, SourceWriteStrategy from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.executor.load.models import LoadExecutionIndexes, LoadExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus @@ -191,3 +192,14 @@ def is_untargeted_self_managed_intermediate( and source_entry.write_strategy is None and loader_function.destination is None ) + + +def row_write_entry(source_entry: SourceEntry) -> SourceEntry: + """Return the entry rows are written with: an intermediate loader defaults to `table`.""" + + if ( + source_entry.meta.get("sqlbuild_loader_node") is True + and source_entry.write_strategy is None + ): + return replace(source_entry, write_strategy=SourceWriteStrategy.TABLE) + return source_entry diff --git a/src/sqlbuild/executor/load/_helpers/loader_invocation.py b/src/sqlbuild/executor/load/_helpers/loader_invocation.py index c07f0fd419..d287c8681c 100644 --- a/src/sqlbuild/executor/load/_helpers/loader_invocation.py +++ b/src/sqlbuild/executor/load/_helpers/loader_invocation.py @@ -18,6 +18,7 @@ from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.execution import ( is_untargeted_self_managed_intermediate, + row_write_entry, ) from sqlbuild.executor.load._helpers.relation_refs import ( build_loader_relation_refs, @@ -238,7 +239,7 @@ def interpret_loader_return( duration_ms=int((time.monotonic() - start) * 1000), lifecycle_events=statement_recorder.snapshot(), ) - if source_entry.write_strategy is None: + if row_write_entry(source_entry).write_strategy is None: raise ExecutorInputError( f"Source '{source_entry.name}' loader '{loader_function.name}' returned rows " "but source has no write_strategy" diff --git a/src/sqlbuild/executor/load/main/_execute.py b/src/sqlbuild/executor/load/main/_execute.py index f4ccaa0360..eef2c5c56b 100644 --- a/src/sqlbuild/executor/load/main/_execute.py +++ b/src/sqlbuild/executor/load/main/_execute.py @@ -26,7 +26,7 @@ infer_cursor_kind, load_staging_cursor_bounds, ) -from sqlbuild.executor.load._helpers.execution import load_resource_kind +from sqlbuild.executor.load._helpers.execution import load_resource_kind, row_write_entry from sqlbuild.executor.load._helpers.external import execute_external_source_load from sqlbuild.executor.load._helpers.loader_invocation import ( build_loader_context, @@ -176,6 +176,7 @@ def _execute_source_load( ) if early_result is not None: return early_result + source_entry = row_write_entry(source_entry) rows_loaded: int = write_loader_rows_to_staging( loader_return_value=raw_rows, source_entry=source_entry, diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py index 39097300b7..f64d2156b0 100644 --- a/src/sqlbuild/executor/load/models.py +++ b/src/sqlbuild/executor/load/models.py @@ -17,13 +17,12 @@ from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) -from sqlbuild.compiler.python_nodes.types import PythonNodeKind, SkipMode +from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.constants import LOADER_RELATION_QUALIFIER_SEPARATOR from sqlbuild.executor.load.types import LoadProgressCallback -from sqlbuild.executor.node_results.models import NodeResultEnvelope +from sqlbuild.executor.node_results.classes.dependency_result_reader import DependencyResultReader from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard -from sqlbuild.executor.python_nodes.constants import MISSING_DEFAULT from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container from sqlbuild.python_nodes.models import SqlResourceRef @@ -95,7 +94,7 @@ class LoaderResult: @dataclass(frozen=True) -class LoaderContext: +class LoaderContext(DependencyResultReader): """Runtime context passed to a source loader function.""" adapter: BaseAdapter @@ -164,44 +163,6 @@ def result( materialized=materialized, ) - def result_of( - self, - *, - node_function: Callable[..., object], - run_id: str | None = None, - default: object = MISSING_DEFAULT, - ) -> NodeResultEnvelope | object: - """Return the latest persisted upstream result by Python node function reference.""" - - if self.result_store is None: - if default is not MISSING_DEFAULT: - return default - raise ExecutorInputError("No Python node result store is available") - node_type, node_name = self._result_dependency_identity(node_function) - return self.result_store.result_of( - node_type=node_type, - node_name=node_name, - run_id=run_id, - default=default, - ) - - def results_of( - self, - *, - node_function: Callable[..., object], - limit: int, - ) -> tuple[NodeResultEnvelope, ...]: - """Return persisted successful upstream result history, newest first.""" - - if self.result_store is None: - return () - node_type, node_name = self._result_dependency_identity(node_function) - return self.result_store.results_of( - node_type=node_type, - node_name=node_name, - limit=limit, - ) - def qualify_name( self, *, @@ -251,24 +212,6 @@ def _record_resolved(self, relation_ref: LoaderRelationRef) -> None: SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=relation_ref.name) ) - def _result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: - task_definition: object = getattr(node_function, "__sqlbuild_task__", None) - if task_definition is not None: - return PythonNodeKind.TASK.value, self._definition_name(task_definition) - asset_definition: object = getattr(node_function, "__sqlbuild_asset__", None) - if asset_definition is not None: - return PythonNodeKind.ASSET.value, self._definition_name(asset_definition) - loader_definition: object = getattr(node_function, "__sqlbuild_loader__", None) - if loader_definition is not None: - return PythonNodeKind.LOADER.value, self._definition_name(loader_definition) - raise ExecutorInputError("Python node result dependency must be a task, asset, or loader") - - def _definition_name(self, definition: object) -> str: - name: object = getattr(definition, "name", None) - if isinstance(name, str): - return name - raise ExecutorInputError("Python node result dependency has no resolved name") - @dataclass(frozen=True) class LoadExecutionResult: diff --git a/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py b/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py new file mode 100644 index 0000000000..7e5a36393f --- /dev/null +++ b/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py @@ -0,0 +1,61 @@ +"""Reads of a Python node's upstream results, looked up by the upstream's decorated function.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.node_results.constants import RESULT_DEPENDENCY_ATTRIBUTES +from sqlbuild.executor.node_results.models import NodeResultEnvelope +from sqlbuild.executor.python_nodes.constants import MISSING_DEFAULT + + +class DependencyResultReader: + """Mixin for node contexts that read their upstream results from a `result_store`.""" + + result_store: Any | None + + def result_of( + self, + *, + node_function: Callable[..., object], + run_id: str | None = None, + default: object = MISSING_DEFAULT, + ) -> NodeResultEnvelope | object: + """Return the latest persisted upstream result by Python node function reference.""" + + if self.result_store is None: + if default is not MISSING_DEFAULT: + return default + raise ExecutorInputError("No Python node result store is available") + node_type, node_name = self.result_dependency_identity(node_function) + return self.result_store.result_of( + node_type=node_type, node_name=node_name, run_id=run_id, default=default + ) + + def results_of( + self, + *, + node_function: Callable[..., object], + limit: int, + ) -> tuple[NodeResultEnvelope, ...]: + """Return persisted successful upstream result history, newest first.""" + + if self.result_store is None: + return () + node_type, node_name = self.result_dependency_identity(node_function) + return self.result_store.results_of(node_type=node_type, node_name=node_name, limit=limit) + + def result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: + """Return the `(kind, name)` a dependency's results are stored under.""" + + for attribute, kind in RESULT_DEPENDENCY_ATTRIBUTES: + definition: object = getattr(node_function, attribute, None) + if definition is None: + continue + name: object = getattr(definition, "name", None) + if not isinstance(name, str): + raise ExecutorInputError("Python node result dependency has no resolved name") + return kind, name + raise ExecutorInputError("Python node result dependency must be a task, asset, or loader") diff --git a/src/sqlbuild/executor/node_results/constants.py b/src/sqlbuild/executor/node_results/constants.py index b2e7444998..3b2d792021 100644 --- a/src/sqlbuild/executor/node_results/constants.py +++ b/src/sqlbuild/executor/node_results/constants.py @@ -1,6 +1,7 @@ """Runtime node result storage constants.""" from sqlbuild._native import StateSqlValueType +from sqlbuild.compiler.python_nodes.types import PythonNodeKind NODE_RESULT_MATERIALIZED_TRUE_VALUE: str = "true" @@ -36,3 +37,9 @@ **{column: StateSqlValueType.STRING for column in NODE_RESULT_COLUMNS}, COLUMN_TIMESTAMP: StateSqlValueType.TEXT_TIMESTAMP, } + +RESULT_DEPENDENCY_ATTRIBUTES: tuple[tuple[str, str], ...] = ( + ("__sqlbuild_task__", PythonNodeKind.TASK.value), + ("__sqlbuild_asset__", PythonNodeKind.ASSET.value), + ("__sqlbuild_loader__", PythonNodeKind.LOADER.value), +) diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 43bd63753c..dafc8e882a 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -189,6 +189,18 @@ def _execute_ready_node_in_cost_scope( result_store=result_store, warnings=warnings, ) + missing: str | None = _missing_upstream_result( + node=node, node_kind=node_kind, context=context, upstream_results=upstream_results + ) + if missing is not None: + result = PythonNodeExecutionResult( + node_name=node.name, + kind=node_kind, + status=PythonNodeStatus.FAILED, + error_message=missing, + ) + _persist_python_node_result(result_store=result_store, result=result, run_id=run_id) + return result, None try: result: PythonNodeExecutionResult = _call_node_with_retry( node=node, @@ -344,6 +356,37 @@ def _call_node_with_retry( attempt += 1 +def _missing_upstream_result( + *, + node: ExecutablePythonNode, + node_kind: PythonNodeKind, + context: TaskContext | AssetContext, + upstream_results: tuple[PythonNodeExecutionResult, ...], +) -> str | None: + """Why the node cannot run: a Python upstream outside this run that has never succeeded.""" + + python_dependencies: tuple[Callable[..., object], ...] = tuple( + dependency for dependency in node.depends_on if not isinstance(dependency, SqlResourceRef) + ) + if len(upstream_results) == len(python_dependencies): + return None + in_run: frozenset[str] = frozenset(result.node_name for result in upstream_results) + for dependency in python_dependencies: + kind, name = context.result_dependency_identity(dependency) + if name in in_run or context.result_of(node_function=dependency, default=None) is not None: + continue + both: str = f"'sqb build --select +{node_kind.value}:{node.name}'" + if kind == PythonNodeKind.LOADER.value: + return ( + f"Upstream loader '{name}' has no successful result to read. Build it with {both}." + ) + return ( + f"Upstream {kind} '{name}' has no successful result to read. Build it first with " + f"'sqb build --select {kind}:{name}', or build both with {both}." + ) + return None + + def _build_context( *, node: ExecutablePythonNode, diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index e41a47874f..4d7ffae694 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -21,10 +21,9 @@ ) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.node_results.models import NodeResultEnvelope +from sqlbuild.executor.node_results.classes.dependency_result_reader import DependencyResultReader from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard from sqlbuild.executor.python_nodes.constants import ( - MISSING_DEFAULT, PYTHON_NODE_RELATION_QUALIFIER_SEPARATOR, ) from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder @@ -165,7 +164,7 @@ class IngressCallbacks: @dataclass(frozen=True, kw_only=True) -class BasePythonNodeContext: +class BasePythonNodeContext(DependencyResultReader): """Shared runtime helpers for framework-owned Python nodes.""" adapter: BaseAdapter @@ -254,62 +253,6 @@ def skip( metadata={} if metadata is None else metadata, ) - def result_of( - self, - *, - node_function: Callable[..., object], - run_id: str | None = None, - default: object = MISSING_DEFAULT, - ) -> NodeResultEnvelope | object: - """Return the latest persisted upstream result by Python node function reference.""" - - if self.result_store is None: - if default is not MISSING_DEFAULT: - return default - raise ExecutorInputError("No Python node result store is available") - node_type, node_name = self._result_dependency_identity(node_function) - return self.result_store.result_of( - node_type=node_type, - node_name=node_name, - run_id=run_id, - default=default, - ) - - def results_of( - self, - *, - node_function: Callable[..., object], - limit: int, - ) -> tuple[NodeResultEnvelope, ...]: - """Return persisted successful upstream result history, newest first.""" - - if self.result_store is None: - return () - node_type, node_name = self._result_dependency_identity(node_function) - return self.result_store.results_of( - node_type=node_type, - node_name=node_name, - limit=limit, - ) - - def _result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: - task_definition: object = getattr(node_function, "__sqlbuild_task__", None) - if task_definition is not None: - return PythonNodeKind.TASK.value, self._definition_name(task_definition) - asset_definition: object = getattr(node_function, "__sqlbuild_asset__", None) - if asset_definition is not None: - return PythonNodeKind.ASSET.value, self._definition_name(asset_definition) - loader_definition: object = getattr(node_function, "__sqlbuild_loader__", None) - if loader_definition is not None: - return PythonNodeKind.LOADER.value, self._definition_name(loader_definition) - raise ExecutorInputError("Python node result dependency must be a task, asset, or loader") - - def _definition_name(self, definition: object) -> str: - name: object = getattr(definition, "name", None) - if isinstance(name, str): - return name - raise ExecutorInputError("Python node result dependency must have a string name") - @dataclass(frozen=True, kw_only=True) class TaskContext(BasePythonNodeContext): diff --git a/src/sqlbuild/provider/_helpers/injection.py b/src/sqlbuild/provider/_helpers/injection.py index e7680a16ef..4718700410 100644 --- a/src/sqlbuild/provider/_helpers/injection.py +++ b/src/sqlbuild/provider/_helpers/injection.py @@ -73,7 +73,12 @@ def call_with_provider_injection( if not context_bound and parameter.default is inspect.Parameter.empty: kwargs[parameter.name] = context context_bound = True - return function(**kwargs) + positional: list[object] = [] + for parameter in signature.parameters.values(): + if parameter.kind != inspect.Parameter.POSITIONAL_ONLY or parameter.name not in kwargs: + break + positional.append(kwargs.pop(parameter.name)) + return function(*positional, **kwargs) def _provider_for_parameter( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 4cb4545bfb..56620025a9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1622,3 +1622,13 @@ class ChangedTwoUpE2ETestCase: warehouse: ChangedWarehouse expected_crm_run: frozenset[str] expected_downstream: frozenset[str] + + +@dataclass(frozen=True) +class PythonUpstreamResultE2ETestCase: + """A Python node selected without its Python upstream, before and after the upstream builds.""" + + description: str + select: str + upstream_select: str + expected_error: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 774bce1499..712fe48b83 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -904,6 +904,18 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: 'MODEL (description "Test model order_summary.", ' 'materialized table);\nSELECT count(*) AS order_count FROM __ref("orders")\n' ), + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py" + ).read_text(), + "python/tasks/reports.py": ( + "from python.tasks.counts import count_orders\nfrom sqlbuild.tasks import task\n\n\n" + "@task(depends_on=count_orders)\n" + "def report_order_count(ctx):\n" + " '''Test task report_order_count.'''\n" + " upstream = ctx.result_of(node_function=count_orders)\n" + ' ctx.execute_sql(f"CREATE OR REPLACE TABLE main.report_order_count_result AS "\n' + " f\"SELECT '{upstream.status}' AS status\")\n" + ), } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py index bfc5361cf4..5e38ffcc8e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py @@ -18,3 +18,4 @@ class RunE2ETestCase: expected_query_results: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] = field( default_factory=tuple ) + select: tuple[str, ...] = field(default_factory=tuple) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index 3100cd5be9..3af739ac24 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -2185,15 +2185,41 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou "test_case", [ RunE2ETestCase( - description="run rejects selected check node", + description="build rejects a selected check whose task is not selected", expected_exit_code=1, expected_table_names=(), expected_view_names=(), - ) + expected_output_fragments=( + "Python check 'check_orders_export' checks task:prepare_orders, which the " + "selection does not include; select it too, or narrow the selector so it no " + "longer matches the check.", + ), + select=("check:check_orders_export",), + ), + RunE2ETestCase( + description="build runs a selected check with its selected task", + expected_exit_code=0, + expected_table_names=(), + expected_view_names=(), + expected_output_fragments=("check check_orders_export START", "PASS=2"), + select=("check:check_orders_export", "task:prepare_orders"), + ), + RunE2ETestCase( + description="build rejects a selected check on a source no loader writes", + expected_exit_code=1, + expected_table_names=(), + expected_view_names=(), + expected_output_fragments=( + "Python check 'check_raw_events' checks source:raw_events, which a build does " + "not load, so the check would not run; run it with sqb check, or narrow the " + "selector so it no longer matches the check.", + ), + select=("check:check_raw_events", "source:raw_events"), + ), ], ids=lambda case: case.description, ) -def test_given_check_selector_when_running_run_then_command_rejects_check( +def test_given_check_selector_when_running_build_then_check_needs_its_checked_node( test_case: RunE2ETestCase, tmp_path: Path, ) -> None: @@ -2220,6 +2246,16 @@ def test_given_check_selector_when_running_run_then_command_rejects_check( "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n return ctx.result()\n" ), + "python/checks/events.py": ( + "from sqlbuild.checks import check\n" + "from sqlbuild.refs import source\n\n" + "@check(depends_on=source('raw_events'))\n" + "def check_raw_events(ctx):\n" + " '''Test check check_raw_events.'''\n return ctx.pass_()\n" + ), + "sources/raw.yml": ( + "sources:\n - name: raw_events\n description: Test source raw_events.\n" + ), }, ) @@ -2230,14 +2266,12 @@ def test_given_check_selector_when_running_run_then_command_rejects_check( "--no-tests", "--no-audits", "--select", - "check:check_orders_export", + *test_case.select, ), project_dir=project_dir, ) - assert result.returncode == test_case.expected_exit_code combined_output: str = result.stdout + result.stderr - assert ( - "Python checks are not selectable here: " - "check_orders_export. Use sqb check instead." in combined_output - ) + assert result.returncode == test_case.expected_exit_code, combined_output + for fragment in test_case.expected_output_fragments: + assert fragment in combined_output diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py index c792b221af..80803f98f7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py @@ -10,6 +10,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( PythonNodeSelectionE2ETestCase, + PythonUpstreamResultE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( PYTHON_NODE_SELECTION_DATABASE, @@ -103,6 +104,30 @@ expected_row_counts={"orders": 2}, expected_node_counts={"orders_extract": 2}, ), + PythonNodeSelectionE2ETestCase( + description="a job selecting only Python nodes builds them", + select=("job:customer_counts",), + expected_exit_code=0, + expected_output_patterns=(r"count_customers\s+OK", r"customers_extract\s+OK"), + expected_row_counts={"raw_customers": 3}, + expected_node_counts={"count_customers": 3, "customers_extract": 3}, + ), + PythonNodeSelectionE2ETestCase( + description="a job named without job: builds its Python nodes", + select=("customer_counts",), + expected_exit_code=0, + expected_output_patterns=(r"count_customers\s+OK", r"customers_extract\s+OK"), + expected_row_counts={"raw_customers": 3}, + expected_node_counts={"count_customers": 3, "customers_extract": 3}, + ), + PythonNodeSelectionE2ETestCase( + description="a job selecting a task with its upstream model builds both", + select=("job:order_counts",), + expected_exit_code=0, + expected_output_patterns=(r"count_orders\s+OK",), + expected_row_counts={"orders": 2}, + expected_node_counts={"count_orders": 2}, + ), PythonNodeSelectionE2ETestCase( description="task is skipped when its selected upstream model fails", select=("+task:count_orders",), @@ -174,3 +199,51 @@ def test_given_python_node_selection_when_building_then_it_gates_only_on_schedul table_exists(db_path=database, table_name=table) for table in test_case.expected_missing_tables ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonUpstreamResultE2ETestCase( + description="a task reads its unselected upstream task's latest result", + select="task:report_order_count", + upstream_select="task:count_orders", + expected_error=( + "Upstream task 'count_orders' has no successful result to read. Build it first " + "with 'sqb build --select task:count_orders', or build both with " + "'sqb build --select +task:report_order_count'." + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_task_selected_without_its_python_upstream_when_building_then_it_reads_the_persisted_result( + test_case: PythonUpstreamResultE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="order_nodes", + repo_files=python_node_selection_project_files( + orders_sql=PythonNodeSelectionE2ETestCase.orders_sql + ), + ) + database: Path = project_dir / PYTHON_NODE_SELECTION_DATABASE + execute_duckdb(db_path=database, sql=_EXISTING_ROWS + _EXISTING_ORDERS) + report: tuple[str, ...] = ("--no-color", "build", "--select", test_case.select) + + never_built: subprocess.CompletedProcess[str] = run_sqb(command=report, project_dir=project_dir) + upstream: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", test_case.upstream_select), + project_dir=project_dir, + ) + rebuilt: subprocess.CompletedProcess[str] = run_sqb(command=report, project_dir=project_dir) + + output: str = never_built.stdout + never_built.stderr + assert never_built.returncode == 1, test_case.description + output + assert test_case.expected_error in output, test_case.description + assert upstream.returncode == 0, upstream.stdout + upstream.stderr + assert rebuilt.returncode == 0, rebuilt.stdout + rebuilt.stderr + node: str = test_case.select.split(":", 1)[1] + assert query_duckdb(db_path=database, sql=f"SELECT status FROM main.{node}_result") == [ + ("success",) + ] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py index 79bad71942..7a71fd98f2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py @@ -127,3 +127,11 @@ class LoaderRelationGuardE2ETestCase: enforce_explicit: bool expected_warnings: tuple[str, ...] expected_final_line_prefix: str + + +@dataclass(frozen=True) +class IntermediateLoaderDefaultE2ETestCase: + description: str + command: tuple[str, ...] + expected_intermediate_rows: tuple[tuple[object, ...], ...] + expected_terminal_rows: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py index 40ffb4dc45..483e1c7853 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py @@ -10,6 +10,7 @@ ChainedLoaderFailureE2ETestCase, ChainedLoaderPruningE2ETestCase, ChainedLoaderSelectionE2ETestCase, + IntermediateLoaderDefaultE2ETestCase, IntermediateLoaderStrategyE2ETestCase, LoaderWaffleShopE2ETestCase, SourceLoaderErrorE2ETestCase, @@ -1191,6 +1192,65 @@ def test_given_chained_loader_project_when_intermediate_uses_strategy_then_appli assert tuple(terminal_rows) == test_case.expected_terminal_rows +@pytest.mark.parametrize( + "test_case", + [ + IntermediateLoaderDefaultE2ETestCase( + description="sqb load writes the docs' intermediate loader as a table", + command=("--no-color", "load", "--select", "+raw_account_metrics"), + expected_intermediate_rows=((1, "Pawnee Parks"), (2, "Eagleton")), + expected_terminal_rows=((1, "active"), (2, "active")), + ), + IntermediateLoaderDefaultE2ETestCase( + description="sqb build writes the docs' intermediate loader as a table", + command=("--no-color", "build", "--select", "+source:raw_account_metrics"), + expected_intermediate_rows=((1, "Pawnee Parks"), (2, "Eagleton")), + expected_terminal_rows=((1, "active"), (2, "active")), + ), + ], + ids=lambda case: case.description, +) +def test_given_intermediate_loader_without_write_strategy_when_loading_twice_then_rows_replace_a_table( + tmp_path: Path, + test_case: IntermediateLoaderDefaultE2ETestCase, +) -> None: + edge_cases: Path = Path("benchmarks/duckshop/edge_cases/loaders") + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="source_loader_schema_behavior", + repo_files=build_schema_behavior_project_files( + source_yaml=(edge_cases / "intermediate_default_accounts.yml").read_text(), + loader_py=(edge_cases / "intermediate_default_accounts.py").read_text(), + ), + ) + db_path: Path = project_dir / "source_loader_schema_behavior.duckdb" + + results: list[subprocess.CompletedProcess[str]] = [ + run_sqb(command=test_case.command, project_dir=project_dir) for _ in range(2) + ] + + for result in results: + assert result.returncode == 0, test_case.description + result.stdout + result.stderr + assert ( + tuple( + query_duckdb( + db_path=db_path, + sql="SELECT account_id, account_name FROM __loader__raw_accounts ORDER BY 1", + ) + ) + == test_case.expected_intermediate_rows + ) + assert ( + tuple( + query_duckdb( + db_path=db_path, + sql="SELECT account_id, metric FROM raw_account_metrics ORDER BY 1", + ) + ) + == test_case.expected_terminal_rows + ) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index f9fc57f343..fa3f650f96 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -262,6 +262,41 @@ class SnapshotProvidersTestCase: expected_providers: dict[str, object] | None +@dataclass(frozen=True) +class PythonJobSelectionTestCase: + description: str + select: str + expected_selection: tuple[str, ...] + + +@dataclass(frozen=True) +class PythonJobGroupTestCase: + description: str + jobs: tuple[tuple[str, str], ...] + expected_groups: dict[str, dict[str, list[str]]] + + +@dataclass(frozen=True) +class PythonJobGroupCycleTestCase: + description: str + jobs: tuple[tuple[str, str], ...] + expected_message: str + + +@dataclass(frozen=True) +class PythonCheckJobTestCase: + description: str + select: str + expected_selection: list[str] + + +@dataclass(frozen=True) +class PythonCheckJobRejectionTestCase: + description: str + select: str + expected_message: str + + @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 22a57cf538..d114213a62 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -26,6 +26,7 @@ LocalConfig, MeasurementThresholdBound, MeasurementThresholds, + NativeOrchestrationSnapshot, ProjectConfig, SourceColumnEntry, SqlLexicalSyntax, @@ -44,6 +45,8 @@ CompiledModelView, CompiledProjectView, ) +from sqlbuild.orchestration import compile_snapshot +from sqlbuild.orchestration_declarations.models.authoring import Job, JobGroup from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files if TYPE_CHECKING: @@ -66,6 +69,26 @@ "documents": {}, } +PYTHON_PIPELINE_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', tags [orders]);\n\nSELECT 1 AS order_id\n" + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', tags [orders]);\n\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + "python/orders/partner_export/node.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py" + ).read_text(), + "python/fulfillment/carrier_sync/node.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py" + ).read_text(), + "python/checks/orders_present.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py" + ).read_text(), +} + TYPE_PROOF_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRCONTRACT105"]\n\n' @@ -728,3 +751,23 @@ def published_texts(test_case: SnapshotExpressionRangeTestCase, tmp_path: Path) column: compiled[slice(*metadata["expression_range"])].decode() for column, metadata in columns.items() } + + +def python_pipeline_snapshot( + tmp_path: Path, declarations: list[Job | JobGroup] +) -> NativeOrchestrationSnapshot: + """Publish the Python pipeline project with the given job declarations.""" + write_files(project_dir=tmp_path, files=PYTHON_PIPELINE_FILES) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + return compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=declarations, + discovered_inputs=discovered, + ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index a217b17815..de0a584346 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -132,6 +132,12 @@ class WorkerExecutionTestCase: ) +@dataclass(frozen=True) +class ClaimedLoadTestCase(WorkerExecutionTestCase): + description: str + expected_write_set: frozenset[str] = frozenset() + + @dataclass(frozen=True) class UnclaimedLoadTestCase: description: str @@ -372,6 +378,13 @@ class ProjectRolesTestCase: expected_denial: str +@dataclass(frozen=True) +class ProjectRoleRunTestCase: + description: str + role: str + expected_response: tuple[int, Any] + + @dataclass(frozen=True, kw_only=True) class ChangedJobTestCase(WorkerExecutionTestCase): description: str @@ -396,6 +409,8 @@ class AdhocVerbTestCase: # Files the worker's checkout gains after it verified the final run, which must not run. rewrites: dict[str, str] = field(default_factory=dict) expected_status: str = "success" + # Files that replace the published project's own before publication. + files: dict[str, str] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 2b37e4fa84..d29d97e19f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -1606,6 +1606,22 @@ def hold_write_lock(database: Path) -> duckdb.DuckDBPyConnection: } +INTERMEDIATE_LOADER_FILES: dict[str, str] = { + "sources/shipments.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml" + ).read_text(), + "python/loaders/shipments.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py" + ).read_text(), + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql" + ).read_text(), + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" + ).read_text(), +} + + def without_loads(control: sqlite3.Connection, node_key: str) -> None: """Drop a published node's `loads` in place; its content address and the worker's check stay.""" for node_hash, body in control.execute( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index 512b7a78dd..8357d5d100 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -265,6 +265,22 @@ def person_request( return json.load(response) +def request_person_run( + api: SnapshotApi, session: tuple[str, str], body: dict[str, Any] +) -> tuple[int, Any]: + """A person's run request: the response on success, else the denial without its decision.""" + request: Request = Request( + api.url + "/api/control/runs", + data=json.dumps(body).encode(), + headers=person_headers(session), + ) + try: + with urlopen(request, timeout=5) as response: + return 200, json.load(response) + except HTTPError as denied: + return denied.code, json.load(denied)["error"].split("; decision ")[0] + + def person_login(api: SnapshotApi, name: str) -> tuple[str, str]: request: Request = Request( api.url + "/api/auth/password/login", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py index cd3ce745b0..b7247fd0a6 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py @@ -260,6 +260,34 @@ def test_given_an_ad_hoc_command_when_checked_and_launched_then_the_worker_runs_ ], rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, ), + AdhocVerbTestCase( + description="test: runs only the selected verified SQL test", + prerequisites=[], + command=["test", "--select", "test:customers_have_ids"], + expected_selection=["sql_test:customers_have_ids"], + expected_run_nodes=[("sql_test:customers_have_ids", "node", "success")], + rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, + ), + AdhocVerbTestCase( + description="build test: runs and reports only the selected verified SQL test", + prerequisites=[], + command=["build", "--select", "test:customers_have_ids"], + expected_selection=["sql_test:customers_have_ids"], + expected_run_nodes=[("sql_test:customers_have_ids", "node", "success")], + rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, + ), + AdhocVerbTestCase( + description="test of a model and its failing SQL test fails both", + prerequisites=[], + command=["test", "--select", "customers", "test:customers_have_ids"], + expected_selection=["model:customers", "sql_test:customers_have_ids"], + expected_run_nodes=[ + ("model:customers", "node", "failed"), + ("sql_test:customers_have_ids", "node", "failed"), + ], + files={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, + expected_status="failed", + ), AdhocVerbTestCase( description="freshness observes the verified sources upstream of the selection", prerequisites=[], @@ -295,7 +323,8 @@ def test_given_an_ad_hoc_verb_when_the_checkout_changes_after_verification_then_ "seeds/seeds.yml": SEEDS_YML.format(region_type="VARCHAR"), "sources/raw.yml": SOURCES_YML.format(table="raw_orders"), "python/rewrite_checkout.py": REWRITE_CHECKOUT, - }, + } + | test_case.files, ) with scheduled_worker( tmp_path, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py index fc6221eb23..0215796c88 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py @@ -3,13 +3,16 @@ import subprocess from pathlib import Path from typing import Any +from uuid import uuid4 import pytest from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ProjectRoleRunTestCase, ProjectRolesTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + INTERMEDIATE_LOADER_FILES, publish_generation, read_person_sql, snapshot_api, @@ -19,6 +22,7 @@ person_login, person_request, provision_person, + request_person_run, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -100,3 +104,72 @@ def test_given_project_roles_when_published_then_person_sql_follows_selector_cov assert rewritten.returncode == 0, rewritten.stderr assert read_person_sql(api, session, "model:customers") == (200, "model:customers") assert read_person_sql(api, session, "model:orders")[0] == 403 + + +@pytest.mark.parametrize( + "test_case", + [ + ProjectRoleRunTestCase( + description="a role granting the loader chain runs the job", + role="shipments_runner", + expected_response=(200, {"applied": True}), + ), + ProjectRoleRunTestCase( + description="a role missing the intermediate loader is refused", + role="orders_runner", + expected_response=(403, "missing job.run permission for this target"), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_job_loading_an_intermediate_loader_when_a_role_runs_it_then_the_grant_must_cover_it( + test_case: ProjectRoleRunTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + "roles.toml": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml" + ).read_text(), + } + | INTERMEDIATE_LOADER_FILES, + ) + with snapshot_api( + tmp_path / "control", + server_arguments=("--password-origin", "https://control.example.com"), + ) as api: + published: subprocess.CompletedProcess[str] = publish_generation(project, api, 1) + assert published.returncode == 0, published.stderr + admin: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-runner" + ) + assert person_request( + api, + "/api/admin/roles", + body={ + "principal_id": person["principal_id"], + "role": test_case.role, + "location_id": api.location_id, + "target": "development", + }, + session=person_login(api, admin["name"]), + ) == {"applied": True} + request: dict[str, Any] = { + "run_id": str(uuid4()), + "location_id": api.location_id, + "target": "development", + "job_key": "annual_orders", + "reason": "Refresh shipments", + "priority": 0, + } + session: tuple[str, str] = person_login(api, person["name"]) + assert request_person_run(api, session, request) == test_case.expected_response diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index fd3cb32046..b1c6be9526 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -30,6 +30,7 @@ ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( CheckExecutionTestCase, + ClaimedLoadTestCase, ClaimedWorker, GroupExecutionTestCase, SnapshotWorkerTestCase, @@ -37,6 +38,7 @@ WorkerExecutionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + INTERMEDIATE_LOADER_FILES, LOADED_ORDER_FILES, cancel_running_statement, claimed_worker, @@ -924,26 +926,55 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re @pytest.mark.parametrize( "test_case", [ - WorkerExecutionTestCase( + ClaimedLoadTestCase( "a tag-selected job claims its loader chain and hook-read source, not unread ones", "", "SELECT amount FROM orders", "success", [(7,)], files=LOADED_ORDER_FILES, - ) + expected_write_set=frozenset( + {"model:orders", "source:raw_orders", "source:raw_customers", "source:raw_returns"} + ), + ), + ClaimedLoadTestCase( + "a job claims an intermediate loader no source declares", + "", + "SELECT amount FROM orders", + "success", + [(9,)], + files=INTERMEDIATE_LOADER_FILES, + expected_write_set=frozenset( + {"model:orders", "source:raw_shipments", "source:stage_shipments"} + ), + ), + ClaimedLoadTestCase( + "a job reaching a managed source through + runs the source's loader", + "", + "SELECT amount FROM orders", + "success", + [(9,)], + files=INTERMEDIATE_LOADER_FILES + | { + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py" + ).read_text() + }, + expected_node_count=3, + expected_write_set=frozenset( + {"model:orders", "source:raw_shipments", "source:stage_shipments"} + ), + ), ], ids=lambda case: case.description, ) def test_given_loads_when_a_worker_executes_then_the_run_claims_exactly_what_it_loads( - test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + test_case: ClaimedLoadTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: run: dict[str, Any] = worker_result(worker, test_case) assert run["status"] == test_case.expected_status, worker.log.read_text() - assert run_write_set(worker.control_store, run["run_id"]) == frozenset( - {"model:orders", "source:raw_orders", "source:raw_customers", "source:raw_returns"} - ) + assert run_write_set(worker.control_store, run["run_id"]) == test_case.expected_write_set @pytest.mark.parametrize( @@ -958,27 +989,6 @@ def test_given_loads_when_a_worker_executes_then_the_run_claims_exactly_what_it_ {"model:orders", "source:raw_orders", "source:raw_customers"} ), ), - UnclaimedLoadTestCase( - # A known gap: no snapshot node exists for an intermediate loader, so `loads` leaves it out. - "an intermediate loader no source declares is never claimed, so the run still fails", - files={ - "sources/shipments.yml": Path( - "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml" - ).read_text(), - "python/loaders/shipments.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py" - ).read_text(), - "models/orders.sql": Path( - "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql" - ).read_text(), - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" - ).read_text(), - }, - store_edit=lambda _: None, - refused_writer="source:stage_shipments", - expected_write_set=frozenset({"model:orders", "source:raw_shipments"}), - ), ], ids=lambda case: case.description, ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 770f4cbd4d..179999d185 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -32,8 +32,13 @@ schedule, sensor, ) -from sqlbuild.orchestration_declarations.models.authoring import Job +from sqlbuild.orchestration_declarations.models.authoring import Job, JobGroup from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + PythonCheckJobRejectionTestCase, + PythonCheckJobTestCase, + PythonJobGroupCycleTestCase, + PythonJobGroupTestCase, + PythonJobSelectionTestCase, SnapshotCatalogTestCase, SnapshotDeclarationRejectionTestCase, SnapshotDiscoveryRejectionTestCase, @@ -50,8 +55,10 @@ FUNCTION_PROJECT_FILES, METADATA, PARITY_FILES, + PYTHON_PIPELINE_FILES, TYPE_PROOF_FILES, published_texts, + python_pipeline_snapshot, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -1787,7 +1794,7 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects .replace('("orders",)', '("unknown_*",)') }, expected_error=ValueError, - expected_message="unknown selector pattern", + expected_message="executed_successfully: unknown selector name 'unknown_\\*'", ), SnapshotDiscoveryRejectionTestCase( description="invalid execution selector syntax fails closed", @@ -1846,3 +1853,196 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho project_dir=tmp_path, discovered_inputs=discovered, ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonJobSelectionTestCase( + description="a model selector selects models", + select="+orders", + expected_selection=("model:orders", "model:stg_orders"), + ), + PythonJobSelectionTestCase( + description="a task selector selects only the task", + select="task:request_carrier_collection", + expected_selection=("task:request_carrier_collection",), + ), + PythonJobSelectionTestCase( + description="typed selectors select the task and the asset it depends on", + select="task:request_carrier_collection asset:partner_orders_file", + expected_selection=("asset:partner_orders_file", "task:request_carrier_collection"), + ), + PythonJobSelectionTestCase( + description="upstream of a task crosses its Python and SQL dependencies", + select="+task:request_carrier_collection", + expected_selection=( + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ), + ), + PythonJobSelectionTestCase( + description="a python folder selects every node under it", + select="python/orders", + expected_selection=("asset:partner_orders_file",), + ), + PythonJobSelectionTestCase( + description="a tag shared by models and Python nodes selects both", + select="tag:orders", + expected_selection=( + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_python_selectors_when_publishing_jobs_then_selections_match_the_cli( + tmp_path: Path, test_case: PythonJobSelectionTestCase +) -> None: + write_files(project_dir=tmp_path, files=PYTHON_PIPELINE_FILES) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + + snapshot: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=[job("orders_job", description="Orders.", select=test_case.select)], + discovered_inputs=discovered, + ).to_json() + ) + + assert snapshot["documents"]["jobs"]["orders_job"]["selection"] == list( + test_case.expected_selection + ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonJobGroupTestCase( + description="a job waits for the job building the Python node its task depends on", + jobs=( + ("orders_export_job", "python/orders/partner_export"), + ("carrier_sync_job", "python/fulfillment/carrier_sync"), + ), + expected_groups={ + "orders_nightly": { + "carrier_sync_job": ["orders_export_job"], + "orders_export_job": [], + } + }, + ), + PythonJobGroupTestCase( + description="a Python node's SQL dependency orders it after the model's job", + jobs=(("orders_export_job", "asset:partner_orders_file"), ("models_job", "+orders")), + expected_groups={ + "orders_nightly": {"models_job": [], "orders_export_job": ["models_job"]} + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_dependencies_across_jobs_when_publishing_a_group_then_jobs_wait_for_them( + tmp_path: Path, test_case: PythonJobGroupTestCase +) -> None: + jobs: list[Job] = [ + job(name, description="Orders.", select=select) for name, select in test_case.jobs + ] + group: JobGroup = job_group("orders_nightly", description="Nightly orders.", jobs=jobs) + + snapshot: dict[str, Any] = json.loads( + python_pipeline_snapshot(tmp_path, [*jobs, group]).to_json() + ) + + assert { + name: definition["jobs"] for name, definition in snapshot["documents"]["groups"].items() + } == test_case.expected_groups + + +@pytest.mark.parametrize( + "test_case", + [ + PythonJobGroupCycleTestCase( + description="a Python edge that closes a cycle names both jobs and the edges", + jobs=( + ("staging_job", "stg_orders task:request_carrier_collection"), + ("export_job", "asset:partner_orders_file"), + ), + expected_message=( + "group dependencies contain a cycle in job group orders_nightly: " + "export_job waits for staging_job (model:orders reads model:stg_orders); " + "staging_job waits for export_job " + "(task:request_carrier_collection reads asset:partner_orders_file)" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_python_dependency_cycle_across_jobs_when_publishing_a_group_then_it_is_rejected( + tmp_path: Path, test_case: PythonJobGroupCycleTestCase +) -> None: + jobs: list[Job] = [ + job(name, description="Orders.", select=select) for name, select in test_case.jobs + ] + group: JobGroup = job_group("orders_nightly", description="Nightly orders.", jobs=jobs) + + with pytest.raises(ValueError, match=re.escape(test_case.expected_message)): + python_pipeline_snapshot(tmp_path, [*jobs, group]) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonCheckJobTestCase( + description="a job matching a check and the model it checks publishes the model", + select="tag:order_checks orders", + expected_selection=["model:orders"], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_job_matching_a_check_and_its_node_when_publishing_then_the_check_runs_with_it( + tmp_path: Path, test_case: PythonCheckJobTestCase +) -> None: + nightly: Job = job("nightly_orders", description="Orders.", select=test_case.select) + + snapshot: dict[str, Any] = json.loads(python_pipeline_snapshot(tmp_path, [nightly]).to_json()) + + assert snapshot["documents"]["jobs"]["nightly_orders"]["selection"] == ( + test_case.expected_selection + ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonCheckJobRejectionTestCase( + description="a job matching a check without the model it checks names both", + select="tag:order_checks", + expected_message=( + "job nightly_orders: Python check 'orders_present' checks model:orders, which " + "the selection does not include; select it too, or narrow the selector so it no " + "longer matches the check." + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_job_matching_a_check_without_its_node_when_publishing_then_it_is_rejected( + tmp_path: Path, test_case: PythonCheckJobRejectionTestCase +) -> None: + nightly: Job = job("nightly_orders", description="Orders.", select=test_case.select) + + with pytest.raises(ValueError, match=re.escape(test_case.expected_message)): + python_pipeline_snapshot(tmp_path, [nightly]) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py index c36f6956be..f42a9bed93 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py @@ -35,6 +35,20 @@ expected_sql_names=frozenset(), expected_python_node_names=frozenset({"prepare_orders", "export_orders"}), ), + PythonSqlSelectorTestCase( + description="selects a Python asset without its unselected Python dependency", + select=("path:python/assets",), + exclude=(), + expected_sql_names=frozenset(), + expected_python_node_names=frozenset({"export_orders"}), + ), + PythonSqlSelectorTestCase( + description="accepts a check whose checked node the selection also matches", + select=("tag:exports",), + exclude=(), + expected_sql_names=frozenset(), + expected_python_node_names=frozenset({"export_orders"}), + ), PythonSqlSelectorTestCase( description="selects expanded source terminal loader requirement for run", select=("+source:raw_orders",), @@ -66,26 +80,13 @@ def test_given_run_selectors_when_resolving_then_excludes_python_checks( "test_case", [ PythonSqlSelectorErrorTestCase( - description="rejects explicit check selector for run", + description="rejects a check whose checked node the selection leaves out", select=("check:check_orders_export",), exclude=(), expected_error_type=ValueError, - expected_error_fragment=("Python checks are not selectable here: check_orders_export"), - ), - PythonSqlSelectorErrorTestCase( - description="rejects tag selector that includes a check for run", - select=("tag:exports",), - exclude=(), - expected_error_type=ValueError, - expected_error_fragment=("Python checks are not selectable here: check_orders_export"), - ), - PythonSqlSelectorErrorTestCase( - description="rejects direct asset path with unselected Python dependency for run", - select=("path:python/assets",), - exclude=(), - expected_error_type=ValueError, expected_error_fragment=( - "Python node 'export_orders' depends on unselected Python node 'prepare_orders'" + "Python check 'check_orders_export' checks asset:export_orders, which the " + "selection does not include" ), ), ], diff --git a/tests/unit/src/sqlbuild/provider/_helpers/helpers.py b/tests/unit/src/sqlbuild/provider/_helpers/helpers.py index 26a1bbdb83..23b22ab3c4 100644 --- a/tests/unit/src/sqlbuild/provider/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/provider/_helpers/helpers.py @@ -30,6 +30,14 @@ def context_and_provider(ctx: object, slack_provider: SlackProvider) -> tuple[ob return ctx, slack_provider.label +def positional_only_context(ctx: object, /, *, slack_provider: SlackProvider) -> tuple[object, str]: + return ctx, slack_provider.label + + +def keyword_only_provider(ctx: object, *, slack_provider: SlackProvider) -> tuple[object, str]: + return ctx, slack_provider.label + + def provider_only(slack_provider: SlackProvider) -> str: return slack_provider.label diff --git a/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py b/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py index a930e53846..0ba9419ac2 100644 --- a/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py +++ b/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py @@ -20,9 +20,11 @@ SlackProvider, context_and_provider, context_only, + keyword_only_provider, load_module, mismatched_provider, missing_provider, + positional_only_context, provider_only, reserved_context_conflict, unannotated_provider, @@ -65,6 +67,16 @@ def test_given_context_only_function_when_calling_with_provider_injection_then_c function=provider_only, expected_result="slack", ), + ProviderInjectionTestCase( + description="keyword-only provider is injected by name", + function=keyword_only_provider, + expected_result=("runtime-context", "slack"), + ), + ProviderInjectionTestCase( + description="positional-only context is passed by position", + function=positional_only_context, + expected_result=("runtime-context", "slack"), + ), ), ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx index dd93a67b34..17a34a1947 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx @@ -109,7 +109,11 @@ Checks do not support `columns`, `column_lineage`, or `retry`. ## Running checks -Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. To run checks on their own, use [`sqb check`](/docs/cli/check): +Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. + +A `sqb build` selector that matches a check must also select the nodes the check depends on. Otherwise the build fails with an error that names the check and the missing node; select that node too, or narrow the selector so it no longer matches the check. A check on a source that no loader writes never runs in a build, so a selector that matches it fails too; run that check with `sqb check`. Jobs and server runs resolve selectors the same way. + +To run checks on their own, use [`sqb check`](/docs/cli/check): ```bash # Run all checks diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 3f20e3ab26..9d215b702c 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -127,7 +127,7 @@ def export_orders(ctx): - `ctx.skip(reason, mode=...)` skips the node. `mode` accepts `"soft"` (default, skips only this node) or `"hard"` (also blocks dependents), as a string or the `SkipMode` enum from `sqlbuild.tasks`/`sqlbuild.assets`. - Assets may pass `materialized=True`/`False` to record whether an artifact was produced. -Downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. +Within one run, downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. ## Result persistence @@ -141,7 +141,7 @@ Python nodes are selected like SQL resources, by bare name or typed selector: sqb build --select export_orders # bare name sqb build --select task:export_orders # typed sqb build --select asset:orders_export -sqb build --select check:check_orders +sqb build --select export_orders check:check_orders_exported # a check with the node it checks sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` @@ -161,6 +161,18 @@ dependency fails or is skipped. A dependency that is not part of the run (not se unmanaged source) is read as it currently exists, so `sqb build --select task:export_orders` runs the task against the relations already in the warehouse. +Python dependencies follow the same rule. A task or asset selected without its Python upstream +reads that upstream's latest successful result through `ctx.result_of`. If the upstream has never +succeeded for the target, the node fails with an error that names the upstream, for example: + +```text +Upstream task 'prepare_orders' has no successful result to read. Build it first with +'sqb build --select task:prepare_orders', or build both with 'sqb build --select +asset:export_orders'. +``` + +Jobs, sensors, roles and ad-hoc runs on the server resolve selectors the same way, so a job can +select one Python node and a job group orders it after the job that builds its upstream. + The commands differ in what they include by default: | Command | SQL | Loaders / tasks / assets | Checks | Audits | diff --git a/website/src/content/docs/docs/concepts/selectors.mdx b/website/src/content/docs/docs/concepts/selectors.mdx index a10fac80ae..7f598d7a33 100644 --- a/website/src/content/docs/docs/concepts/selectors.mdx +++ b/website/src/content/docs/docs/concepts/selectors.mdx @@ -69,7 +69,7 @@ sqb build --select source:raw__orders ### Job -Select the nodes a job or job group declared under `orchestration/` builds. Job names are globally +Select the SQL and Python nodes a job or job group declared under `orchestration/` builds. Job names are globally unique, so a bare name works; `job:` is the explicit form and accepts glob patterns: ```bash