From aadbc98ff95b3e16417a407138cddd315b374cc3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:44:23 +0000 Subject: [PATCH 1/7] test: reproduce Python selectors in published job selections --- .../compiler/compiled_project/_test_types.py | 7 +++ .../compiler/compiled_project/helpers.py | 22 +++++++ .../test_orchestration_snapshot.py | 59 +++++++++++++++++++ 3 files changed, 88 insertions(+) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index f9fc57f343..048a0e9102 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -262,6 +262,13 @@ class SnapshotProvidersTestCase: expected_providers: dict[str, object] | None +@dataclass(frozen=True) +class PythonJobSelectionTestCase: + description: str + select: str + expected_selection: tuple[str, ...] + + @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 22a57cf538..03050f1519 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -66,6 +66,28 @@ "documents": {}, } +PYTHON_PIPELINE_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', tags [orders]);\n\nSELECT 1 AS order_id\n" + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', tags [orders]);\n\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + "python/orders/partner_export/node.py": ( + "from sqlbuild.assets import asset\nfrom sqlbuild.refs import model\n\n\n" + "@asset(depends_on=model('orders'), tags=['orders'], description='Partner order file.')\n" + "def partner_orders_file(ctx):\n return None\n" + ), + "python/fulfillment/carrier_sync/node.py": ( + "from python.orders.partner_export.node import partner_orders_file\n" + "from sqlbuild.tasks import task\n\n\n" + "@task(depends_on=partner_orders_file, tags=['orders'], description='Book collection.')\n" + "def request_carrier_collection(ctx):\n return None\n" + ), +} + TYPE_PROOF_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRCONTRACT105"]\n\n' diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 770f4cbd4d..10556cef24 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -44,12 +44,14 @@ SnapshotLineageTestCase, SnapshotProvidersTestCase, SnapshotRejectionTestCase, + PythonJobSelectionTestCase, SnapshotScheduleTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( FUNCTION_PROJECT_FILES, METADATA, PARITY_FILES, + PYTHON_PIPELINE_FILES, TYPE_PROOF_FILES, published_texts, ) @@ -1846,3 +1848,60 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho project_dir=tmp_path, discovered_inputs=discovered, ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonJobSelectionTestCase( + description="a model selector selects models", + select="+orders", + expected_selection=("model:orders", "model:stg_orders"), + ), + PythonJobSelectionTestCase( + description="a task selector selects the task", + select="task:request_carrier_collection", + expected_selection=("task:request_carrier_collection",), + ), + PythonJobSelectionTestCase( + description="a python folder selects every node under it", + select="python/orders", + expected_selection=("asset:partner_orders_file",), + ), + PythonJobSelectionTestCase( + description="a tag shared by models and Python nodes selects both", + select="tag:orders", + expected_selection=( + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_python_selectors_when_publishing_jobs_then_selections_match_the_cli( + tmp_path: Path, test_case: PythonJobSelectionTestCase +) -> None: + write_files(project_dir=tmp_path, files=PYTHON_PIPELINE_FILES) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + + snapshot: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=[job("orders_job", description="Orders.", select=test_case.select)], + discovered_inputs=discovered, + ).to_json() + ) + + assert snapshot["documents"]["jobs"]["orders_job"]["selection"] == list( + test_case.expected_selection + ) From 889f3d499fc74f6cc0acec466eaceeea8f64acb8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 02:11:12 +0000 Subject: [PATCH 2/7] wip: resolve published Python selectors with the CLI's unified resolver Work in progress, not ready for review. - Publish-time job, sensor-condition and role selections resolve with the same unified SQL/Python resolver `sqb build --select` uses. - Python snapshot nodes publish deps (Python and SQL) and tags; an intermediate loader publishes the source node it writes, and loader chains list it in `loads`. - Server-side ad-hoc selection rebuilds the Python selection graph from the snapshot and resolves with the same resolver. - The worker selects Python nodes by typed key. Validated so far: test_orchestration_snapshot.py passes (99 tests, including the Python selector reproduction). The Rust unit tests for the ad-hoc planner and planner scope have not finished yet; no review and no full gate. --- .../src/planner_scope/main/node_loads.rs | 3 +- .../src/planner_scope/tests/test_loaders.rs | 9 +- .../src/python_selection/_helpers/unified.rs | 32 ++++- .../src/python_selection/main/mod.rs | 2 + .../main/published_python_graph.rs | 52 ++++++++ .../main/resolve_python_sql_nodes.rs | 21 +++ .../src/python_selection/models.rs | 13 ++ .../src/adhoc/_helpers/planner.rs | 75 ++++++++++- .../sqlbuild-cli/src/adhoc/tests/helpers.rs | 1 + .../src/adhoc/tests/test_adhoc_planner.rs | 123 ++++++++++++++++++ .../src/runs/_helpers/requests/adhoc.rs | 21 ++- .../sqlbuild-orchestration/src/runs/models.rs | 3 +- .../compiled_project/snapshot_declarations.rs | 15 +-- .../compiled_project/snapshot_roles.rs | 23 +--- .../_helpers/graph/python_selection.rs | 89 +++++++++++++ .../_helpers/location_command/snapshot.rs | 88 ++++++++++++- .../_helpers/location_command/worker.rs | 10 +- .../_helpers/native_host/constants.rs | 2 - .../native_host/orchestration_triggers.rs | 47 ++----- .../test_orchestration_snapshot.py | 18 ++- 20 files changed, 555 insertions(+), 92 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs b/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs index 4d2f33a411..e80d101d53 100644 --- a/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs +++ b/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs @@ -5,7 +5,7 @@ use crate::planner_scope::constants::{MODEL, SOURCE}; use crate::planner_scope::models::LoaderGraph; use std::collections::{BTreeMap, BTreeSet}; -/// Managed sources each model's execution deps read and each managed source's declared loader chain, as `expand_loaders` keys them. +/// Managed sources each model's execution deps read and each managed source's loader chain, intermediate loaders included, as `expand_loaders` keys them. pub fn node_loads( graph: &LoaderGraph, upstream: &[(ObjectKey, Vec)], @@ -24,7 +24,6 @@ pub fn node_loads( graph .upstream(chain) .iter() - .filter(|name| !graph.intermediate(name)) .map(|name| graph.key(name)) .collect() } else { diff --git a/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs index b90277e273..8fd13b6768 100644 --- a/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs +++ b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs @@ -147,8 +147,8 @@ fn given_loader_chains_and_unmanaged_reads_when_listing_node_loads_then_only_dec expected_loads: vec![("model:orders", vec!["source:raw_returns"])], }, NodeLoadsTestCase { - // stage_orders has no declaring source, so a run that loads it still fails in the worker. - description: "a chain keeps declared loaders behind an intermediate one and drops the intermediate", + // stage_orders has no declaring source; the snapshot publishes it as source:stage_orders. + description: "a chain keeps an intermediate loader and the declared loaders behind it", loaders: vec![ ("raw_orders", vec!["stage_orders"]), ("stage_orders", vec!["raw_customers"]), @@ -162,7 +162,10 @@ fn given_loader_chains_and_unmanaged_reads_when_listing_node_loads_then_only_dec ("source:raw_orders", vec![]), ("source:raw_customers", vec![]), ], - expected_loads: vec![("source:raw_orders", vec!["source:raw_customers"])], + expected_loads: vec![( + "source:raw_orders", + vec!["source:stage_orders", "source:raw_customers"], + )], }, ]; let key = |text: &str| { diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index b98e48dca6..7bd1a63517 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -102,15 +102,20 @@ pub(crate) fn run_selection( .filter(|atom| !is_check(atom)) .collect(); validate_selected_dependencies(context, &runnable)?; + Ok(selection(context, sql_keys(&runnable), &runnable)) +} + +/// The SQL keys, with the source each selected intermediate loader writes, and the selected Python node names. +fn selection(context: &SelectionContext<'_>, sql: Vec, atoms: &Atoms) -> RunSelection { let terminal: HashSet<&str> = context .sql .sources .iter() .filter_map(|(_, loader)| loader.as_deref()) .collect(); - let mut keys: BTreeSet = sql_keys(&runnable).into_iter().collect(); + let mut keys: BTreeSet = sql.into_iter().collect(); let mut names: Vec = Vec::new(); - for atom in &runnable { + for atom in atoms { let Atom::Python(name) = atom else { continue; }; @@ -119,10 +124,29 @@ pub(crate) fn run_selection( } names.push(name.clone()); } - Ok(RunSelection { + RunSelection { sql_keys: keys.into_iter().collect(), python_node_names: names, - }) + } +} + +/// The SQL keys with their build functions and the Python node names, checks included, that selectors name; nothing is refused. +pub(crate) fn node_selection( + context: &SelectionContext<'_>, + select: &[String], + exclude: &[String], +) -> Result { + validate_boundaries(context)?; + let selected: Atoms = groups(context, select)? + .difference(&groups(context, exclude)?) + .cloned() + .collect(); + let sql: Vec = build_resources( + context.sql.graph, + &sql_keys(&selected), + BuildResources::SELECTION, + ); + Ok(selection(context, sql, &selected)) } fn groups(context: &SelectionContext<'_>, raw_groups: &[String]) -> Resolved { diff --git a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs index 884bfa0b85..3dace0806a 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs @@ -1,4 +1,6 @@ +pub mod published_python_graph; pub mod python_node_graph_indexes; pub mod resolve_python_node_excludes; pub mod resolve_python_node_selectors; +pub mod resolve_python_sql_nodes; pub mod resolve_python_sql_run_selection; diff --git a/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs b/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs new file mode 100644 index 0000000000..482d1689f5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs @@ -0,0 +1,52 @@ +use crate::definition_changes::constants::PYTHON_NODE_KINDS; +use crate::python_selection::_helpers::inventory::indexes; +use crate::python_selection::models::{ + InventoryIndexes, InventoryNode, PublishedPythonNode, PythonSelectionGraph, PythonSelectionNode, +}; + +/// The selection graph of Python nodes rebuilt from a published snapshot, so the server selects them as the CLI does. +pub fn published_python_graph(nodes: &[PublishedPythonNode]) -> PythonSelectionGraph { + let is_python = |kind: &str| PYTHON_NODE_KINDS.contains(&kind); + let inventory: Vec = nodes + .iter() + .map(|node| InventoryNode { + name: node.name.clone(), + tags: node.tags.clone(), + relative_path: String::new(), + upstream: node + .deps + .iter() + .filter(|(kind, _)| is_python(kind)) + .map(|(_, name)| name.clone()) + .collect(), + }) + .collect(); + let InventoryIndexes { + upstream, + downstream, + tags, + .. + } = indexes(&inventory); + PythonSelectionGraph { + nodes: nodes + .iter() + .map(|node| PythonSelectionNode { + name: node.name.clone(), + kind: node.kind.clone(), + sql_deps: node + .deps + .iter() + .filter(|(kind, _)| !is_python(kind)) + .cloned() + .collect(), + }) + .collect(), + upstream, + downstream, + tags, + paths: nodes + .iter() + .map(|node| (node.name.clone(), node.folder.clone())) + .collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs new file mode 100644 index 0000000000..6a2d99a3cb --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs @@ -0,0 +1,21 @@ +use crate::graph::errors::SelectorError; +use crate::python_selection::_helpers::nodes::sql_names; +use crate::python_selection::_helpers::unified::node_selection; +use crate::python_selection::models::{ + NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, +}; + +/// The SQL resources and Python nodes, checks included, that selectors name, as grants and sensor conditions read them. +pub fn resolve_python_sql_nodes( + sql: SqlSelectionGraph<'_>, + python: &PythonSelectionGraph, + select: &[String], + exclude: &[String], +) -> Result { + let context: SelectionContext<'_> = SelectionContext { + sql, + nodes: NodeIndex::new(python), + sql_names: sql_names(sql.graph), + }; + node_selection(&context, select, exclude) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/models.rs b/crates/sqlbuild-analysis/src/python_selection/models.rs index b0d615e1dd..63b7aacf07 100644 --- a/crates/sqlbuild-analysis/src/python_selection/models.rs +++ b/crates/sqlbuild-analysis/src/python_selection/models.rs @@ -52,6 +52,19 @@ pub struct InventoryNode { pub upstream: Vec, } +/// One Python node as a published snapshot lists it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PublishedPythonNode { + pub name: String, + /// `task`, `asset`, `loader` or `check`. + pub kind: String, + pub tags: Vec, + /// The folder of the node's file relative to the project, in POSIX form. + pub folder: String, + /// The Python nodes and SQL resources it depends on, as published keys. + pub deps: Vec, +} + /// Python's `PythonNodeGraph` edges and indexes, each in Python's dict order. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct InventoryIndexes { diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs index 93c34ee625..fd215a452f 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs @@ -3,11 +3,14 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::{Value as Json, json}; +use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; use sqlbuild_analysis::graph::models::{GraphResource, JobSelection}; +use sqlbuild_analysis::python_selection::main::published_python_graph::published_python_graph; +use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection; +use sqlbuild_analysis::python_selection::models::{PublishedPythonNode, SqlSelectionGraph}; use sqlbuild_core::text::main::python_str_repr::python_str_repr; use sqlbuild_orchestration::runs::models::{CommandPlan, PlannerResource}; use sqlbuild_orchestration::runs::types::{CommandPlanner, PublishedJobs}; @@ -203,19 +206,81 @@ fn resolved( select: &[String], exclude: &[String], ) -> Result, String> { - let graph_resources: Vec = resources.iter().filter_map(graph_resource).collect(); + let (python, sql): (Vec<&PlannerResource>, Vec<&PlannerResource>) = resources + .iter() + .partition(|resource| python_kind(&resource.key).is_some()); + let graph_resources: Vec = sql + .iter() + .filter_map(|resource| graph_resource(resource)) + .collect(); let mut graph = build_project_graph(&graph_resources); let selectors: Vec = select.iter().chain(exclude).cloned().collect(); - if selectors_need_jobs(&graph, &selectors, &[]) { + let node_names: Vec<&str> = python + .iter() + .filter_map(|resource| object_key(&resource.key).map(|_| name_of(&resource.key))) + .collect(); + if selectors_need_jobs(&graph, &selectors, &node_names) { graph = attach_jobs(graph, published_jobs(jobs)); } - Ok(resolve_selectors(&graph, select, exclude) - .map_err(|error| format!("{}: {}", error.code, error.message))? + let sources: Vec<(String, Option)> = sql + .iter() + .filter(|resource| resource.key.starts_with("source:")) + .map(|resource| (name_of(&resource.key).to_owned(), resource.loader.clone())) + .collect(); + let nodes: Vec = python + .iter() + .filter_map(|resource| { + Some(PublishedPythonNode { + name: name_of(&resource.key).to_owned(), + kind: python_kind(&resource.key)?.to_owned(), + tags: resource.tags.clone(), + folder: resource.folder.clone().unwrap_or_else(|| ".".to_owned()), + deps: resource + .deps + .iter() + .filter_map(|dep| object_key(dep)) + .collect(), + }) + }) + .collect(); + let python_graph = published_python_graph(&nodes); + let selection = resolve_python_sql_run_selection( + SqlSelectionGraph { + graph: &graph, + sources: &sources, + }, + &python_graph, + select, + exclude, + ) + .map_err(|error| format!("{}: {}", error.code, error.message))?; + let kinds: BTreeMap<&str, &str> = nodes + .iter() + .map(|node| (node.name.as_str(), node.kind.as_str())) + .collect(); + Ok(selection + .sql_keys .into_iter() .map(|(kind, name)| format!("{kind}:{name}")) + .chain( + selection + .python_node_names + .iter() + .filter_map(|name| Some(format!("{}:{name}", kinds.get(name.as_str())?))), + ) .collect()) } +/// The Python node kind of a published key, such as `task` for `task:export_orders`. +fn python_kind(key: &str) -> Option<&'static str> { + let (kind, _) = key.split_once(':')?; + PYTHON_NODE_KINDS.into_iter().find(|python| *python == kind) +} + +fn name_of(key: &str) -> &str { + key.split_once(':').map_or(key, |(_, name)| name) +} + /// Published jobs as already-resolved graph selections. fn published_jobs(jobs: &PublishedJobs) -> BTreeMap { let mut selections: BTreeMap = BTreeMap::new(); diff --git a/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs b/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs index 0725ac9380..3cd4087f5e 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs @@ -13,6 +13,7 @@ pub(super) fn resource( deps: deps.iter().map(|dep| (*dep).to_owned()).collect(), tags: tags.iter().map(|tag| (*tag).to_owned()).collect(), folder: folder.map(str::to_owned), + loader: None, } } diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index 8631f17666..47d3753118 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -289,3 +289,126 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select ); } } + +#[test] +fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_command_line() { + let resources = [ + resource("model:stg_orders", &[], &["orders"], Some("models/staging")), + resource( + "model:orders", + &["model:stg_orders"], + &["orders"], + Some("models/marts"), + ), + resource("model:customers", &[], &[], Some("models/marts")), + resource( + "asset:partner_orders_file", + &["model:orders"], + &["orders"], + Some("python/orders/partner_export"), + ), + resource( + "task:request_carrier_collection", + &["asset:partner_orders_file"], + &["orders"], + Some("python/fulfillment/carrier_sync"), + ), + ]; + let jobs: PublishedJobs = BTreeMap::new(); + let test_cases = [ + AdhocPlanTestCase { + description: "typed selectors select the Python nodes", + command: &[ + "build", + "--select", + "task:request_carrier_collection", + "asset:partner_orders_file", + ], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "a task without the Python node it depends on is refused, as on the command line", + command: &["build", "--select", "task:request_carrier_collection"], + expected_plan: Err("depends on unselected Python node 'partner_orders_file'"), + }, + AdhocPlanTestCase { + description: "a python folder selects the nodes under it", + command: &["build", "--select", "python/orders"], + expected_plan: Ok((&["asset:partner_orders_file"], json!({}))), + }, + AdhocPlanTestCase { + description: "a tag shared by models and Python nodes selects both", + command: &["build", "--select", "tag:orders"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "upstream of a task crosses its Python and SQL dependencies", + command: &["build", "--select", "+task:request_carrier_collection"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "downstream of a model reaches the Python nodes that read it", + command: &["build", "--select", "orders+", "--exclude", "stg_orders"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + ]; + let planner = adhoc_planner(); + for test_case in test_cases { + let command: Vec = test_case + .command + .iter() + .map(|token| (*token).to_owned()) + .collect(); + let actual = planner + .plan(&command, (&resources, &jobs)) + .map(|plan| { + ( + plan.selection.into_iter().collect::>(), + json!(plan.settings), + ) + }) + .map_err(|error| { + test_case + .expected_plan + .as_ref() + .err() + .filter(|fragment| error.contains(**fragment)) + .map_or(error.clone(), |fragment| (*fragment).to_owned()) + }); + assert_eq!( + actual, + owned_plan(&test_case.expected_plan), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index bcb1eddb60..6de44e69d1 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -30,6 +30,10 @@ use crate::store::models::{ControlTransaction, Parameter}; /// Graph kinds in the order the CLI builds its project graph: models, sources, seeds, then functions. const GRAPH_KINDS: [&str; 5] = ["model", "source", "seed", "udf", "table_fn"]; +/// Python node kinds, in the order the CLI discovers them. +const PYTHON_NODE_KINDS: [&str; 4] = ["task", "asset", "loader", "check"]; +/// Marks the source node an intermediate loader writes, which the CLI's SQL graph does not list. +const INTERMEDIATE_LOADER_FIELD: &str = "intermediate_loader"; /// The frozen request and its verb, or why the command cannot run; `run_id` comes from the caller. pub(crate) async fn frozen_adhoc_request( @@ -141,23 +145,26 @@ pub(crate) async fn publication_plan( )) } -/// The selector graph rebuilt from published definitions, in the CLI's kind order. +/// The selector graph rebuilt from published definitions, in the CLI's kind order; sources intermediate loaders write are selected through their loader. fn planner_resources( nodes: &std::collections::BTreeMap, ) -> Vec { let mut resources: Vec = Vec::new(); - for kind in GRAPH_KINDS { + for kind in GRAPH_KINDS.iter().chain(&PYTHON_NODE_KINDS) { for (key, node) in nodes { - if node.kind != kind { + let definition = &node.definition; + if node.kind != *kind || definition.get(INTERMEDIATE_LOADER_FIELD).is_some() { continue; } - let definition = &node.definition; - let (tags, folder) = match kind { + let (tags, folder) = match *kind { "model" => ( definition.pointer("/config/tags"), Some(model_folder(definition)), ), "seed" => (definition.pointer("/schema/tags"), None), + _ if PYTHON_NODE_KINDS.contains(kind) => { + (definition.get("tags"), Some(model_folder(definition))) + } _ => (definition.get("tags"), None), }; resources.push(PlannerResource { @@ -165,6 +172,10 @@ fn planner_resources( deps: texts(definition.get("deps")), tags: texts(tags), folder, + loader: definition + .pointer("/entry/loader") + .and_then(Value::as_str) + .map(str::to_owned), }); } } diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 6b92161618..48ac16d019 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -760,13 +760,14 @@ pub struct AdhocCheckRequest { pub priority: i64, } -/// One graph resource selectors resolve over: its `kind:name` key, upstream keys, tags and model folder. +/// One SQL resource or Python node selectors resolve over: its `kind:name` key, upstream keys, tags, file folder and a source's loader. #[derive(Clone, Debug, PartialEq, Eq)] pub struct PlannerResource { pub key: String, pub deps: Vec, pub tags: Vec, pub folder: Option, + pub loader: Option, } /// A parsed command: its verb, the exact node keys it selects and the flags it sets, by parser destination. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index a63c08cd12..ca9901107b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -6,7 +6,6 @@ use std::path::Path; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use serde_json::Value; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; @@ -20,6 +19,7 @@ use sqlbuild_orchestration::declarations::models::{ }; use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::native_host::orchestration_declarations::{ AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, declared_at, module_jobs, run_controls, @@ -60,7 +60,7 @@ pub(crate) fn discover_declarations<'py>( pub(crate) fn compile_declarations( py: Python<'_>, mut input: SnapshotInput, - (project_dir, graph): (&Path, &ProjectGraph), + (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), declarations: Vec, ) -> PyResult { if declarations.is_empty() { @@ -113,12 +113,9 @@ pub(crate) fn compile_declarations( job.name ))); } - let selection: BTreeSet = - resolve_selectors(graph, &tokens(&job.select), &tokens(&job.exclude)) - .map_err(|error| PyValueError::new_err(error.message))? - .into_iter() - .map(|(kind, name)| format!("{kind}:{name}")) - .collect(); + let selection: BTreeSet = selected + .run_keys(graph, &tokens(&job.select), &tokens(&job.exclude)) + .map_err(|error| PyValueError::new_err(format!("job {}: {error}", job.name)))?; let declared_at: Option = declared_at(project_dir, job.declared_at)?; let controls = run_controls( &job.name, @@ -229,7 +226,7 @@ pub(crate) fn compile_declarations( .. } = compile_sensors( py, - (project_dir, &input, graph), + (project_dir, &input, (graph, selected)), authored_sensors, compile_calendars( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs index 4867f861ab..26e7bc1266 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs @@ -1,35 +1,23 @@ //! Project roles from the reviewed `roles.toml` file, their selectors resolved against this snapshot. -use std::collections::BTreeSet; use std::path::Path; use pyo3::exceptions::PyValueError; use pyo3::prelude::{PyResult, Python}; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_orchestration::authorization::main::compile_roles::compile_roles; use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::native_host::host::write_console; /// The project-root file that defines project roles. pub(crate) const ROLES_FILE: &str = "roles.toml"; -/// Node keys of one grant selector, in the job selection grammar. -fn selector_nodes(graph: &ProjectGraph, selector: &str) -> Result, String> { - let tokens: Vec = selector.split_whitespace().map(str::to_owned).collect(); - let keys = resolve_selectors(graph, &tokens, &[]).map_err(|error| error.message)?; - let mut nodes: BTreeSet = BTreeSet::new(); - for (kind, name) in keys { - nodes.insert(format!("{kind}:{name}")); - } - Ok(nodes) -} - pub(crate) fn compile_project_roles( py: Python<'_>, mut input: SnapshotInput, - (project_dir, graph): (&Path, &ProjectGraph), + (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), ) -> PyResult { let path = project_dir.join(ROLES_FILE); if !path.is_file() { @@ -42,8 +30,11 @@ pub(crate) fn compile_project_roles( } let text = std::fs::read_to_string(&path) .map_err(|error| PyValueError::new_err(format!("roles.toml: {error}")))?; - let (document, warnings) = compile_roles(&text, |selector| selector_nodes(graph, selector)) - .map_err(PyValueError::new_err)?; + let (document, warnings) = compile_roles(&text, |selector| { + let tokens: Vec = selector.split_whitespace().map(str::to_owned).collect(); + selected.node_keys(graph, &tokens, &[]) + }) + .map_err(PyValueError::new_err)?; for warning in warnings { write_console(py, "stderr", &format!("Warning: {warning}\n"))?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 1dd9d85d85..3859130776 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -1,5 +1,7 @@ //! Python node and unified SQL/Python selector resolution, and the Python node graph indexes. +use std::collections::BTreeSet; + use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -7,9 +9,11 @@ use pyo3::types::{PyDict, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::errors::SelectorError; +use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; +use sqlbuild_analysis::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection as run_selection; use sqlbuild_analysis::python_selection::models::{ InventoryIndexes, InventoryNode, PythonSelectionGraph, PythonSelectionNode, RunSelection, @@ -19,6 +23,7 @@ use sqlbuild_analysis::python_selection::models::{ use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compiled_project::job_catalog::graph_with_jobs; use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_items; @@ -128,6 +133,90 @@ fn project_sources(project: &Bound<'_, PyAny>) -> PyResult)>, + pub(crate) nodes: PythonSelectionGraph, +} + +impl ProjectSelection { + pub(crate) fn new( + project: &Bound<'_, PyAny>, + discovered_inputs: &Bound<'_, PyAny>, + ) -> PyResult { + Ok(Self { + sources: project_sources(project)?, + nodes: selection_graph(&python_node_graph(discovered_inputs)?)?, + }) + } + + /// The `kind:name` keys `sqb build --select` runs for these selectors. + pub(crate) fn run_keys( + &self, + graph: &ProjectGraph, + select: &[String], + exclude: &[String], + ) -> Result, String> { + self.keys(run_selection(self.sql(graph), &self.nodes, select, exclude)) + } + + /// The `kind:name` keys these selectors name, Python checks included. + pub(crate) fn node_keys( + &self, + graph: &ProjectGraph, + select: &[String], + exclude: &[String], + ) -> Result, String> { + self.keys(resolve_python_sql_nodes( + self.sql(graph), + &self.nodes, + select, + exclude, + )) + } + + /// The published key of a Python node, such as `task:export_orders`. + pub(crate) fn node_key(&self, name: &str) -> Option { + self.nodes + .nodes + .iter() + .rev() + .find(|node| node.name == name) + .map(|node| format!("{}:{name}", node.kind)) + } + + /// Whether a loader writes no declared source and so runs only as part of another loader's chain. + pub(crate) fn intermediate_loader(&self, name: &str) -> bool { + !self + .sources + .iter() + .any(|(_, loader)| loader.as_deref() == Some(name)) + } + + fn sql<'graph>(&'graph self, graph: &'graph ProjectGraph) -> SqlSelectionGraph<'graph> { + SqlSelectionGraph { + graph, + sources: &self.sources, + } + } + + fn keys( + &self, + selection: Result, + ) -> Result, String> { + let selection: RunSelection = selection.map_err(|error| error.message)?; + let mut keys: BTreeSet = selection + .sql_keys + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + for name in &selection.python_node_names { + keys.extend(self.node_key(name)); + } + Ok(keys) + } +} + /// The Python node names `select` minus `exclude` resolve to; every node when unselected. pub(crate) fn python_node_names<'py>( graph: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 830e43a841..40a0682a21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -1,6 +1,6 @@ //! Location publication assembles catalog data and planner-owned local identities. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; use pyo3::exceptions::PyValueError; @@ -24,6 +24,7 @@ use crate::bindings::_helpers::compiled_project::snapshot_roles::compile_project use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::discovery::project_facts::{named_node_rows, project_facts}; use crate::bindings::_helpers::discovery::project_validation::validation_error; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; use crate::bindings::_helpers::native_host::orchestration_declarations::{ @@ -36,6 +37,9 @@ use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; use crate::bindings::_helpers::planner_scope::values::project_graph; use crate::bindings::_helpers::records::fields::field; +/// The Python node kind whose functions load sources. +const LOADER_KIND: &str = "loader"; + /// Where declarations passed to the snapshot directly are reported as declared. const DECLARATIONS_ARGUMENT: &str = "declarations argument"; @@ -111,6 +115,9 @@ pub(crate) fn orchestration_snapshot<'py>( } input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; drop(owner); + let selected = + ProjectSelection::new((&project).into_pyobject(py)?.as_any(), &discovered_inputs)?; + input.nodes = python_nodes(&selected, input.nodes)?; if let Some(providers) = providers_document(&discovered_inputs, &input.nodes)? { input.documents.insert("providers".into(), providers); } @@ -137,11 +144,15 @@ pub(crate) fn orchestration_snapshot<'py>( input = compile_declarations( py, input, - (&project_dir, graph.bind(py).get().graph()), + (&project_dir, graph.bind(py).get().graph(), &selected), authored, )?; } - input = compile_project_roles(py, input, (&project_dir, graph.bind(py).get().graph()))?; + input = compile_project_roles( + py, + input, + (&project_dir, graph.bind(py).get().graph(), &selected), + )?; let project = project.into_pyobject(py)?; if let Some(lineage) = column_lineage_document(project.as_any())? { input.documents.insert("lineage".into(), lineage); @@ -151,6 +162,77 @@ pub(crate) fn orchestration_snapshot<'py>( Ok(NativeOrchestrationSnapshot { input }) } +/// Python nodes publish their `deps` and tags, and each intermediate loader publishes the source it writes, so the server orders, selects and grants them as the CLI does. +fn python_nodes( + selected: &ProjectSelection, + mut nodes: BTreeMap, +) -> PyResult> { + let graph = &selected.nodes; + let mut tags: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + for (tag, names) in &graph.tags { + for name in names { + tags.entry(name.as_str()).or_default().push(tag.as_str()); + } + } + let upstream: BTreeMap<&str, &[String]> = graph + .upstream + .iter() + .map(|(name, names)| (name.as_str(), names.as_slice())) + .collect(); + let mut outputs: Vec<(String, SnapshotNode)> = Vec::new(); + for node in &graph.nodes { + let key = format!("{}:{}", node.kind, node.name); + let Some(published) = nodes.get_mut(&key) else { + return Err(PyValueError::new_err(format!( + "Python node {key} is missing from the snapshot" + ))); + }; + let deps: BTreeSet = upstream + .get(node.name.as_str()) + .into_iter() + .flat_map(|names| names.iter()) + .filter_map(|name| selected.node_key(name)) + .chain( + node.sql_deps + .iter() + .map(|(kind, name)| format!("{kind}:{name}")), + ) + .collect(); + published.definition["deps"] = json!(deps); + published.definition["tags"] = + json!(tags.get(node.name.as_str()).cloned().unwrap_or_default()); + if node.kind == LOADER_KIND && selected.intermediate_loader(&node.name) { + let mut definition = json!({ + "name": node.name, + "entry": {"loader": node.name}, + "intermediate_loader": true, + "deps": [key], + }); + if let Some(path) = published.definition.get("path") { + definition["path"] = path.clone(); + } + outputs.push(( + format!("source:{}", node.name), + SnapshotNode { + kind: "source".into(), + definition, + raw_sql: None, + compiled_sql: None, + python_source: None, + }, + )); + } + } + for (key, node) in outputs { + if nodes.insert(key.clone(), node).is_some() { + return Err(PyValueError::new_err(format!( + "intermediate loader {key} collides with a published node" + ))); + } + } + Ok(nodes) +} + fn identity_nodes( project: &Bound<'_, PyAny>, mut nodes: BTreeMap, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 4d28d969ae..ec62b33a15 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -8,6 +8,7 @@ use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyString; use serde_json::Value; +use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; use sqlbuild_cli::argv::main::command_settings_arguments::command_settings_arguments; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; @@ -148,10 +149,15 @@ fn command_request<'py>( if execution.skipped_nodes.contains_key(key) { continue; } - let (_, name) = key + let (kind, name) = key .split_once(':') .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; - argv.extend([PyString::new(py, "--select"), PyString::new(py, name)]); + let selector = if PYTHON_NODE_KINDS.contains(&kind) { + key.as_str() + } else { + name + }; + argv.extend([PyString::new(py, "--select"), PyString::new(py, selector)]); } let parsed = parse_cli_invocation(py, argv, false)?; let args = field(&parsed, "args")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index d19d662071..d0ef09dcb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -4,8 +4,6 @@ pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; pub(crate) const RUN_EVENT_CONDITION: &str = "run_event"; -pub(crate) const NODE_EVENT_CONDITION: &str = "node_event"; -pub(crate) const CHECK_KIND: &str = "check"; pub(crate) const SOURCE_NEW_DATA_CONDITION: &str = "source_new_data"; pub(crate) const SOURCE_STALE_CONDITION: &str = "source_stale"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 8779ef8608..b8923950a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -1,6 +1,6 @@ //! Authored trigger controls and bounded calendars evaluated only in the location interpreter. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; use std::path::Path; use chrono::{DateTime, TimeDelta, Timelike, Utc}; @@ -10,21 +10,20 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use serde_json::{Value, json}; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; use sqlbuild_orchestration::sensors::models::SensorDefinition; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::native_host::constants::{ - CALENDAR_WINDOW_PARAMETER, CHECK_KIND, NODE_EVENT_CONDITION, SOURCE_NEW_DATA_CONDITION, - SOURCE_STALE_CONDITION, + CALENDAR_WINDOW_PARAMETER, SOURCE_NEW_DATA_CONDITION, SOURCE_STALE_CONDITION, }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_declarations::{ @@ -253,7 +252,7 @@ pub(crate) fn compile_calendars( pub(crate) fn compile_sensors( py: Python<'_>, - (project_dir, input, graph): (&Path, &SnapshotInput, &ProjectGraph), + (project_dir, input, graph): (&Path, &SnapshotInput, (&ProjectGraph, &ProjectSelection)), authored: Vec, mut definitions: Definitions, ) -> PyResult { @@ -281,12 +280,11 @@ pub(crate) fn compile_sensors( if let Some(select) = when.select { let selectors: Vec = select.split_whitespace().map(str::to_owned).collect(); - condition["selection"] = - sensor_nodes(graph, &input.nodes, &selectors, &when.kind)?; + condition["selection"] = sensor_nodes(graph, &selectors, &when.kind)?; } if let Some(selectors) = when.executed_successfully { condition["executed_successfully"] = - sensor_nodes(graph, &input.nodes, &selectors, "executed_successfully")?; + sensor_nodes(graph, &selectors, "executed_successfully")?; } condition } @@ -314,8 +312,7 @@ pub(crate) fn compile_sensors( } fn sensor_nodes( - graph: &ProjectGraph, - available: &BTreeMap, + (graph, selected): (&ProjectGraph, &ProjectSelection), selectors: &[String], field: &str, ) -> PyResult { @@ -324,31 +321,9 @@ fn sensor_nodes( "{field} needs nonempty selectors" ))); } - let mut nodes: BTreeSet = BTreeSet::new(); - let mut sql_selectors: Vec = Vec::new(); - for selector in selectors { - if field == NODE_EVENT_CONDITION && selector.starts_with("check:") { - if !available - .get(selector) - .is_some_and(|node| node.kind == CHECK_KIND) - { - return Err(PyValueError::new_err(format!( - "unknown check node {selector}" - ))); - } - nodes.insert(selector.clone()); - } else { - sql_selectors.push(selector.clone()); - } - } - if !sql_selectors.is_empty() { - nodes.extend( - resolve_selectors(graph, &sql_selectors, &[]) - .map_err(|error| PyValueError::new_err(error.message))? - .into_iter() - .map(|(kind, name)| format!("{kind}:{name}")), - ); - } + let nodes: BTreeSet = selected + .node_keys(graph, selectors, &[]) + .map_err(|error| PyValueError::new_err(format!("{field}: {error}")))?; if nodes.is_empty() { return Err(PyValueError::new_err(format!( "{field} needs nonempty selectors" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 10556cef24..9cce168cc4 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -1789,7 +1789,7 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects .replace('("orders",)', '("unknown_*",)') }, expected_error=ValueError, - expected_message="unknown selector pattern", + expected_message="executed_successfully: unknown selector name 'unknown_\\*'", ), SnapshotDiscoveryRejectionTestCase( description="invalid execution selector syntax fails closed", @@ -1859,9 +1859,19 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho expected_selection=("model:orders", "model:stg_orders"), ), PythonJobSelectionTestCase( - description="a task selector selects the task", - select="task:request_carrier_collection", - expected_selection=("task:request_carrier_collection",), + description="typed selectors select the task and the asset it depends on", + select="task:request_carrier_collection asset:partner_orders_file", + expected_selection=("asset:partner_orders_file", "task:request_carrier_collection"), + ), + PythonJobSelectionTestCase( + description="upstream of a task crosses its Python and SQL dependencies", + select="+task:request_carrier_collection", + expected_selection=( + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ), ), PythonJobSelectionTestCase( description="a python folder selects every node under it", From cc4a6c59b08c71ba2d38d6d1fa40dc159fd91018 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:22:56 +0000 Subject: [PATCH 3/7] feat: let a selected Python node read an unselected upstream's result A Python node can now be selected without its Python upstream in every path that uses the shared resolver: sqb build --select, published jobs, sensors, roles and ad-hoc runs. It reads the upstream's latest persisted result, and fails at run time with an error naming the upstream and the command to build it when the upstream has never succeeded. An intermediate loader without a write_strategy that returns rows now writes them to __loader__ as a table, as the loaders docs describe. A self-managed intermediate that returns nothing keeps its behaviour. The published snapshot rebuilds the Python selection graph from node deps, and job groups order jobs across Python and Python-to-SQL edges. Validated locally: python_selection and adhoc planner Rust tests, clippy on both crates, fensu check, the worker claim tests, orchestration snapshot, Python node selection E2E, load E2E (sqb load and sqb build on the docs example) and the python_nodes and executor/load unit tests. Not yet run: make check-ci, the full suite, review. --- .../loaders/intermediate_default_accounts.py | 25 ++++++ .../loaders/intermediate_default_accounts.yml | 5 ++ .../intermediate_loader_sources.py | 7 +- .../python_selection/_helpers/inventory.rs | 17 ++++ .../python_selection/_helpers/validation.rs | 15 +--- .../main/published_python_graph.rs | 42 ++++----- .../src/python_selection/tests/helpers.rs | 46 +++++++++- .../tests/test_python_selection.rs | 25 +++--- .../src/adhoc/_helpers/planner.rs | 30 ++++--- .../src/adhoc/tests/test_adhoc_planner.rs | 4 +- .../executor/load/_helpers/execution.py | 14 ++- .../load/_helpers/loader_invocation.py | 3 +- src/sqlbuild/executor/load/main/_execute.py | 3 +- .../python_nodes/_helpers/execution.py | 43 ++++++++++ src/sqlbuild/executor/python_nodes/models.py | 8 +- .../cli/commands/main/build/_test_types.py | 10 +++ .../cli/commands/main/build/helpers.py | 9 ++ .../main/build/test_python_node_selection.py | 49 +++++++++++ .../cli/commands/main/load/_test_types.py | 8 ++ .../cli/commands/main/load/test_load.py | 60 +++++++++++++ .../compiler/compiled_project/_test_types.py | 14 +++ .../compiler/compiled_project/helpers.py | 23 +++++ .../publication/_test_types.py | 6 ++ .../compiled_project/publication/helpers.py | 16 ++++ .../publication/test_worker.py | 47 +++++----- .../test_orchestration_snapshot.py | 86 ++++++++++++++++++- .../_helpers/test_run_selectors.py | 16 ++-- .../docs/concepts/python-nodes/overview.mdx | 14 ++- 28 files changed, 528 insertions(+), 117 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.py create mode 100644 benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.yml diff --git a/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.py b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.py new file mode 100644 index 0000000000..6b01434f43 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.py @@ -0,0 +1,25 @@ +"""The loader-dependencies example from the loaders docs, as written. + +raw_accounts is an intermediate loader without a write_strategy that returns rows, so it writes +them to __loader__raw_accounts as a table. +""" + +from sqlbuild.executor.load.models import LoaderContext +from sqlbuild.loaders import loader + + +@loader +def raw_accounts(ctx: LoaderContext) -> list[dict[str, object]]: + """Load customer accounts.""" + return [ + {"account_id": 1, "account_name": "Pawnee Parks"}, + {"account_id": 2, "account_name": "Eagleton"}, + ] + + +@loader(depends_on=[raw_accounts]) +def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]: + """Account metrics.""" + accounts = ctx.loader(raw_accounts) + rows = ctx.query(f"SELECT account_id FROM {accounts.destination}") + return [{"account_id": row[0], "metric": "active"} for row in rows.fetchall()] diff --git a/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.yml b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.yml new file mode 100644 index 0000000000..90e7a2605a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.yml @@ -0,0 +1,5 @@ +sources: + - name: raw_account_metrics + description: Account metrics read from the intermediate raw_accounts loader. + managed: true + write_strategy: table diff --git a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py index 9512d788c5..70db157dde 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py +++ b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py @@ -1,9 +1,12 @@ -"""A source loader that depends on an intermediate loader no source declares.""" +"""A source loader that depends on an intermediate loader no source declares. + +A job that selects the model must claim the intermediate loader's relation as well as the source. +""" from sqlbuild.loaders import loader -@loader +@loader(write_strategy="table") def stage_shipments(ctx): """Stage synthetic shipments in the intermediate loader's own relation.""" return [{"order_id": 9}] diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs index 9a416f0c44..944eb5294f 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs @@ -2,6 +2,9 @@ use std::collections::HashMap; +use crate::assembly::project::types::ObjectKey; +use crate::definition_changes::constants::PYTHON_NODE_KINDS; + use crate::python_selection::constants::{CURRENT_DIRECTORY, ROOT_DIRECTORY}; use crate::python_selection::models::{InventoryIndexes, InventoryNode}; @@ -83,3 +86,17 @@ pub(crate) fn indexes(nodes: &[InventoryNode]) -> InventoryIndexes { paths: paths.rows, } } + +/// A published node's dependency keys split into its Python upstream names and its SQL keys. +pub(crate) fn split_published_deps(deps: &[ObjectKey]) -> (Vec, Vec) { + let mut python: Vec = Vec::new(); + let mut sql: Vec = Vec::new(); + for (kind, name) in deps { + if PYTHON_NODE_KINDS.contains(&kind.as_str()) { + python.push(name.clone()); + } else { + sql.push((kind.clone(), name.clone())); + } + } + (python, sql) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs index ddacd27837..9bed1e9624 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs @@ -157,7 +157,7 @@ fn validate_loader_upstreams(context: &SelectionContext<'_>) -> Result<(), Selec Ok(()) } -/// Python's `_validate_selected_dependencies`: selected sources and nodes have their loaders. +/// Selected managed sources have their loaders selected; a Python node may run without its Python upstream, whose latest persisted result it reads. pub(crate) fn validate_selected_dependencies( context: &SelectionContext<'_>, atoms: &BTreeSet, @@ -188,18 +188,5 @@ pub(crate) fn validate_selected_dependencies( select it directly or use upstream expansion" ))); } - for atom in atoms { - let Atom::Python(name) = atom else { - continue; - }; - for upstream in context.nodes.edges(name, EdgeDirection::Upstream) { - if !selected.contains(upstream.as_str()) { - return Err(failure(format!( - "Python node '{name}' depends on unselected Python node '{upstream}'; \ - select it directly or use upstream expansion" - ))); - } - } - } Ok(()) } diff --git a/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs b/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs index 482d1689f5..695361c4e6 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs @@ -1,26 +1,26 @@ -use crate::definition_changes::constants::PYTHON_NODE_KINDS; -use crate::python_selection::_helpers::inventory::indexes; +use crate::python_selection::_helpers::inventory::{indexes, split_published_deps}; use crate::python_selection::models::{ InventoryIndexes, InventoryNode, PublishedPythonNode, PythonSelectionGraph, PythonSelectionNode, }; /// The selection graph of Python nodes rebuilt from a published snapshot, so the server selects them as the CLI does. pub fn published_python_graph(nodes: &[PublishedPythonNode]) -> PythonSelectionGraph { - let is_python = |kind: &str| PYTHON_NODE_KINDS.contains(&kind); - let inventory: Vec = nodes - .iter() - .map(|node| InventoryNode { + let mut inventory: Vec = Vec::with_capacity(nodes.len()); + let mut selection_nodes: Vec = Vec::with_capacity(nodes.len()); + for node in nodes { + let (upstream, sql_deps) = split_published_deps(&node.deps); + inventory.push(InventoryNode { name: node.name.clone(), tags: node.tags.clone(), relative_path: String::new(), - upstream: node - .deps - .iter() - .filter(|(kind, _)| is_python(kind)) - .map(|(_, name)| name.clone()) - .collect(), - }) - .collect(); + upstream, + }); + selection_nodes.push(PythonSelectionNode { + name: node.name.clone(), + kind: node.kind.clone(), + sql_deps, + }); + } let InventoryIndexes { upstream, downstream, @@ -28,19 +28,7 @@ pub fn published_python_graph(nodes: &[PublishedPythonNode]) -> PythonSelectionG .. } = indexes(&inventory); PythonSelectionGraph { - nodes: nodes - .iter() - .map(|node| PythonSelectionNode { - name: node.name.clone(), - kind: node.kind.clone(), - sql_deps: node - .deps - .iter() - .filter(|(kind, _)| !is_python(kind)) - .cloned() - .collect(), - }) - .collect(), + nodes: selection_nodes, upstream, downstream, tags, diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs index 92b68329c3..2ac7b7de0e 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs @@ -2,13 +2,14 @@ use crate::assembly::project::types::ObjectKey; use crate::graph::errors::SelectorError; use crate::graph::main::build_project_graph::build_project_graph; use crate::graph::models::{GraphResource, ProjectGraph}; +use crate::python_selection::main::published_python_graph::published_python_graph; use crate::python_selection::main::python_node_graph_indexes::python_node_graph_indexes; use crate::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use crate::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; use crate::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection; use crate::python_selection::models::{ - InventoryIndexes, InventoryNode, PythonSelectionGraph, PythonSelectionNode, RunSelection, - SqlSelectionGraph, + InventoryIndexes, InventoryNode, PublishedPythonNode, PythonSelectionGraph, + PythonSelectionNode, RunSelection, SqlSelectionGraph, }; use crate::python_selection::tests::test_types::{ExpectedRun, SelectorFailure}; @@ -130,6 +131,36 @@ fn orders_sql_graph() -> ProjectGraph { ]) } +/// The orders Python node graph as the server rebuilds it from published snapshot nodes. +pub(super) fn published_orders_graph() -> PythonSelectionGraph { + let nodes: Vec = ORDERS_NODES + .iter() + .map( + |(name, kind, path, tags, upstream, sql_deps)| PublishedPythonNode { + name: (*name).to_owned(), + kind: (*kind).to_owned(), + tags: owned(tags), + folder: path + .rsplit_once('/') + .map_or(".", |(folder, _)| folder) + .to_owned(), + deps: upstream + .iter() + .map(|upstream| { + let kind: &str = ORDERS_NODES + .iter() + .find(|node| node.0 == *upstream) + .map_or("task", |node| node.1); + key(&(kind, upstream)) + }) + .chain(sql_deps.iter().map(key)) + .collect(), + }, + ) + .collect(); + published_python_graph(&nodes) +} + /// The Python node names the selectors resolve to over the orders graph. pub(super) fn node_names(select: &[&str], exclude: &[&str]) -> Result, OwnedFailure> { resolve_python_node_selectors(&orders_python_graph(), &owned(select), &owned(exclude)) @@ -138,6 +169,15 @@ pub(super) fn node_names(select: &[&str], exclude: &[&str]) -> Result Result { + run_over(&orders_python_graph(), select, exclude) +} + +/// The run selection over the orders SQL graph and the given Python graph. +pub(super) fn run_over( + python: &PythonSelectionGraph, + select: &[&str], + exclude: &[&str], +) -> Result { let graph: ProjectGraph = orders_sql_graph(); let sources: Vec<(String, Option)> = vec![("raw_orders".to_owned(), Some("raw_orders".to_owned()))]; @@ -146,7 +186,7 @@ pub(super) fn run(select: &[&str], exclude: &[&str]) -> Result = select.iter().chain(exclude).cloned().collect(); let node_names: Vec<&str> = python .iter() - .filter_map(|resource| object_key(&resource.key).map(|_| name_of(&resource.key))) + .map(|resource| name_of(&resource.key)) .collect(); if selectors_need_jobs(&graph, &selectors, &node_names) { graph = attach_jobs(graph, published_jobs(jobs)); @@ -229,19 +229,7 @@ fn resolved( .collect(); let nodes: Vec = python .iter() - .filter_map(|resource| { - Some(PublishedPythonNode { - name: name_of(&resource.key).to_owned(), - kind: python_kind(&resource.key)?.to_owned(), - tags: resource.tags.clone(), - folder: resource.folder.clone().unwrap_or_else(|| ".".to_owned()), - deps: resource - .deps - .iter() - .filter_map(|dep| object_key(dep)) - .collect(), - }) - }) + .filter_map(|resource| published_python_node(resource)) .collect(); let python_graph = published_python_graph(&nodes); let selection = resolve_python_sql_run_selection( @@ -277,6 +265,20 @@ fn python_kind(key: &str) -> Option<&'static str> { PYTHON_NODE_KINDS.into_iter().find(|python| *python == kind) } +fn published_python_node(resource: &PlannerResource) -> Option { + Some(PublishedPythonNode { + name: name_of(&resource.key).to_owned(), + kind: python_kind(&resource.key)?.to_owned(), + tags: resource.tags.clone(), + folder: resource.folder.clone().unwrap_or_else(|| ".".to_owned()), + deps: resource + .deps + .iter() + .filter_map(|dep| object_key(dep)) + .collect(), + }) +} + fn name_of(key: &str) -> &str { key.split_once(':').map_or(key, |(_, name)| name) } diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index 47d3753118..9855c4cbc3 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -333,9 +333,9 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co )), }, AdhocPlanTestCase { - description: "a task without the Python node it depends on is refused, as on the command line", + description: "a task runs without the Python node it depends on, as on the command line", command: &["build", "--select", "task:request_carrier_collection"], - expected_plan: Err("depends on unselected Python node 'partner_orders_file'"), + expected_plan: Ok((&["task:request_carrier_collection"], json!({}))), }, AdhocPlanTestCase { description: "a python folder selects the nodes under it", diff --git a/src/sqlbuild/executor/load/_helpers/execution.py b/src/sqlbuild/executor/load/_helpers/execution.py index 955a267d89..673e81e4af 100644 --- a/src/sqlbuild/executor/load/_helpers/execution.py +++ b/src/sqlbuild/executor/load/_helpers/execution.py @@ -3,8 +3,9 @@ from __future__ import annotations from collections.abc import Callable +from dataclasses import replace -from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry, SourceWriteStrategy from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.executor.load.models import LoadExecutionIndexes, LoadExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus @@ -191,3 +192,14 @@ def is_untargeted_self_managed_intermediate( and source_entry.write_strategy is None and loader_function.destination is None ) + + +def row_write_entry(source_entry: SourceEntry) -> SourceEntry: + """Return the entry rows are written with: an intermediate loader defaults to `table`.""" + + if ( + source_entry.meta.get("sqlbuild_loader_node") is True + and source_entry.write_strategy is None + ): + return replace(source_entry, write_strategy=SourceWriteStrategy.TABLE) + return source_entry diff --git a/src/sqlbuild/executor/load/_helpers/loader_invocation.py b/src/sqlbuild/executor/load/_helpers/loader_invocation.py index c07f0fd419..d287c8681c 100644 --- a/src/sqlbuild/executor/load/_helpers/loader_invocation.py +++ b/src/sqlbuild/executor/load/_helpers/loader_invocation.py @@ -18,6 +18,7 @@ from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.execution import ( is_untargeted_self_managed_intermediate, + row_write_entry, ) from sqlbuild.executor.load._helpers.relation_refs import ( build_loader_relation_refs, @@ -238,7 +239,7 @@ def interpret_loader_return( duration_ms=int((time.monotonic() - start) * 1000), lifecycle_events=statement_recorder.snapshot(), ) - if source_entry.write_strategy is None: + if row_write_entry(source_entry).write_strategy is None: raise ExecutorInputError( f"Source '{source_entry.name}' loader '{loader_function.name}' returned rows " "but source has no write_strategy" diff --git a/src/sqlbuild/executor/load/main/_execute.py b/src/sqlbuild/executor/load/main/_execute.py index f4ccaa0360..eef2c5c56b 100644 --- a/src/sqlbuild/executor/load/main/_execute.py +++ b/src/sqlbuild/executor/load/main/_execute.py @@ -26,7 +26,7 @@ infer_cursor_kind, load_staging_cursor_bounds, ) -from sqlbuild.executor.load._helpers.execution import load_resource_kind +from sqlbuild.executor.load._helpers.execution import load_resource_kind, row_write_entry from sqlbuild.executor.load._helpers.external import execute_external_source_load from sqlbuild.executor.load._helpers.loader_invocation import ( build_loader_context, @@ -176,6 +176,7 @@ def _execute_source_load( ) if early_result is not None: return early_result + source_entry = row_write_entry(source_entry) rows_loaded: int = write_loader_rows_to_staging( loader_return_value=raw_rows, source_entry=source_entry, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 43bd63753c..dafc8e882a 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -189,6 +189,18 @@ def _execute_ready_node_in_cost_scope( result_store=result_store, warnings=warnings, ) + missing: str | None = _missing_upstream_result( + node=node, node_kind=node_kind, context=context, upstream_results=upstream_results + ) + if missing is not None: + result = PythonNodeExecutionResult( + node_name=node.name, + kind=node_kind, + status=PythonNodeStatus.FAILED, + error_message=missing, + ) + _persist_python_node_result(result_store=result_store, result=result, run_id=run_id) + return result, None try: result: PythonNodeExecutionResult = _call_node_with_retry( node=node, @@ -344,6 +356,37 @@ def _call_node_with_retry( attempt += 1 +def _missing_upstream_result( + *, + node: ExecutablePythonNode, + node_kind: PythonNodeKind, + context: TaskContext | AssetContext, + upstream_results: tuple[PythonNodeExecutionResult, ...], +) -> str | None: + """Why the node cannot run: a Python upstream outside this run that has never succeeded.""" + + python_dependencies: tuple[Callable[..., object], ...] = tuple( + dependency for dependency in node.depends_on if not isinstance(dependency, SqlResourceRef) + ) + if len(upstream_results) == len(python_dependencies): + return None + in_run: frozenset[str] = frozenset(result.node_name for result in upstream_results) + for dependency in python_dependencies: + kind, name = context.result_dependency_identity(dependency) + if name in in_run or context.result_of(node_function=dependency, default=None) is not None: + continue + both: str = f"'sqb build --select +{node_kind.value}:{node.name}'" + if kind == PythonNodeKind.LOADER.value: + return ( + f"Upstream loader '{name}' has no successful result to read. Build it with {both}." + ) + return ( + f"Upstream {kind} '{name}' has no successful result to read. Build it first with " + f"'sqb build --select {kind}:{name}', or build both with {both}." + ) + return None + + def _build_context( *, node: ExecutablePythonNode, diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index e41a47874f..245fab2a91 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -267,7 +267,7 @@ def result_of( if default is not MISSING_DEFAULT: return default raise ExecutorInputError("No Python node result store is available") - node_type, node_name = self._result_dependency_identity(node_function) + node_type, node_name = self.result_dependency_identity(node_function) return self.result_store.result_of( node_type=node_type, node_name=node_name, @@ -285,14 +285,16 @@ def results_of( if self.result_store is None: return () - node_type, node_name = self._result_dependency_identity(node_function) + node_type, node_name = self.result_dependency_identity(node_function) return self.result_store.results_of( node_type=node_type, node_name=node_name, limit=limit, ) - def _result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: + def result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: + """Return the `(kind, name)` a dependency's results are stored under.""" + task_definition: object = getattr(node_function, "__sqlbuild_task__", None) if task_definition is not None: return PythonNodeKind.TASK.value, self._definition_name(task_definition) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index ae15ae5ad1..64837288d3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1614,3 +1614,13 @@ class ChangedTwoUpE2ETestCase: warehouse: ChangedWarehouse expected_crm_run: frozenset[str] expected_downstream: frozenset[str] + + +@dataclass(frozen=True) +class PythonUpstreamResultE2ETestCase: + """A Python node selected without its Python upstream, before and after the upstream builds.""" + + description: str + select: str + upstream_select: str + expected_error: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index fc8da19a0e..95f42565c1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -895,6 +895,15 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: 'MODEL (description "Test model order_summary.", ' 'materialized table);\nSELECT count(*) AS order_count FROM __ref("orders")\n' ), + "python/tasks/reports.py": ( + "from python.tasks.counts import count_orders\nfrom sqlbuild.tasks import task\n\n\n" + "@task(depends_on=count_orders)\n" + "def report_order_count(ctx):\n" + " '''Test task report_order_count.'''\n" + " upstream = ctx.result_of(node_function=count_orders)\n" + ' ctx.execute_sql(f"CREATE OR REPLACE TABLE main.report_order_count_result AS "\n' + " f\"SELECT '{upstream.status}' AS status\")\n" + ), } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py index c792b221af..1ec298fe6b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py @@ -10,6 +10,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( PythonNodeSelectionE2ETestCase, + PythonUpstreamResultE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( PYTHON_NODE_SELECTION_DATABASE, @@ -174,3 +175,51 @@ def test_given_python_node_selection_when_building_then_it_gates_only_on_schedul table_exists(db_path=database, table_name=table) for table in test_case.expected_missing_tables ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonUpstreamResultE2ETestCase( + description="a task reads its unselected upstream task's latest result", + select="task:report_order_count", + upstream_select="task:count_orders", + expected_error=( + "Upstream task 'count_orders' has no successful result to read. Build it first " + "with 'sqb build --select task:count_orders', or build both with " + "'sqb build --select +task:report_order_count'." + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_task_selected_without_its_python_upstream_when_building_then_it_reads_the_persisted_result( + test_case: PythonUpstreamResultE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="order_nodes", + repo_files=python_node_selection_project_files( + orders_sql=PythonNodeSelectionE2ETestCase.orders_sql + ), + ) + database: Path = project_dir / PYTHON_NODE_SELECTION_DATABASE + execute_duckdb(db_path=database, sql=_EXISTING_ROWS + _EXISTING_ORDERS) + report: tuple[str, ...] = ("--no-color", "build", "--select", test_case.select) + + never_built: subprocess.CompletedProcess[str] = run_sqb(command=report, project_dir=project_dir) + upstream: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", test_case.upstream_select), + project_dir=project_dir, + ) + rebuilt: subprocess.CompletedProcess[str] = run_sqb(command=report, project_dir=project_dir) + + output: str = never_built.stdout + never_built.stderr + assert never_built.returncode == 1, test_case.description + output + assert test_case.expected_error in output, test_case.description + assert upstream.returncode == 0, upstream.stdout + upstream.stderr + assert rebuilt.returncode == 0, rebuilt.stdout + rebuilt.stderr + node: str = test_case.select.split(":", 1)[1] + assert query_duckdb(db_path=database, sql=f"SELECT status FROM main.{node}_result") == [ + ("success",) + ] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py index 79bad71942..7a71fd98f2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py @@ -127,3 +127,11 @@ class LoaderRelationGuardE2ETestCase: enforce_explicit: bool expected_warnings: tuple[str, ...] expected_final_line_prefix: str + + +@dataclass(frozen=True) +class IntermediateLoaderDefaultE2ETestCase: + description: str + command: tuple[str, ...] + expected_intermediate_rows: tuple[tuple[object, ...], ...] + expected_terminal_rows: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py index 40ffb4dc45..483e1c7853 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py @@ -10,6 +10,7 @@ ChainedLoaderFailureE2ETestCase, ChainedLoaderPruningE2ETestCase, ChainedLoaderSelectionE2ETestCase, + IntermediateLoaderDefaultE2ETestCase, IntermediateLoaderStrategyE2ETestCase, LoaderWaffleShopE2ETestCase, SourceLoaderErrorE2ETestCase, @@ -1191,6 +1192,65 @@ def test_given_chained_loader_project_when_intermediate_uses_strategy_then_appli assert tuple(terminal_rows) == test_case.expected_terminal_rows +@pytest.mark.parametrize( + "test_case", + [ + IntermediateLoaderDefaultE2ETestCase( + description="sqb load writes the docs' intermediate loader as a table", + command=("--no-color", "load", "--select", "+raw_account_metrics"), + expected_intermediate_rows=((1, "Pawnee Parks"), (2, "Eagleton")), + expected_terminal_rows=((1, "active"), (2, "active")), + ), + IntermediateLoaderDefaultE2ETestCase( + description="sqb build writes the docs' intermediate loader as a table", + command=("--no-color", "build", "--select", "+source:raw_account_metrics"), + expected_intermediate_rows=((1, "Pawnee Parks"), (2, "Eagleton")), + expected_terminal_rows=((1, "active"), (2, "active")), + ), + ], + ids=lambda case: case.description, +) +def test_given_intermediate_loader_without_write_strategy_when_loading_twice_then_rows_replace_a_table( + tmp_path: Path, + test_case: IntermediateLoaderDefaultE2ETestCase, +) -> None: + edge_cases: Path = Path("benchmarks/duckshop/edge_cases/loaders") + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="source_loader_schema_behavior", + repo_files=build_schema_behavior_project_files( + source_yaml=(edge_cases / "intermediate_default_accounts.yml").read_text(), + loader_py=(edge_cases / "intermediate_default_accounts.py").read_text(), + ), + ) + db_path: Path = project_dir / "source_loader_schema_behavior.duckdb" + + results: list[subprocess.CompletedProcess[str]] = [ + run_sqb(command=test_case.command, project_dir=project_dir) for _ in range(2) + ] + + for result in results: + assert result.returncode == 0, test_case.description + result.stdout + result.stderr + assert ( + tuple( + query_duckdb( + db_path=db_path, + sql="SELECT account_id, account_name FROM __loader__raw_accounts ORDER BY 1", + ) + ) + == test_case.expected_intermediate_rows + ) + assert ( + tuple( + query_duckdb( + db_path=db_path, + sql="SELECT account_id, metric FROM raw_account_metrics ORDER BY 1", + ) + ) + == test_case.expected_terminal_rows + ) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 048a0e9102..01776fdad9 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -269,6 +269,20 @@ class PythonJobSelectionTestCase: expected_selection: tuple[str, ...] +@dataclass(frozen=True) +class PythonJobGroupTestCase: + description: str + jobs: tuple[tuple[str, str], ...] + expected_groups: dict[str, dict[str, list[str]]] + + +@dataclass(frozen=True) +class PythonJobGroupCycleTestCase: + description: str + jobs: tuple[tuple[str, str], ...] + expected_message: str + + @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 03050f1519..57ceccc803 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -26,6 +26,7 @@ LocalConfig, MeasurementThresholdBound, MeasurementThresholds, + NativeOrchestrationSnapshot, ProjectConfig, SourceColumnEntry, SqlLexicalSyntax, @@ -44,6 +45,8 @@ CompiledModelView, CompiledProjectView, ) +from sqlbuild.orchestration import compile_snapshot +from sqlbuild.orchestration_declarations.models.authoring import Job, JobGroup from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files if TYPE_CHECKING: @@ -750,3 +753,23 @@ def published_texts(test_case: SnapshotExpressionRangeTestCase, tmp_path: Path) column: compiled[slice(*metadata["expression_range"])].decode() for column, metadata in columns.items() } + + +def python_pipeline_snapshot( + tmp_path: Path, declarations: list[Job | JobGroup] +) -> NativeOrchestrationSnapshot: + """Publish the Python pipeline project with the given job declarations.""" + write_files(project_dir=tmp_path, files=PYTHON_PIPELINE_FILES) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + return compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=declarations, + discovered_inputs=discovered, + ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index b230cebb88..4f1c568195 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -125,6 +125,12 @@ class WorkerExecutionTestCase: warehouse_setup: Callable[[Path], None] = lambda _: None +@dataclass(frozen=True) +class ClaimedLoadTestCase(WorkerExecutionTestCase): + description: str + expected_write_set: frozenset[str] = frozenset() + + @dataclass(frozen=True) class UnclaimedLoadTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index abf17ff923..60fb29529b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -1569,6 +1569,22 @@ def hold_write_lock(database: Path) -> duckdb.DuckDBPyConnection: } +INTERMEDIATE_LOADER_FILES: dict[str, str] = { + "sources/shipments.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml" + ).read_text(), + "python/loaders/shipments.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py" + ).read_text(), + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql" + ).read_text(), + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" + ).read_text(), +} + + def without_loads(control: sqlite3.Connection, node_key: str) -> None: """Drop a published node's `loads` in place; its content address and the worker's check stay.""" for node_hash, body in control.execute( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index ff3cfb341e..48db2d8bd6 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -30,6 +30,7 @@ ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( CheckExecutionTestCase, + ClaimedLoadTestCase, ClaimedWorker, GroupExecutionTestCase, SnapshotWorkerTestCase, @@ -37,6 +38,7 @@ WorkerExecutionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + INTERMEDIATE_LOADER_FILES, LOADED_ORDER_FILES, cancel_running_statement, claimed_worker, @@ -605,26 +607,38 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re @pytest.mark.parametrize( "test_case", [ - WorkerExecutionTestCase( + ClaimedLoadTestCase( "a tag-selected job claims its loader chain and hook-read source, not unread ones", "", "SELECT amount FROM orders", "success", [(7,)], files=LOADED_ORDER_FILES, - ) + expected_write_set=frozenset( + {"model:orders", "source:raw_orders", "source:raw_customers", "source:raw_returns"} + ), + ), + ClaimedLoadTestCase( + "a job claims an intermediate loader no source declares", + "", + "SELECT amount FROM orders", + "success", + [(9,)], + files=INTERMEDIATE_LOADER_FILES, + expected_write_set=frozenset( + {"model:orders", "source:raw_shipments", "source:stage_shipments"} + ), + ), ], ids=lambda case: case.description, ) def test_given_loads_when_a_worker_executes_then_the_run_claims_exactly_what_it_loads( - test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + test_case: ClaimedLoadTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: run: dict[str, Any] = worker_result(worker, test_case) assert run["status"] == test_case.expected_status, worker.log.read_text() - assert run_write_set(worker.control_store, run["run_id"]) == frozenset( - {"model:orders", "source:raw_orders", "source:raw_customers", "source:raw_returns"} - ) + assert run_write_set(worker.control_store, run["run_id"]) == test_case.expected_write_set @pytest.mark.parametrize( @@ -639,27 +653,6 @@ def test_given_loads_when_a_worker_executes_then_the_run_claims_exactly_what_it_ {"model:orders", "source:raw_orders", "source:raw_customers"} ), ), - UnclaimedLoadTestCase( - # A known gap: no snapshot node exists for an intermediate loader, so `loads` leaves it out. - "an intermediate loader no source declares is never claimed, so the run still fails", - files={ - "sources/shipments.yml": Path( - "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml" - ).read_text(), - "python/loaders/shipments.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py" - ).read_text(), - "models/orders.sql": Path( - "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql" - ).read_text(), - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" - ).read_text(), - }, - store_edit=lambda _: None, - refused_writer="source:stage_shipments", - expected_write_set=frozenset({"model:orders", "source:raw_shipments"}), - ), ], ids=lambda case: case.description, ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 9cce168cc4..76d37d2d1b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -32,8 +32,11 @@ schedule, sensor, ) -from sqlbuild.orchestration_declarations.models.authoring import Job +from sqlbuild.orchestration_declarations.models.authoring import Job, JobGroup from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + PythonJobGroupCycleTestCase, + PythonJobGroupTestCase, + PythonJobSelectionTestCase, SnapshotCatalogTestCase, SnapshotDeclarationRejectionTestCase, SnapshotDiscoveryRejectionTestCase, @@ -44,7 +47,6 @@ SnapshotLineageTestCase, SnapshotProvidersTestCase, SnapshotRejectionTestCase, - PythonJobSelectionTestCase, SnapshotScheduleTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( @@ -54,6 +56,7 @@ PYTHON_PIPELINE_FILES, TYPE_PROOF_FILES, published_texts, + python_pipeline_snapshot, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -1858,6 +1861,11 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho select="+orders", expected_selection=("model:orders", "model:stg_orders"), ), + PythonJobSelectionTestCase( + description="a task selector selects only the task", + select="task:request_carrier_collection", + expected_selection=("task:request_carrier_collection",), + ), PythonJobSelectionTestCase( description="typed selectors select the task and the asset it depends on", select="task:request_carrier_collection asset:partner_orders_file", @@ -1915,3 +1923,77 @@ def test_given_python_selectors_when_publishing_jobs_then_selections_match_the_c assert snapshot["documents"]["jobs"]["orders_job"]["selection"] == list( test_case.expected_selection ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonJobGroupTestCase( + description="a job waits for the job building the Python node its task depends on", + jobs=( + ("orders_export_job", "python/orders/partner_export"), + ("carrier_sync_job", "python/fulfillment/carrier_sync"), + ), + expected_groups={ + "orders_nightly": { + "carrier_sync_job": ["orders_export_job"], + "orders_export_job": [], + } + }, + ), + PythonJobGroupTestCase( + description="a Python node's SQL dependency orders it after the model's job", + jobs=(("orders_export_job", "asset:partner_orders_file"), ("models_job", "+orders")), + expected_groups={ + "orders_nightly": {"models_job": [], "orders_export_job": ["models_job"]} + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_dependencies_across_jobs_when_publishing_a_group_then_jobs_wait_for_them( + tmp_path: Path, test_case: PythonJobGroupTestCase +) -> None: + jobs: list[Job] = [ + job(name, description="Orders.", select=select) for name, select in test_case.jobs + ] + group: JobGroup = job_group("orders_nightly", description="Nightly orders.", jobs=jobs) + + snapshot: dict[str, Any] = json.loads( + python_pipeline_snapshot(tmp_path, [*jobs, group]).to_json() + ) + + assert { + name: definition["jobs"] for name, definition in snapshot["documents"]["groups"].items() + } == test_case.expected_groups + + +@pytest.mark.parametrize( + "test_case", + [ + PythonJobGroupCycleTestCase( + description="a Python edge that closes a cycle names both jobs and the edges", + jobs=( + ("staging_job", "stg_orders task:request_carrier_collection"), + ("export_job", "asset:partner_orders_file"), + ), + expected_message=( + "group dependencies contain a cycle in job group orders_nightly: " + "export_job waits for staging_job (model:orders reads model:stg_orders); " + "staging_job waits for export_job " + "(task:request_carrier_collection reads asset:partner_orders_file)" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_python_dependency_cycle_across_jobs_when_publishing_a_group_then_it_is_rejected( + tmp_path: Path, test_case: PythonJobGroupCycleTestCase +) -> None: + jobs: list[Job] = [ + job(name, description="Orders.", select=select) for name, select in test_case.jobs + ] + group: JobGroup = job_group("orders_nightly", description="Nightly orders.", jobs=jobs) + + with pytest.raises(ValueError, match=re.escape(test_case.expected_message)): + python_pipeline_snapshot(tmp_path, [*jobs, group]) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py index c36f6956be..ae3937e7da 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py @@ -35,6 +35,13 @@ expected_sql_names=frozenset(), expected_python_node_names=frozenset({"prepare_orders", "export_orders"}), ), + PythonSqlSelectorTestCase( + description="selects a Python asset without its unselected Python dependency", + select=("path:python/assets",), + exclude=(), + expected_sql_names=frozenset(), + expected_python_node_names=frozenset({"export_orders"}), + ), PythonSqlSelectorTestCase( description="selects expanded source terminal loader requirement for run", select=("+source:raw_orders",), @@ -79,15 +86,6 @@ def test_given_run_selectors_when_resolving_then_excludes_python_checks( expected_error_type=ValueError, expected_error_fragment=("Python checks are not selectable here: check_orders_export"), ), - PythonSqlSelectorErrorTestCase( - description="rejects direct asset path with unselected Python dependency for run", - select=("path:python/assets",), - exclude=(), - expected_error_type=ValueError, - expected_error_fragment=( - "Python node 'export_orders' depends on unselected Python node 'prepare_orders'" - ), - ), ], ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 3f20e3ab26..ddc395623b 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -127,7 +127,7 @@ def export_orders(ctx): - `ctx.skip(reason, mode=...)` skips the node. `mode` accepts `"soft"` (default, skips only this node) or `"hard"` (also blocks dependents), as a string or the `SkipMode` enum from `sqlbuild.tasks`/`sqlbuild.assets`. - Assets may pass `materialized=True`/`False` to record whether an artifact was produced. -Downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. +Within one run, downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. ## Result persistence @@ -161,6 +161,18 @@ dependency fails or is skipped. A dependency that is not part of the run (not se unmanaged source) is read as it currently exists, so `sqb build --select task:export_orders` runs the task against the relations already in the warehouse. +Python dependencies follow the same rule. A task or asset selected without its Python upstream +reads that upstream's latest successful result through `ctx.result_of`. If the upstream has never +succeeded for the target, the node fails with an error that names the upstream, for example: + +```text +Upstream task 'prepare_orders' has no successful result to read. Build it first with +'sqb build --select task:prepare_orders', or build both with 'sqb build --select +asset:export_orders'. +``` + +Jobs, sensors, roles and ad-hoc runs on the server resolve selectors the same way, so a job can +select one Python node and a job group orders it after the job that builds its upstream. + The commands differ in what they include by default: | Command | SQL | Loaders / tasks / assets | Checks | Audits | From 0e665a98b2eeb2b5025f3eaccd813594be32ff93 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:20:11 +0000 Subject: [PATCH 4/7] feat: select Python-only jobs and SQL tests across CLI and server runs - `job:` and bare job names resolve jobs through the Python-aware resolver publication uses, so a job that selects only Python nodes builds them. - Server ad-hoc runs accept `test:` selectors; a selected SQL test reports as a run node rather than as an attached check. - A provider function whose context is positional-only (`ctx, /`) receives it by position instead of failing at run time. - A role must grant the intermediate loader a job loads to run that job. --- .../intermediate_loader_roles.toml | 15 ++++ .../orchestration/python_node_jobs.py | 10 +++ .../python_pipeline_carrier_sync.py | 10 +++ .../python_pipeline_partner_export.py | 9 +++ .../src/graph/_helpers/jobs.rs | 14 +++- .../src/graph/main/attach_jobs_with.rs | 18 +++++ .../sqlbuild-analysis/src/graph/main/mod.rs | 1 + .../src/python_selection/_helpers/unified.rs | 48 +++++++++++- .../main/attach_python_jobs.rs | 40 ++++++++++ .../src/python_selection/main/mod.rs | 1 + .../tests/test_python_selection.rs | 2 +- .../src/adhoc/_helpers/planner.rs | 59 ++++++++++++++- crates/sqlbuild-cli/src/adhoc/constants.rs | 6 ++ .../src/adhoc/tests/test_adhoc_planner.rs | 58 ++++++++++++++- .../src/runs/_helpers/requests/adhoc.rs | 24 ++++++ .../_helpers/compiled_project/job_catalog.rs | 20 +++-- .../compiled_project/snapshot_execution.rs | 8 +- .../_helpers/graph/python_selection.rs | 17 ++++- .../_helpers/location_command/constants.rs | 3 + .../_helpers/location_command/worker.rs | 13 ++-- src/sqlbuild/provider/_helpers/injection.py | 7 +- .../cli/commands/main/build/helpers.py | 3 + .../main/build/test_python_node_selection.py | 24 ++++++ .../compiler/compiled_project/helpers.py | 17 ++--- .../publication/_test_types.py | 7 ++ .../publication/person/helpers.py | 16 ++++ .../publication/test_adhoc.py | 8 ++ .../publication/test_project_roles.py | 73 +++++++++++++++++++ .../src/sqlbuild/provider/_helpers/helpers.py | 8 ++ .../provider/_helpers/test_injection.py | 12 +++ .../content/docs/docs/concepts/selectors.mdx | 2 +- 31 files changed, 517 insertions(+), 36 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py create mode 100644 crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml new file mode 100644 index 0000000000..78acf58022 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml @@ -0,0 +1,15 @@ +# Roles over a loader chain whose middle loader no source declares: running the job needs a grant +# on the intermediate loader's node as well as the model and the source it feeds. +[roles.shipments_runner] +description = "Run the shipments build, including its staging loader" + +[[roles.shipments_runner.grants]] +nodes = ["orders", "source:raw_shipments", "loader:stage_shipments"] +permissions = ["job.run"] + +[roles.orders_runner] +description = "Run the orders build without the staging loader" + +[[roles.orders_runner.grants]] +nodes = ["orders", "source:raw_shipments"] +permissions = ["job.run"] diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py new file mode 100644 index 0000000000..d153e434ff --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py @@ -0,0 +1,10 @@ +"""Jobs whose selections are Python nodes, which `--select job:` must select as published.""" + +from sqlbuild.orchestration import job + +customer_counts = job( + "customer_counts", + description="Count customers.", + select="task:count_customers asset:customers_extract", +) +order_counts = job("order_counts", description="Count orders.", select="+task:count_orders") diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py new file mode 100644 index 0000000000..221447d1c8 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py @@ -0,0 +1,10 @@ +"""A task whose Python upstream another job builds, so a job group must order the two jobs.""" + +from python.orders.partner_export.node import partner_orders_file + +from sqlbuild.tasks import task + + +@task(depends_on=partner_orders_file, tags=["orders"], description="Book collection.") +def request_carrier_collection(ctx): + return None diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py new file mode 100644 index 0000000000..dfe6194d20 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py @@ -0,0 +1,9 @@ +"""An asset one job builds that a task in another job of the same group depends on.""" + +from sqlbuild.assets import asset +from sqlbuild.refs import model + + +@asset(depends_on=model("orders"), tags=["orders"], description="Partner order file.") +def partner_orders_file(ctx): + return None diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs b/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs index 329bbbc646..f1c9480ace 100644 --- a/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs +++ b/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs @@ -8,17 +8,23 @@ use crate::graph::errors::SelectorError; use crate::graph::models::{JobSelection, ProjectGraph}; /// Resolve every job on the graph without jobs attached, as publication does, then attach them. -pub(crate) fn attach( +pub(crate) fn attach(graph: ProjectGraph, catalog: BTreeMap) -> ProjectGraph { + attach_with(graph, catalog, |graph, select, exclude| { + resolve(graph, select, exclude).map(|keys| keys.into_iter().collect()) + }) +} + +/// [`attach`] with each job's selectors resolved by `resolve_job` on the graph without jobs. +pub(crate) fn attach_with( mut graph: ProjectGraph, catalog: BTreeMap, + resolve_job: impl Fn(&ProjectGraph, &[String], &[String]) -> Result, ) -> ProjectGraph { graph.jobs = None; let mut resolved: BTreeMap> = BTreeMap::new(); for (name, selection) in &catalog { let keys: Result = match selection { - JobSelection::Job { select, exclude } => { - resolve(&graph, select, exclude).map(|keys| keys.into_iter().collect()) - } + JobSelection::Job { select, exclude } => resolve_job(&graph, select, exclude), JobSelection::Resolved { keys } => Ok(keys.iter().cloned().collect()), JobSelection::Group { .. } => continue, }; diff --git a/crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs b/crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs new file mode 100644 index 0000000000..091081bc50 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs @@ -0,0 +1,18 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::models::{JobSelection, ProjectGraph}; + +/// [`super::attach_jobs::attach_jobs`] with each job's selectors resolved by `resolve_job`. +pub fn attach_jobs_with( + graph: ProjectGraph, + catalog: BTreeMap, + resolve_job: impl Fn( + &ProjectGraph, + &[String], + &[String], + ) -> Result, SelectorError>, +) -> ProjectGraph { + crate::graph::_helpers::jobs::attach_with(graph, catalog, resolve_job) +} diff --git a/crates/sqlbuild-analysis/src/graph/main/mod.rs b/crates/sqlbuild-analysis/src/graph/main/mod.rs index ccd21dc172..05d0272f4d 100644 --- a/crates/sqlbuild-analysis/src/graph/main/mod.rs +++ b/crates/sqlbuild-analysis/src/graph/main/mod.rs @@ -1,4 +1,5 @@ pub mod attach_jobs; +pub mod attach_jobs_with; pub mod build_project_graph; pub mod build_resources; pub mod downstream_edges; diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index 7bd1a63517..4c94cd2f87 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -7,6 +7,7 @@ use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; use sqlbuild_core::text::main::glob_match::glob_match; use crate::assembly::project::types::ObjectKey; +use crate::definition_changes::constants::PYTHON_NODE_KINDS; use crate::graph::errors::SelectorError; use crate::graph::main::build_resources::build_resources; use crate::graph::main::edge_keys::edge_keys; @@ -282,7 +283,8 @@ fn single_side(context: &SelectionContext<'_>, raw: &str) -> Resolved { let atoms: Atoms = match kind.as_str() { TEST_KIND => return Err(unit_test_error(raw)), PATH_KIND => resolve_path(context, raw, &value)?, - SEED_KIND | SOURCE_KIND | JOB_KIND => resolve_sql(context, raw)?, + SEED_KIND | SOURCE_KIND => resolve_sql(context, raw)?, + JOB_KIND => resolve_job(context, raw)?, TASK_KIND | ASSET_KIND | LOADER_KIND | CHECK_KIND => { return resolve_typed(context, raw, (&kind, &value)); } @@ -314,6 +316,48 @@ fn resolve_sql(context: &SelectionContext<'_>, raw: &str) -> Resolved { )?)) } +/// A job's SQL keys and Python nodes, with `+` also expanding them over the Python node graph. +fn resolve_job(context: &SelectionContext<'_>, raw: &str) -> Resolved { + let ParsedSelector::Kind { + upstream, + downstream, + .. + } = parse_selector(raw)? + else { + return resolve_sql(context, raw); + }; + let mut atoms: Atoms = Atoms::new(); + let mut members: Vec = Vec::new(); + for atom in resolve_sql(context, raw)? { + match atom { + Atom::Sql((kind, name)) + if PYTHON_NODE_KINDS.contains(&kind.as_str()) + && context.nodes.node(&name).is_some() => + { + members.push(name); + } + atom => { + atoms.insert(atom); + } + } + } + let mut names: Names = members.iter().cloned().collect(); + for (expand, direction) in [ + (upstream, EdgeDirection::Upstream), + (downstream, EdgeDirection::Downstream), + ] { + if expand { + names.extend( + context + .nodes + .closure(members.iter().map(String::as_str), direction), + ); + } + } + atoms.extend(python_atoms(names)); + Ok(atoms) +} + fn resolve_python(context: &SelectionContext<'_>, raw: &str) -> Resolved { Ok(python_atoms(select_nodes( &context.nodes, @@ -407,7 +451,7 @@ fn resolve_name(context: &SelectionContext<'_>, raw: &str, value: &str) -> Resol (true, true) => Err(both_error(value)), (true, false) => resolve_sql(context, raw), (false, true) => resolve_python(context, raw), - (false, false) if is_job(context, value) => resolve_sql(context, raw), + (false, false) if is_job(context, value) => resolve_job(context, raw), (false, false) => Err(unknown_name(value)), }; } diff --git a/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs new file mode 100644 index 0000000000..6e3095f6f7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs @@ -0,0 +1,40 @@ +use std::collections::BTreeMap; + +use crate::graph::main::attach_jobs_with::attach_jobs_with; +use crate::graph::models::{JobSelection, ProjectGraph}; +use crate::python_selection::_helpers::nodes::sql_names; +use crate::python_selection::_helpers::unified::run_selection; +use crate::python_selection::models::{ + NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, +}; + +/// Attach jobs as publication resolves them, keeping each job's Python nodes as `(kind, name)` keys. +pub fn attach_python_jobs( + sql: SqlSelectionGraph<'_>, + python: &PythonSelectionGraph, + catalog: BTreeMap, +) -> ProjectGraph { + let kinds: BTreeMap<&str, &str> = python + .nodes + .iter() + .map(|node| (node.name.as_str(), node.kind.as_str())) + .collect(); + attach_jobs_with(sql.graph.clone(), catalog, |graph, select, exclude| { + let context: SelectionContext<'_> = SelectionContext { + sql: SqlSelectionGraph { + graph, + sources: sql.sources, + }, + nodes: NodeIndex::new(python), + sql_names: sql_names(graph), + }; + let RunSelection { + sql_keys, + python_node_names, + } = run_selection(&context, select, exclude)?; + let python_keys = python_node_names + .into_iter() + .filter_map(|name| Some((kinds.get(name.as_str())?.to_string(), name))); + Ok(sql_keys.into_iter().chain(python_keys).collect()) + }) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs index 3dace0806a..487d121f6d 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs @@ -1,3 +1,4 @@ +pub mod attach_python_jobs; pub mod published_python_graph; pub mod python_node_graph_indexes; pub mod resolve_python_node_excludes; diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs index 27bc2914bd..7a2c24b305 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs @@ -1,5 +1,5 @@ use crate::python_selection::tests::helpers::{ - excluded_names, expected_names, expected_run, indexed_folder, node_names, orders_python_graph, + excluded_names, expected_names, expected_run, indexed_folder, node_names, published_orders_graph, run, run_over, }; use crate::python_selection::tests::test_types::{ diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs index 389c0fcd79..ddb5193499 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs @@ -8,6 +8,8 @@ use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; use sqlbuild_analysis::graph::models::{GraphResource, JobSelection}; +use sqlbuild_analysis::planner_selection::main::split_unit_test_selectors::split_unit_test_selectors; +use sqlbuild_analysis::planner_selection::models::{SelectorGroups, UnitTestSelectorNames}; use sqlbuild_analysis::python_selection::main::published_python_graph::published_python_graph; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection; use sqlbuild_analysis::python_selection::models::{PublishedPythonNode, SqlSelectionGraph}; @@ -18,7 +20,7 @@ use sqlbuild_orchestration::runs::types::{CommandPlanner, PublishedJobs}; use crate::adhoc::constants::{ ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, CHANGED_COUNTS_DEST, CHANGED_DEST, COMMAND_DEST, FRESHNESS_VERB, INTEGER_CONVERTERS, LOAD_SOURCES_DEST, POSITIVE_INTEGER_CONVERTER, - REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, + REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, SQL_TEST_KIND, UNIT_TEST_VERBS, }; use crate::argv::main::command_settings_arguments::command_settings_arguments; use crate::argv::main::job_settings_arguments::job_settings_arguments; @@ -84,7 +86,7 @@ impl CommandPlanner for CliPlanner { seeds } FRESHNESS_VERB => freshness_sources(publication, &select, &exclude)?, - _ => resolve(publication, &select, &exclude)?, + _ => resolve_with_tests(verb, publication, &select, &exclude)?, }; Ok(CommandPlan { verb: verb.clone(), @@ -201,6 +203,58 @@ fn resolve( Ok(selected) } +/// [`resolve`] with `test:` and unit-test names selecting `sql_test:` keys, as `sqb test` does. +fn resolve_with_tests( + verb: &str, + (resources, jobs): (&[PlannerResource], &PublishedJobs), + select: &[String], + exclude: &[String], +) -> Result, String> { + let names = UnitTestSelectorNames { + tests: resources + .iter() + .filter(|resource| is_sql_test(resource)) + .map(|resource| name_of(&resource.key).to_owned()) + .collect(), + resources: resources + .iter() + .filter(|resource| !is_sql_test(resource)) + .map(|resource| name_of(&resource.key).to_owned()) + .chain(jobs.keys().cloned()) + .collect(), + }; + let split = split_unit_test_selectors( + SelectorGroups { select, exclude }, + &names, + UNIT_TEST_VERBS.contains(&verb), + ) + .map_err(|error| format!("{}: {}", error.code, error.message))?; + if !split.excluded_test_names.is_empty() { + return Err( + "an ad-hoc run cannot exclude unit tests; select the unit tests to run instead".into(), + ); + } + let mut keys: BTreeSet = if split.models_selected { + resolved((resources, jobs), &split.select, &split.exclude)? + } else { + BTreeSet::new() + }; + keys.extend( + split + .test_names + .iter() + .map(|name| format!("{SQL_TEST_KIND}:{name}")), + ); + if keys.is_empty() { + return Err("the selectors match no nodes in the current publication".into()); + } + Ok(keys) +} + +fn is_sql_test(resource: &PlannerResource) -> bool { + resource.key.split_once(':').map(|(kind, _)| kind) == Some(SQL_TEST_KIND) +} + fn resolved( (resources, jobs): (&[PlannerResource], &PublishedJobs), select: &[String], @@ -208,6 +262,7 @@ fn resolved( ) -> Result, String> { let (python, sql): (Vec<&PlannerResource>, Vec<&PlannerResource>) = resources .iter() + .filter(|resource| !is_sql_test(resource)) .partition(|resource| python_kind(&resource.key).is_some()); let graph_resources: Vec = sql .iter() diff --git a/crates/sqlbuild-cli/src/adhoc/constants.rs b/crates/sqlbuild-cli/src/adhoc/constants.rs index c4fc1ca671..0de8284749 100644 --- a/crates/sqlbuild-cli/src/adhoc/constants.rs +++ b/crates/sqlbuild-cli/src/adhoc/constants.rs @@ -6,6 +6,12 @@ pub(crate) const ADHOC_VERBS: [&str; 5] = ["build", "test", "seed", "audit", "fr /// The verb whose settings must also form valid job settings. pub(crate) const BUILD_VERB: &str = "build"; +/// Verbs that select SQL unit tests by `test:` or by name, as on the command line. +pub(crate) const UNIT_TEST_VERBS: [&str; 2] = ["build", "test"]; + +/// The published kind of a SQL unit test. +pub(crate) const SQL_TEST_KIND: &str = "sql_test"; + /// The verb that runs only the seeds of its selection. pub(crate) const SEED_VERB: &str = "seed"; diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index 9855c4cbc3..fe0b38f40a 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -28,6 +28,7 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select resource("model:customers", &[], &[], Some("models/staging")), resource("source:shop.raw_orders", &[], &[], None), resource("seed:currencies", &[], &[], None), + resource("sql_test:orders_keep_amounts", &["model:orders"], &[], None), ]; let jobs: PublishedJobs = BTreeMap::from([ ( @@ -229,6 +230,40 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select command: &["test", "-s", "orders", "--case", "happy_path"], expected_plan: Ok((&["model:orders"], json!({"case": "happy_path"}))), }, + AdhocPlanTestCase { + description: "test: selects one unit test", + command: &["test", "--select", "test:orders_keep_amounts"], + expected_plan: Ok((&["sql_test:orders_keep_amounts"], json!({}))), + }, + AdhocPlanTestCase { + description: "a unit test's bare name selects it, as on the command line", + command: &["test", "--select", "orders_keep_amounts"], + expected_plan: Ok((&["sql_test:orders_keep_amounts"], json!({}))), + }, + AdhocPlanTestCase { + description: "build selects a model and a unit test together", + command: &["build", "--select", "customers", "test:orders_keep_amounts"], + expected_plan: Ok(( + &["model:customers", "sql_test:orders_keep_amounts"], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "excluding a unit test is refused, since a run freezes only what it selects", + command: &[ + "test", + "--select", + "orders", + "--exclude", + "test:orders_keep_amounts", + ], + expected_plan: Err("an ad-hoc run cannot exclude unit tests"), + }, + AdhocPlanTestCase { + description: "audit refuses unit-test selectors as the command line does", + command: &["audit", "--select", "test:orders_keep_amounts"], + expected_plan: Err("selects a unit test"), + }, AdhocPlanTestCase { description: "audit settings come from the audit parser", command: &["audit", "-s", "orders", "--concurrency", "2"], @@ -314,8 +349,29 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co Some("python/fulfillment/carrier_sync"), ), ]; - let jobs: PublishedJobs = BTreeMap::new(); + let jobs: PublishedJobs = BTreeMap::from([( + "carrier_sync".to_owned(), + BTreeSet::from(["task:request_carrier_collection".to_owned()]), + )]); let test_cases = [ + AdhocPlanTestCase { + description: "a published job selects its Python nodes", + command: &["build", "--select", "job:carrier_sync"], + expected_plan: Ok((&["task:request_carrier_collection"], json!({}))), + }, + AdhocPlanTestCase { + description: "upstream of a published job crosses its Python node's dependencies", + command: &["build", "--select", "+job:carrier_sync"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, AdhocPlanTestCase { description: "typed selectors select the Python nodes", command: &[ diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index 6de44e69d1..45ca1649da 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -32,6 +32,10 @@ use crate::store::models::{ControlTransaction, Parameter}; const GRAPH_KINDS: [&str; 5] = ["model", "source", "seed", "udf", "table_fn"]; /// Python node kinds, in the order the CLI discovers them. const PYTHON_NODE_KINDS: [&str; 4] = ["task", "asset", "loader", "check"]; +/// SQL unit tests, which `test:` selects; each lists the models it tests. +const SQL_TEST_KIND: &str = "sql_test"; +/// Where a SQL unit test's definition lists the models it tests. +const SQL_TEST_MODELS_POINTER: &str = "/test/payload/expected_model_names"; /// Marks the source node an intermediate loader writes, which the CLI's SQL graph does not list. const INTERMEDIATE_LOADER_FIELD: &str = "intermediate_loader"; @@ -179,6 +183,18 @@ fn planner_resources( }); } } + resources.extend( + nodes + .iter() + .filter(|(_, node)| node.kind == SQL_TEST_KIND) + .map(|(key, node)| PlannerResource { + key: key.clone(), + deps: tested_models(&node.definition), + tags: Vec::new(), + folder: None, + loader: None, + }), + ); resources } @@ -193,6 +209,14 @@ fn model_folder(definition: &Value) -> String { .map_or_else(|| ".".to_owned(), |(folder, _)| folder.to_owned()) } +/// The model keys a SQL unit test tests. +fn tested_models(definition: &Value) -> Vec { + texts(definition.pointer(SQL_TEST_MODELS_POINTER)) + .into_iter() + .map(|model| format!("model:{model}")) + .collect() +} + fn texts(value: Option<&Value>) -> Vec { value .and_then(Value::as_array) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs index f5122b631c..79b2b2e30c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs @@ -61,15 +61,25 @@ pub(crate) fn graph_with_jobs<'g>( selectors: &[String], also_known: &[&str], ) -> PyResult> { - if graph.jobs.is_some() || !selectors_need_jobs(graph, selectors, also_known) { - return Ok(Cow::Borrowed(graph)); - } - Ok(match job_catalog(py)? { - Some((catalog, _)) => Cow::Owned(attach_jobs(graph.clone(), catalog)), + Ok(match needed_jobs(py, graph, selectors, also_known)? { + Some(catalog) => Cow::Owned(attach_jobs(graph.clone(), catalog)), None => Cow::Borrowed(graph), }) } +/// The command's jobs when `selectors` name one and `graph` has none attached yet. +pub(crate) fn needed_jobs( + py: Python<'_>, + graph: &ProjectGraph, + selectors: &[String], + also_known: &[&str], +) -> PyResult>> { + if graph.jobs.is_some() || !selectors_need_jobs(graph, selectors, also_known) { + return Ok(None); + } + Ok(job_catalog(py)?.map(|(catalog, _)| catalog)) +} + /// The command's job and job group names, loaded on first use; empty outside a command. pub(crate) fn job_names(py: Python<'_>) -> PyResult> { Ok(job_catalog(py)? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index 509d5b7530..a9392f573d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -260,12 +260,15 @@ impl WorkerBuild { self.complete(py, key, (outcome, None, (None, None))) } - /// A test or audit the snapshot records has started; the node it attaches to starts with it. + /// A test or audit the snapshot records has started; a selected SQL test reports as a node. pub(crate) fn start_attached(&self, py: Python<'_>, key: &str) -> PyResult<()> { self.check_cancelled(py)?; let Some(nodes) = self.attached.get(key) else { return Ok(()); }; + if self.execution.claim.request.selection.contains(key) { + return self.start_key(py, key); + } if self .checks .lock() @@ -313,6 +316,9 @@ impl WorkerBuild { } else { outcome }; + if self.execution.claim.request.selection.contains(key) { + return self.complete(py, key, (outcome, None, (None, None))); + } self.finish_check(py, key, outcome)?; if outcome == NodeOutcome::Skipped { return Ok(()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 3859130776..3a5d3b1e6c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -1,5 +1,6 @@ //! Python node and unified SQL/Python selector resolution, and the Python node graph indexes. +use std::borrow::Cow; use std::collections::BTreeSet; use pyo3::prelude::{ @@ -10,6 +11,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::errors::SelectorError; use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_analysis::python_selection::main::attach_python_jobs::attach_python_jobs; use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; @@ -21,7 +23,7 @@ use sqlbuild_analysis::python_selection::models::{ }; use crate::bindings::_helpers::boundary::panics::compiler_guard; -use crate::bindings::_helpers::compiled_project::job_catalog::graph_with_jobs; +use crate::bindings::_helpers::compiled_project::job_catalog::needed_jobs; use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::records::construct::error_class; @@ -272,7 +274,18 @@ pub(crate) fn resolve_python_sql_run_selection( let nodes: PythonSelectionGraph = selection_graph(python_graph)?; let node_names: Vec<&str> = nodes.nodes.iter().map(|node| node.name.as_str()).collect(); let selectors: Vec = select.iter().chain(&exclude).cloned().collect(); - let graph = graph_with_jobs(py, native.get().graph(), &selectors, &node_names)?; + let graph: Cow<'_, ProjectGraph> = + match needed_jobs(py, native.get().graph(), &selectors, &node_names)? { + Some(catalog) => Cow::Owned(attach_python_jobs( + SqlSelectionGraph { + graph: native.get().graph(), + sources: &sources, + }, + &nodes, + catalog, + )), + None => Cow::Borrowed(native.get().graph()), + }; let selection: RunSelection = compiler_guard(|| { Ok(run_selection( SqlSelectionGraph { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index b9e42eaa95..fb1c70fdfc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -33,3 +33,6 @@ pub(crate) const REACH_SETTING: &str = "reach"; pub(crate) const JOB_RUN_REACH: &str = "all"; /// What a job run with `changed: True` counts when `[changes] counts` is not set: every change. pub(crate) const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; + +/// A SQL unit test's published kind; a frozen run selects it back with `test:`. +pub(crate) const SQL_TEST_NODE_KIND: &str = "sql_test"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index ec62b33a15..889158c12d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -34,6 +34,7 @@ use crate::bindings::_helpers::discovery::project_config::configured_changes; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::constants::{ CHANGED_SETTING, COMMAND_IMPLEMENTATIONS, JOB_RUN_COUNTS, JOB_RUN_REACH, REACH_SETTING, + SQL_TEST_NODE_KIND, }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::recall::console::write_stderr; @@ -59,7 +60,7 @@ fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap definition + SQL_TEST_NODE_KIND => definition .pointer("/test/payload/expected_model_names") .and_then(Value::as_array) .into_iter() @@ -152,12 +153,14 @@ fn command_request<'py>( let (kind, name) = key .split_once(':') .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; - let selector = if PYTHON_NODE_KINDS.contains(&kind) { - key.as_str() + let selector: String = if PYTHON_NODE_KINDS.contains(&kind) { + key.clone() + } else if kind == SQL_TEST_NODE_KIND { + format!("test:{name}") } else { - name + name.to_owned() }; - argv.extend([PyString::new(py, "--select"), PyString::new(py, selector)]); + argv.extend([PyString::new(py, "--select"), PyString::new(py, &selector)]); } let parsed = parse_cli_invocation(py, argv, false)?; let args = field(&parsed, "args")?; diff --git a/src/sqlbuild/provider/_helpers/injection.py b/src/sqlbuild/provider/_helpers/injection.py index e7680a16ef..4718700410 100644 --- a/src/sqlbuild/provider/_helpers/injection.py +++ b/src/sqlbuild/provider/_helpers/injection.py @@ -73,7 +73,12 @@ def call_with_provider_injection( if not context_bound and parameter.default is inspect.Parameter.empty: kwargs[parameter.name] = context context_bound = True - return function(**kwargs) + positional: list[object] = [] + for parameter in signature.parameters.values(): + if parameter.kind != inspect.Parameter.POSITIONAL_ONLY or parameter.name not in kwargs: + break + positional.append(kwargs.pop(parameter.name)) + return function(*positional, **kwargs) def _provider_for_parameter( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 95f42565c1..c1e0f92112 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -895,6 +895,9 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: 'MODEL (description "Test model order_summary.", ' 'materialized table);\nSELECT count(*) AS order_count FROM __ref("orders")\n' ), + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py" + ).read_text(), "python/tasks/reports.py": ( "from python.tasks.counts import count_orders\nfrom sqlbuild.tasks import task\n\n\n" "@task(depends_on=count_orders)\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py index 1ec298fe6b..80803f98f7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py @@ -104,6 +104,30 @@ expected_row_counts={"orders": 2}, expected_node_counts={"orders_extract": 2}, ), + PythonNodeSelectionE2ETestCase( + description="a job selecting only Python nodes builds them", + select=("job:customer_counts",), + expected_exit_code=0, + expected_output_patterns=(r"count_customers\s+OK", r"customers_extract\s+OK"), + expected_row_counts={"raw_customers": 3}, + expected_node_counts={"count_customers": 3, "customers_extract": 3}, + ), + PythonNodeSelectionE2ETestCase( + description="a job named without job: builds its Python nodes", + select=("customer_counts",), + expected_exit_code=0, + expected_output_patterns=(r"count_customers\s+OK", r"customers_extract\s+OK"), + expected_row_counts={"raw_customers": 3}, + expected_node_counts={"count_customers": 3, "customers_extract": 3}, + ), + PythonNodeSelectionE2ETestCase( + description="a job selecting a task with its upstream model builds both", + select=("job:order_counts",), + expected_exit_code=0, + expected_output_patterns=(r"count_orders\s+OK",), + expected_row_counts={"orders": 2}, + expected_node_counts={"count_orders": 2}, + ), PythonNodeSelectionE2ETestCase( description="task is skipped when its selected upstream model fails", select=("+task:count_orders",), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 57ceccc803..bbaca4925f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -78,17 +78,12 @@ "MODEL (description 'Orders.', tags [orders]);\n\n" 'SELECT order_id FROM __ref("stg_orders")\n' ), - "python/orders/partner_export/node.py": ( - "from sqlbuild.assets import asset\nfrom sqlbuild.refs import model\n\n\n" - "@asset(depends_on=model('orders'), tags=['orders'], description='Partner order file.')\n" - "def partner_orders_file(ctx):\n return None\n" - ), - "python/fulfillment/carrier_sync/node.py": ( - "from python.orders.partner_export.node import partner_orders_file\n" - "from sqlbuild.tasks import task\n\n\n" - "@task(depends_on=partner_orders_file, tags=['orders'], description='Book collection.')\n" - "def request_carrier_collection(ctx):\n return None\n" - ), + "python/orders/partner_export/node.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py" + ).read_text(), + "python/fulfillment/carrier_sync/node.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py" + ).read_text(), } TYPE_PROOF_FILES: dict[str, str] = { diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 4f1c568195..ddeb7ebdee 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -371,6 +371,13 @@ class ProjectRolesTestCase: expected_denial: str +@dataclass(frozen=True) +class ProjectRoleRunTestCase: + description: str + role: str + expected_response: tuple[int, Any] + + @dataclass(frozen=True, kw_only=True) class ChangedJobTestCase(WorkerExecutionTestCase): description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index 512b7a78dd..8357d5d100 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -265,6 +265,22 @@ def person_request( return json.load(response) +def request_person_run( + api: SnapshotApi, session: tuple[str, str], body: dict[str, Any] +) -> tuple[int, Any]: + """A person's run request: the response on success, else the denial without its decision.""" + request: Request = Request( + api.url + "/api/control/runs", + data=json.dumps(body).encode(), + headers=person_headers(session), + ) + try: + with urlopen(request, timeout=5) as response: + return 200, json.load(response) + except HTTPError as denied: + return denied.code, json.load(denied)["error"].split("; decision ")[0] + + def person_login(api: SnapshotApi, name: str) -> tuple[str, str]: request: Request = Request( api.url + "/api/auth/password/login", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py index cd3ce745b0..703856d119 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py @@ -260,6 +260,14 @@ def test_given_an_ad_hoc_command_when_checked_and_launched_then_the_worker_runs_ ], rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, ), + AdhocVerbTestCase( + description="test: runs only the selected verified SQL test", + prerequisites=[], + command=["test", "--select", "test:customers_have_ids"], + expected_selection=["sql_test:customers_have_ids"], + expected_run_nodes=[("sql_test:customers_have_ids", "node", "success")], + rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, + ), AdhocVerbTestCase( description="freshness observes the verified sources upstream of the selection", prerequisites=[], diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py index fc6221eb23..0215796c88 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py @@ -3,13 +3,16 @@ import subprocess from pathlib import Path from typing import Any +from uuid import uuid4 import pytest from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ProjectRoleRunTestCase, ProjectRolesTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + INTERMEDIATE_LOADER_FILES, publish_generation, read_person_sql, snapshot_api, @@ -19,6 +22,7 @@ person_login, person_request, provision_person, + request_person_run, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -100,3 +104,72 @@ def test_given_project_roles_when_published_then_person_sql_follows_selector_cov assert rewritten.returncode == 0, rewritten.stderr assert read_person_sql(api, session, "model:customers") == (200, "model:customers") assert read_person_sql(api, session, "model:orders")[0] == 403 + + +@pytest.mark.parametrize( + "test_case", + [ + ProjectRoleRunTestCase( + description="a role granting the loader chain runs the job", + role="shipments_runner", + expected_response=(200, {"applied": True}), + ), + ProjectRoleRunTestCase( + description="a role missing the intermediate loader is refused", + role="orders_runner", + expected_response=(403, "missing job.run permission for this target"), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_job_loading_an_intermediate_loader_when_a_role_runs_it_then_the_grant_must_cover_it( + test_case: ProjectRoleRunTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + "roles.toml": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml" + ).read_text(), + } + | INTERMEDIATE_LOADER_FILES, + ) + with snapshot_api( + tmp_path / "control", + server_arguments=("--password-origin", "https://control.example.com"), + ) as api: + published: subprocess.CompletedProcess[str] = publish_generation(project, api, 1) + assert published.returncode == 0, published.stderr + admin: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-runner" + ) + assert person_request( + api, + "/api/admin/roles", + body={ + "principal_id": person["principal_id"], + "role": test_case.role, + "location_id": api.location_id, + "target": "development", + }, + session=person_login(api, admin["name"]), + ) == {"applied": True} + request: dict[str, Any] = { + "run_id": str(uuid4()), + "location_id": api.location_id, + "target": "development", + "job_key": "annual_orders", + "reason": "Refresh shipments", + "priority": 0, + } + session: tuple[str, str] = person_login(api, person["name"]) + assert request_person_run(api, session, request) == test_case.expected_response diff --git a/tests/unit/src/sqlbuild/provider/_helpers/helpers.py b/tests/unit/src/sqlbuild/provider/_helpers/helpers.py index 26a1bbdb83..23b22ab3c4 100644 --- a/tests/unit/src/sqlbuild/provider/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/provider/_helpers/helpers.py @@ -30,6 +30,14 @@ def context_and_provider(ctx: object, slack_provider: SlackProvider) -> tuple[ob return ctx, slack_provider.label +def positional_only_context(ctx: object, /, *, slack_provider: SlackProvider) -> tuple[object, str]: + return ctx, slack_provider.label + + +def keyword_only_provider(ctx: object, *, slack_provider: SlackProvider) -> tuple[object, str]: + return ctx, slack_provider.label + + def provider_only(slack_provider: SlackProvider) -> str: return slack_provider.label diff --git a/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py b/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py index a930e53846..0ba9419ac2 100644 --- a/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py +++ b/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py @@ -20,9 +20,11 @@ SlackProvider, context_and_provider, context_only, + keyword_only_provider, load_module, mismatched_provider, missing_provider, + positional_only_context, provider_only, reserved_context_conflict, unannotated_provider, @@ -65,6 +67,16 @@ def test_given_context_only_function_when_calling_with_provider_injection_then_c function=provider_only, expected_result="slack", ), + ProviderInjectionTestCase( + description="keyword-only provider is injected by name", + function=keyword_only_provider, + expected_result=("runtime-context", "slack"), + ), + ProviderInjectionTestCase( + description="positional-only context is passed by position", + function=positional_only_context, + expected_result=("runtime-context", "slack"), + ), ), ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/concepts/selectors.mdx b/website/src/content/docs/docs/concepts/selectors.mdx index a10fac80ae..7f598d7a33 100644 --- a/website/src/content/docs/docs/concepts/selectors.mdx +++ b/website/src/content/docs/docs/concepts/selectors.mdx @@ -69,7 +69,7 @@ sqb build --select source:raw__orders ### Job -Select the nodes a job or job group declared under `orchestration/` builds. Job names are globally +Select the SQL and Python nodes a job or job group declared under `orchestration/` builds. Job names are globally unique, so a bare name works; `job:` is the explicit form and accepts glob patterns: ```bash From a96db64de6743be9b1ba8155a6bf93b8896eb9a1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:10:19 +0000 Subject: [PATCH 5/7] fix: keep Python selections runnable on workers and require checked nodes Selections now publish a managed source instead of its loader, ad-hoc test, audit, seed and freshness runs resolve SQL only, a selected SQL test reports its own outcome, and a selected model folds in its failing test. A selector that matches a Python check must also select the nodes the check depends on, in jobs, roles and the CLI alike. --- .../python_pipeline_orders_check.py | 16 +++++ .../orchestration/upstream_loaded_orders.py | 15 +++++ .../src/python_selection/_helpers/unified.rs | 44 +++++++++++--- .../src/python_selection/main/mod.rs | 1 + .../python_selection/main/python_run_key.rs | 13 +++++ .../tests/test_python_selection.rs | 13 ++++- .../src/adhoc/_helpers/planner.rs | 46 ++++++++++----- crates/sqlbuild-cli/src/adhoc/constants.rs | 3 + .../src/adhoc/tests/test_adhoc_planner.rs | 58 +++++++++++++++++++ .../compiled_project/snapshot_execution.rs | 23 +++++--- .../_helpers/graph/python_selection.rs | 14 ++++- .../build/no_tests_no_audits/_test_types.py | 1 + .../test_build_no_tests_no_audits.py | 30 +++++++--- .../compiler/compiled_project/_test_types.py | 14 +++++ .../compiler/compiled_project/helpers.py | 3 + .../publication/_test_types.py | 2 + .../publication/test_adhoc.py | 23 +++++++- .../publication/test_worker.py | 17 ++++++ .../test_orchestration_snapshot.py | 49 ++++++++++++++++ .../_helpers/test_run_selectors.py | 21 ++++--- .../docs/concepts/python-nodes/checks.mdx | 6 +- .../docs/concepts/python-nodes/overview.mdx | 2 +- 22 files changed, 357 insertions(+), 57 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/python_run_key.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py new file mode 100644 index 0000000000..2f8b9b8329 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py @@ -0,0 +1,16 @@ +"""A Python check on the orders model that a job's tag selector matches. + +A job matching the check and the model it checks publishes and runs the check with the model. A job +matching the check alone fails publication, naming the check and the model it leaves out. +""" + +from sqlbuild.checks import check +from sqlbuild.refs import model + + +@check(depends_on=model("orders"), tags=["order_checks"]) +def orders_present(ctx): + """Orders are present.""" + orders = ctx.relation(model("orders")) + count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0] + return ctx.pass_() if count else ctx.fail(message="no orders") diff --git a/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py b/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py new file mode 100644 index 0000000000..0723fa403a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py @@ -0,0 +1,15 @@ +"""A job that reaches a managed source through `+`, so its run selects the source's loader chain. + +`+orders` selects the model, its managed source raw_shipments, the raw_shipments loader and the +intermediate stage_shipments loader it depends on. The run records each loader as the source it +writes, which is the node its load reports under. +""" + +from sqlbuild.orchestration import job, schedule + +annual_orders = job( + "annual_orders", + description="Build orders and everything upstream of it, loaders included.", + select="+orders", + schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), +) diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index 4c94cd2f87..06eaa10482 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -89,14 +89,10 @@ pub(crate) fn run_selection( Atom::Sql(_) => None, }) .collect(); - if !select.is_empty() && !checks.is_empty() { - return Err(error( - PLANNER_DEFAULT_CODE, - format!( - "Python checks are not selectable here: {}. Use sqb check instead.", - checks.join(", ") - ), - )); + if !select.is_empty() + && let Some(message) = unselected_checked_nodes(context, &selected, &checks) + { + return Err(error(PLANNER_DEFAULT_CODE, message)); } let runnable: Atoms = selected .into_iter() @@ -106,6 +102,38 @@ pub(crate) fn run_selection( Ok(selection(context, sql_keys(&runnable), &runnable)) } +/// A matched check runs with the nodes it checks, so each of them must be selected too. +fn unselected_checked_nodes( + context: &SelectionContext<'_>, + selected: &Atoms, + checks: &[&str], +) -> Option { + checks.iter().find_map(|check| { + let python = context + .nodes + .edges(check, EdgeDirection::Upstream) + .iter() + .filter(|name| !selected.contains(&Atom::Python((*name).clone()))) + .filter_map(|name| Some(format!("{}:{name}", context.nodes.node(name)?.kind))); + let sql = context + .nodes + .node(check) + .map_or(&[][..], |node| &node.sql_deps); + let sql = sql + .iter() + .filter(|key| !selected.contains(&Atom::Sql((*key).clone()))) + .map(|(kind, name)| format!("{kind}:{name}")); + let missing: Vec = python.chain(sql).collect(); + (!missing.is_empty()).then(|| { + format!( + "Python check '{check}' checks {}, which the selection does not include; \ + select it too, or narrow the selector so it no longer matches the check.", + missing.join(", ") + ) + }) + }) +} + /// The SQL keys, with the source each selected intermediate loader writes, and the selected Python node names. fn selection(context: &SelectionContext<'_>, sql: Vec, atoms: &Atoms) -> RunSelection { let terminal: HashSet<&str> = context diff --git a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs index 487d121f6d..4630bc0f8c 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs @@ -1,6 +1,7 @@ pub mod attach_python_jobs; pub mod published_python_graph; pub mod python_node_graph_indexes; +pub mod python_run_key; pub mod resolve_python_node_excludes; pub mod resolve_python_node_selectors; pub mod resolve_python_sql_nodes; diff --git a/crates/sqlbuild-analysis/src/python_selection/main/python_run_key.rs b/crates/sqlbuild-analysis/src/python_selection/main/python_run_key.rs new file mode 100644 index 0000000000..a23f1075fe --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/python_run_key.rs @@ -0,0 +1,13 @@ +use crate::python_selection::constants::LOADER_KIND; + +/// The key a selected Python node's run reports under; a loader reports as the source it writes. +pub fn python_run_key(sources: &[(String, Option)], (kind, name): (&str, &str)) -> String { + if kind != LOADER_KIND { + return format!("{kind}:{name}"); + } + let source = sources + .iter() + .find(|(_, loader)| loader.as_deref() == Some(name)) + .map_or(name, |(source, _)| source.as_str()); + format!("source:{source}") +} diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs index 7a2c24b305..6e89230f2c 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs @@ -139,13 +139,20 @@ fn given_run_selectors_when_resolving_then_selection_matches_python() { expected_selection: Ok((&[("model", "orders")], &["export_orders", "prepare_orders"])), }, RunSelectionTestCase { - description: "checks are rejected", + description: "a check whose checked node is selected runs with it", + select: &["tag:exports"], + exclude: &[], + expected_selection: Ok((&[], &["export_orders"])), + }, + RunSelectionTestCase { + description: "a check whose checked node is unselected names both", select: &["check:check_orders_export"], exclude: &[], expected_selection: Err(( "S000", - "Python checks are not selectable here: check_orders_export. \ - Use sqb check instead.", + "Python check 'check_orders_export' checks asset:export_orders, which the \ + selection does not include; select it too, or narrow the selector so it no \ + longer matches the check.", )), }, RunSelectionTestCase { diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs index ddb5193499..241a2eaf88 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs @@ -11,6 +11,7 @@ use sqlbuild_analysis::graph::models::{GraphResource, JobSelection}; use sqlbuild_analysis::planner_selection::main::split_unit_test_selectors::split_unit_test_selectors; use sqlbuild_analysis::planner_selection::models::{SelectorGroups, UnitTestSelectorNames}; use sqlbuild_analysis::python_selection::main::published_python_graph::published_python_graph; +use sqlbuild_analysis::python_selection::main::python_run_key::python_run_key; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection; use sqlbuild_analysis::python_selection::models::{PublishedPythonNode, SqlSelectionGraph}; use sqlbuild_core::text::main::python_str_repr::python_str_repr; @@ -19,8 +20,9 @@ use sqlbuild_orchestration::runs::types::{CommandPlanner, PublishedJobs}; use crate::adhoc::constants::{ ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, CHANGED_COUNTS_DEST, CHANGED_DEST, COMMAND_DEST, - FRESHNESS_VERB, INTEGER_CONVERTERS, LOAD_SOURCES_DEST, POSITIVE_INTEGER_CONVERTER, - REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, SQL_TEST_KIND, UNIT_TEST_VERBS, + FRESHNESS_VERB, INCLUDE_PYTHON_DEST, INTEGER_CONVERTERS, LOAD_SOURCES_DEST, + POSITIVE_INTEGER_CONVERTER, REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, SQL_TEST_KIND, + UNIT_TEST_VERBS, }; use crate::argv::main::command_settings_arguments::command_settings_arguments; use crate::argv::main::job_settings_arguments::job_settings_arguments; @@ -86,7 +88,11 @@ impl CommandPlanner for CliPlanner { seeds } FRESHNESS_VERB => freshness_sources(publication, &select, &exclude)?, - _ => resolve_with_tests(verb, publication, &select, &exclude)?, + _ => { + let python = verb == BUILD_VERB + && settings.get(INCLUDE_PYTHON_DEST) != Some(&Json::Bool(false)); + resolve_with_tests((verb, python), publication, &select, &exclude)? + } }; Ok(CommandPlan { verb: verb.clone(), @@ -196,7 +202,7 @@ fn resolve( select: &[String], exclude: &[String], ) -> Result, String> { - let selected: BTreeSet = resolved(publication, select, exclude)?; + let selected: BTreeSet = resolved(publication, select, exclude, false)?; if selected.is_empty() { return Err("the selectors match no nodes in the current publication".into()); } @@ -205,7 +211,7 @@ fn resolve( /// [`resolve`] with `test:` and unit-test names selecting `sql_test:` keys, as `sqb test` does. fn resolve_with_tests( - verb: &str, + (verb, python): (&str, bool), (resources, jobs): (&[PlannerResource], &PublishedJobs), select: &[String], exclude: &[String], @@ -235,7 +241,7 @@ fn resolve_with_tests( ); } let mut keys: BTreeSet = if split.models_selected { - resolved((resources, jobs), &split.select, &split.exclude)? + resolved((resources, jobs), &split.select, &split.exclude, python)? } else { BTreeSet::new() }; @@ -255,15 +261,20 @@ fn is_sql_test(resource: &PlannerResource) -> bool { resource.key.split_once(':').map(|(kind, _)| kind) == Some(SQL_TEST_KIND) } +/// The published keys the selectors match; Python nodes only when the command runs them. fn resolved( (resources, jobs): (&[PlannerResource], &PublishedJobs), select: &[String], exclude: &[String], + include_python: bool, ) -> Result, String> { - let (python, sql): (Vec<&PlannerResource>, Vec<&PlannerResource>) = resources + let (mut python, sql): (Vec<&PlannerResource>, Vec<&PlannerResource>) = resources .iter() .filter(|resource| !is_sql_test(resource)) .partition(|resource| python_kind(&resource.key).is_some()); + if !include_python { + python.clear(); + } let graph_resources: Vec = sql .iter() .filter_map(|resource| graph_resource(resource)) @@ -275,7 +286,7 @@ fn resolved( .map(|resource| name_of(&resource.key)) .collect(); if selectors_need_jobs(&graph, &selectors, &node_names) { - graph = attach_jobs(graph, published_jobs(jobs)); + graph = attach_jobs(graph, published_jobs(jobs, include_python)); } let sources: Vec<(String, Option)> = sql .iter() @@ -306,10 +317,9 @@ fn resolved( .into_iter() .map(|(kind, name)| format!("{kind}:{name}")) .chain( - selection - .python_node_names - .iter() - .filter_map(|name| Some(format!("{}:{name}", kinds.get(name.as_str())?))), + selection.python_node_names.iter().filter_map(|name| { + Some(python_run_key(&sources, (kinds.get(name.as_str())?, name))) + }), ) .collect()) } @@ -339,10 +349,14 @@ fn name_of(key: &str) -> &str { } /// Published jobs as already-resolved graph selections. -fn published_jobs(jobs: &PublishedJobs) -> BTreeMap { +fn published_jobs(jobs: &PublishedJobs, include_python: bool) -> BTreeMap { let mut selections: BTreeMap = BTreeMap::new(); for (name, keys) in jobs { - let keys: Vec<(String, String)> = keys.iter().filter_map(|key| object_key(key)).collect(); + let keys: Vec<(String, String)> = keys + .iter() + .filter(|key| include_python || python_kind(key).is_none()) + .filter_map(|key| object_key(key)) + .collect(); selections.insert(name.clone(), JobSelection::Resolved { keys }); } selections @@ -355,11 +369,11 @@ fn freshness_sources( exclude: &[String], ) -> Result, String> { let selected: BTreeSet = - upstream_sources(publication.0, &resolved(publication, select, &[])?); + upstream_sources(publication.0, &resolved(publication, select, &[], false)?); let excluded: BTreeSet = if exclude.is_empty() { BTreeSet::new() } else { - upstream_sources(publication.0, &resolved(publication, exclude, &[])?) + upstream_sources(publication.0, &resolved(publication, exclude, &[], false)?) }; let sources: BTreeSet = selected.difference(&excluded).cloned().collect(); if sources.is_empty() { diff --git a/crates/sqlbuild-cli/src/adhoc/constants.rs b/crates/sqlbuild-cli/src/adhoc/constants.rs index 0de8284749..dc93f3bcba 100644 --- a/crates/sqlbuild-cli/src/adhoc/constants.rs +++ b/crates/sqlbuild-cli/src/adhoc/constants.rs @@ -27,6 +27,9 @@ pub(crate) const CHANGED_COUNTS_DEST: &str = "changed_counts"; /// The job setting `--changed` becomes. pub(crate) const CHANGED_DEST: &str = "changed"; +/// The destination `--no-python` clears; only a build with Python nodes resolves them. +pub(crate) const INCLUDE_PYTHON_DEST: &str = "include_python"; + /// The destination `--load` and `--no-load` share; only `--no-load` is accepted. pub(crate) const LOAD_SOURCES_DEST: &str = "load_sources"; diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index fe0b38f40a..df7501c9c1 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::json; +use sqlbuild_orchestration::runs::models::PlannerResource; use sqlbuild_orchestration::runs::types::PublishedJobs; use crate::adhoc::main::adhoc_planner::adhoc_planner; @@ -348,6 +349,23 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co &["orders"], Some("python/fulfillment/carrier_sync"), ), + PlannerResource { + loader: Some("raw_shipments".to_owned()), + ..resource("source:raw_shipments", &[], &[], None) + }, + resource( + "model:shipments", + &["source:raw_shipments"], + &[], + Some("models/marts"), + ), + resource( + "loader:raw_shipments", + &["loader:stage_shipments"], + &[], + Some("python/loaders"), + ), + resource("loader:stage_shipments", &[], &[], Some("python/loaders")), ]; let jobs: PublishedJobs = BTreeMap::from([( "carrier_sync".to_owned(), @@ -436,6 +454,46 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co json!({}), )), }, + AdhocPlanTestCase { + description: "upstream of a model runs its source's loader chain as the sources they write", + command: &["build", "--select", "+shipments"], + expected_plan: Ok(( + &[ + "model:shipments", + "source:raw_shipments", + "source:stage_shipments", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "an intermediate loader runs as the source it writes", + command: &["build", "--select", "loader:stage_shipments"], + expected_plan: Ok((&["source:stage_shipments"], json!({}))), + }, + AdhocPlanTestCase { + description: "test selects only the SQL nodes a tag shared with Python nodes matches", + command: &["test", "--select", "tag:orders"], + expected_plan: Ok((&["model:orders", "model:stg_orders"], json!({}))), + }, + AdhocPlanTestCase { + description: "test downstream of a model stops at the Python nodes that read it", + command: &["test", "--select", "orders+", "--exclude", "stg_orders"], + expected_plan: Ok((&["model:orders"], json!({}))), + }, + AdhocPlanTestCase { + description: "a build without Python nodes selects only the SQL nodes a shared tag matches", + command: &["build", "--no-python", "--select", "tag:orders"], + expected_plan: Ok(( + &["model:orders", "model:stg_orders"], + json!({"include_python": false}), + )), + }, + AdhocPlanTestCase { + description: "test of a job with only Python nodes has nothing to test", + command: &["test", "--select", "job:carrier_sync"], + expected_plan: Err("the selectors match no nodes in the current publication"), + }, ]; let planner = adhoc_planner(); for test_case in test_cases { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index b27d9b4b27..033b08c4cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -357,9 +357,10 @@ impl WorkerBuild { outcome }; if self.execution.claim.request.selection.contains(key) { - return self.complete(py, key, (outcome, None, (None, None))); + self.complete(py, key, (outcome, None, (None, None)))?; + } else { + self.finish_check(py, key, outcome)?; } - self.finish_check(py, key, outcome)?; if outcome == NodeOutcome::Skipped { return Ok(()); } @@ -395,15 +396,12 @@ impl WorkerBuild { Ok(None) } - /// A failing SQL test, found by its compiled name such as `parent [case]`, fails the models it blocks with its error. + /// A finished SQL test completes itself when selected; a failing one fails the models it blocks. fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { let py = result.py(); let outcome: String = field(result, "outcome")?.str()?.extract()?; - if SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass) - || self.worker.bind(py).get().refresh_cancellation(py)? - { - return Ok(()); - } + let passed = SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass); + let cancelled = !passed && self.worker.bind(py).get().refresh_cancellation(py)?; let name: String = field(result, "test_name")?.extract()?; let key = format!("{SQL_TEST_KEY_PREFIX}{name}"); let error: Option = match result.getattr_opt("error_code")? { @@ -414,6 +412,15 @@ impl WorkerBuild { Some(value) => value.extract()?, None => None, }; + let node = match (passed, cancelled) { + (true, _) => NodeOutcome::Success, + (false, true) => NodeOutcome::Cancelled, + (false, false) => NodeOutcome::Failed, + }; + self.complete(py, &key, (node, None, (error.clone(), message.clone())))?; + if passed || cancelled { + return Ok(()); + } for model in self.blocked.get(&key).into_iter().flatten() { self.complete( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 3a5d3b1e6c..f9d1b4e03f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -13,6 +13,7 @@ use sqlbuild_analysis::graph::errors::SelectorError; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_analysis::python_selection::main::attach_python_jobs::attach_python_jobs; use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; +use sqlbuild_analysis::python_selection::main::python_run_key::python_run_key; use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; use sqlbuild_analysis::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; @@ -187,6 +188,17 @@ impl ProjectSelection { .map(|node| format!("{}:{name}", node.kind)) } + /// The key a selected Python node's run reports under; a loader reports as the source it writes. + fn run_key(&self, name: &str) -> Option { + let node = self + .nodes + .nodes + .iter() + .rev() + .find(|node| node.name == name)?; + Some(python_run_key(&self.sources, (&node.kind, name))) + } + /// Whether a loader writes no declared source and so runs only as part of another loader's chain. pub(crate) fn intermediate_loader(&self, name: &str) -> bool { !self @@ -213,7 +225,7 @@ impl ProjectSelection { .map(|(kind, name)| format!("{kind}:{name}")) .collect(); for name in &selection.python_node_names { - keys.extend(self.node_key(name)); + keys.extend(self.run_key(name)); } Ok(keys) } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py index bfc5361cf4..5e38ffcc8e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py @@ -18,3 +18,4 @@ class RunE2ETestCase: expected_query_results: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] = field( default_factory=tuple ) + select: tuple[str, ...] = field(default_factory=tuple) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index 3100cd5be9..b3114af936 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -2185,15 +2185,29 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou "test_case", [ RunE2ETestCase( - description="run rejects selected check node", + description="build rejects a selected check whose task is not selected", expected_exit_code=1, expected_table_names=(), expected_view_names=(), - ) + expected_output_fragments=( + "Python check 'check_orders_export' checks task:prepare_orders, which the " + "selection does not include; select it too, or narrow the selector so it no " + "longer matches the check.", + ), + select=("check:check_orders_export",), + ), + RunE2ETestCase( + description="build runs a selected check with its selected task", + expected_exit_code=0, + expected_table_names=(), + expected_view_names=(), + expected_output_fragments=("check check_orders_export START", "PASS=2"), + select=("check:check_orders_export", "task:prepare_orders"), + ), ], ids=lambda case: case.description, ) -def test_given_check_selector_when_running_run_then_command_rejects_check( +def test_given_check_selector_when_running_build_then_check_needs_its_checked_node( test_case: RunE2ETestCase, tmp_path: Path, ) -> None: @@ -2230,14 +2244,12 @@ def test_given_check_selector_when_running_run_then_command_rejects_check( "--no-tests", "--no-audits", "--select", - "check:check_orders_export", + *test_case.select, ), project_dir=project_dir, ) - assert result.returncode == test_case.expected_exit_code combined_output: str = result.stdout + result.stderr - assert ( - "Python checks are not selectable here: " - "check_orders_export. Use sqb check instead." in combined_output - ) + assert result.returncode == test_case.expected_exit_code, combined_output + for fragment in test_case.expected_output_fragments: + assert fragment in combined_output diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 01776fdad9..fa3f650f96 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -283,6 +283,20 @@ class PythonJobGroupCycleTestCase: expected_message: str +@dataclass(frozen=True) +class PythonCheckJobTestCase: + description: str + select: str + expected_selection: list[str] + + +@dataclass(frozen=True) +class PythonCheckJobRejectionTestCase: + description: str + select: str + expected_message: str + + @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index bbaca4925f..d114213a62 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -84,6 +84,9 @@ "python/fulfillment/carrier_sync/node.py": Path( "benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py" ).read_text(), + "python/checks/orders_present.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py" + ).read_text(), } TYPE_PROOF_FILES: dict[str, str] = { diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 1da98aee95..be2c128601 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -406,6 +406,8 @@ class AdhocVerbTestCase: # Files the worker's checkout gains after it verified the final run, which must not run. rewrites: dict[str, str] = field(default_factory=dict) expected_status: str = "success" + # Files that replace the published project's own before publication. + files: dict[str, str] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py index 703856d119..b7247fd0a6 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py @@ -268,6 +268,26 @@ def test_given_an_ad_hoc_command_when_checked_and_launched_then_the_worker_runs_ expected_run_nodes=[("sql_test:customers_have_ids", "node", "success")], rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, ), + AdhocVerbTestCase( + description="build test: runs and reports only the selected verified SQL test", + prerequisites=[], + command=["build", "--select", "test:customers_have_ids"], + expected_selection=["sql_test:customers_have_ids"], + expected_run_nodes=[("sql_test:customers_have_ids", "node", "success")], + rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, + ), + AdhocVerbTestCase( + description="test of a model and its failing SQL test fails both", + prerequisites=[], + command=["test", "--select", "customers", "test:customers_have_ids"], + expected_selection=["model:customers", "sql_test:customers_have_ids"], + expected_run_nodes=[ + ("model:customers", "node", "failed"), + ("sql_test:customers_have_ids", "node", "failed"), + ], + files={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, + expected_status="failed", + ), AdhocVerbTestCase( description="freshness observes the verified sources upstream of the selection", prerequisites=[], @@ -303,7 +323,8 @@ def test_given_an_ad_hoc_verb_when_the_checkout_changes_after_verification_then_ "seeds/seeds.yml": SEEDS_YML.format(region_type="VARCHAR"), "sources/raw.yml": SOURCES_YML.format(table="raw_orders"), "python/rewrite_checkout.py": REWRITE_CHECKOUT, - }, + } + | test_case.files, ) with scheduled_worker( tmp_path, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 0169dd24b0..4231bf9a7d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -851,6 +851,23 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re {"model:orders", "source:raw_shipments", "source:stage_shipments"} ), ), + ClaimedLoadTestCase( + "a job reaching a managed source through + runs the source's loader", + "", + "SELECT amount FROM orders", + "success", + [(9,)], + files=INTERMEDIATE_LOADER_FILES + | { + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py" + ).read_text() + }, + expected_node_count=3, + expected_write_set=frozenset( + {"model:orders", "source:raw_shipments", "source:stage_shipments"} + ), + ), ], ids=lambda case: case.description, ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 76d37d2d1b..179999d185 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -34,6 +34,8 @@ ) from sqlbuild.orchestration_declarations.models.authoring import Job, JobGroup from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + PythonCheckJobRejectionTestCase, + PythonCheckJobTestCase, PythonJobGroupCycleTestCase, PythonJobGroupTestCase, PythonJobSelectionTestCase, @@ -1997,3 +1999,50 @@ def test_given_a_python_dependency_cycle_across_jobs_when_publishing_a_group_the with pytest.raises(ValueError, match=re.escape(test_case.expected_message)): python_pipeline_snapshot(tmp_path, [*jobs, group]) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonCheckJobTestCase( + description="a job matching a check and the model it checks publishes the model", + select="tag:order_checks orders", + expected_selection=["model:orders"], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_job_matching_a_check_and_its_node_when_publishing_then_the_check_runs_with_it( + tmp_path: Path, test_case: PythonCheckJobTestCase +) -> None: + nightly: Job = job("nightly_orders", description="Orders.", select=test_case.select) + + snapshot: dict[str, Any] = json.loads(python_pipeline_snapshot(tmp_path, [nightly]).to_json()) + + assert snapshot["documents"]["jobs"]["nightly_orders"]["selection"] == ( + test_case.expected_selection + ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonCheckJobRejectionTestCase( + description="a job matching a check without the model it checks names both", + select="tag:order_checks", + expected_message=( + "job nightly_orders: Python check 'orders_present' checks model:orders, which " + "the selection does not include; select it too, or narrow the selector so it no " + "longer matches the check." + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_job_matching_a_check_without_its_node_when_publishing_then_it_is_rejected( + tmp_path: Path, test_case: PythonCheckJobRejectionTestCase +) -> None: + nightly: Job = job("nightly_orders", description="Orders.", select=test_case.select) + + with pytest.raises(ValueError, match=re.escape(test_case.expected_message)): + python_pipeline_snapshot(tmp_path, [nightly]) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py index ae3937e7da..f42a9bed93 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py @@ -42,6 +42,13 @@ expected_sql_names=frozenset(), expected_python_node_names=frozenset({"export_orders"}), ), + PythonSqlSelectorTestCase( + description="accepts a check whose checked node the selection also matches", + select=("tag:exports",), + exclude=(), + expected_sql_names=frozenset(), + expected_python_node_names=frozenset({"export_orders"}), + ), PythonSqlSelectorTestCase( description="selects expanded source terminal loader requirement for run", select=("+source:raw_orders",), @@ -73,18 +80,14 @@ def test_given_run_selectors_when_resolving_then_excludes_python_checks( "test_case", [ PythonSqlSelectorErrorTestCase( - description="rejects explicit check selector for run", + description="rejects a check whose checked node the selection leaves out", select=("check:check_orders_export",), exclude=(), expected_error_type=ValueError, - expected_error_fragment=("Python checks are not selectable here: check_orders_export"), - ), - PythonSqlSelectorErrorTestCase( - description="rejects tag selector that includes a check for run", - select=("tag:exports",), - exclude=(), - expected_error_type=ValueError, - expected_error_fragment=("Python checks are not selectable here: check_orders_export"), + expected_error_fragment=( + "Python check 'check_orders_export' checks asset:export_orders, which the " + "selection does not include" + ), ), ], ids=lambda case: case.description, diff --git a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx index dd93a67b34..c1acdf2f86 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx @@ -109,7 +109,11 @@ Checks do not support `columns`, `column_lineage`, or `retry`. ## Running checks -Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. To run checks on their own, use [`sqb check`](/docs/cli/check): +Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. + +A `sqb build` selector that matches a check must also select the nodes the check depends on. Otherwise the build fails with an error that names the check and the missing node; select that node too, or narrow the selector so it no longer matches the check. Jobs, roles and server runs resolve selectors the same way. + +To run checks on their own, use [`sqb check`](/docs/cli/check): ```bash # Run all checks diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index ddc395623b..9d215b702c 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -141,7 +141,7 @@ Python nodes are selected like SQL resources, by bare name or typed selector: sqb build --select export_orders # bare name sqb build --select task:export_orders # typed sqb build --select asset:orders_export -sqb build --select check:check_orders +sqb build --select export_orders check:check_orders_exported # a check with the node it checks sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` From 879f9316b6be7d9a9aae48d9e0ca2d9504fed148 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:31:15 +0000 Subject: [PATCH 6/7] fix: refuse selecting a Python check on a source no loader writes A build never loads such a source, so a matched check on it would be accepted and then never run. The resolver now names the check and the source and points to sqb check. --- .../src/python_selection/_helpers/unified.rs | 79 +++++++++++++------ .../src/python_selection/tests/helpers.rs | 14 ++++ .../tests/test_python_selection.rs | 12 +++ .../test_build_no_tests_no_audits.py | 22 ++++++ .../docs/concepts/python-nodes/checks.mdx | 2 +- 5 files changed, 104 insertions(+), 25 deletions(-) diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index 06eaa10482..56bf52afce 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -102,35 +102,66 @@ pub(crate) fn run_selection( Ok(selection(context, sql_keys(&runnable), &runnable)) } -/// A matched check runs with the nodes it checks, so each of them must be selected too. +/// The first matched check a build cannot run after the nodes it checks, as an error message. fn unselected_checked_nodes( context: &SelectionContext<'_>, selected: &Atoms, checks: &[&str], ) -> Option { - checks.iter().find_map(|check| { - let python = context - .nodes - .edges(check, EdgeDirection::Upstream) - .iter() - .filter(|name| !selected.contains(&Atom::Python((*name).clone()))) - .filter_map(|name| Some(format!("{}:{name}", context.nodes.node(name)?.kind))); - let sql = context - .nodes - .node(check) - .map_or(&[][..], |node| &node.sql_deps); - let sql = sql - .iter() - .filter(|key| !selected.contains(&Atom::Sql((*key).clone()))) - .map(|(kind, name)| format!("{kind}:{name}")); - let missing: Vec = python.chain(sql).collect(); - (!missing.is_empty()).then(|| { - format!( - "Python check '{check}' checks {}, which the selection does not include; \ - select it too, or narrow the selector so it no longer matches the check.", - missing.join(", ") - ) - }) + let loaded: HashSet<&str> = source_loaders(context.sql) + .into_iter() + .map(|(source, _)| source) + .collect(); + for check in checks { + if let Some(message) = unrunnable_check(context, selected, &loaded, check) { + return Some(message); + } + } + None +} + +/// Why a build cannot run `check`: it checks a source no loader writes, or an unselected node. +fn unrunnable_check( + context: &SelectionContext<'_>, + selected: &Atoms, + loaded: &HashSet<&str>, + check: &str, +) -> Option { + let sql: &[ObjectKey] = context + .nodes + .node(check) + .map_or(&[][..], |node| &node.sql_deps); + let unloaded: Vec = sql + .iter() + .filter(|(kind, name)| kind == SOURCE_RESOURCE && !loaded.contains(name.as_str())) + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + if !unloaded.is_empty() { + return Some(format!( + "Python check '{check}' checks {}, which a build does not load, so the check \ + would not run; run it with sqb check, or narrow the selector so it no longer \ + matches the check.", + unloaded.join(", ") + )); + } + let missing: Vec = context + .nodes + .edges(check, EdgeDirection::Upstream) + .iter() + .filter(|name| !selected.contains(&Atom::Python((*name).clone()))) + .filter_map(|name| Some(format!("{}:{name}", context.nodes.node(name)?.kind))) + .chain( + sql.iter() + .filter(|key| !selected.contains(&Atom::Sql((*key).clone()))) + .map(|(kind, name)| format!("{kind}:{name}")), + ) + .collect(); + (!missing.is_empty()).then(|| { + format!( + "Python check '{check}' checks {}, which the selection does not include; \ + select it too, or narrow the selector so it no longer matches the check.", + missing.join(", ") + ) }) } diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs index 2ac7b7de0e..2e8c3134f8 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs @@ -67,6 +67,14 @@ const ORDERS_NODES: &[NodeSpec] = &[ &[], &[], ), + ( + "check_raw_customers", + "check", + "python/checks/customers.py", + &["customer_checks"], + &[], + &[("source", "raw_customers")], + ), ]; fn owned(values: &[&str]) -> Vec { @@ -128,6 +136,12 @@ fn orders_sql_graph() -> ProjectGraph { tags: Vec::new(), folder: None, }, + GraphResource { + key: key(&("source", "raw_customers")), + deps: Vec::new(), + tags: Vec::new(), + folder: None, + }, ]) } diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs index 6e89230f2c..fad17b2ca9 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs @@ -113,6 +113,7 @@ fn given_run_selectors_when_resolving_then_selection_matches_python() { &[ ("model", "orders"), ("source", "load_events"), + ("source", "raw_customers"), ("source", "raw_orders"), ], &[ @@ -144,6 +145,17 @@ fn given_run_selectors_when_resolving_then_selection_matches_python() { exclude: &[], expected_selection: Ok((&[], &["export_orders"])), }, + RunSelectionTestCase { + description: "a check on a source a build does not load is refused even when selected", + select: &["tag:customer_checks", "source:raw_customers"], + exclude: &[], + expected_selection: Err(( + "S000", + "Python check 'check_raw_customers' checks source:raw_customers, which a build \ + does not load, so the check would not run; run it with sqb check, or narrow the \ + selector so it no longer matches the check.", + )), + }, RunSelectionTestCase { description: "a check whose checked node is unselected names both", select: &["check:check_orders_export"], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index b3114af936..3af739ac24 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -2204,6 +2204,18 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou expected_output_fragments=("check check_orders_export START", "PASS=2"), select=("check:check_orders_export", "task:prepare_orders"), ), + RunE2ETestCase( + description="build rejects a selected check on a source no loader writes", + expected_exit_code=1, + expected_table_names=(), + expected_view_names=(), + expected_output_fragments=( + "Python check 'check_raw_events' checks source:raw_events, which a build does " + "not load, so the check would not run; run it with sqb check, or narrow the " + "selector so it no longer matches the check.", + ), + select=("check:check_raw_events", "source:raw_events"), + ), ], ids=lambda case: case.description, ) @@ -2234,6 +2246,16 @@ def test_given_check_selector_when_running_build_then_check_needs_its_checked_no "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n return ctx.result()\n" ), + "python/checks/events.py": ( + "from sqlbuild.checks import check\n" + "from sqlbuild.refs import source\n\n" + "@check(depends_on=source('raw_events'))\n" + "def check_raw_events(ctx):\n" + " '''Test check check_raw_events.'''\n return ctx.pass_()\n" + ), + "sources/raw.yml": ( + "sources:\n - name: raw_events\n description: Test source raw_events.\n" + ), }, ) diff --git a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx index c1acdf2f86..17a34a1947 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx @@ -111,7 +111,7 @@ Checks do not support `columns`, `column_lineage`, or `retry`. Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. -A `sqb build` selector that matches a check must also select the nodes the check depends on. Otherwise the build fails with an error that names the check and the missing node; select that node too, or narrow the selector so it no longer matches the check. Jobs, roles and server runs resolve selectors the same way. +A `sqb build` selector that matches a check must also select the nodes the check depends on. Otherwise the build fails with an error that names the check and the missing node; select that node too, or narrow the selector so it no longer matches the check. A check on a source that no loader writes never runs in a build, so a selector that matches it fails too; run that check with `sqb check`. Jobs and server runs resolve selectors the same way. To run checks on their own, use [`sqb check`](/docs/cli/check): From 059f8cdb7380c82e8d9e3a4449b1dfa517f1c724 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:56:00 +0000 Subject: [PATCH 7/7] refactor: share upstream result reads between loader and node contexts Loader and Python node contexts read upstream results through one mixin, and the selection entry points build their context in one place. Regenerates the packaged docs skill for the selector and check pages. --- .../src/python_selection/_helpers/nodes.rs | 17 ++++- .../main/attach_python_jobs.rs | 12 ++-- .../main/resolve_python_sql_nodes.rs | 10 +-- .../main/resolve_python_sql_run_selection.rs | 10 +-- .../docs/concepts/python-nodes/checks.md | 6 +- .../docs/concepts/python-nodes/overview.md | 16 ++++- .../references/docs/concepts/selectors.md | 2 +- src/sqlbuild/executor/load/models.py | 63 +------------------ .../classes/dependency_result_reader.py | 61 ++++++++++++++++++ .../executor/node_results/constants.py | 7 +++ src/sqlbuild/executor/python_nodes/models.py | 63 +------------------ 11 files changed, 118 insertions(+), 149 deletions(-) create mode 100644 src/sqlbuild/executor/node_results/classes/dependency_result_reader.py diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs index e2390a571b..7fc0e9cd4a 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs @@ -17,7 +17,9 @@ use crate::python_selection::constants::{ UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, UNMAPPED_KIND_CODE, }; -use crate::python_selection::models::{NodeIndex, PythonSelectionGraph, PythonSelectionNode}; +use crate::python_selection::models::{ + NodeIndex, PythonSelectionGraph, PythonSelectionNode, SelectionContext, SqlSelectionGraph, +}; use sqlbuild_core::constants::PYTHON_NODE_ROOT as PYTHON_NODE_PATH_ROOT; pub(crate) type Names = BTreeSet; @@ -59,6 +61,19 @@ fn keyed(rows: &[(String, Vec)]) -> HashMap<&str, &[String]> { .collect() } +impl<'graph> SelectionContext<'graph> { + pub(crate) fn new( + sql: SqlSelectionGraph<'graph>, + python: &'graph PythonSelectionGraph, + ) -> Self { + Self { + sql, + nodes: NodeIndex::new(python), + sql_names: sql_names(sql.graph), + } + } +} + impl<'graph> NodeIndex<'graph> { pub(crate) fn new(graph: &'graph PythonSelectionGraph) -> Self { Self { diff --git a/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs index 6e3095f6f7..21edb1f7eb 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs @@ -2,10 +2,9 @@ use std::collections::BTreeMap; use crate::graph::main::attach_jobs_with::attach_jobs_with; use crate::graph::models::{JobSelection, ProjectGraph}; -use crate::python_selection::_helpers::nodes::sql_names; use crate::python_selection::_helpers::unified::run_selection; use crate::python_selection::models::{ - NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, + PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, }; /// Attach jobs as publication resolves them, keeping each job's Python nodes as `(kind, name)` keys. @@ -20,14 +19,13 @@ pub fn attach_python_jobs( .map(|node| (node.name.as_str(), node.kind.as_str())) .collect(); attach_jobs_with(sql.graph.clone(), catalog, |graph, select, exclude| { - let context: SelectionContext<'_> = SelectionContext { - sql: SqlSelectionGraph { + let context: SelectionContext<'_> = SelectionContext::new( + SqlSelectionGraph { graph, sources: sql.sources, }, - nodes: NodeIndex::new(python), - sql_names: sql_names(graph), - }; + python, + ); let RunSelection { sql_keys, python_node_names, diff --git a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs index 6a2d99a3cb..b482df85e7 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs @@ -1,8 +1,7 @@ use crate::graph::errors::SelectorError; -use crate::python_selection::_helpers::nodes::sql_names; use crate::python_selection::_helpers::unified::node_selection; use crate::python_selection::models::{ - NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, + PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, }; /// The SQL resources and Python nodes, checks included, that selectors name, as grants and sensor conditions read them. @@ -12,10 +11,5 @@ pub fn resolve_python_sql_nodes( select: &[String], exclude: &[String], ) -> Result { - let context: SelectionContext<'_> = SelectionContext { - sql, - nodes: NodeIndex::new(python), - sql_names: sql_names(sql.graph), - }; - node_selection(&context, select, exclude) + node_selection(&SelectionContext::new(sql, python), select, exclude) } diff --git a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs index a820fd41d7..fef9983c7b 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs @@ -1,8 +1,7 @@ use crate::graph::errors::SelectorError; -use crate::python_selection::_helpers::nodes::sql_names; use crate::python_selection::_helpers::unified::run_selection; use crate::python_selection::models::{ - NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, + PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, }; /// Python's `resolve_python_sql_run_selectors`: run selectors over SQL resources and Python nodes. @@ -12,10 +11,5 @@ pub fn resolve_python_sql_run_selection( select: &[String], exclude: &[String], ) -> Result { - let context: SelectionContext<'_> = SelectionContext { - sql, - nodes: NodeIndex::new(python), - sql_names: sql_names(sql.graph), - }; - run_selection(&context, select, exclude) + run_selection(&SelectionContext::new(sql, python), select, exclude) } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md index 4dc3e69973..fe54d4880a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md @@ -120,7 +120,11 @@ Checks do not support `columns`, `column_lineage`, or `retry`. ## Running checks -Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. To run checks on their own, use [`sqb check`](../../cli/check.md): +Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. + +A `sqb build` selector that matches a check must also select the nodes the check depends on. Otherwise the build fails with an error that names the check and the missing node; select that node too, or narrow the selector so it no longer matches the check. A check on a source that no loader writes never runs in a build, so a selector that matches it fails too; run that check with `sqb check`. Jobs and server runs resolve selectors the same way. + +To run checks on their own, use [`sqb check`](../../cli/check.md): ```bash # Run all checks diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index 17cdf8438d..c4c2db12e9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -143,7 +143,7 @@ def export_orders(ctx): - `ctx.skip(reason, mode=...)` skips the node. `mode` accepts `"soft"` (default, skips only this node) or `"hard"` (also blocks dependents), as a string or the `SkipMode` enum from `sqlbuild.tasks`/`sqlbuild.assets`. - Assets may pass `materialized=True`/`False` to record whether an artifact was produced. -Downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. +Within one run, downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. ## Result persistence @@ -157,7 +157,7 @@ Python nodes are selected like SQL resources, by bare name or typed selector: sqb build --select export_orders # bare name sqb build --select task:export_orders # typed sqb build --select asset:orders_export -sqb build --select check:check_orders +sqb build --select export_orders check:check_orders_exported # a check with the node it checks sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` @@ -177,6 +177,18 @@ dependency fails or is skipped. A dependency that is not part of the run (not se unmanaged source) is read as it currently exists, so `sqb build --select task:export_orders` runs the task against the relations already in the warehouse. +Python dependencies follow the same rule. A task or asset selected without its Python upstream +reads that upstream's latest successful result through `ctx.result_of`. If the upstream has never +succeeded for the target, the node fails with an error that names the upstream, for example: + +```text +Upstream task 'prepare_orders' has no successful result to read. Build it first with +'sqb build --select task:prepare_orders', or build both with 'sqb build --select +asset:export_orders'. +``` + +Jobs, sensors, roles and ad-hoc runs on the server resolve selectors the same way, so a job can +select one Python node and a job group orders it after the job that builds its upstream. + The commands differ in what they include by default: | Command | SQL | Loaders / tasks / assets | Checks | Audits | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md index cf563c84a9..09e66562eb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md @@ -82,7 +82,7 @@ sqb build --select source:raw__orders ### Job -Select the nodes a job or job group declared under `orchestration/` builds. Job names are globally +Select the SQL and Python nodes a job or job group declared under `orchestration/` builds. Job names are globally unique, so a bare name works; `job:` is the explicit form and accepts glob patterns: ```bash diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py index 39097300b7..f64d2156b0 100644 --- a/src/sqlbuild/executor/load/models.py +++ b/src/sqlbuild/executor/load/models.py @@ -17,13 +17,12 @@ from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) -from sqlbuild.compiler.python_nodes.types import PythonNodeKind, SkipMode +from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.constants import LOADER_RELATION_QUALIFIER_SEPARATOR from sqlbuild.executor.load.types import LoadProgressCallback -from sqlbuild.executor.node_results.models import NodeResultEnvelope +from sqlbuild.executor.node_results.classes.dependency_result_reader import DependencyResultReader from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard -from sqlbuild.executor.python_nodes.constants import MISSING_DEFAULT from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container from sqlbuild.python_nodes.models import SqlResourceRef @@ -95,7 +94,7 @@ class LoaderResult: @dataclass(frozen=True) -class LoaderContext: +class LoaderContext(DependencyResultReader): """Runtime context passed to a source loader function.""" adapter: BaseAdapter @@ -164,44 +163,6 @@ def result( materialized=materialized, ) - def result_of( - self, - *, - node_function: Callable[..., object], - run_id: str | None = None, - default: object = MISSING_DEFAULT, - ) -> NodeResultEnvelope | object: - """Return the latest persisted upstream result by Python node function reference.""" - - if self.result_store is None: - if default is not MISSING_DEFAULT: - return default - raise ExecutorInputError("No Python node result store is available") - node_type, node_name = self._result_dependency_identity(node_function) - return self.result_store.result_of( - node_type=node_type, - node_name=node_name, - run_id=run_id, - default=default, - ) - - def results_of( - self, - *, - node_function: Callable[..., object], - limit: int, - ) -> tuple[NodeResultEnvelope, ...]: - """Return persisted successful upstream result history, newest first.""" - - if self.result_store is None: - return () - node_type, node_name = self._result_dependency_identity(node_function) - return self.result_store.results_of( - node_type=node_type, - node_name=node_name, - limit=limit, - ) - def qualify_name( self, *, @@ -251,24 +212,6 @@ def _record_resolved(self, relation_ref: LoaderRelationRef) -> None: SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=relation_ref.name) ) - def _result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: - task_definition: object = getattr(node_function, "__sqlbuild_task__", None) - if task_definition is not None: - return PythonNodeKind.TASK.value, self._definition_name(task_definition) - asset_definition: object = getattr(node_function, "__sqlbuild_asset__", None) - if asset_definition is not None: - return PythonNodeKind.ASSET.value, self._definition_name(asset_definition) - loader_definition: object = getattr(node_function, "__sqlbuild_loader__", None) - if loader_definition is not None: - return PythonNodeKind.LOADER.value, self._definition_name(loader_definition) - raise ExecutorInputError("Python node result dependency must be a task, asset, or loader") - - def _definition_name(self, definition: object) -> str: - name: object = getattr(definition, "name", None) - if isinstance(name, str): - return name - raise ExecutorInputError("Python node result dependency has no resolved name") - @dataclass(frozen=True) class LoadExecutionResult: diff --git a/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py b/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py new file mode 100644 index 0000000000..7e5a36393f --- /dev/null +++ b/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py @@ -0,0 +1,61 @@ +"""Reads of a Python node's upstream results, looked up by the upstream's decorated function.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.node_results.constants import RESULT_DEPENDENCY_ATTRIBUTES +from sqlbuild.executor.node_results.models import NodeResultEnvelope +from sqlbuild.executor.python_nodes.constants import MISSING_DEFAULT + + +class DependencyResultReader: + """Mixin for node contexts that read their upstream results from a `result_store`.""" + + result_store: Any | None + + def result_of( + self, + *, + node_function: Callable[..., object], + run_id: str | None = None, + default: object = MISSING_DEFAULT, + ) -> NodeResultEnvelope | object: + """Return the latest persisted upstream result by Python node function reference.""" + + if self.result_store is None: + if default is not MISSING_DEFAULT: + return default + raise ExecutorInputError("No Python node result store is available") + node_type, node_name = self.result_dependency_identity(node_function) + return self.result_store.result_of( + node_type=node_type, node_name=node_name, run_id=run_id, default=default + ) + + def results_of( + self, + *, + node_function: Callable[..., object], + limit: int, + ) -> tuple[NodeResultEnvelope, ...]: + """Return persisted successful upstream result history, newest first.""" + + if self.result_store is None: + return () + node_type, node_name = self.result_dependency_identity(node_function) + return self.result_store.results_of(node_type=node_type, node_name=node_name, limit=limit) + + def result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: + """Return the `(kind, name)` a dependency's results are stored under.""" + + for attribute, kind in RESULT_DEPENDENCY_ATTRIBUTES: + definition: object = getattr(node_function, attribute, None) + if definition is None: + continue + name: object = getattr(definition, "name", None) + if not isinstance(name, str): + raise ExecutorInputError("Python node result dependency has no resolved name") + return kind, name + raise ExecutorInputError("Python node result dependency must be a task, asset, or loader") diff --git a/src/sqlbuild/executor/node_results/constants.py b/src/sqlbuild/executor/node_results/constants.py index b2e7444998..3b2d792021 100644 --- a/src/sqlbuild/executor/node_results/constants.py +++ b/src/sqlbuild/executor/node_results/constants.py @@ -1,6 +1,7 @@ """Runtime node result storage constants.""" from sqlbuild._native import StateSqlValueType +from sqlbuild.compiler.python_nodes.types import PythonNodeKind NODE_RESULT_MATERIALIZED_TRUE_VALUE: str = "true" @@ -36,3 +37,9 @@ **{column: StateSqlValueType.STRING for column in NODE_RESULT_COLUMNS}, COLUMN_TIMESTAMP: StateSqlValueType.TEXT_TIMESTAMP, } + +RESULT_DEPENDENCY_ATTRIBUTES: tuple[tuple[str, str], ...] = ( + ("__sqlbuild_task__", PythonNodeKind.TASK.value), + ("__sqlbuild_asset__", PythonNodeKind.ASSET.value), + ("__sqlbuild_loader__", PythonNodeKind.LOADER.value), +) diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index 245fab2a91..4d7ffae694 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -21,10 +21,9 @@ ) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.node_results.models import NodeResultEnvelope +from sqlbuild.executor.node_results.classes.dependency_result_reader import DependencyResultReader from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard from sqlbuild.executor.python_nodes.constants import ( - MISSING_DEFAULT, PYTHON_NODE_RELATION_QUALIFIER_SEPARATOR, ) from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder @@ -165,7 +164,7 @@ class IngressCallbacks: @dataclass(frozen=True, kw_only=True) -class BasePythonNodeContext: +class BasePythonNodeContext(DependencyResultReader): """Shared runtime helpers for framework-owned Python nodes.""" adapter: BaseAdapter @@ -254,64 +253,6 @@ def skip( metadata={} if metadata is None else metadata, ) - def result_of( - self, - *, - node_function: Callable[..., object], - run_id: str | None = None, - default: object = MISSING_DEFAULT, - ) -> NodeResultEnvelope | object: - """Return the latest persisted upstream result by Python node function reference.""" - - if self.result_store is None: - if default is not MISSING_DEFAULT: - return default - raise ExecutorInputError("No Python node result store is available") - node_type, node_name = self.result_dependency_identity(node_function) - return self.result_store.result_of( - node_type=node_type, - node_name=node_name, - run_id=run_id, - default=default, - ) - - def results_of( - self, - *, - node_function: Callable[..., object], - limit: int, - ) -> tuple[NodeResultEnvelope, ...]: - """Return persisted successful upstream result history, newest first.""" - - if self.result_store is None: - return () - node_type, node_name = self.result_dependency_identity(node_function) - return self.result_store.results_of( - node_type=node_type, - node_name=node_name, - limit=limit, - ) - - def result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: - """Return the `(kind, name)` a dependency's results are stored under.""" - - task_definition: object = getattr(node_function, "__sqlbuild_task__", None) - if task_definition is not None: - return PythonNodeKind.TASK.value, self._definition_name(task_definition) - asset_definition: object = getattr(node_function, "__sqlbuild_asset__", None) - if asset_definition is not None: - return PythonNodeKind.ASSET.value, self._definition_name(asset_definition) - loader_definition: object = getattr(node_function, "__sqlbuild_loader__", None) - if loader_definition is not None: - return PythonNodeKind.LOADER.value, self._definition_name(loader_definition) - raise ExecutorInputError("Python node result dependency must be a task, asset, or loader") - - def _definition_name(self, definition: object) -> str: - name: object = getattr(definition, "name", None) - if isinstance(name, str): - return name - raise ExecutorInputError("Python node result dependency must have a string name") - @dataclass(frozen=True, kw_only=True) class TaskContext(BasePythonNodeContext):