From aadbc98ff95b3e16417a407138cddd315b374cc3 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 01:44:23 +0000 Subject: [PATCH 01/19] test: reproduce Python selectors in published job selections --- .../compiler/compiled_project/_test_types.py | 7 +++ .../compiler/compiled_project/helpers.py | 22 +++++++ .../test_orchestration_snapshot.py | 59 +++++++++++++++++++ 3 files changed, 88 insertions(+) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index f9fc57f343..048a0e9102 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -262,6 +262,13 @@ class SnapshotProvidersTestCase: expected_providers: dict[str, object] | None +@dataclass(frozen=True) +class PythonJobSelectionTestCase: + description: str + select: str + expected_selection: tuple[str, ...] + + @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 22a57cf538..03050f1519 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -66,6 +66,28 @@ "documents": {}, } +PYTHON_PIPELINE_FILES: dict[str, str] = { + "sqlbuild_project.toml": 'name = "orders"\nadapter = "duckdb"\n', + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', tags [orders]);\n\nSELECT 1 AS order_id\n" + ), + "models/orders.sql": ( + "MODEL (description 'Orders.', tags [orders]);\n\n" + 'SELECT order_id FROM __ref("stg_orders")\n' + ), + "python/orders/partner_export/node.py": ( + "from sqlbuild.assets import asset\nfrom sqlbuild.refs import model\n\n\n" + "@asset(depends_on=model('orders'), tags=['orders'], description='Partner order file.')\n" + "def partner_orders_file(ctx):\n return None\n" + ), + "python/fulfillment/carrier_sync/node.py": ( + "from python.orders.partner_export.node import partner_orders_file\n" + "from sqlbuild.tasks import task\n\n\n" + "@task(depends_on=partner_orders_file, tags=['orders'], description='Book collection.')\n" + "def request_carrier_collection(ctx):\n return None\n" + ), +} + TYPE_PROOF_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "orders"\nadapter = "duckdb"\n\n[rules]\nselect = ["SQBRCONTRACT105"]\n\n' diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 770f4cbd4d..10556cef24 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -44,12 +44,14 @@ SnapshotLineageTestCase, SnapshotProvidersTestCase, SnapshotRejectionTestCase, + PythonJobSelectionTestCase, SnapshotScheduleTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( FUNCTION_PROJECT_FILES, METADATA, PARITY_FILES, + PYTHON_PIPELINE_FILES, TYPE_PROOF_FILES, published_texts, ) @@ -1846,3 +1848,60 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho project_dir=tmp_path, discovered_inputs=discovered, ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonJobSelectionTestCase( + description="a model selector selects models", + select="+orders", + expected_selection=("model:orders", "model:stg_orders"), + ), + PythonJobSelectionTestCase( + description="a task selector selects the task", + select="task:request_carrier_collection", + expected_selection=("task:request_carrier_collection",), + ), + PythonJobSelectionTestCase( + description="a python folder selects every node under it", + select="python/orders", + expected_selection=("asset:partner_orders_file",), + ), + PythonJobSelectionTestCase( + description="a tag shared by models and Python nodes selects both", + select="tag:orders", + expected_selection=( + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_python_selectors_when_publishing_jobs_then_selections_match_the_cli( + tmp_path: Path, test_case: PythonJobSelectionTestCase +) -> None: + write_files(project_dir=tmp_path, files=PYTHON_PIPELINE_FILES) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + + snapshot: dict[str, Any] = json.loads( + compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=[job("orders_job", description="Orders.", select=test_case.select)], + discovered_inputs=discovered, + ).to_json() + ) + + assert snapshot["documents"]["jobs"]["orders_job"]["selection"] == list( + test_case.expected_selection + ) From 889f3d499fc74f6cc0acec466eaceeea8f64acb8 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 02:11:12 +0000 Subject: [PATCH 02/19] wip: resolve published Python selectors with the CLI's unified resolver Work in progress, not ready for review. - Publish-time job, sensor-condition and role selections resolve with the same unified SQL/Python resolver `sqb build --select` uses. - Python snapshot nodes publish deps (Python and SQL) and tags; an intermediate loader publishes the source node it writes, and loader chains list it in `loads`. - Server-side ad-hoc selection rebuilds the Python selection graph from the snapshot and resolves with the same resolver. - The worker selects Python nodes by typed key. Validated so far: test_orchestration_snapshot.py passes (99 tests, including the Python selector reproduction). The Rust unit tests for the ad-hoc planner and planner scope have not finished yet; no review and no full gate. --- .../src/planner_scope/main/node_loads.rs | 3 +- .../src/planner_scope/tests/test_loaders.rs | 9 +- .../src/python_selection/_helpers/unified.rs | 32 ++++- .../src/python_selection/main/mod.rs | 2 + .../main/published_python_graph.rs | 52 ++++++++ .../main/resolve_python_sql_nodes.rs | 21 +++ .../src/python_selection/models.rs | 13 ++ .../src/adhoc/_helpers/planner.rs | 75 ++++++++++- .../sqlbuild-cli/src/adhoc/tests/helpers.rs | 1 + .../src/adhoc/tests/test_adhoc_planner.rs | 123 ++++++++++++++++++ .../src/runs/_helpers/requests/adhoc.rs | 21 ++- .../sqlbuild-orchestration/src/runs/models.rs | 3 +- .../compiled_project/snapshot_declarations.rs | 15 +-- .../compiled_project/snapshot_roles.rs | 23 +--- .../_helpers/graph/python_selection.rs | 89 +++++++++++++ .../_helpers/location_command/snapshot.rs | 88 ++++++++++++- .../_helpers/location_command/worker.rs | 10 +- .../_helpers/native_host/constants.rs | 2 - .../native_host/orchestration_triggers.rs | 47 ++----- .../test_orchestration_snapshot.py | 18 ++- 20 files changed, 555 insertions(+), 92 deletions(-) create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs b/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs index 4d2f33a411..e80d101d53 100644 --- a/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs +++ b/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs @@ -5,7 +5,7 @@ use crate::planner_scope::constants::{MODEL, SOURCE}; use crate::planner_scope::models::LoaderGraph; use std::collections::{BTreeMap, BTreeSet}; -/// Managed sources each model's execution deps read and each managed source's declared loader chain, as `expand_loaders` keys them. +/// Managed sources each model's execution deps read and each managed source's loader chain, intermediate loaders included, as `expand_loaders` keys them. pub fn node_loads( graph: &LoaderGraph, upstream: &[(ObjectKey, Vec)], @@ -24,7 +24,6 @@ pub fn node_loads( graph .upstream(chain) .iter() - .filter(|name| !graph.intermediate(name)) .map(|name| graph.key(name)) .collect() } else { diff --git a/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs index b90277e273..8fd13b6768 100644 --- a/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs +++ b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs @@ -147,8 +147,8 @@ fn given_loader_chains_and_unmanaged_reads_when_listing_node_loads_then_only_dec expected_loads: vec![("model:orders", vec!["source:raw_returns"])], }, NodeLoadsTestCase { - // stage_orders has no declaring source, so a run that loads it still fails in the worker. - description: "a chain keeps declared loaders behind an intermediate one and drops the intermediate", + // stage_orders has no declaring source; the snapshot publishes it as source:stage_orders. + description: "a chain keeps an intermediate loader and the declared loaders behind it", loaders: vec![ ("raw_orders", vec!["stage_orders"]), ("stage_orders", vec!["raw_customers"]), @@ -162,7 +162,10 @@ fn given_loader_chains_and_unmanaged_reads_when_listing_node_loads_then_only_dec ("source:raw_orders", vec![]), ("source:raw_customers", vec![]), ], - expected_loads: vec![("source:raw_orders", vec!["source:raw_customers"])], + expected_loads: vec![( + "source:raw_orders", + vec!["source:stage_orders", "source:raw_customers"], + )], }, ]; let key = |text: &str| { diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index b98e48dca6..7bd1a63517 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -102,15 +102,20 @@ pub(crate) fn run_selection( .filter(|atom| !is_check(atom)) .collect(); validate_selected_dependencies(context, &runnable)?; + Ok(selection(context, sql_keys(&runnable), &runnable)) +} + +/// The SQL keys, with the source each selected intermediate loader writes, and the selected Python node names. +fn selection(context: &SelectionContext<'_>, sql: Vec, atoms: &Atoms) -> RunSelection { let terminal: HashSet<&str> = context .sql .sources .iter() .filter_map(|(_, loader)| loader.as_deref()) .collect(); - let mut keys: BTreeSet = sql_keys(&runnable).into_iter().collect(); + let mut keys: BTreeSet = sql.into_iter().collect(); let mut names: Vec = Vec::new(); - for atom in &runnable { + for atom in atoms { let Atom::Python(name) = atom else { continue; }; @@ -119,10 +124,29 @@ pub(crate) fn run_selection( } names.push(name.clone()); } - Ok(RunSelection { + RunSelection { sql_keys: keys.into_iter().collect(), python_node_names: names, - }) + } +} + +/// The SQL keys with their build functions and the Python node names, checks included, that selectors name; nothing is refused. +pub(crate) fn node_selection( + context: &SelectionContext<'_>, + select: &[String], + exclude: &[String], +) -> Result { + validate_boundaries(context)?; + let selected: Atoms = groups(context, select)? + .difference(&groups(context, exclude)?) + .cloned() + .collect(); + let sql: Vec = build_resources( + context.sql.graph, + &sql_keys(&selected), + BuildResources::SELECTION, + ); + Ok(selection(context, sql, &selected)) } fn groups(context: &SelectionContext<'_>, raw_groups: &[String]) -> Resolved { diff --git a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs index 884bfa0b85..3dace0806a 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs @@ -1,4 +1,6 @@ +pub mod published_python_graph; pub mod python_node_graph_indexes; pub mod resolve_python_node_excludes; pub mod resolve_python_node_selectors; +pub mod resolve_python_sql_nodes; pub mod resolve_python_sql_run_selection; diff --git a/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs b/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs new file mode 100644 index 0000000000..482d1689f5 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs @@ -0,0 +1,52 @@ +use crate::definition_changes::constants::PYTHON_NODE_KINDS; +use crate::python_selection::_helpers::inventory::indexes; +use crate::python_selection::models::{ + InventoryIndexes, InventoryNode, PublishedPythonNode, PythonSelectionGraph, PythonSelectionNode, +}; + +/// The selection graph of Python nodes rebuilt from a published snapshot, so the server selects them as the CLI does. +pub fn published_python_graph(nodes: &[PublishedPythonNode]) -> PythonSelectionGraph { + let is_python = |kind: &str| PYTHON_NODE_KINDS.contains(&kind); + let inventory: Vec = nodes + .iter() + .map(|node| InventoryNode { + name: node.name.clone(), + tags: node.tags.clone(), + relative_path: String::new(), + upstream: node + .deps + .iter() + .filter(|(kind, _)| is_python(kind)) + .map(|(_, name)| name.clone()) + .collect(), + }) + .collect(); + let InventoryIndexes { + upstream, + downstream, + tags, + .. + } = indexes(&inventory); + PythonSelectionGraph { + nodes: nodes + .iter() + .map(|node| PythonSelectionNode { + name: node.name.clone(), + kind: node.kind.clone(), + sql_deps: node + .deps + .iter() + .filter(|(kind, _)| !is_python(kind)) + .cloned() + .collect(), + }) + .collect(), + upstream, + downstream, + tags, + paths: nodes + .iter() + .map(|node| (node.name.clone(), node.folder.clone())) + .collect(), + } +} diff --git a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs new file mode 100644 index 0000000000..6a2d99a3cb --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs @@ -0,0 +1,21 @@ +use crate::graph::errors::SelectorError; +use crate::python_selection::_helpers::nodes::sql_names; +use crate::python_selection::_helpers::unified::node_selection; +use crate::python_selection::models::{ + NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, +}; + +/// The SQL resources and Python nodes, checks included, that selectors name, as grants and sensor conditions read them. +pub fn resolve_python_sql_nodes( + sql: SqlSelectionGraph<'_>, + python: &PythonSelectionGraph, + select: &[String], + exclude: &[String], +) -> Result { + let context: SelectionContext<'_> = SelectionContext { + sql, + nodes: NodeIndex::new(python), + sql_names: sql_names(sql.graph), + }; + node_selection(&context, select, exclude) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/models.rs b/crates/sqlbuild-analysis/src/python_selection/models.rs index b0d615e1dd..63b7aacf07 100644 --- a/crates/sqlbuild-analysis/src/python_selection/models.rs +++ b/crates/sqlbuild-analysis/src/python_selection/models.rs @@ -52,6 +52,19 @@ pub struct InventoryNode { pub upstream: Vec, } +/// One Python node as a published snapshot lists it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PublishedPythonNode { + pub name: String, + /// `task`, `asset`, `loader` or `check`. + pub kind: String, + pub tags: Vec, + /// The folder of the node's file relative to the project, in POSIX form. + pub folder: String, + /// The Python nodes and SQL resources it depends on, as published keys. + pub deps: Vec, +} + /// Python's `PythonNodeGraph` edges and indexes, each in Python's dict order. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct InventoryIndexes { diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs index 93c34ee625..fd215a452f 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs @@ -3,11 +3,14 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::{Value as Json, json}; +use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; use sqlbuild_analysis::graph::models::{GraphResource, JobSelection}; +use sqlbuild_analysis::python_selection::main::published_python_graph::published_python_graph; +use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection; +use sqlbuild_analysis::python_selection::models::{PublishedPythonNode, SqlSelectionGraph}; use sqlbuild_core::text::main::python_str_repr::python_str_repr; use sqlbuild_orchestration::runs::models::{CommandPlan, PlannerResource}; use sqlbuild_orchestration::runs::types::{CommandPlanner, PublishedJobs}; @@ -203,19 +206,81 @@ fn resolved( select: &[String], exclude: &[String], ) -> Result, String> { - let graph_resources: Vec = resources.iter().filter_map(graph_resource).collect(); + let (python, sql): (Vec<&PlannerResource>, Vec<&PlannerResource>) = resources + .iter() + .partition(|resource| python_kind(&resource.key).is_some()); + let graph_resources: Vec = sql + .iter() + .filter_map(|resource| graph_resource(resource)) + .collect(); let mut graph = build_project_graph(&graph_resources); let selectors: Vec = select.iter().chain(exclude).cloned().collect(); - if selectors_need_jobs(&graph, &selectors, &[]) { + let node_names: Vec<&str> = python + .iter() + .filter_map(|resource| object_key(&resource.key).map(|_| name_of(&resource.key))) + .collect(); + if selectors_need_jobs(&graph, &selectors, &node_names) { graph = attach_jobs(graph, published_jobs(jobs)); } - Ok(resolve_selectors(&graph, select, exclude) - .map_err(|error| format!("{}: {}", error.code, error.message))? + let sources: Vec<(String, Option)> = sql + .iter() + .filter(|resource| resource.key.starts_with("source:")) + .map(|resource| (name_of(&resource.key).to_owned(), resource.loader.clone())) + .collect(); + let nodes: Vec = python + .iter() + .filter_map(|resource| { + Some(PublishedPythonNode { + name: name_of(&resource.key).to_owned(), + kind: python_kind(&resource.key)?.to_owned(), + tags: resource.tags.clone(), + folder: resource.folder.clone().unwrap_or_else(|| ".".to_owned()), + deps: resource + .deps + .iter() + .filter_map(|dep| object_key(dep)) + .collect(), + }) + }) + .collect(); + let python_graph = published_python_graph(&nodes); + let selection = resolve_python_sql_run_selection( + SqlSelectionGraph { + graph: &graph, + sources: &sources, + }, + &python_graph, + select, + exclude, + ) + .map_err(|error| format!("{}: {}", error.code, error.message))?; + let kinds: BTreeMap<&str, &str> = nodes + .iter() + .map(|node| (node.name.as_str(), node.kind.as_str())) + .collect(); + Ok(selection + .sql_keys .into_iter() .map(|(kind, name)| format!("{kind}:{name}")) + .chain( + selection + .python_node_names + .iter() + .filter_map(|name| Some(format!("{}:{name}", kinds.get(name.as_str())?))), + ) .collect()) } +/// The Python node kind of a published key, such as `task` for `task:export_orders`. +fn python_kind(key: &str) -> Option<&'static str> { + let (kind, _) = key.split_once(':')?; + PYTHON_NODE_KINDS.into_iter().find(|python| *python == kind) +} + +fn name_of(key: &str) -> &str { + key.split_once(':').map_or(key, |(_, name)| name) +} + /// Published jobs as already-resolved graph selections. fn published_jobs(jobs: &PublishedJobs) -> BTreeMap { let mut selections: BTreeMap = BTreeMap::new(); diff --git a/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs b/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs index 0725ac9380..3cd4087f5e 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/helpers.rs @@ -13,6 +13,7 @@ pub(super) fn resource( deps: deps.iter().map(|dep| (*dep).to_owned()).collect(), tags: tags.iter().map(|tag| (*tag).to_owned()).collect(), folder: folder.map(str::to_owned), + loader: None, } } diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index 8631f17666..47d3753118 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -289,3 +289,126 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select ); } } + +#[test] +fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_command_line() { + let resources = [ + resource("model:stg_orders", &[], &["orders"], Some("models/staging")), + resource( + "model:orders", + &["model:stg_orders"], + &["orders"], + Some("models/marts"), + ), + resource("model:customers", &[], &[], Some("models/marts")), + resource( + "asset:partner_orders_file", + &["model:orders"], + &["orders"], + Some("python/orders/partner_export"), + ), + resource( + "task:request_carrier_collection", + &["asset:partner_orders_file"], + &["orders"], + Some("python/fulfillment/carrier_sync"), + ), + ]; + let jobs: PublishedJobs = BTreeMap::new(); + let test_cases = [ + AdhocPlanTestCase { + description: "typed selectors select the Python nodes", + command: &[ + "build", + "--select", + "task:request_carrier_collection", + "asset:partner_orders_file", + ], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "a task without the Python node it depends on is refused, as on the command line", + command: &["build", "--select", "task:request_carrier_collection"], + expected_plan: Err("depends on unselected Python node 'partner_orders_file'"), + }, + AdhocPlanTestCase { + description: "a python folder selects the nodes under it", + command: &["build", "--select", "python/orders"], + expected_plan: Ok((&["asset:partner_orders_file"], json!({}))), + }, + AdhocPlanTestCase { + description: "a tag shared by models and Python nodes selects both", + command: &["build", "--select", "tag:orders"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "upstream of a task crosses its Python and SQL dependencies", + command: &["build", "--select", "+task:request_carrier_collection"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "downstream of a model reaches the Python nodes that read it", + command: &["build", "--select", "orders+", "--exclude", "stg_orders"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, + ]; + let planner = adhoc_planner(); + for test_case in test_cases { + let command: Vec = test_case + .command + .iter() + .map(|token| (*token).to_owned()) + .collect(); + let actual = planner + .plan(&command, (&resources, &jobs)) + .map(|plan| { + ( + plan.selection.into_iter().collect::>(), + json!(plan.settings), + ) + }) + .map_err(|error| { + test_case + .expected_plan + .as_ref() + .err() + .filter(|fragment| error.contains(**fragment)) + .map_or(error.clone(), |fragment| (*fragment).to_owned()) + }); + assert_eq!( + actual, + owned_plan(&test_case.expected_plan), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index bcb1eddb60..6de44e69d1 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -30,6 +30,10 @@ use crate::store::models::{ControlTransaction, Parameter}; /// Graph kinds in the order the CLI builds its project graph: models, sources, seeds, then functions. const GRAPH_KINDS: [&str; 5] = ["model", "source", "seed", "udf", "table_fn"]; +/// Python node kinds, in the order the CLI discovers them. +const PYTHON_NODE_KINDS: [&str; 4] = ["task", "asset", "loader", "check"]; +/// Marks the source node an intermediate loader writes, which the CLI's SQL graph does not list. +const INTERMEDIATE_LOADER_FIELD: &str = "intermediate_loader"; /// The frozen request and its verb, or why the command cannot run; `run_id` comes from the caller. pub(crate) async fn frozen_adhoc_request( @@ -141,23 +145,26 @@ pub(crate) async fn publication_plan( )) } -/// The selector graph rebuilt from published definitions, in the CLI's kind order. +/// The selector graph rebuilt from published definitions, in the CLI's kind order; sources intermediate loaders write are selected through their loader. fn planner_resources( nodes: &std::collections::BTreeMap, ) -> Vec { let mut resources: Vec = Vec::new(); - for kind in GRAPH_KINDS { + for kind in GRAPH_KINDS.iter().chain(&PYTHON_NODE_KINDS) { for (key, node) in nodes { - if node.kind != kind { + let definition = &node.definition; + if node.kind != *kind || definition.get(INTERMEDIATE_LOADER_FIELD).is_some() { continue; } - let definition = &node.definition; - let (tags, folder) = match kind { + let (tags, folder) = match *kind { "model" => ( definition.pointer("/config/tags"), Some(model_folder(definition)), ), "seed" => (definition.pointer("/schema/tags"), None), + _ if PYTHON_NODE_KINDS.contains(kind) => { + (definition.get("tags"), Some(model_folder(definition))) + } _ => (definition.get("tags"), None), }; resources.push(PlannerResource { @@ -165,6 +172,10 @@ fn planner_resources( deps: texts(definition.get("deps")), tags: texts(tags), folder, + loader: definition + .pointer("/entry/loader") + .and_then(Value::as_str) + .map(str::to_owned), }); } } diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 6b92161618..48ac16d019 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -760,13 +760,14 @@ pub struct AdhocCheckRequest { pub priority: i64, } -/// One graph resource selectors resolve over: its `kind:name` key, upstream keys, tags and model folder. +/// One SQL resource or Python node selectors resolve over: its `kind:name` key, upstream keys, tags, file folder and a source's loader. #[derive(Clone, Debug, PartialEq, Eq)] pub struct PlannerResource { pub key: String, pub deps: Vec, pub tags: Vec, pub folder: Option, + pub loader: Option, } /// A parsed command: its verb, the exact node keys it selects and the flags it sets, by parser destination. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index a63c08cd12..ca9901107b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -6,7 +6,6 @@ use std::path::Path; use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use serde_json::Value; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_core::constants::PYTHON_INIT_MODULE_STEM; @@ -20,6 +19,7 @@ use sqlbuild_orchestration::declarations::models::{ }; use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::native_host::orchestration_declarations::{ AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, declared_at, module_jobs, run_controls, @@ -60,7 +60,7 @@ pub(crate) fn discover_declarations<'py>( pub(crate) fn compile_declarations( py: Python<'_>, mut input: SnapshotInput, - (project_dir, graph): (&Path, &ProjectGraph), + (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), declarations: Vec, ) -> PyResult { if declarations.is_empty() { @@ -113,12 +113,9 @@ pub(crate) fn compile_declarations( job.name ))); } - let selection: BTreeSet = - resolve_selectors(graph, &tokens(&job.select), &tokens(&job.exclude)) - .map_err(|error| PyValueError::new_err(error.message))? - .into_iter() - .map(|(kind, name)| format!("{kind}:{name}")) - .collect(); + let selection: BTreeSet = selected + .run_keys(graph, &tokens(&job.select), &tokens(&job.exclude)) + .map_err(|error| PyValueError::new_err(format!("job {}: {error}", job.name)))?; let declared_at: Option = declared_at(project_dir, job.declared_at)?; let controls = run_controls( &job.name, @@ -229,7 +226,7 @@ pub(crate) fn compile_declarations( .. } = compile_sensors( py, - (project_dir, &input, graph), + (project_dir, &input, (graph, selected)), authored_sensors, compile_calendars( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs index 4867f861ab..26e7bc1266 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_roles.rs @@ -1,35 +1,23 @@ //! Project roles from the reviewed `roles.toml` file, their selectors resolved against this snapshot. -use std::collections::BTreeSet; use std::path::Path; use pyo3::exceptions::PyValueError; use pyo3::prelude::{PyResult, Python}; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_orchestration::authorization::main::compile_roles::compile_roles; use sqlbuild_orchestration::snapshots::models::SnapshotInput; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::native_host::host::write_console; /// The project-root file that defines project roles. pub(crate) const ROLES_FILE: &str = "roles.toml"; -/// Node keys of one grant selector, in the job selection grammar. -fn selector_nodes(graph: &ProjectGraph, selector: &str) -> Result, String> { - let tokens: Vec = selector.split_whitespace().map(str::to_owned).collect(); - let keys = resolve_selectors(graph, &tokens, &[]).map_err(|error| error.message)?; - let mut nodes: BTreeSet = BTreeSet::new(); - for (kind, name) in keys { - nodes.insert(format!("{kind}:{name}")); - } - Ok(nodes) -} - pub(crate) fn compile_project_roles( py: Python<'_>, mut input: SnapshotInput, - (project_dir, graph): (&Path, &ProjectGraph), + (project_dir, graph, selected): (&Path, &ProjectGraph, &ProjectSelection), ) -> PyResult { let path = project_dir.join(ROLES_FILE); if !path.is_file() { @@ -42,8 +30,11 @@ pub(crate) fn compile_project_roles( } let text = std::fs::read_to_string(&path) .map_err(|error| PyValueError::new_err(format!("roles.toml: {error}")))?; - let (document, warnings) = compile_roles(&text, |selector| selector_nodes(graph, selector)) - .map_err(PyValueError::new_err)?; + let (document, warnings) = compile_roles(&text, |selector| { + let tokens: Vec = selector.split_whitespace().map(str::to_owned).collect(); + selected.node_keys(graph, &tokens, &[]) + }) + .map_err(PyValueError::new_err)?; for warning in warnings { write_console(py, "stderr", &format!("Warning: {warning}\n"))?; } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 1dd9d85d85..3859130776 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -1,5 +1,7 @@ //! Python node and unified SQL/Python selector resolution, and the Python node graph indexes. +use std::collections::BTreeSet; + use pyo3::prelude::{ Bound, PyAny, PyAnyMethods, PyErr, PyModule, PyModuleMethods, PyResult, Python, }; @@ -7,9 +9,11 @@ use pyo3::types::{PyDict, PyFrozenSet}; use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::errors::SelectorError; +use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; +use sqlbuild_analysis::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection as run_selection; use sqlbuild_analysis::python_selection::models::{ InventoryIndexes, InventoryNode, PythonSelectionGraph, PythonSelectionNode, RunSelection, @@ -19,6 +23,7 @@ use sqlbuild_analysis::python_selection::models::{ use crate::bindings::_helpers::boundary::panics::compiler_guard; use crate::bindings::_helpers::compiled_project::job_catalog::graph_with_jobs; use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; +use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::records::construct::error_class; use crate::bindings::_helpers::records::fields::field as native_field; use crate::bindings::_helpers::records::fields::mapping_items; @@ -128,6 +133,90 @@ fn project_sources(project: &Bound<'_, PyAny>) -> PyResult)>, + pub(crate) nodes: PythonSelectionGraph, +} + +impl ProjectSelection { + pub(crate) fn new( + project: &Bound<'_, PyAny>, + discovered_inputs: &Bound<'_, PyAny>, + ) -> PyResult { + Ok(Self { + sources: project_sources(project)?, + nodes: selection_graph(&python_node_graph(discovered_inputs)?)?, + }) + } + + /// The `kind:name` keys `sqb build --select` runs for these selectors. + pub(crate) fn run_keys( + &self, + graph: &ProjectGraph, + select: &[String], + exclude: &[String], + ) -> Result, String> { + self.keys(run_selection(self.sql(graph), &self.nodes, select, exclude)) + } + + /// The `kind:name` keys these selectors name, Python checks included. + pub(crate) fn node_keys( + &self, + graph: &ProjectGraph, + select: &[String], + exclude: &[String], + ) -> Result, String> { + self.keys(resolve_python_sql_nodes( + self.sql(graph), + &self.nodes, + select, + exclude, + )) + } + + /// The published key of a Python node, such as `task:export_orders`. + pub(crate) fn node_key(&self, name: &str) -> Option { + self.nodes + .nodes + .iter() + .rev() + .find(|node| node.name == name) + .map(|node| format!("{}:{name}", node.kind)) + } + + /// Whether a loader writes no declared source and so runs only as part of another loader's chain. + pub(crate) fn intermediate_loader(&self, name: &str) -> bool { + !self + .sources + .iter() + .any(|(_, loader)| loader.as_deref() == Some(name)) + } + + fn sql<'graph>(&'graph self, graph: &'graph ProjectGraph) -> SqlSelectionGraph<'graph> { + SqlSelectionGraph { + graph, + sources: &self.sources, + } + } + + fn keys( + &self, + selection: Result, + ) -> Result, String> { + let selection: RunSelection = selection.map_err(|error| error.message)?; + let mut keys: BTreeSet = selection + .sql_keys + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + for name in &selection.python_node_names { + keys.extend(self.node_key(name)); + } + Ok(keys) + } +} + /// The Python node names `select` minus `exclude` resolve to; every node when unselected. pub(crate) fn python_node_names<'py>( graph: &Bound<'py, PyAny>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 830e43a841..40a0682a21 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -1,6 +1,6 @@ //! Location publication assembles catalog data and planner-owned local identities. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; use pyo3::exceptions::PyValueError; @@ -24,6 +24,7 @@ use crate::bindings::_helpers::compiled_project::snapshot_roles::compile_project use crate::bindings::_helpers::compiled_views::project_view::NativeProjectView; use crate::bindings::_helpers::discovery::project_facts::{named_node_rows, project_facts}; use crate::bindings::_helpers::discovery::project_validation::validation_error; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::location_command::snapshot_lineage::column_lineage_document; use crate::bindings::_helpers::location_command::snapshot_providers::providers_document; use crate::bindings::_helpers::native_host::orchestration_declarations::{ @@ -36,6 +37,9 @@ use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; use crate::bindings::_helpers::planner_scope::values::project_graph; use crate::bindings::_helpers::records::fields::field; +/// The Python node kind whose functions load sources. +const LOADER_KIND: &str = "loader"; + /// Where declarations passed to the snapshot directly are reported as declared. const DECLARATIONS_ARGUMENT: &str = "declarations argument"; @@ -111,6 +115,9 @@ pub(crate) fn orchestration_snapshot<'py>( } input.nodes = resource_nodes(py, &project, &owner, input.nodes)?; drop(owner); + let selected = + ProjectSelection::new((&project).into_pyobject(py)?.as_any(), &discovered_inputs)?; + input.nodes = python_nodes(&selected, input.nodes)?; if let Some(providers) = providers_document(&discovered_inputs, &input.nodes)? { input.documents.insert("providers".into(), providers); } @@ -137,11 +144,15 @@ pub(crate) fn orchestration_snapshot<'py>( input = compile_declarations( py, input, - (&project_dir, graph.bind(py).get().graph()), + (&project_dir, graph.bind(py).get().graph(), &selected), authored, )?; } - input = compile_project_roles(py, input, (&project_dir, graph.bind(py).get().graph()))?; + input = compile_project_roles( + py, + input, + (&project_dir, graph.bind(py).get().graph(), &selected), + )?; let project = project.into_pyobject(py)?; if let Some(lineage) = column_lineage_document(project.as_any())? { input.documents.insert("lineage".into(), lineage); @@ -151,6 +162,77 @@ pub(crate) fn orchestration_snapshot<'py>( Ok(NativeOrchestrationSnapshot { input }) } +/// Python nodes publish their `deps` and tags, and each intermediate loader publishes the source it writes, so the server orders, selects and grants them as the CLI does. +fn python_nodes( + selected: &ProjectSelection, + mut nodes: BTreeMap, +) -> PyResult> { + let graph = &selected.nodes; + let mut tags: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); + for (tag, names) in &graph.tags { + for name in names { + tags.entry(name.as_str()).or_default().push(tag.as_str()); + } + } + let upstream: BTreeMap<&str, &[String]> = graph + .upstream + .iter() + .map(|(name, names)| (name.as_str(), names.as_slice())) + .collect(); + let mut outputs: Vec<(String, SnapshotNode)> = Vec::new(); + for node in &graph.nodes { + let key = format!("{}:{}", node.kind, node.name); + let Some(published) = nodes.get_mut(&key) else { + return Err(PyValueError::new_err(format!( + "Python node {key} is missing from the snapshot" + ))); + }; + let deps: BTreeSet = upstream + .get(node.name.as_str()) + .into_iter() + .flat_map(|names| names.iter()) + .filter_map(|name| selected.node_key(name)) + .chain( + node.sql_deps + .iter() + .map(|(kind, name)| format!("{kind}:{name}")), + ) + .collect(); + published.definition["deps"] = json!(deps); + published.definition["tags"] = + json!(tags.get(node.name.as_str()).cloned().unwrap_or_default()); + if node.kind == LOADER_KIND && selected.intermediate_loader(&node.name) { + let mut definition = json!({ + "name": node.name, + "entry": {"loader": node.name}, + "intermediate_loader": true, + "deps": [key], + }); + if let Some(path) = published.definition.get("path") { + definition["path"] = path.clone(); + } + outputs.push(( + format!("source:{}", node.name), + SnapshotNode { + kind: "source".into(), + definition, + raw_sql: None, + compiled_sql: None, + python_source: None, + }, + )); + } + } + for (key, node) in outputs { + if nodes.insert(key.clone(), node).is_some() { + return Err(PyValueError::new_err(format!( + "intermediate loader {key} collides with a published node" + ))); + } + } + Ok(nodes) +} + fn identity_nodes( project: &Bound<'_, PyAny>, mut nodes: BTreeMap, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index 4d28d969ae..ec62b33a15 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -8,6 +8,7 @@ use pyo3::exceptions::{PyRuntimeError, PyValueError}; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::PyString; use serde_json::Value; +use sqlbuild_analysis::definition_changes::constants::PYTHON_NODE_KINDS; use sqlbuild_cli::argv::main::command_settings_arguments::command_settings_arguments; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; use sqlbuild_cli::dispatch::main::command_dispatch::command_dispatch; @@ -148,10 +149,15 @@ fn command_request<'py>( if execution.skipped_nodes.contains_key(key) { continue; } - let (_, name) = key + let (kind, name) = key .split_once(':') .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; - argv.extend([PyString::new(py, "--select"), PyString::new(py, name)]); + let selector = if PYTHON_NODE_KINDS.contains(&kind) { + key.as_str() + } else { + name + }; + argv.extend([PyString::new(py, "--select"), PyString::new(py, selector)]); } let parsed = parse_cli_invocation(py, argv, false)?; let args = field(&parsed, "args")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index d19d662071..d0ef09dcb8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -4,8 +4,6 @@ pub(crate) const CALENDAR_WINDOW_PARAMETER: &str = "window"; pub(crate) const RUN_EVENT_CONDITION: &str = "run_event"; -pub(crate) const NODE_EVENT_CONDITION: &str = "node_event"; -pub(crate) const CHECK_KIND: &str = "check"; pub(crate) const SOURCE_NEW_DATA_CONDITION: &str = "source_new_data"; pub(crate) const SOURCE_STALE_CONDITION: &str = "source_stale"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index 8779ef8608..b8923950a0 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -1,6 +1,6 @@ //! Authored trigger controls and bounded calendars evaluated only in the location interpreter. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; use std::path::Path; use chrono::{DateTime, TimeDelta, Timelike, Utc}; @@ -10,21 +10,20 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods}; use serde_json::{Value, json}; -use sqlbuild_analysis::graph::main::resolve_selectors::resolve_selectors; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cursor::algebra::main::parse_duration::parse_duration; use sqlbuild_host::entries::models::UserCodeHost; use sqlbuild_orchestration::declarations::models::{Definitions, Schedule, TriggerDefinition}; use sqlbuild_orchestration::scheduler::constants::MAX_TICKS_PER_BATCH; use sqlbuild_orchestration::sensors::models::SensorDefinition; -use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; +use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::boundary::python_calls::{call_with, imported, within}; use crate::bindings::_helpers::discovery::imported_modules::{provider_by_name, provider_usages}; use crate::bindings::_helpers::discovery::imported_providers::discover_provider_classes; +use crate::bindings::_helpers::graph::python_selection::ProjectSelection; use crate::bindings::_helpers::native_host::constants::{ - CALENDAR_WINDOW_PARAMETER, CHECK_KIND, NODE_EVENT_CONDITION, SOURCE_NEW_DATA_CONDITION, - SOURCE_STALE_CONDITION, + CALENDAR_WINDOW_PARAMETER, SOURCE_NEW_DATA_CONDITION, SOURCE_STALE_CONDITION, }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::native_host::orchestration_declarations::{ @@ -253,7 +252,7 @@ pub(crate) fn compile_calendars( pub(crate) fn compile_sensors( py: Python<'_>, - (project_dir, input, graph): (&Path, &SnapshotInput, &ProjectGraph), + (project_dir, input, graph): (&Path, &SnapshotInput, (&ProjectGraph, &ProjectSelection)), authored: Vec, mut definitions: Definitions, ) -> PyResult { @@ -281,12 +280,11 @@ pub(crate) fn compile_sensors( if let Some(select) = when.select { let selectors: Vec = select.split_whitespace().map(str::to_owned).collect(); - condition["selection"] = - sensor_nodes(graph, &input.nodes, &selectors, &when.kind)?; + condition["selection"] = sensor_nodes(graph, &selectors, &when.kind)?; } if let Some(selectors) = when.executed_successfully { condition["executed_successfully"] = - sensor_nodes(graph, &input.nodes, &selectors, "executed_successfully")?; + sensor_nodes(graph, &selectors, "executed_successfully")?; } condition } @@ -314,8 +312,7 @@ pub(crate) fn compile_sensors( } fn sensor_nodes( - graph: &ProjectGraph, - available: &BTreeMap, + (graph, selected): (&ProjectGraph, &ProjectSelection), selectors: &[String], field: &str, ) -> PyResult { @@ -324,31 +321,9 @@ fn sensor_nodes( "{field} needs nonempty selectors" ))); } - let mut nodes: BTreeSet = BTreeSet::new(); - let mut sql_selectors: Vec = Vec::new(); - for selector in selectors { - if field == NODE_EVENT_CONDITION && selector.starts_with("check:") { - if !available - .get(selector) - .is_some_and(|node| node.kind == CHECK_KIND) - { - return Err(PyValueError::new_err(format!( - "unknown check node {selector}" - ))); - } - nodes.insert(selector.clone()); - } else { - sql_selectors.push(selector.clone()); - } - } - if !sql_selectors.is_empty() { - nodes.extend( - resolve_selectors(graph, &sql_selectors, &[]) - .map_err(|error| PyValueError::new_err(error.message))? - .into_iter() - .map(|(kind, name)| format!("{kind}:{name}")), - ); - } + let nodes: BTreeSet = selected + .node_keys(graph, selectors, &[]) + .map_err(|error| PyValueError::new_err(format!("{field}: {error}")))?; if nodes.is_empty() { return Err(PyValueError::new_err(format!( "{field} needs nonempty selectors" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 10556cef24..9cce168cc4 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -1789,7 +1789,7 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects .replace('("orders",)', '("unknown_*",)') }, expected_error=ValueError, - expected_message="unknown selector pattern", + expected_message="executed_successfully: unknown selector name 'unknown_\\*'", ), SnapshotDiscoveryRejectionTestCase( description="invalid execution selector syntax fails closed", @@ -1859,9 +1859,19 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho expected_selection=("model:orders", "model:stg_orders"), ), PythonJobSelectionTestCase( - description="a task selector selects the task", - select="task:request_carrier_collection", - expected_selection=("task:request_carrier_collection",), + description="typed selectors select the task and the asset it depends on", + select="task:request_carrier_collection asset:partner_orders_file", + expected_selection=("asset:partner_orders_file", "task:request_carrier_collection"), + ), + PythonJobSelectionTestCase( + description="upstream of a task crosses its Python and SQL dependencies", + select="+task:request_carrier_collection", + expected_selection=( + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ), ), PythonJobSelectionTestCase( description="a python folder selects every node under it", From cc4a6c59b08c71ba2d38d6d1fa40dc159fd91018 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:22:56 +0000 Subject: [PATCH 03/19] feat: let a selected Python node read an unselected upstream's result A Python node can now be selected without its Python upstream in every path that uses the shared resolver: sqb build --select, published jobs, sensors, roles and ad-hoc runs. It reads the upstream's latest persisted result, and fails at run time with an error naming the upstream and the command to build it when the upstream has never succeeded. An intermediate loader without a write_strategy that returns rows now writes them to __loader__ as a table, as the loaders docs describe. A self-managed intermediate that returns nothing keeps its behaviour. The published snapshot rebuilds the Python selection graph from node deps, and job groups order jobs across Python and Python-to-SQL edges. Validated locally: python_selection and adhoc planner Rust tests, clippy on both crates, fensu check, the worker claim tests, orchestration snapshot, Python node selection E2E, load E2E (sqb load and sqb build on the docs example) and the python_nodes and executor/load unit tests. Not yet run: make check-ci, the full suite, review. --- .../loaders/intermediate_default_accounts.py | 25 ++++++ .../loaders/intermediate_default_accounts.yml | 5 ++ .../intermediate_loader_sources.py | 7 +- .../python_selection/_helpers/inventory.rs | 17 ++++ .../python_selection/_helpers/validation.rs | 15 +--- .../main/published_python_graph.rs | 42 ++++----- .../src/python_selection/tests/helpers.rs | 46 +++++++++- .../tests/test_python_selection.rs | 25 +++--- .../src/adhoc/_helpers/planner.rs | 30 ++++--- .../src/adhoc/tests/test_adhoc_planner.rs | 4 +- .../executor/load/_helpers/execution.py | 14 ++- .../load/_helpers/loader_invocation.py | 3 +- src/sqlbuild/executor/load/main/_execute.py | 3 +- .../python_nodes/_helpers/execution.py | 43 ++++++++++ src/sqlbuild/executor/python_nodes/models.py | 8 +- .../cli/commands/main/build/_test_types.py | 10 +++ .../cli/commands/main/build/helpers.py | 9 ++ .../main/build/test_python_node_selection.py | 49 +++++++++++ .../cli/commands/main/load/_test_types.py | 8 ++ .../cli/commands/main/load/test_load.py | 60 +++++++++++++ .../compiler/compiled_project/_test_types.py | 14 +++ .../compiler/compiled_project/helpers.py | 23 +++++ .../publication/_test_types.py | 6 ++ .../compiled_project/publication/helpers.py | 16 ++++ .../publication/test_worker.py | 47 +++++----- .../test_orchestration_snapshot.py | 86 ++++++++++++++++++- .../_helpers/test_run_selectors.py | 16 ++-- .../docs/concepts/python-nodes/overview.mdx | 14 ++- 28 files changed, 528 insertions(+), 117 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.py create mode 100644 benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.yml diff --git a/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.py b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.py new file mode 100644 index 0000000000..6b01434f43 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.py @@ -0,0 +1,25 @@ +"""The loader-dependencies example from the loaders docs, as written. + +raw_accounts is an intermediate loader without a write_strategy that returns rows, so it writes +them to __loader__raw_accounts as a table. +""" + +from sqlbuild.executor.load.models import LoaderContext +from sqlbuild.loaders import loader + + +@loader +def raw_accounts(ctx: LoaderContext) -> list[dict[str, object]]: + """Load customer accounts.""" + return [ + {"account_id": 1, "account_name": "Pawnee Parks"}, + {"account_id": 2, "account_name": "Eagleton"}, + ] + + +@loader(depends_on=[raw_accounts]) +def raw_account_metrics(ctx: LoaderContext) -> list[dict[str, object]]: + """Account metrics.""" + accounts = ctx.loader(raw_accounts) + rows = ctx.query(f"SELECT account_id FROM {accounts.destination}") + return [{"account_id": row[0], "metric": "active"} for row in rows.fetchall()] diff --git a/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.yml b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.yml new file mode 100644 index 0000000000..90e7a2605a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/intermediate_default_accounts.yml @@ -0,0 +1,5 @@ +sources: + - name: raw_account_metrics + description: Account metrics read from the intermediate raw_accounts loader. + managed: true + write_strategy: table diff --git a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py index 9512d788c5..70db157dde 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py +++ b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py @@ -1,9 +1,12 @@ -"""A source loader that depends on an intermediate loader no source declares.""" +"""A source loader that depends on an intermediate loader no source declares. + +A job that selects the model must claim the intermediate loader's relation as well as the source. +""" from sqlbuild.loaders import loader -@loader +@loader(write_strategy="table") def stage_shipments(ctx): """Stage synthetic shipments in the intermediate loader's own relation.""" return [{"order_id": 9}] diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs index 9a416f0c44..944eb5294f 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/inventory.rs @@ -2,6 +2,9 @@ use std::collections::HashMap; +use crate::assembly::project::types::ObjectKey; +use crate::definition_changes::constants::PYTHON_NODE_KINDS; + use crate::python_selection::constants::{CURRENT_DIRECTORY, ROOT_DIRECTORY}; use crate::python_selection::models::{InventoryIndexes, InventoryNode}; @@ -83,3 +86,17 @@ pub(crate) fn indexes(nodes: &[InventoryNode]) -> InventoryIndexes { paths: paths.rows, } } + +/// A published node's dependency keys split into its Python upstream names and its SQL keys. +pub(crate) fn split_published_deps(deps: &[ObjectKey]) -> (Vec, Vec) { + let mut python: Vec = Vec::new(); + let mut sql: Vec = Vec::new(); + for (kind, name) in deps { + if PYTHON_NODE_KINDS.contains(&kind.as_str()) { + python.push(name.clone()); + } else { + sql.push((kind.clone(), name.clone())); + } + } + (python, sql) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs index ddacd27837..9bed1e9624 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/validation.rs @@ -157,7 +157,7 @@ fn validate_loader_upstreams(context: &SelectionContext<'_>) -> Result<(), Selec Ok(()) } -/// Python's `_validate_selected_dependencies`: selected sources and nodes have their loaders. +/// Selected managed sources have their loaders selected; a Python node may run without its Python upstream, whose latest persisted result it reads. pub(crate) fn validate_selected_dependencies( context: &SelectionContext<'_>, atoms: &BTreeSet, @@ -188,18 +188,5 @@ pub(crate) fn validate_selected_dependencies( select it directly or use upstream expansion" ))); } - for atom in atoms { - let Atom::Python(name) = atom else { - continue; - }; - for upstream in context.nodes.edges(name, EdgeDirection::Upstream) { - if !selected.contains(upstream.as_str()) { - return Err(failure(format!( - "Python node '{name}' depends on unselected Python node '{upstream}'; \ - select it directly or use upstream expansion" - ))); - } - } - } Ok(()) } diff --git a/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs b/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs index 482d1689f5..695361c4e6 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/published_python_graph.rs @@ -1,26 +1,26 @@ -use crate::definition_changes::constants::PYTHON_NODE_KINDS; -use crate::python_selection::_helpers::inventory::indexes; +use crate::python_selection::_helpers::inventory::{indexes, split_published_deps}; use crate::python_selection::models::{ InventoryIndexes, InventoryNode, PublishedPythonNode, PythonSelectionGraph, PythonSelectionNode, }; /// The selection graph of Python nodes rebuilt from a published snapshot, so the server selects them as the CLI does. pub fn published_python_graph(nodes: &[PublishedPythonNode]) -> PythonSelectionGraph { - let is_python = |kind: &str| PYTHON_NODE_KINDS.contains(&kind); - let inventory: Vec = nodes - .iter() - .map(|node| InventoryNode { + let mut inventory: Vec = Vec::with_capacity(nodes.len()); + let mut selection_nodes: Vec = Vec::with_capacity(nodes.len()); + for node in nodes { + let (upstream, sql_deps) = split_published_deps(&node.deps); + inventory.push(InventoryNode { name: node.name.clone(), tags: node.tags.clone(), relative_path: String::new(), - upstream: node - .deps - .iter() - .filter(|(kind, _)| is_python(kind)) - .map(|(_, name)| name.clone()) - .collect(), - }) - .collect(); + upstream, + }); + selection_nodes.push(PythonSelectionNode { + name: node.name.clone(), + kind: node.kind.clone(), + sql_deps, + }); + } let InventoryIndexes { upstream, downstream, @@ -28,19 +28,7 @@ pub fn published_python_graph(nodes: &[PublishedPythonNode]) -> PythonSelectionG .. } = indexes(&inventory); PythonSelectionGraph { - nodes: nodes - .iter() - .map(|node| PythonSelectionNode { - name: node.name.clone(), - kind: node.kind.clone(), - sql_deps: node - .deps - .iter() - .filter(|(kind, _)| !is_python(kind)) - .cloned() - .collect(), - }) - .collect(), + nodes: selection_nodes, upstream, downstream, tags, diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs index 92b68329c3..2ac7b7de0e 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs @@ -2,13 +2,14 @@ use crate::assembly::project::types::ObjectKey; use crate::graph::errors::SelectorError; use crate::graph::main::build_project_graph::build_project_graph; use crate::graph::models::{GraphResource, ProjectGraph}; +use crate::python_selection::main::published_python_graph::published_python_graph; use crate::python_selection::main::python_node_graph_indexes::python_node_graph_indexes; use crate::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use crate::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; use crate::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection; use crate::python_selection::models::{ - InventoryIndexes, InventoryNode, PythonSelectionGraph, PythonSelectionNode, RunSelection, - SqlSelectionGraph, + InventoryIndexes, InventoryNode, PublishedPythonNode, PythonSelectionGraph, + PythonSelectionNode, RunSelection, SqlSelectionGraph, }; use crate::python_selection::tests::test_types::{ExpectedRun, SelectorFailure}; @@ -130,6 +131,36 @@ fn orders_sql_graph() -> ProjectGraph { ]) } +/// The orders Python node graph as the server rebuilds it from published snapshot nodes. +pub(super) fn published_orders_graph() -> PythonSelectionGraph { + let nodes: Vec = ORDERS_NODES + .iter() + .map( + |(name, kind, path, tags, upstream, sql_deps)| PublishedPythonNode { + name: (*name).to_owned(), + kind: (*kind).to_owned(), + tags: owned(tags), + folder: path + .rsplit_once('/') + .map_or(".", |(folder, _)| folder) + .to_owned(), + deps: upstream + .iter() + .map(|upstream| { + let kind: &str = ORDERS_NODES + .iter() + .find(|node| node.0 == *upstream) + .map_or("task", |node| node.1); + key(&(kind, upstream)) + }) + .chain(sql_deps.iter().map(key)) + .collect(), + }, + ) + .collect(); + published_python_graph(&nodes) +} + /// The Python node names the selectors resolve to over the orders graph. pub(super) fn node_names(select: &[&str], exclude: &[&str]) -> Result, OwnedFailure> { resolve_python_node_selectors(&orders_python_graph(), &owned(select), &owned(exclude)) @@ -138,6 +169,15 @@ pub(super) fn node_names(select: &[&str], exclude: &[&str]) -> Result Result { + run_over(&orders_python_graph(), select, exclude) +} + +/// The run selection over the orders SQL graph and the given Python graph. +pub(super) fn run_over( + python: &PythonSelectionGraph, + select: &[&str], + exclude: &[&str], +) -> Result { let graph: ProjectGraph = orders_sql_graph(); let sources: Vec<(String, Option)> = vec![("raw_orders".to_owned(), Some("raw_orders".to_owned()))]; @@ -146,7 +186,7 @@ pub(super) fn run(select: &[&str], exclude: &[&str]) -> Result = select.iter().chain(exclude).cloned().collect(); let node_names: Vec<&str> = python .iter() - .filter_map(|resource| object_key(&resource.key).map(|_| name_of(&resource.key))) + .map(|resource| name_of(&resource.key)) .collect(); if selectors_need_jobs(&graph, &selectors, &node_names) { graph = attach_jobs(graph, published_jobs(jobs)); @@ -229,19 +229,7 @@ fn resolved( .collect(); let nodes: Vec = python .iter() - .filter_map(|resource| { - Some(PublishedPythonNode { - name: name_of(&resource.key).to_owned(), - kind: python_kind(&resource.key)?.to_owned(), - tags: resource.tags.clone(), - folder: resource.folder.clone().unwrap_or_else(|| ".".to_owned()), - deps: resource - .deps - .iter() - .filter_map(|dep| object_key(dep)) - .collect(), - }) - }) + .filter_map(|resource| published_python_node(resource)) .collect(); let python_graph = published_python_graph(&nodes); let selection = resolve_python_sql_run_selection( @@ -277,6 +265,20 @@ fn python_kind(key: &str) -> Option<&'static str> { PYTHON_NODE_KINDS.into_iter().find(|python| *python == kind) } +fn published_python_node(resource: &PlannerResource) -> Option { + Some(PublishedPythonNode { + name: name_of(&resource.key).to_owned(), + kind: python_kind(&resource.key)?.to_owned(), + tags: resource.tags.clone(), + folder: resource.folder.clone().unwrap_or_else(|| ".".to_owned()), + deps: resource + .deps + .iter() + .filter_map(|dep| object_key(dep)) + .collect(), + }) +} + fn name_of(key: &str) -> &str { key.split_once(':').map_or(key, |(_, name)| name) } diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index 47d3753118..9855c4cbc3 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -333,9 +333,9 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co )), }, AdhocPlanTestCase { - description: "a task without the Python node it depends on is refused, as on the command line", + description: "a task runs without the Python node it depends on, as on the command line", command: &["build", "--select", "task:request_carrier_collection"], - expected_plan: Err("depends on unselected Python node 'partner_orders_file'"), + expected_plan: Ok((&["task:request_carrier_collection"], json!({}))), }, AdhocPlanTestCase { description: "a python folder selects the nodes under it", diff --git a/src/sqlbuild/executor/load/_helpers/execution.py b/src/sqlbuild/executor/load/_helpers/execution.py index 955a267d89..673e81e4af 100644 --- a/src/sqlbuild/executor/load/_helpers/execution.py +++ b/src/sqlbuild/executor/load/_helpers/execution.py @@ -3,8 +3,9 @@ from __future__ import annotations from collections.abc import Callable +from dataclasses import replace -from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry +from sqlbuild._native import DiscoveredLoaderFunction, SourceEntry, SourceWriteStrategy from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.executor.load.models import LoadExecutionIndexes, LoadExecutionResult from sqlbuild.executor.scheduling.types import ExecutionStatus @@ -191,3 +192,14 @@ def is_untargeted_self_managed_intermediate( and source_entry.write_strategy is None and loader_function.destination is None ) + + +def row_write_entry(source_entry: SourceEntry) -> SourceEntry: + """Return the entry rows are written with: an intermediate loader defaults to `table`.""" + + if ( + source_entry.meta.get("sqlbuild_loader_node") is True + and source_entry.write_strategy is None + ): + return replace(source_entry, write_strategy=SourceWriteStrategy.TABLE) + return source_entry diff --git a/src/sqlbuild/executor/load/_helpers/loader_invocation.py b/src/sqlbuild/executor/load/_helpers/loader_invocation.py index c07f0fd419..d287c8681c 100644 --- a/src/sqlbuild/executor/load/_helpers/loader_invocation.py +++ b/src/sqlbuild/executor/load/_helpers/loader_invocation.py @@ -18,6 +18,7 @@ from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load._helpers.execution import ( is_untargeted_self_managed_intermediate, + row_write_entry, ) from sqlbuild.executor.load._helpers.relation_refs import ( build_loader_relation_refs, @@ -238,7 +239,7 @@ def interpret_loader_return( duration_ms=int((time.monotonic() - start) * 1000), lifecycle_events=statement_recorder.snapshot(), ) - if source_entry.write_strategy is None: + if row_write_entry(source_entry).write_strategy is None: raise ExecutorInputError( f"Source '{source_entry.name}' loader '{loader_function.name}' returned rows " "but source has no write_strategy" diff --git a/src/sqlbuild/executor/load/main/_execute.py b/src/sqlbuild/executor/load/main/_execute.py index f4ccaa0360..eef2c5c56b 100644 --- a/src/sqlbuild/executor/load/main/_execute.py +++ b/src/sqlbuild/executor/load/main/_execute.py @@ -26,7 +26,7 @@ infer_cursor_kind, load_staging_cursor_bounds, ) -from sqlbuild.executor.load._helpers.execution import load_resource_kind +from sqlbuild.executor.load._helpers.execution import load_resource_kind, row_write_entry from sqlbuild.executor.load._helpers.external import execute_external_source_load from sqlbuild.executor.load._helpers.loader_invocation import ( build_loader_context, @@ -176,6 +176,7 @@ def _execute_source_load( ) if early_result is not None: return early_result + source_entry = row_write_entry(source_entry) rows_loaded: int = write_loader_rows_to_staging( loader_return_value=raw_rows, source_entry=source_entry, diff --git a/src/sqlbuild/executor/python_nodes/_helpers/execution.py b/src/sqlbuild/executor/python_nodes/_helpers/execution.py index 43bd63753c..dafc8e882a 100644 --- a/src/sqlbuild/executor/python_nodes/_helpers/execution.py +++ b/src/sqlbuild/executor/python_nodes/_helpers/execution.py @@ -189,6 +189,18 @@ def _execute_ready_node_in_cost_scope( result_store=result_store, warnings=warnings, ) + missing: str | None = _missing_upstream_result( + node=node, node_kind=node_kind, context=context, upstream_results=upstream_results + ) + if missing is not None: + result = PythonNodeExecutionResult( + node_name=node.name, + kind=node_kind, + status=PythonNodeStatus.FAILED, + error_message=missing, + ) + _persist_python_node_result(result_store=result_store, result=result, run_id=run_id) + return result, None try: result: PythonNodeExecutionResult = _call_node_with_retry( node=node, @@ -344,6 +356,37 @@ def _call_node_with_retry( attempt += 1 +def _missing_upstream_result( + *, + node: ExecutablePythonNode, + node_kind: PythonNodeKind, + context: TaskContext | AssetContext, + upstream_results: tuple[PythonNodeExecutionResult, ...], +) -> str | None: + """Why the node cannot run: a Python upstream outside this run that has never succeeded.""" + + python_dependencies: tuple[Callable[..., object], ...] = tuple( + dependency for dependency in node.depends_on if not isinstance(dependency, SqlResourceRef) + ) + if len(upstream_results) == len(python_dependencies): + return None + in_run: frozenset[str] = frozenset(result.node_name for result in upstream_results) + for dependency in python_dependencies: + kind, name = context.result_dependency_identity(dependency) + if name in in_run or context.result_of(node_function=dependency, default=None) is not None: + continue + both: str = f"'sqb build --select +{node_kind.value}:{node.name}'" + if kind == PythonNodeKind.LOADER.value: + return ( + f"Upstream loader '{name}' has no successful result to read. Build it with {both}." + ) + return ( + f"Upstream {kind} '{name}' has no successful result to read. Build it first with " + f"'sqb build --select {kind}:{name}', or build both with {both}." + ) + return None + + def _build_context( *, node: ExecutablePythonNode, diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index e41a47874f..245fab2a91 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -267,7 +267,7 @@ def result_of( if default is not MISSING_DEFAULT: return default raise ExecutorInputError("No Python node result store is available") - node_type, node_name = self._result_dependency_identity(node_function) + node_type, node_name = self.result_dependency_identity(node_function) return self.result_store.result_of( node_type=node_type, node_name=node_name, @@ -285,14 +285,16 @@ def results_of( if self.result_store is None: return () - node_type, node_name = self._result_dependency_identity(node_function) + node_type, node_name = self.result_dependency_identity(node_function) return self.result_store.results_of( node_type=node_type, node_name=node_name, limit=limit, ) - def _result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: + def result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: + """Return the `(kind, name)` a dependency's results are stored under.""" + task_definition: object = getattr(node_function, "__sqlbuild_task__", None) if task_definition is not None: return PythonNodeKind.TASK.value, self._definition_name(task_definition) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index ae15ae5ad1..64837288d3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1614,3 +1614,13 @@ class ChangedTwoUpE2ETestCase: warehouse: ChangedWarehouse expected_crm_run: frozenset[str] expected_downstream: frozenset[str] + + +@dataclass(frozen=True) +class PythonUpstreamResultE2ETestCase: + """A Python node selected without its Python upstream, before and after the upstream builds.""" + + description: str + select: str + upstream_select: str + expected_error: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index fc8da19a0e..95f42565c1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -895,6 +895,15 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: 'MODEL (description "Test model order_summary.", ' 'materialized table);\nSELECT count(*) AS order_count FROM __ref("orders")\n' ), + "python/tasks/reports.py": ( + "from python.tasks.counts import count_orders\nfrom sqlbuild.tasks import task\n\n\n" + "@task(depends_on=count_orders)\n" + "def report_order_count(ctx):\n" + " '''Test task report_order_count.'''\n" + " upstream = ctx.result_of(node_function=count_orders)\n" + ' ctx.execute_sql(f"CREATE OR REPLACE TABLE main.report_order_count_result AS "\n' + " f\"SELECT '{upstream.status}' AS status\")\n" + ), } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py index c792b221af..1ec298fe6b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py @@ -10,6 +10,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( PythonNodeSelectionE2ETestCase, + PythonUpstreamResultE2ETestCase, ) from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( PYTHON_NODE_SELECTION_DATABASE, @@ -174,3 +175,51 @@ def test_given_python_node_selection_when_building_then_it_gates_only_on_schedul table_exists(db_path=database, table_name=table) for table in test_case.expected_missing_tables ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonUpstreamResultE2ETestCase( + description="a task reads its unselected upstream task's latest result", + select="task:report_order_count", + upstream_select="task:count_orders", + expected_error=( + "Upstream task 'count_orders' has no successful result to read. Build it first " + "with 'sqb build --select task:count_orders', or build both with " + "'sqb build --select +task:report_order_count'." + ), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_task_selected_without_its_python_upstream_when_building_then_it_reads_the_persisted_result( + test_case: PythonUpstreamResultE2ETestCase, tmp_path: Path +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="order_nodes", + repo_files=python_node_selection_project_files( + orders_sql=PythonNodeSelectionE2ETestCase.orders_sql + ), + ) + database: Path = project_dir / PYTHON_NODE_SELECTION_DATABASE + execute_duckdb(db_path=database, sql=_EXISTING_ROWS + _EXISTING_ORDERS) + report: tuple[str, ...] = ("--no-color", "build", "--select", test_case.select) + + never_built: subprocess.CompletedProcess[str] = run_sqb(command=report, project_dir=project_dir) + upstream: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build", "--select", test_case.upstream_select), + project_dir=project_dir, + ) + rebuilt: subprocess.CompletedProcess[str] = run_sqb(command=report, project_dir=project_dir) + + output: str = never_built.stdout + never_built.stderr + assert never_built.returncode == 1, test_case.description + output + assert test_case.expected_error in output, test_case.description + assert upstream.returncode == 0, upstream.stdout + upstream.stderr + assert rebuilt.returncode == 0, rebuilt.stdout + rebuilt.stderr + node: str = test_case.select.split(":", 1)[1] + assert query_duckdb(db_path=database, sql=f"SELECT status FROM main.{node}_result") == [ + ("success",) + ] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py index 79bad71942..7a71fd98f2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/_test_types.py @@ -127,3 +127,11 @@ class LoaderRelationGuardE2ETestCase: enforce_explicit: bool expected_warnings: tuple[str, ...] expected_final_line_prefix: str + + +@dataclass(frozen=True) +class IntermediateLoaderDefaultE2ETestCase: + description: str + command: tuple[str, ...] + expected_intermediate_rows: tuple[tuple[object, ...], ...] + expected_terminal_rows: tuple[tuple[object, ...], ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py index 40ffb4dc45..483e1c7853 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/load/test_load.py @@ -10,6 +10,7 @@ ChainedLoaderFailureE2ETestCase, ChainedLoaderPruningE2ETestCase, ChainedLoaderSelectionE2ETestCase, + IntermediateLoaderDefaultE2ETestCase, IntermediateLoaderStrategyE2ETestCase, LoaderWaffleShopE2ETestCase, SourceLoaderErrorE2ETestCase, @@ -1191,6 +1192,65 @@ def test_given_chained_loader_project_when_intermediate_uses_strategy_then_appli assert tuple(terminal_rows) == test_case.expected_terminal_rows +@pytest.mark.parametrize( + "test_case", + [ + IntermediateLoaderDefaultE2ETestCase( + description="sqb load writes the docs' intermediate loader as a table", + command=("--no-color", "load", "--select", "+raw_account_metrics"), + expected_intermediate_rows=((1, "Pawnee Parks"), (2, "Eagleton")), + expected_terminal_rows=((1, "active"), (2, "active")), + ), + IntermediateLoaderDefaultE2ETestCase( + description="sqb build writes the docs' intermediate loader as a table", + command=("--no-color", "build", "--select", "+source:raw_account_metrics"), + expected_intermediate_rows=((1, "Pawnee Parks"), (2, "Eagleton")), + expected_terminal_rows=((1, "active"), (2, "active")), + ), + ], + ids=lambda case: case.description, +) +def test_given_intermediate_loader_without_write_strategy_when_loading_twice_then_rows_replace_a_table( + tmp_path: Path, + test_case: IntermediateLoaderDefaultE2ETestCase, +) -> None: + edge_cases: Path = Path("benchmarks/duckshop/edge_cases/loaders") + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="source_loader_schema_behavior", + repo_files=build_schema_behavior_project_files( + source_yaml=(edge_cases / "intermediate_default_accounts.yml").read_text(), + loader_py=(edge_cases / "intermediate_default_accounts.py").read_text(), + ), + ) + db_path: Path = project_dir / "source_loader_schema_behavior.duckdb" + + results: list[subprocess.CompletedProcess[str]] = [ + run_sqb(command=test_case.command, project_dir=project_dir) for _ in range(2) + ] + + for result in results: + assert result.returncode == 0, test_case.description + result.stdout + result.stderr + assert ( + tuple( + query_duckdb( + db_path=db_path, + sql="SELECT account_id, account_name FROM __loader__raw_accounts ORDER BY 1", + ) + ) + == test_case.expected_intermediate_rows + ) + assert ( + tuple( + query_duckdb( + db_path=db_path, + sql="SELECT account_id, metric FROM raw_account_metrics ORDER BY 1", + ) + ) + == test_case.expected_terminal_rows + ) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 048a0e9102..01776fdad9 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -269,6 +269,20 @@ class PythonJobSelectionTestCase: expected_selection: tuple[str, ...] +@dataclass(frozen=True) +class PythonJobGroupTestCase: + description: str + jobs: tuple[tuple[str, str], ...] + expected_groups: dict[str, dict[str, list[str]]] + + +@dataclass(frozen=True) +class PythonJobGroupCycleTestCase: + description: str + jobs: tuple[tuple[str, str], ...] + expected_message: str + + @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 03050f1519..57ceccc803 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -26,6 +26,7 @@ LocalConfig, MeasurementThresholdBound, MeasurementThresholds, + NativeOrchestrationSnapshot, ProjectConfig, SourceColumnEntry, SqlLexicalSyntax, @@ -44,6 +45,8 @@ CompiledModelView, CompiledProjectView, ) +from sqlbuild.orchestration import compile_snapshot +from sqlbuild.orchestration_declarations.models.authoring import Job, JobGroup from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files if TYPE_CHECKING: @@ -750,3 +753,23 @@ def published_texts(test_case: SnapshotExpressionRangeTestCase, tmp_path: Path) column: compiled[slice(*metadata["expression_range"])].decode() for column, metadata in columns.items() } + + +def python_pipeline_snapshot( + tmp_path: Path, declarations: list[Job | JobGroup] +) -> NativeOrchestrationSnapshot: + """Publish the Python pipeline project with the given job declarations.""" + write_files(project_dir=tmp_path, files=PYTHON_PIPELINE_FILES) + discovered: DiscoveredProjectInputs = discover_project_inputs(project_dir=tmp_path) + project: CompiledProjectView = build_compiled_project( + discovered_inputs=discovered, + adapter=DuckDbAdapter(), + resolved_connection={"database": "warehouse", "schema": "main"}, + ) + return compile_snapshot( + project=project, + metadata_json=json.dumps(METADATA), + project_dir=tmp_path, + declarations=declarations, + discovered_inputs=discovered, + ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index b230cebb88..4f1c568195 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -125,6 +125,12 @@ class WorkerExecutionTestCase: warehouse_setup: Callable[[Path], None] = lambda _: None +@dataclass(frozen=True) +class ClaimedLoadTestCase(WorkerExecutionTestCase): + description: str + expected_write_set: frozenset[str] = frozenset() + + @dataclass(frozen=True) class UnclaimedLoadTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index abf17ff923..60fb29529b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -1569,6 +1569,22 @@ def hold_write_lock(database: Path) -> duckdb.DuckDBPyConnection: } +INTERMEDIATE_LOADER_FILES: dict[str, str] = { + "sources/shipments.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml" + ).read_text(), + "python/loaders/shipments.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py" + ).read_text(), + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql" + ).read_text(), + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" + ).read_text(), +} + + def without_loads(control: sqlite3.Connection, node_key: str) -> None: """Drop a published node's `loads` in place; its content address and the worker's check stay.""" for node_hash, body in control.execute( diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index ff3cfb341e..48db2d8bd6 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -30,6 +30,7 @@ ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( CheckExecutionTestCase, + ClaimedLoadTestCase, ClaimedWorker, GroupExecutionTestCase, SnapshotWorkerTestCase, @@ -37,6 +38,7 @@ WorkerExecutionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + INTERMEDIATE_LOADER_FILES, LOADED_ORDER_FILES, cancel_running_statement, claimed_worker, @@ -605,26 +607,38 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re @pytest.mark.parametrize( "test_case", [ - WorkerExecutionTestCase( + ClaimedLoadTestCase( "a tag-selected job claims its loader chain and hook-read source, not unread ones", "", "SELECT amount FROM orders", "success", [(7,)], files=LOADED_ORDER_FILES, - ) + expected_write_set=frozenset( + {"model:orders", "source:raw_orders", "source:raw_customers", "source:raw_returns"} + ), + ), + ClaimedLoadTestCase( + "a job claims an intermediate loader no source declares", + "", + "SELECT amount FROM orders", + "success", + [(9,)], + files=INTERMEDIATE_LOADER_FILES, + expected_write_set=frozenset( + {"model:orders", "source:raw_shipments", "source:stage_shipments"} + ), + ), ], ids=lambda case: case.description, ) def test_given_loads_when_a_worker_executes_then_the_run_claims_exactly_what_it_loads( - test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + test_case: ClaimedLoadTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: run: dict[str, Any] = worker_result(worker, test_case) assert run["status"] == test_case.expected_status, worker.log.read_text() - assert run_write_set(worker.control_store, run["run_id"]) == frozenset( - {"model:orders", "source:raw_orders", "source:raw_customers", "source:raw_returns"} - ) + assert run_write_set(worker.control_store, run["run_id"]) == test_case.expected_write_set @pytest.mark.parametrize( @@ -639,27 +653,6 @@ def test_given_loads_when_a_worker_executes_then_the_run_claims_exactly_what_it_ {"model:orders", "source:raw_orders", "source:raw_customers"} ), ), - UnclaimedLoadTestCase( - # A known gap: no snapshot node exists for an intermediate loader, so `loads` leaves it out. - "an intermediate loader no source declares is never claimed, so the run still fails", - files={ - "sources/shipments.yml": Path( - "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml" - ).read_text(), - "python/loaders/shipments.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py" - ).read_text(), - "models/orders.sql": Path( - "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql" - ).read_text(), - "orchestration/orders.py": Path( - "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" - ).read_text(), - }, - store_edit=lambda _: None, - refused_writer="source:stage_shipments", - expected_write_set=frozenset({"model:orders", "source:raw_shipments"}), - ), ], ids=lambda case: case.description, ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 9cce168cc4..76d37d2d1b 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -32,8 +32,11 @@ schedule, sensor, ) -from sqlbuild.orchestration_declarations.models.authoring import Job +from sqlbuild.orchestration_declarations.models.authoring import Job, JobGroup from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + PythonJobGroupCycleTestCase, + PythonJobGroupTestCase, + PythonJobSelectionTestCase, SnapshotCatalogTestCase, SnapshotDeclarationRejectionTestCase, SnapshotDiscoveryRejectionTestCase, @@ -44,7 +47,6 @@ SnapshotLineageTestCase, SnapshotProvidersTestCase, SnapshotRejectionTestCase, - PythonJobSelectionTestCase, SnapshotScheduleTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( @@ -54,6 +56,7 @@ PYTHON_PIPELINE_FILES, TYPE_PROOF_FILES, published_texts, + python_pipeline_snapshot, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -1858,6 +1861,11 @@ def test_given_broken_declaration_modules_when_compiling_then_no_partial_snapsho select="+orders", expected_selection=("model:orders", "model:stg_orders"), ), + PythonJobSelectionTestCase( + description="a task selector selects only the task", + select="task:request_carrier_collection", + expected_selection=("task:request_carrier_collection",), + ), PythonJobSelectionTestCase( description="typed selectors select the task and the asset it depends on", select="task:request_carrier_collection asset:partner_orders_file", @@ -1915,3 +1923,77 @@ def test_given_python_selectors_when_publishing_jobs_then_selections_match_the_c assert snapshot["documents"]["jobs"]["orders_job"]["selection"] == list( test_case.expected_selection ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonJobGroupTestCase( + description="a job waits for the job building the Python node its task depends on", + jobs=( + ("orders_export_job", "python/orders/partner_export"), + ("carrier_sync_job", "python/fulfillment/carrier_sync"), + ), + expected_groups={ + "orders_nightly": { + "carrier_sync_job": ["orders_export_job"], + "orders_export_job": [], + } + }, + ), + PythonJobGroupTestCase( + description="a Python node's SQL dependency orders it after the model's job", + jobs=(("orders_export_job", "asset:partner_orders_file"), ("models_job", "+orders")), + expected_groups={ + "orders_nightly": {"models_job": [], "orders_export_job": ["models_job"]} + }, + ), + ], + ids=lambda case: case.description, +) +def test_given_python_dependencies_across_jobs_when_publishing_a_group_then_jobs_wait_for_them( + tmp_path: Path, test_case: PythonJobGroupTestCase +) -> None: + jobs: list[Job] = [ + job(name, description="Orders.", select=select) for name, select in test_case.jobs + ] + group: JobGroup = job_group("orders_nightly", description="Nightly orders.", jobs=jobs) + + snapshot: dict[str, Any] = json.loads( + python_pipeline_snapshot(tmp_path, [*jobs, group]).to_json() + ) + + assert { + name: definition["jobs"] for name, definition in snapshot["documents"]["groups"].items() + } == test_case.expected_groups + + +@pytest.mark.parametrize( + "test_case", + [ + PythonJobGroupCycleTestCase( + description="a Python edge that closes a cycle names both jobs and the edges", + jobs=( + ("staging_job", "stg_orders task:request_carrier_collection"), + ("export_job", "asset:partner_orders_file"), + ), + expected_message=( + "group dependencies contain a cycle in job group orders_nightly: " + "export_job waits for staging_job (model:orders reads model:stg_orders); " + "staging_job waits for export_job " + "(task:request_carrier_collection reads asset:partner_orders_file)" + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_python_dependency_cycle_across_jobs_when_publishing_a_group_then_it_is_rejected( + tmp_path: Path, test_case: PythonJobGroupCycleTestCase +) -> None: + jobs: list[Job] = [ + job(name, description="Orders.", select=select) for name, select in test_case.jobs + ] + group: JobGroup = job_group("orders_nightly", description="Nightly orders.", jobs=jobs) + + with pytest.raises(ValueError, match=re.escape(test_case.expected_message)): + python_pipeline_snapshot(tmp_path, [*jobs, group]) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py index c36f6956be..ae3937e7da 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py @@ -35,6 +35,13 @@ expected_sql_names=frozenset(), expected_python_node_names=frozenset({"prepare_orders", "export_orders"}), ), + PythonSqlSelectorTestCase( + description="selects a Python asset without its unselected Python dependency", + select=("path:python/assets",), + exclude=(), + expected_sql_names=frozenset(), + expected_python_node_names=frozenset({"export_orders"}), + ), PythonSqlSelectorTestCase( description="selects expanded source terminal loader requirement for run", select=("+source:raw_orders",), @@ -79,15 +86,6 @@ def test_given_run_selectors_when_resolving_then_excludes_python_checks( expected_error_type=ValueError, expected_error_fragment=("Python checks are not selectable here: check_orders_export"), ), - PythonSqlSelectorErrorTestCase( - description="rejects direct asset path with unselected Python dependency for run", - select=("path:python/assets",), - exclude=(), - expected_error_type=ValueError, - expected_error_fragment=( - "Python node 'export_orders' depends on unselected Python node 'prepare_orders'" - ), - ), ], ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 3f20e3ab26..ddc395623b 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -127,7 +127,7 @@ def export_orders(ctx): - `ctx.skip(reason, mode=...)` skips the node. `mode` accepts `"soft"` (default, skips only this node) or `"hard"` (also blocks dependents), as a string or the `SkipMode` enum from `sqlbuild.tasks`/`sqlbuild.assets`. - Assets may pass `materialized=True`/`False` to record whether an artifact was produced. -Downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. +Within one run, downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. ## Result persistence @@ -161,6 +161,18 @@ dependency fails or is skipped. A dependency that is not part of the run (not se unmanaged source) is read as it currently exists, so `sqb build --select task:export_orders` runs the task against the relations already in the warehouse. +Python dependencies follow the same rule. A task or asset selected without its Python upstream +reads that upstream's latest successful result through `ctx.result_of`. If the upstream has never +succeeded for the target, the node fails with an error that names the upstream, for example: + +```text +Upstream task 'prepare_orders' has no successful result to read. Build it first with +'sqb build --select task:prepare_orders', or build both with 'sqb build --select +asset:export_orders'. +``` + +Jobs, sensors, roles and ad-hoc runs on the server resolve selectors the same way, so a job can +select one Python node and a job group orders it after the job that builds its upstream. + The commands differ in what they include by default: | Command | SQL | Loaders / tasks / assets | Checks | Audits | From 0e665a98b2eeb2b5025f3eaccd813594be32ff93 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:20:11 +0000 Subject: [PATCH 04/19] feat: select Python-only jobs and SQL tests across CLI and server runs - `job:` and bare job names resolve jobs through the Python-aware resolver publication uses, so a job that selects only Python nodes builds them. - Server ad-hoc runs accept `test:` selectors; a selected SQL test reports as a run node rather than as an attached check. - A provider function whose context is positional-only (`ctx, /`) receives it by position instead of failing at run time. - A role must grant the intermediate loader a job loads to run that job. --- .../intermediate_loader_roles.toml | 15 ++++ .../orchestration/python_node_jobs.py | 10 +++ .../python_pipeline_carrier_sync.py | 10 +++ .../python_pipeline_partner_export.py | 9 +++ .../src/graph/_helpers/jobs.rs | 14 +++- .../src/graph/main/attach_jobs_with.rs | 18 +++++ .../sqlbuild-analysis/src/graph/main/mod.rs | 1 + .../src/python_selection/_helpers/unified.rs | 48 +++++++++++- .../main/attach_python_jobs.rs | 40 ++++++++++ .../src/python_selection/main/mod.rs | 1 + .../tests/test_python_selection.rs | 2 +- .../src/adhoc/_helpers/planner.rs | 59 ++++++++++++++- crates/sqlbuild-cli/src/adhoc/constants.rs | 6 ++ .../src/adhoc/tests/test_adhoc_planner.rs | 58 ++++++++++++++- .../src/runs/_helpers/requests/adhoc.rs | 24 ++++++ .../_helpers/compiled_project/job_catalog.rs | 20 +++-- .../compiled_project/snapshot_execution.rs | 8 +- .../_helpers/graph/python_selection.rs | 17 ++++- .../_helpers/location_command/constants.rs | 3 + .../_helpers/location_command/worker.rs | 13 ++-- src/sqlbuild/provider/_helpers/injection.py | 7 +- .../cli/commands/main/build/helpers.py | 3 + .../main/build/test_python_node_selection.py | 24 ++++++ .../compiler/compiled_project/helpers.py | 17 ++--- .../publication/_test_types.py | 7 ++ .../publication/person/helpers.py | 16 ++++ .../publication/test_adhoc.py | 8 ++ .../publication/test_project_roles.py | 73 +++++++++++++++++++ .../src/sqlbuild/provider/_helpers/helpers.py | 8 ++ .../provider/_helpers/test_injection.py | 12 +++ .../content/docs/docs/concepts/selectors.mdx | 2 +- 31 files changed, 517 insertions(+), 36 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py create mode 100644 crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml new file mode 100644 index 0000000000..78acf58022 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml @@ -0,0 +1,15 @@ +# Roles over a loader chain whose middle loader no source declares: running the job needs a grant +# on the intermediate loader's node as well as the model and the source it feeds. +[roles.shipments_runner] +description = "Run the shipments build, including its staging loader" + +[[roles.shipments_runner.grants]] +nodes = ["orders", "source:raw_shipments", "loader:stage_shipments"] +permissions = ["job.run"] + +[roles.orders_runner] +description = "Run the orders build without the staging loader" + +[[roles.orders_runner.grants]] +nodes = ["orders", "source:raw_shipments"] +permissions = ["job.run"] diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py new file mode 100644 index 0000000000..d153e434ff --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py @@ -0,0 +1,10 @@ +"""Jobs whose selections are Python nodes, which `--select job:` must select as published.""" + +from sqlbuild.orchestration import job + +customer_counts = job( + "customer_counts", + description="Count customers.", + select="task:count_customers asset:customers_extract", +) +order_counts = job("order_counts", description="Count orders.", select="+task:count_orders") diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py new file mode 100644 index 0000000000..221447d1c8 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py @@ -0,0 +1,10 @@ +"""A task whose Python upstream another job builds, so a job group must order the two jobs.""" + +from python.orders.partner_export.node import partner_orders_file + +from sqlbuild.tasks import task + + +@task(depends_on=partner_orders_file, tags=["orders"], description="Book collection.") +def request_carrier_collection(ctx): + return None diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py new file mode 100644 index 0000000000..dfe6194d20 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py @@ -0,0 +1,9 @@ +"""An asset one job builds that a task in another job of the same group depends on.""" + +from sqlbuild.assets import asset +from sqlbuild.refs import model + + +@asset(depends_on=model("orders"), tags=["orders"], description="Partner order file.") +def partner_orders_file(ctx): + return None diff --git a/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs b/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs index 329bbbc646..f1c9480ace 100644 --- a/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs +++ b/crates/sqlbuild-analysis/src/graph/_helpers/jobs.rs @@ -8,17 +8,23 @@ use crate::graph::errors::SelectorError; use crate::graph::models::{JobSelection, ProjectGraph}; /// Resolve every job on the graph without jobs attached, as publication does, then attach them. -pub(crate) fn attach( +pub(crate) fn attach(graph: ProjectGraph, catalog: BTreeMap) -> ProjectGraph { + attach_with(graph, catalog, |graph, select, exclude| { + resolve(graph, select, exclude).map(|keys| keys.into_iter().collect()) + }) +} + +/// [`attach`] with each job's selectors resolved by `resolve_job` on the graph without jobs. +pub(crate) fn attach_with( mut graph: ProjectGraph, catalog: BTreeMap, + resolve_job: impl Fn(&ProjectGraph, &[String], &[String]) -> Result, ) -> ProjectGraph { graph.jobs = None; let mut resolved: BTreeMap> = BTreeMap::new(); for (name, selection) in &catalog { let keys: Result = match selection { - JobSelection::Job { select, exclude } => { - resolve(&graph, select, exclude).map(|keys| keys.into_iter().collect()) - } + JobSelection::Job { select, exclude } => resolve_job(&graph, select, exclude), JobSelection::Resolved { keys } => Ok(keys.iter().cloned().collect()), JobSelection::Group { .. } => continue, }; diff --git a/crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs b/crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs new file mode 100644 index 0000000000..091081bc50 --- /dev/null +++ b/crates/sqlbuild-analysis/src/graph/main/attach_jobs_with.rs @@ -0,0 +1,18 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use crate::assembly::project::types::ObjectKey; +use crate::graph::errors::SelectorError; +use crate::graph::models::{JobSelection, ProjectGraph}; + +/// [`super::attach_jobs::attach_jobs`] with each job's selectors resolved by `resolve_job`. +pub fn attach_jobs_with( + graph: ProjectGraph, + catalog: BTreeMap, + resolve_job: impl Fn( + &ProjectGraph, + &[String], + &[String], + ) -> Result, SelectorError>, +) -> ProjectGraph { + crate::graph::_helpers::jobs::attach_with(graph, catalog, resolve_job) +} diff --git a/crates/sqlbuild-analysis/src/graph/main/mod.rs b/crates/sqlbuild-analysis/src/graph/main/mod.rs index ccd21dc172..05d0272f4d 100644 --- a/crates/sqlbuild-analysis/src/graph/main/mod.rs +++ b/crates/sqlbuild-analysis/src/graph/main/mod.rs @@ -1,4 +1,5 @@ pub mod attach_jobs; +pub mod attach_jobs_with; pub mod build_project_graph; pub mod build_resources; pub mod downstream_edges; diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index 7bd1a63517..4c94cd2f87 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -7,6 +7,7 @@ use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; use sqlbuild_core::text::main::glob_match::glob_match; use crate::assembly::project::types::ObjectKey; +use crate::definition_changes::constants::PYTHON_NODE_KINDS; use crate::graph::errors::SelectorError; use crate::graph::main::build_resources::build_resources; use crate::graph::main::edge_keys::edge_keys; @@ -282,7 +283,8 @@ fn single_side(context: &SelectionContext<'_>, raw: &str) -> Resolved { let atoms: Atoms = match kind.as_str() { TEST_KIND => return Err(unit_test_error(raw)), PATH_KIND => resolve_path(context, raw, &value)?, - SEED_KIND | SOURCE_KIND | JOB_KIND => resolve_sql(context, raw)?, + SEED_KIND | SOURCE_KIND => resolve_sql(context, raw)?, + JOB_KIND => resolve_job(context, raw)?, TASK_KIND | ASSET_KIND | LOADER_KIND | CHECK_KIND => { return resolve_typed(context, raw, (&kind, &value)); } @@ -314,6 +316,48 @@ fn resolve_sql(context: &SelectionContext<'_>, raw: &str) -> Resolved { )?)) } +/// A job's SQL keys and Python nodes, with `+` also expanding them over the Python node graph. +fn resolve_job(context: &SelectionContext<'_>, raw: &str) -> Resolved { + let ParsedSelector::Kind { + upstream, + downstream, + .. + } = parse_selector(raw)? + else { + return resolve_sql(context, raw); + }; + let mut atoms: Atoms = Atoms::new(); + let mut members: Vec = Vec::new(); + for atom in resolve_sql(context, raw)? { + match atom { + Atom::Sql((kind, name)) + if PYTHON_NODE_KINDS.contains(&kind.as_str()) + && context.nodes.node(&name).is_some() => + { + members.push(name); + } + atom => { + atoms.insert(atom); + } + } + } + let mut names: Names = members.iter().cloned().collect(); + for (expand, direction) in [ + (upstream, EdgeDirection::Upstream), + (downstream, EdgeDirection::Downstream), + ] { + if expand { + names.extend( + context + .nodes + .closure(members.iter().map(String::as_str), direction), + ); + } + } + atoms.extend(python_atoms(names)); + Ok(atoms) +} + fn resolve_python(context: &SelectionContext<'_>, raw: &str) -> Resolved { Ok(python_atoms(select_nodes( &context.nodes, @@ -407,7 +451,7 @@ fn resolve_name(context: &SelectionContext<'_>, raw: &str, value: &str) -> Resol (true, true) => Err(both_error(value)), (true, false) => resolve_sql(context, raw), (false, true) => resolve_python(context, raw), - (false, false) if is_job(context, value) => resolve_sql(context, raw), + (false, false) if is_job(context, value) => resolve_job(context, raw), (false, false) => Err(unknown_name(value)), }; } diff --git a/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs new file mode 100644 index 0000000000..6e3095f6f7 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs @@ -0,0 +1,40 @@ +use std::collections::BTreeMap; + +use crate::graph::main::attach_jobs_with::attach_jobs_with; +use crate::graph::models::{JobSelection, ProjectGraph}; +use crate::python_selection::_helpers::nodes::sql_names; +use crate::python_selection::_helpers::unified::run_selection; +use crate::python_selection::models::{ + NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, +}; + +/// Attach jobs as publication resolves them, keeping each job's Python nodes as `(kind, name)` keys. +pub fn attach_python_jobs( + sql: SqlSelectionGraph<'_>, + python: &PythonSelectionGraph, + catalog: BTreeMap, +) -> ProjectGraph { + let kinds: BTreeMap<&str, &str> = python + .nodes + .iter() + .map(|node| (node.name.as_str(), node.kind.as_str())) + .collect(); + attach_jobs_with(sql.graph.clone(), catalog, |graph, select, exclude| { + let context: SelectionContext<'_> = SelectionContext { + sql: SqlSelectionGraph { + graph, + sources: sql.sources, + }, + nodes: NodeIndex::new(python), + sql_names: sql_names(graph), + }; + let RunSelection { + sql_keys, + python_node_names, + } = run_selection(&context, select, exclude)?; + let python_keys = python_node_names + .into_iter() + .filter_map(|name| Some((kinds.get(name.as_str())?.to_string(), name))); + Ok(sql_keys.into_iter().chain(python_keys).collect()) + }) +} diff --git a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs index 3dace0806a..487d121f6d 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs @@ -1,3 +1,4 @@ +pub mod attach_python_jobs; pub mod published_python_graph; pub mod python_node_graph_indexes; pub mod resolve_python_node_excludes; diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs index 27bc2914bd..7a2c24b305 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs @@ -1,5 +1,5 @@ use crate::python_selection::tests::helpers::{ - excluded_names, expected_names, expected_run, indexed_folder, node_names, orders_python_graph, + excluded_names, expected_names, expected_run, indexed_folder, node_names, published_orders_graph, run, run_over, }; use crate::python_selection::tests::test_types::{ diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs index 389c0fcd79..ddb5193499 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs @@ -8,6 +8,8 @@ use sqlbuild_analysis::graph::main::attach_jobs::attach_jobs; use sqlbuild_analysis::graph::main::build_project_graph::build_project_graph; use sqlbuild_analysis::graph::main::selectors_need_jobs::selectors_need_jobs; use sqlbuild_analysis::graph::models::{GraphResource, JobSelection}; +use sqlbuild_analysis::planner_selection::main::split_unit_test_selectors::split_unit_test_selectors; +use sqlbuild_analysis::planner_selection::models::{SelectorGroups, UnitTestSelectorNames}; use sqlbuild_analysis::python_selection::main::published_python_graph::published_python_graph; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection; use sqlbuild_analysis::python_selection::models::{PublishedPythonNode, SqlSelectionGraph}; @@ -18,7 +20,7 @@ use sqlbuild_orchestration::runs::types::{CommandPlanner, PublishedJobs}; use crate::adhoc::constants::{ ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, CHANGED_COUNTS_DEST, CHANGED_DEST, COMMAND_DEST, FRESHNESS_VERB, INTEGER_CONVERTERS, LOAD_SOURCES_DEST, POSITIVE_INTEGER_CONVERTER, - REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, + REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, SQL_TEST_KIND, UNIT_TEST_VERBS, }; use crate::argv::main::command_settings_arguments::command_settings_arguments; use crate::argv::main::job_settings_arguments::job_settings_arguments; @@ -84,7 +86,7 @@ impl CommandPlanner for CliPlanner { seeds } FRESHNESS_VERB => freshness_sources(publication, &select, &exclude)?, - _ => resolve(publication, &select, &exclude)?, + _ => resolve_with_tests(verb, publication, &select, &exclude)?, }; Ok(CommandPlan { verb: verb.clone(), @@ -201,6 +203,58 @@ fn resolve( Ok(selected) } +/// [`resolve`] with `test:` and unit-test names selecting `sql_test:` keys, as `sqb test` does. +fn resolve_with_tests( + verb: &str, + (resources, jobs): (&[PlannerResource], &PublishedJobs), + select: &[String], + exclude: &[String], +) -> Result, String> { + let names = UnitTestSelectorNames { + tests: resources + .iter() + .filter(|resource| is_sql_test(resource)) + .map(|resource| name_of(&resource.key).to_owned()) + .collect(), + resources: resources + .iter() + .filter(|resource| !is_sql_test(resource)) + .map(|resource| name_of(&resource.key).to_owned()) + .chain(jobs.keys().cloned()) + .collect(), + }; + let split = split_unit_test_selectors( + SelectorGroups { select, exclude }, + &names, + UNIT_TEST_VERBS.contains(&verb), + ) + .map_err(|error| format!("{}: {}", error.code, error.message))?; + if !split.excluded_test_names.is_empty() { + return Err( + "an ad-hoc run cannot exclude unit tests; select the unit tests to run instead".into(), + ); + } + let mut keys: BTreeSet = if split.models_selected { + resolved((resources, jobs), &split.select, &split.exclude)? + } else { + BTreeSet::new() + }; + keys.extend( + split + .test_names + .iter() + .map(|name| format!("{SQL_TEST_KIND}:{name}")), + ); + if keys.is_empty() { + return Err("the selectors match no nodes in the current publication".into()); + } + Ok(keys) +} + +fn is_sql_test(resource: &PlannerResource) -> bool { + resource.key.split_once(':').map(|(kind, _)| kind) == Some(SQL_TEST_KIND) +} + fn resolved( (resources, jobs): (&[PlannerResource], &PublishedJobs), select: &[String], @@ -208,6 +262,7 @@ fn resolved( ) -> Result, String> { let (python, sql): (Vec<&PlannerResource>, Vec<&PlannerResource>) = resources .iter() + .filter(|resource| !is_sql_test(resource)) .partition(|resource| python_kind(&resource.key).is_some()); let graph_resources: Vec = sql .iter() diff --git a/crates/sqlbuild-cli/src/adhoc/constants.rs b/crates/sqlbuild-cli/src/adhoc/constants.rs index c4fc1ca671..0de8284749 100644 --- a/crates/sqlbuild-cli/src/adhoc/constants.rs +++ b/crates/sqlbuild-cli/src/adhoc/constants.rs @@ -6,6 +6,12 @@ pub(crate) const ADHOC_VERBS: [&str; 5] = ["build", "test", "seed", "audit", "fr /// The verb whose settings must also form valid job settings. pub(crate) const BUILD_VERB: &str = "build"; +/// Verbs that select SQL unit tests by `test:` or by name, as on the command line. +pub(crate) const UNIT_TEST_VERBS: [&str; 2] = ["build", "test"]; + +/// The published kind of a SQL unit test. +pub(crate) const SQL_TEST_KIND: &str = "sql_test"; + /// The verb that runs only the seeds of its selection. pub(crate) const SEED_VERB: &str = "seed"; diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index 9855c4cbc3..fe0b38f40a 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -28,6 +28,7 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select resource("model:customers", &[], &[], Some("models/staging")), resource("source:shop.raw_orders", &[], &[], None), resource("seed:currencies", &[], &[], None), + resource("sql_test:orders_keep_amounts", &["model:orders"], &[], None), ]; let jobs: PublishedJobs = BTreeMap::from([ ( @@ -229,6 +230,40 @@ fn given_ad_hoc_commands_when_planned_then_flags_follow_the_allowlist_and_select command: &["test", "-s", "orders", "--case", "happy_path"], expected_plan: Ok((&["model:orders"], json!({"case": "happy_path"}))), }, + AdhocPlanTestCase { + description: "test: selects one unit test", + command: &["test", "--select", "test:orders_keep_amounts"], + expected_plan: Ok((&["sql_test:orders_keep_amounts"], json!({}))), + }, + AdhocPlanTestCase { + description: "a unit test's bare name selects it, as on the command line", + command: &["test", "--select", "orders_keep_amounts"], + expected_plan: Ok((&["sql_test:orders_keep_amounts"], json!({}))), + }, + AdhocPlanTestCase { + description: "build selects a model and a unit test together", + command: &["build", "--select", "customers", "test:orders_keep_amounts"], + expected_plan: Ok(( + &["model:customers", "sql_test:orders_keep_amounts"], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "excluding a unit test is refused, since a run freezes only what it selects", + command: &[ + "test", + "--select", + "orders", + "--exclude", + "test:orders_keep_amounts", + ], + expected_plan: Err("an ad-hoc run cannot exclude unit tests"), + }, + AdhocPlanTestCase { + description: "audit refuses unit-test selectors as the command line does", + command: &["audit", "--select", "test:orders_keep_amounts"], + expected_plan: Err("selects a unit test"), + }, AdhocPlanTestCase { description: "audit settings come from the audit parser", command: &["audit", "-s", "orders", "--concurrency", "2"], @@ -314,8 +349,29 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co Some("python/fulfillment/carrier_sync"), ), ]; - let jobs: PublishedJobs = BTreeMap::new(); + let jobs: PublishedJobs = BTreeMap::from([( + "carrier_sync".to_owned(), + BTreeSet::from(["task:request_carrier_collection".to_owned()]), + )]); let test_cases = [ + AdhocPlanTestCase { + description: "a published job selects its Python nodes", + command: &["build", "--select", "job:carrier_sync"], + expected_plan: Ok((&["task:request_carrier_collection"], json!({}))), + }, + AdhocPlanTestCase { + description: "upstream of a published job crosses its Python node's dependencies", + command: &["build", "--select", "+job:carrier_sync"], + expected_plan: Ok(( + &[ + "asset:partner_orders_file", + "model:orders", + "model:stg_orders", + "task:request_carrier_collection", + ], + json!({}), + )), + }, AdhocPlanTestCase { description: "typed selectors select the Python nodes", command: &[ diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index 6de44e69d1..45ca1649da 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -32,6 +32,10 @@ use crate::store::models::{ControlTransaction, Parameter}; const GRAPH_KINDS: [&str; 5] = ["model", "source", "seed", "udf", "table_fn"]; /// Python node kinds, in the order the CLI discovers them. const PYTHON_NODE_KINDS: [&str; 4] = ["task", "asset", "loader", "check"]; +/// SQL unit tests, which `test:` selects; each lists the models it tests. +const SQL_TEST_KIND: &str = "sql_test"; +/// Where a SQL unit test's definition lists the models it tests. +const SQL_TEST_MODELS_POINTER: &str = "/test/payload/expected_model_names"; /// Marks the source node an intermediate loader writes, which the CLI's SQL graph does not list. const INTERMEDIATE_LOADER_FIELD: &str = "intermediate_loader"; @@ -179,6 +183,18 @@ fn planner_resources( }); } } + resources.extend( + nodes + .iter() + .filter(|(_, node)| node.kind == SQL_TEST_KIND) + .map(|(key, node)| PlannerResource { + key: key.clone(), + deps: tested_models(&node.definition), + tags: Vec::new(), + folder: None, + loader: None, + }), + ); resources } @@ -193,6 +209,14 @@ fn model_folder(definition: &Value) -> String { .map_or_else(|| ".".to_owned(), |(folder, _)| folder.to_owned()) } +/// The model keys a SQL unit test tests. +fn tested_models(definition: &Value) -> Vec { + texts(definition.pointer(SQL_TEST_MODELS_POINTER)) + .into_iter() + .map(|model| format!("model:{model}")) + .collect() +} + fn texts(value: Option<&Value>) -> Vec { value .and_then(Value::as_array) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs index f5122b631c..79b2b2e30c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/job_catalog.rs @@ -61,15 +61,25 @@ pub(crate) fn graph_with_jobs<'g>( selectors: &[String], also_known: &[&str], ) -> PyResult> { - if graph.jobs.is_some() || !selectors_need_jobs(graph, selectors, also_known) { - return Ok(Cow::Borrowed(graph)); - } - Ok(match job_catalog(py)? { - Some((catalog, _)) => Cow::Owned(attach_jobs(graph.clone(), catalog)), + Ok(match needed_jobs(py, graph, selectors, also_known)? { + Some(catalog) => Cow::Owned(attach_jobs(graph.clone(), catalog)), None => Cow::Borrowed(graph), }) } +/// The command's jobs when `selectors` name one and `graph` has none attached yet. +pub(crate) fn needed_jobs( + py: Python<'_>, + graph: &ProjectGraph, + selectors: &[String], + also_known: &[&str], +) -> PyResult>> { + if graph.jobs.is_some() || !selectors_need_jobs(graph, selectors, also_known) { + return Ok(None); + } + Ok(job_catalog(py)?.map(|(catalog, _)| catalog)) +} + /// The command's job and job group names, loaded on first use; empty outside a command. pub(crate) fn job_names(py: Python<'_>) -> PyResult> { Ok(job_catalog(py)? diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index 509d5b7530..a9392f573d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -260,12 +260,15 @@ impl WorkerBuild { self.complete(py, key, (outcome, None, (None, None))) } - /// A test or audit the snapshot records has started; the node it attaches to starts with it. + /// A test or audit the snapshot records has started; a selected SQL test reports as a node. pub(crate) fn start_attached(&self, py: Python<'_>, key: &str) -> PyResult<()> { self.check_cancelled(py)?; let Some(nodes) = self.attached.get(key) else { return Ok(()); }; + if self.execution.claim.request.selection.contains(key) { + return self.start_key(py, key); + } if self .checks .lock() @@ -313,6 +316,9 @@ impl WorkerBuild { } else { outcome }; + if self.execution.claim.request.selection.contains(key) { + return self.complete(py, key, (outcome, None, (None, None))); + } self.finish_check(py, key, outcome)?; if outcome == NodeOutcome::Skipped { return Ok(()); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 3859130776..3a5d3b1e6c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -1,5 +1,6 @@ //! Python node and unified SQL/Python selector resolution, and the Python node graph indexes. +use std::borrow::Cow; use std::collections::BTreeSet; use pyo3::prelude::{ @@ -10,6 +11,7 @@ use pyo3::{pyfunction, wrap_pyfunction}; use sqlbuild_analysis::assembly::project::types::ObjectKey; use sqlbuild_analysis::graph::errors::SelectorError; use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_analysis::python_selection::main::attach_python_jobs::attach_python_jobs; use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; @@ -21,7 +23,7 @@ use sqlbuild_analysis::python_selection::models::{ }; use crate::bindings::_helpers::boundary::panics::compiler_guard; -use crate::bindings::_helpers::compiled_project::job_catalog::graph_with_jobs; +use crate::bindings::_helpers::compiled_project::job_catalog::needed_jobs; use crate::bindings::_helpers::graph::project_graph::{CompiledProjectGraph, NativeProjectGraph}; use crate::bindings::_helpers::graph::python_node_graph::python_node_graph; use crate::bindings::_helpers::records::construct::error_class; @@ -272,7 +274,18 @@ pub(crate) fn resolve_python_sql_run_selection( let nodes: PythonSelectionGraph = selection_graph(python_graph)?; let node_names: Vec<&str> = nodes.nodes.iter().map(|node| node.name.as_str()).collect(); let selectors: Vec = select.iter().chain(&exclude).cloned().collect(); - let graph = graph_with_jobs(py, native.get().graph(), &selectors, &node_names)?; + let graph: Cow<'_, ProjectGraph> = + match needed_jobs(py, native.get().graph(), &selectors, &node_names)? { + Some(catalog) => Cow::Owned(attach_python_jobs( + SqlSelectionGraph { + graph: native.get().graph(), + sources: &sources, + }, + &nodes, + catalog, + )), + None => Cow::Borrowed(native.get().graph()), + }; let selection: RunSelection = compiler_guard(|| { Ok(run_selection( SqlSelectionGraph { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs index b9e42eaa95..fb1c70fdfc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/constants.rs @@ -33,3 +33,6 @@ pub(crate) const REACH_SETTING: &str = "reach"; pub(crate) const JOB_RUN_REACH: &str = "all"; /// What a job run with `changed: True` counts when `[changes] counts` is not set: every change. pub(crate) const JOB_RUN_COUNTS: [&str; 2] = ["definition", "source_data"]; + +/// A SQL unit test's published kind; a frozen run selects it back with `test:`. +pub(crate) const SQL_TEST_NODE_KIND: &str = "sql_test"; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs index ec62b33a15..889158c12d 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/worker.rs @@ -34,6 +34,7 @@ use crate::bindings::_helpers::discovery::project_config::configured_changes; use crate::bindings::_helpers::freshness::freshness_command::run_freshness_command; use crate::bindings::_helpers::location_command::constants::{ CHANGED_SETTING, COMMAND_IMPLEMENTATIONS, JOB_RUN_COUNTS, JOB_RUN_REACH, REACH_SETTING, + SQL_TEST_NODE_KIND, }; use crate::bindings::_helpers::native_host::host::entry; use crate::bindings::_helpers::recall::console::write_stderr; @@ -59,7 +60,7 @@ fn attached_nodes(snapshot: &NativeOrchestrationSnapshot) -> BTreeMap definition + SQL_TEST_NODE_KIND => definition .pointer("/test/payload/expected_model_names") .and_then(Value::as_array) .into_iter() @@ -152,12 +153,14 @@ fn command_request<'py>( let (kind, name) = key .split_once(':') .ok_or_else(|| PyValueError::new_err("frozen run contains an invalid node key"))?; - let selector = if PYTHON_NODE_KINDS.contains(&kind) { - key.as_str() + let selector: String = if PYTHON_NODE_KINDS.contains(&kind) { + key.clone() + } else if kind == SQL_TEST_NODE_KIND { + format!("test:{name}") } else { - name + name.to_owned() }; - argv.extend([PyString::new(py, "--select"), PyString::new(py, selector)]); + argv.extend([PyString::new(py, "--select"), PyString::new(py, &selector)]); } let parsed = parse_cli_invocation(py, argv, false)?; let args = field(&parsed, "args")?; diff --git a/src/sqlbuild/provider/_helpers/injection.py b/src/sqlbuild/provider/_helpers/injection.py index e7680a16ef..4718700410 100644 --- a/src/sqlbuild/provider/_helpers/injection.py +++ b/src/sqlbuild/provider/_helpers/injection.py @@ -73,7 +73,12 @@ def call_with_provider_injection( if not context_bound and parameter.default is inspect.Parameter.empty: kwargs[parameter.name] = context context_bound = True - return function(**kwargs) + positional: list[object] = [] + for parameter in signature.parameters.values(): + if parameter.kind != inspect.Parameter.POSITIONAL_ONLY or parameter.name not in kwargs: + break + positional.append(kwargs.pop(parameter.name)) + return function(*positional, **kwargs) def _provider_for_parameter( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 95f42565c1..c1e0f92112 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -895,6 +895,9 @@ def python_node_selection_project_files(*, orders_sql: str) -> dict[str, str]: 'MODEL (description "Test model order_summary.", ' 'materialized table);\nSELECT count(*) AS order_count FROM __ref("orders")\n' ), + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_node_jobs.py" + ).read_text(), "python/tasks/reports.py": ( "from python.tasks.counts import count_orders\nfrom sqlbuild.tasks import task\n\n\n" "@task(depends_on=count_orders)\n" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py index 1ec298fe6b..80803f98f7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_node_selection.py @@ -104,6 +104,30 @@ expected_row_counts={"orders": 2}, expected_node_counts={"orders_extract": 2}, ), + PythonNodeSelectionE2ETestCase( + description="a job selecting only Python nodes builds them", + select=("job:customer_counts",), + expected_exit_code=0, + expected_output_patterns=(r"count_customers\s+OK", r"customers_extract\s+OK"), + expected_row_counts={"raw_customers": 3}, + expected_node_counts={"count_customers": 3, "customers_extract": 3}, + ), + PythonNodeSelectionE2ETestCase( + description="a job named without job: builds its Python nodes", + select=("customer_counts",), + expected_exit_code=0, + expected_output_patterns=(r"count_customers\s+OK", r"customers_extract\s+OK"), + expected_row_counts={"raw_customers": 3}, + expected_node_counts={"count_customers": 3, "customers_extract": 3}, + ), + PythonNodeSelectionE2ETestCase( + description="a job selecting a task with its upstream model builds both", + select=("job:order_counts",), + expected_exit_code=0, + expected_output_patterns=(r"count_orders\s+OK",), + expected_row_counts={"orders": 2}, + expected_node_counts={"count_orders": 2}, + ), PythonNodeSelectionE2ETestCase( description="task is skipped when its selected upstream model fails", select=("+task:count_orders",), diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 57ceccc803..bbaca4925f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -78,17 +78,12 @@ "MODEL (description 'Orders.', tags [orders]);\n\n" 'SELECT order_id FROM __ref("stg_orders")\n' ), - "python/orders/partner_export/node.py": ( - "from sqlbuild.assets import asset\nfrom sqlbuild.refs import model\n\n\n" - "@asset(depends_on=model('orders'), tags=['orders'], description='Partner order file.')\n" - "def partner_orders_file(ctx):\n return None\n" - ), - "python/fulfillment/carrier_sync/node.py": ( - "from python.orders.partner_export.node import partner_orders_file\n" - "from sqlbuild.tasks import task\n\n\n" - "@task(depends_on=partner_orders_file, tags=['orders'], description='Book collection.')\n" - "def request_carrier_collection(ctx):\n return None\n" - ), + "python/orders/partner_export/node.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_pipeline_partner_export.py" + ).read_text(), + "python/fulfillment/carrier_sync/node.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py" + ).read_text(), } TYPE_PROOF_FILES: dict[str, str] = { diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 4f1c568195..ddeb7ebdee 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -371,6 +371,13 @@ class ProjectRolesTestCase: expected_denial: str +@dataclass(frozen=True) +class ProjectRoleRunTestCase: + description: str + role: str + expected_response: tuple[int, Any] + + @dataclass(frozen=True, kw_only=True) class ChangedJobTestCase(WorkerExecutionTestCase): description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py index 512b7a78dd..8357d5d100 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/person/helpers.py @@ -265,6 +265,22 @@ def person_request( return json.load(response) +def request_person_run( + api: SnapshotApi, session: tuple[str, str], body: dict[str, Any] +) -> tuple[int, Any]: + """A person's run request: the response on success, else the denial without its decision.""" + request: Request = Request( + api.url + "/api/control/runs", + data=json.dumps(body).encode(), + headers=person_headers(session), + ) + try: + with urlopen(request, timeout=5) as response: + return 200, json.load(response) + except HTTPError as denied: + return denied.code, json.load(denied)["error"].split("; decision ")[0] + + def person_login(api: SnapshotApi, name: str) -> tuple[str, str]: request: Request = Request( api.url + "/api/auth/password/login", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py index cd3ce745b0..703856d119 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py @@ -260,6 +260,14 @@ def test_given_an_ad_hoc_command_when_checked_and_launched_then_the_worker_runs_ ], rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, ), + AdhocVerbTestCase( + description="test: runs only the selected verified SQL test", + prerequisites=[], + command=["test", "--select", "test:customers_have_ids"], + expected_selection=["sql_test:customers_have_ids"], + expected_run_nodes=[("sql_test:customers_have_ids", "node", "success")], + rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, + ), AdhocVerbTestCase( description="freshness observes the verified sources upstream of the selection", prerequisites=[], diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py index fc6221eb23..0215796c88 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_project_roles.py @@ -3,13 +3,16 @@ import subprocess from pathlib import Path from typing import Any +from uuid import uuid4 import pytest from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + ProjectRoleRunTestCase, ProjectRolesTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + INTERMEDIATE_LOADER_FILES, publish_generation, read_person_sql, snapshot_api, @@ -19,6 +22,7 @@ person_login, person_request, provision_person, + request_person_run, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -100,3 +104,72 @@ def test_given_project_roles_when_published_then_person_sql_follows_selector_cov assert rewritten.returncode == 0, rewritten.stderr assert read_person_sql(api, session, "model:customers") == (200, "model:customers") assert read_person_sql(api, session, "model:orders")[0] == 403 + + +@pytest.mark.parametrize( + "test_case", + [ + ProjectRoleRunTestCase( + description="a role granting the loader chain runs the job", + role="shipments_runner", + expected_response=(200, {"applied": True}), + ), + ProjectRoleRunTestCase( + description="a role missing the intermediate loader is refused", + role="orders_runner", + expected_response=(403, "missing job.run permission for this target"), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_job_loading_an_intermediate_loader_when_a_role_runs_it_then_the_grant_must_cover_it( + test_case: ProjectRoleRunTestCase, tmp_path: Path +) -> None: + project: Path = tmp_path / "project" + write_files( + project_dir=project, + files={ + "sqlbuild_project.toml": ( + 'name = "orders"\nadapter = "duckdb"\n' + '[connections.local]\ndatabase = "warehouse.duckdb"\n' + '[targets.development]\nschema = "main"\n' + ), + "roles.toml": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_roles.toml" + ).read_text(), + } + | INTERMEDIATE_LOADER_FILES, + ) + with snapshot_api( + tmp_path / "control", + server_arguments=("--password-origin", "https://control.example.com"), + ) as api: + published: subprocess.CompletedProcess[str] = publish_generation(project, api, 1) + assert published.returncode == 0, published.stderr + admin: dict[str, Any] = provision_person( + tmp_path / "control", "create-user", "orders-admin" + ) + person: dict[str, Any] = provision_person( + tmp_path / "control", "create-principal", "orders-runner" + ) + assert person_request( + api, + "/api/admin/roles", + body={ + "principal_id": person["principal_id"], + "role": test_case.role, + "location_id": api.location_id, + "target": "development", + }, + session=person_login(api, admin["name"]), + ) == {"applied": True} + request: dict[str, Any] = { + "run_id": str(uuid4()), + "location_id": api.location_id, + "target": "development", + "job_key": "annual_orders", + "reason": "Refresh shipments", + "priority": 0, + } + session: tuple[str, str] = person_login(api, person["name"]) + assert request_person_run(api, session, request) == test_case.expected_response diff --git a/tests/unit/src/sqlbuild/provider/_helpers/helpers.py b/tests/unit/src/sqlbuild/provider/_helpers/helpers.py index 26a1bbdb83..23b22ab3c4 100644 --- a/tests/unit/src/sqlbuild/provider/_helpers/helpers.py +++ b/tests/unit/src/sqlbuild/provider/_helpers/helpers.py @@ -30,6 +30,14 @@ def context_and_provider(ctx: object, slack_provider: SlackProvider) -> tuple[ob return ctx, slack_provider.label +def positional_only_context(ctx: object, /, *, slack_provider: SlackProvider) -> tuple[object, str]: + return ctx, slack_provider.label + + +def keyword_only_provider(ctx: object, *, slack_provider: SlackProvider) -> tuple[object, str]: + return ctx, slack_provider.label + + def provider_only(slack_provider: SlackProvider) -> str: return slack_provider.label diff --git a/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py b/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py index a930e53846..0ba9419ac2 100644 --- a/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py +++ b/tests/unit/src/sqlbuild/provider/_helpers/test_injection.py @@ -20,9 +20,11 @@ SlackProvider, context_and_provider, context_only, + keyword_only_provider, load_module, mismatched_provider, missing_provider, + positional_only_context, provider_only, reserved_context_conflict, unannotated_provider, @@ -65,6 +67,16 @@ def test_given_context_only_function_when_calling_with_provider_injection_then_c function=provider_only, expected_result="slack", ), + ProviderInjectionTestCase( + description="keyword-only provider is injected by name", + function=keyword_only_provider, + expected_result=("runtime-context", "slack"), + ), + ProviderInjectionTestCase( + description="positional-only context is passed by position", + function=positional_only_context, + expected_result=("runtime-context", "slack"), + ), ), ids=lambda case: case.description, ) diff --git a/website/src/content/docs/docs/concepts/selectors.mdx b/website/src/content/docs/docs/concepts/selectors.mdx index a10fac80ae..7f598d7a33 100644 --- a/website/src/content/docs/docs/concepts/selectors.mdx +++ b/website/src/content/docs/docs/concepts/selectors.mdx @@ -69,7 +69,7 @@ sqb build --select source:raw__orders ### Job -Select the nodes a job or job group declared under `orchestration/` builds. Job names are globally +Select the SQL and Python nodes a job or job group declared under `orchestration/` builds. Job names are globally unique, so a bare name works; `job:` is the explicit form and accepts glob patterns: ```bash From a96db64de6743be9b1ba8155a6bf93b8896eb9a1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:10:19 +0000 Subject: [PATCH 05/19] fix: keep Python selections runnable on workers and require checked nodes Selections now publish a managed source instead of its loader, ad-hoc test, audit, seed and freshness runs resolve SQL only, a selected SQL test reports its own outcome, and a selected model folds in its failing test. A selector that matches a Python check must also select the nodes the check depends on, in jobs, roles and the CLI alike. --- .../python_pipeline_orders_check.py | 16 +++++ .../orchestration/upstream_loaded_orders.py | 15 +++++ .../src/python_selection/_helpers/unified.rs | 44 +++++++++++--- .../src/python_selection/main/mod.rs | 1 + .../python_selection/main/python_run_key.rs | 13 +++++ .../tests/test_python_selection.rs | 13 ++++- .../src/adhoc/_helpers/planner.rs | 46 ++++++++++----- crates/sqlbuild-cli/src/adhoc/constants.rs | 3 + .../src/adhoc/tests/test_adhoc_planner.rs | 58 +++++++++++++++++++ .../compiled_project/snapshot_execution.rs | 23 +++++--- .../_helpers/graph/python_selection.rs | 14 ++++- .../build/no_tests_no_audits/_test_types.py | 1 + .../test_build_no_tests_no_audits.py | 30 +++++++--- .../compiler/compiled_project/_test_types.py | 14 +++++ .../compiler/compiled_project/helpers.py | 3 + .../publication/_test_types.py | 2 + .../publication/test_adhoc.py | 23 +++++++- .../publication/test_worker.py | 17 ++++++ .../test_orchestration_snapshot.py | 49 ++++++++++++++++ .../_helpers/test_run_selectors.py | 21 ++++--- .../docs/concepts/python-nodes/checks.mdx | 6 +- .../docs/concepts/python-nodes/overview.mdx | 2 +- 22 files changed, 357 insertions(+), 57 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py create mode 100644 crates/sqlbuild-analysis/src/python_selection/main/python_run_key.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py new file mode 100644 index 0000000000..2f8b9b8329 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py @@ -0,0 +1,16 @@ +"""A Python check on the orders model that a job's tag selector matches. + +A job matching the check and the model it checks publishes and runs the check with the model. A job +matching the check alone fails publication, naming the check and the model it leaves out. +""" + +from sqlbuild.checks import check +from sqlbuild.refs import model + + +@check(depends_on=model("orders"), tags=["order_checks"]) +def orders_present(ctx): + """Orders are present.""" + orders = ctx.relation(model("orders")) + count = ctx.query(f"SELECT count(*) FROM {orders}").fetchone()[0] + return ctx.pass_() if count else ctx.fail(message="no orders") diff --git a/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py b/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py new file mode 100644 index 0000000000..0723fa403a --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py @@ -0,0 +1,15 @@ +"""A job that reaches a managed source through `+`, so its run selects the source's loader chain. + +`+orders` selects the model, its managed source raw_shipments, the raw_shipments loader and the +intermediate stage_shipments loader it depends on. The run records each loader as the source it +writes, which is the node its load reports under. +""" + +from sqlbuild.orchestration import job, schedule + +annual_orders = job( + "annual_orders", + description="Build orders and everything upstream of it, loaders included.", + select="+orders", + schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), +) diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index 4c94cd2f87..06eaa10482 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -89,14 +89,10 @@ pub(crate) fn run_selection( Atom::Sql(_) => None, }) .collect(); - if !select.is_empty() && !checks.is_empty() { - return Err(error( - PLANNER_DEFAULT_CODE, - format!( - "Python checks are not selectable here: {}. Use sqb check instead.", - checks.join(", ") - ), - )); + if !select.is_empty() + && let Some(message) = unselected_checked_nodes(context, &selected, &checks) + { + return Err(error(PLANNER_DEFAULT_CODE, message)); } let runnable: Atoms = selected .into_iter() @@ -106,6 +102,38 @@ pub(crate) fn run_selection( Ok(selection(context, sql_keys(&runnable), &runnable)) } +/// A matched check runs with the nodes it checks, so each of them must be selected too. +fn unselected_checked_nodes( + context: &SelectionContext<'_>, + selected: &Atoms, + checks: &[&str], +) -> Option { + checks.iter().find_map(|check| { + let python = context + .nodes + .edges(check, EdgeDirection::Upstream) + .iter() + .filter(|name| !selected.contains(&Atom::Python((*name).clone()))) + .filter_map(|name| Some(format!("{}:{name}", context.nodes.node(name)?.kind))); + let sql = context + .nodes + .node(check) + .map_or(&[][..], |node| &node.sql_deps); + let sql = sql + .iter() + .filter(|key| !selected.contains(&Atom::Sql((*key).clone()))) + .map(|(kind, name)| format!("{kind}:{name}")); + let missing: Vec = python.chain(sql).collect(); + (!missing.is_empty()).then(|| { + format!( + "Python check '{check}' checks {}, which the selection does not include; \ + select it too, or narrow the selector so it no longer matches the check.", + missing.join(", ") + ) + }) + }) +} + /// The SQL keys, with the source each selected intermediate loader writes, and the selected Python node names. fn selection(context: &SelectionContext<'_>, sql: Vec, atoms: &Atoms) -> RunSelection { let terminal: HashSet<&str> = context diff --git a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs index 487d121f6d..4630bc0f8c 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/mod.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/mod.rs @@ -1,6 +1,7 @@ pub mod attach_python_jobs; pub mod published_python_graph; pub mod python_node_graph_indexes; +pub mod python_run_key; pub mod resolve_python_node_excludes; pub mod resolve_python_node_selectors; pub mod resolve_python_sql_nodes; diff --git a/crates/sqlbuild-analysis/src/python_selection/main/python_run_key.rs b/crates/sqlbuild-analysis/src/python_selection/main/python_run_key.rs new file mode 100644 index 0000000000..a23f1075fe --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_selection/main/python_run_key.rs @@ -0,0 +1,13 @@ +use crate::python_selection::constants::LOADER_KIND; + +/// The key a selected Python node's run reports under; a loader reports as the source it writes. +pub fn python_run_key(sources: &[(String, Option)], (kind, name): (&str, &str)) -> String { + if kind != LOADER_KIND { + return format!("{kind}:{name}"); + } + let source = sources + .iter() + .find(|(_, loader)| loader.as_deref() == Some(name)) + .map_or(name, |(source, _)| source.as_str()); + format!("source:{source}") +} diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs index 7a2c24b305..6e89230f2c 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs @@ -139,13 +139,20 @@ fn given_run_selectors_when_resolving_then_selection_matches_python() { expected_selection: Ok((&[("model", "orders")], &["export_orders", "prepare_orders"])), }, RunSelectionTestCase { - description: "checks are rejected", + description: "a check whose checked node is selected runs with it", + select: &["tag:exports"], + exclude: &[], + expected_selection: Ok((&[], &["export_orders"])), + }, + RunSelectionTestCase { + description: "a check whose checked node is unselected names both", select: &["check:check_orders_export"], exclude: &[], expected_selection: Err(( "S000", - "Python checks are not selectable here: check_orders_export. \ - Use sqb check instead.", + "Python check 'check_orders_export' checks asset:export_orders, which the \ + selection does not include; select it too, or narrow the selector so it no \ + longer matches the check.", )), }, RunSelectionTestCase { diff --git a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs index ddb5193499..241a2eaf88 100644 --- a/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/_helpers/planner.rs @@ -11,6 +11,7 @@ use sqlbuild_analysis::graph::models::{GraphResource, JobSelection}; use sqlbuild_analysis::planner_selection::main::split_unit_test_selectors::split_unit_test_selectors; use sqlbuild_analysis::planner_selection::models::{SelectorGroups, UnitTestSelectorNames}; use sqlbuild_analysis::python_selection::main::published_python_graph::published_python_graph; +use sqlbuild_analysis::python_selection::main::python_run_key::python_run_key; use sqlbuild_analysis::python_selection::main::resolve_python_sql_run_selection::resolve_python_sql_run_selection; use sqlbuild_analysis::python_selection::models::{PublishedPythonNode, SqlSelectionGraph}; use sqlbuild_core::text::main::python_str_repr::python_str_repr; @@ -19,8 +20,9 @@ use sqlbuild_orchestration::runs::types::{CommandPlanner, PublishedJobs}; use crate::adhoc::constants::{ ADHOC_VERBS, ALLOWED_DESTS, BUILD_VERB, CHANGED_COUNTS_DEST, CHANGED_DEST, COMMAND_DEST, - FRESHNESS_VERB, INTEGER_CONVERTERS, LOAD_SOURCES_DEST, POSITIVE_INTEGER_CONVERTER, - REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, SQL_TEST_KIND, UNIT_TEST_VERBS, + FRESHNESS_VERB, INCLUDE_PYTHON_DEST, INTEGER_CONVERTERS, LOAD_SOURCES_DEST, + POSITIVE_INTEGER_CONVERTER, REFUSED_DESTS, SEED_VERB, SELECTOR_DESTS, SQL_TEST_KIND, + UNIT_TEST_VERBS, }; use crate::argv::main::command_settings_arguments::command_settings_arguments; use crate::argv::main::job_settings_arguments::job_settings_arguments; @@ -86,7 +88,11 @@ impl CommandPlanner for CliPlanner { seeds } FRESHNESS_VERB => freshness_sources(publication, &select, &exclude)?, - _ => resolve_with_tests(verb, publication, &select, &exclude)?, + _ => { + let python = verb == BUILD_VERB + && settings.get(INCLUDE_PYTHON_DEST) != Some(&Json::Bool(false)); + resolve_with_tests((verb, python), publication, &select, &exclude)? + } }; Ok(CommandPlan { verb: verb.clone(), @@ -196,7 +202,7 @@ fn resolve( select: &[String], exclude: &[String], ) -> Result, String> { - let selected: BTreeSet = resolved(publication, select, exclude)?; + let selected: BTreeSet = resolved(publication, select, exclude, false)?; if selected.is_empty() { return Err("the selectors match no nodes in the current publication".into()); } @@ -205,7 +211,7 @@ fn resolve( /// [`resolve`] with `test:` and unit-test names selecting `sql_test:` keys, as `sqb test` does. fn resolve_with_tests( - verb: &str, + (verb, python): (&str, bool), (resources, jobs): (&[PlannerResource], &PublishedJobs), select: &[String], exclude: &[String], @@ -235,7 +241,7 @@ fn resolve_with_tests( ); } let mut keys: BTreeSet = if split.models_selected { - resolved((resources, jobs), &split.select, &split.exclude)? + resolved((resources, jobs), &split.select, &split.exclude, python)? } else { BTreeSet::new() }; @@ -255,15 +261,20 @@ fn is_sql_test(resource: &PlannerResource) -> bool { resource.key.split_once(':').map(|(kind, _)| kind) == Some(SQL_TEST_KIND) } +/// The published keys the selectors match; Python nodes only when the command runs them. fn resolved( (resources, jobs): (&[PlannerResource], &PublishedJobs), select: &[String], exclude: &[String], + include_python: bool, ) -> Result, String> { - let (python, sql): (Vec<&PlannerResource>, Vec<&PlannerResource>) = resources + let (mut python, sql): (Vec<&PlannerResource>, Vec<&PlannerResource>) = resources .iter() .filter(|resource| !is_sql_test(resource)) .partition(|resource| python_kind(&resource.key).is_some()); + if !include_python { + python.clear(); + } let graph_resources: Vec = sql .iter() .filter_map(|resource| graph_resource(resource)) @@ -275,7 +286,7 @@ fn resolved( .map(|resource| name_of(&resource.key)) .collect(); if selectors_need_jobs(&graph, &selectors, &node_names) { - graph = attach_jobs(graph, published_jobs(jobs)); + graph = attach_jobs(graph, published_jobs(jobs, include_python)); } let sources: Vec<(String, Option)> = sql .iter() @@ -306,10 +317,9 @@ fn resolved( .into_iter() .map(|(kind, name)| format!("{kind}:{name}")) .chain( - selection - .python_node_names - .iter() - .filter_map(|name| Some(format!("{}:{name}", kinds.get(name.as_str())?))), + selection.python_node_names.iter().filter_map(|name| { + Some(python_run_key(&sources, (kinds.get(name.as_str())?, name))) + }), ) .collect()) } @@ -339,10 +349,14 @@ fn name_of(key: &str) -> &str { } /// Published jobs as already-resolved graph selections. -fn published_jobs(jobs: &PublishedJobs) -> BTreeMap { +fn published_jobs(jobs: &PublishedJobs, include_python: bool) -> BTreeMap { let mut selections: BTreeMap = BTreeMap::new(); for (name, keys) in jobs { - let keys: Vec<(String, String)> = keys.iter().filter_map(|key| object_key(key)).collect(); + let keys: Vec<(String, String)> = keys + .iter() + .filter(|key| include_python || python_kind(key).is_none()) + .filter_map(|key| object_key(key)) + .collect(); selections.insert(name.clone(), JobSelection::Resolved { keys }); } selections @@ -355,11 +369,11 @@ fn freshness_sources( exclude: &[String], ) -> Result, String> { let selected: BTreeSet = - upstream_sources(publication.0, &resolved(publication, select, &[])?); + upstream_sources(publication.0, &resolved(publication, select, &[], false)?); let excluded: BTreeSet = if exclude.is_empty() { BTreeSet::new() } else { - upstream_sources(publication.0, &resolved(publication, exclude, &[])?) + upstream_sources(publication.0, &resolved(publication, exclude, &[], false)?) }; let sources: BTreeSet = selected.difference(&excluded).cloned().collect(); if sources.is_empty() { diff --git a/crates/sqlbuild-cli/src/adhoc/constants.rs b/crates/sqlbuild-cli/src/adhoc/constants.rs index 0de8284749..dc93f3bcba 100644 --- a/crates/sqlbuild-cli/src/adhoc/constants.rs +++ b/crates/sqlbuild-cli/src/adhoc/constants.rs @@ -27,6 +27,9 @@ pub(crate) const CHANGED_COUNTS_DEST: &str = "changed_counts"; /// The job setting `--changed` becomes. pub(crate) const CHANGED_DEST: &str = "changed"; +/// The destination `--no-python` clears; only a build with Python nodes resolves them. +pub(crate) const INCLUDE_PYTHON_DEST: &str = "include_python"; + /// The destination `--load` and `--no-load` share; only `--no-load` is accepted. pub(crate) const LOAD_SOURCES_DEST: &str = "load_sources"; diff --git a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs index fe0b38f40a..df7501c9c1 100644 --- a/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs +++ b/crates/sqlbuild-cli/src/adhoc/tests/test_adhoc_planner.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::json; +use sqlbuild_orchestration::runs::models::PlannerResource; use sqlbuild_orchestration::runs::types::PublishedJobs; use crate::adhoc::main::adhoc_planner::adhoc_planner; @@ -348,6 +349,23 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co &["orders"], Some("python/fulfillment/carrier_sync"), ), + PlannerResource { + loader: Some("raw_shipments".to_owned()), + ..resource("source:raw_shipments", &[], &[], None) + }, + resource( + "model:shipments", + &["source:raw_shipments"], + &[], + Some("models/marts"), + ), + resource( + "loader:raw_shipments", + &["loader:stage_shipments"], + &[], + Some("python/loaders"), + ), + resource("loader:stage_shipments", &[], &[], Some("python/loaders")), ]; let jobs: PublishedJobs = BTreeMap::from([( "carrier_sync".to_owned(), @@ -436,6 +454,46 @@ fn given_published_python_nodes_when_planned_then_selectors_resolve_as_on_the_co json!({}), )), }, + AdhocPlanTestCase { + description: "upstream of a model runs its source's loader chain as the sources they write", + command: &["build", "--select", "+shipments"], + expected_plan: Ok(( + &[ + "model:shipments", + "source:raw_shipments", + "source:stage_shipments", + ], + json!({}), + )), + }, + AdhocPlanTestCase { + description: "an intermediate loader runs as the source it writes", + command: &["build", "--select", "loader:stage_shipments"], + expected_plan: Ok((&["source:stage_shipments"], json!({}))), + }, + AdhocPlanTestCase { + description: "test selects only the SQL nodes a tag shared with Python nodes matches", + command: &["test", "--select", "tag:orders"], + expected_plan: Ok((&["model:orders", "model:stg_orders"], json!({}))), + }, + AdhocPlanTestCase { + description: "test downstream of a model stops at the Python nodes that read it", + command: &["test", "--select", "orders+", "--exclude", "stg_orders"], + expected_plan: Ok((&["model:orders"], json!({}))), + }, + AdhocPlanTestCase { + description: "a build without Python nodes selects only the SQL nodes a shared tag matches", + command: &["build", "--no-python", "--select", "tag:orders"], + expected_plan: Ok(( + &["model:orders", "model:stg_orders"], + json!({"include_python": false}), + )), + }, + AdhocPlanTestCase { + description: "test of a job with only Python nodes has nothing to test", + command: &["test", "--select", "job:carrier_sync"], + expected_plan: Err("the selectors match no nodes in the current publication"), + }, ]; let planner = adhoc_planner(); for test_case in test_cases { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs index b27d9b4b27..033b08c4cc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_execution.rs @@ -357,9 +357,10 @@ impl WorkerBuild { outcome }; if self.execution.claim.request.selection.contains(key) { - return self.complete(py, key, (outcome, None, (None, None))); + self.complete(py, key, (outcome, None, (None, None)))?; + } else { + self.finish_check(py, key, outcome)?; } - self.finish_check(py, key, outcome)?; if outcome == NodeOutcome::Skipped { return Ok(()); } @@ -395,15 +396,12 @@ impl WorkerBuild { Ok(None) } - /// A failing SQL test, found by its compiled name such as `parent [case]`, fails the models it blocks with its error. + /// A finished SQL test completes itself when selected; a failing one fails the models it blocks. fn finished_test(&self, result: &Bound<'_, PyAny>) -> PyResult<()> { let py = result.py(); let outcome: String = field(result, "outcome")?.str()?.extract()?; - if SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass) - || self.worker.bind(py).get().refresh_cancellation(py)? - { - return Ok(()); - } + let passed = SqlTestOutcome::from_value(&outcome) == Some(SqlTestOutcome::Pass); + let cancelled = !passed && self.worker.bind(py).get().refresh_cancellation(py)?; let name: String = field(result, "test_name")?.extract()?; let key = format!("{SQL_TEST_KEY_PREFIX}{name}"); let error: Option = match result.getattr_opt("error_code")? { @@ -414,6 +412,15 @@ impl WorkerBuild { Some(value) => value.extract()?, None => None, }; + let node = match (passed, cancelled) { + (true, _) => NodeOutcome::Success, + (false, true) => NodeOutcome::Cancelled, + (false, false) => NodeOutcome::Failed, + }; + self.complete(py, &key, (node, None, (error.clone(), message.clone())))?; + if passed || cancelled { + return Ok(()); + } for model in self.blocked.get(&key).into_iter().flatten() { self.complete( py, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs index 3a5d3b1e6c..f9d1b4e03f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/graph/python_selection.rs @@ -13,6 +13,7 @@ use sqlbuild_analysis::graph::errors::SelectorError; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_analysis::python_selection::main::attach_python_jobs::attach_python_jobs; use sqlbuild_analysis::python_selection::main::python_node_graph_indexes::python_node_graph_indexes as graph_indexes; +use sqlbuild_analysis::python_selection::main::python_run_key::python_run_key; use sqlbuild_analysis::python_selection::main::resolve_python_node_excludes::resolve_python_node_excludes; use sqlbuild_analysis::python_selection::main::resolve_python_node_selectors::resolve_python_node_selectors; use sqlbuild_analysis::python_selection::main::resolve_python_sql_nodes::resolve_python_sql_nodes; @@ -187,6 +188,17 @@ impl ProjectSelection { .map(|node| format!("{}:{name}", node.kind)) } + /// The key a selected Python node's run reports under; a loader reports as the source it writes. + fn run_key(&self, name: &str) -> Option { + let node = self + .nodes + .nodes + .iter() + .rev() + .find(|node| node.name == name)?; + Some(python_run_key(&self.sources, (&node.kind, name))) + } + /// Whether a loader writes no declared source and so runs only as part of another loader's chain. pub(crate) fn intermediate_loader(&self, name: &str) -> bool { !self @@ -213,7 +225,7 @@ impl ProjectSelection { .map(|(kind, name)| format!("{kind}:{name}")) .collect(); for name in &selection.python_node_names { - keys.extend(self.node_key(name)); + keys.extend(self.run_key(name)); } Ok(keys) } diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py index bfc5361cf4..5e38ffcc8e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/_test_types.py @@ -18,3 +18,4 @@ class RunE2ETestCase: expected_query_results: tuple[tuple[str, tuple[tuple[object, ...], ...]], ...] = field( default_factory=tuple ) + select: tuple[str, ...] = field(default_factory=tuple) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index 3100cd5be9..b3114af936 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -2185,15 +2185,29 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou "test_case", [ RunE2ETestCase( - description="run rejects selected check node", + description="build rejects a selected check whose task is not selected", expected_exit_code=1, expected_table_names=(), expected_view_names=(), - ) + expected_output_fragments=( + "Python check 'check_orders_export' checks task:prepare_orders, which the " + "selection does not include; select it too, or narrow the selector so it no " + "longer matches the check.", + ), + select=("check:check_orders_export",), + ), + RunE2ETestCase( + description="build runs a selected check with its selected task", + expected_exit_code=0, + expected_table_names=(), + expected_view_names=(), + expected_output_fragments=("check check_orders_export START", "PASS=2"), + select=("check:check_orders_export", "task:prepare_orders"), + ), ], ids=lambda case: case.description, ) -def test_given_check_selector_when_running_run_then_command_rejects_check( +def test_given_check_selector_when_running_build_then_check_needs_its_checked_node( test_case: RunE2ETestCase, tmp_path: Path, ) -> None: @@ -2230,14 +2244,12 @@ def test_given_check_selector_when_running_run_then_command_rejects_check( "--no-tests", "--no-audits", "--select", - "check:check_orders_export", + *test_case.select, ), project_dir=project_dir, ) - assert result.returncode == test_case.expected_exit_code combined_output: str = result.stdout + result.stderr - assert ( - "Python checks are not selectable here: " - "check_orders_export. Use sqb check instead." in combined_output - ) + assert result.returncode == test_case.expected_exit_code, combined_output + for fragment in test_case.expected_output_fragments: + assert fragment in combined_output diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index 01776fdad9..fa3f650f96 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -283,6 +283,20 @@ class PythonJobGroupCycleTestCase: expected_message: str +@dataclass(frozen=True) +class PythonCheckJobTestCase: + description: str + select: str + expected_selection: list[str] + + +@dataclass(frozen=True) +class PythonCheckJobRejectionTestCase: + description: str + select: str + expected_message: str + + @dataclass(frozen=True) class SnapshotScheduleTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index bbaca4925f..d114213a62 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -84,6 +84,9 @@ "python/fulfillment/carrier_sync/node.py": Path( "benchmarks/duckshop/edge_cases/orchestration/python_pipeline_carrier_sync.py" ).read_text(), + "python/checks/orders_present.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/python_pipeline_orders_check.py" + ).read_text(), } TYPE_PROOF_FILES: dict[str, str] = { diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 1da98aee95..be2c128601 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -406,6 +406,8 @@ class AdhocVerbTestCase: # Files the worker's checkout gains after it verified the final run, which must not run. rewrites: dict[str, str] = field(default_factory=dict) expected_status: str = "success" + # Files that replace the published project's own before publication. + files: dict[str, str] = field(default_factory=dict) @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py index 703856d119..b7247fd0a6 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_adhoc.py @@ -268,6 +268,26 @@ def test_given_an_ad_hoc_command_when_checked_and_launched_then_the_worker_runs_ expected_run_nodes=[("sql_test:customers_have_ids", "node", "success")], rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, ), + AdhocVerbTestCase( + description="build test: runs and reports only the selected verified SQL test", + prerequisites=[], + command=["build", "--select", "test:customers_have_ids"], + expected_selection=["sql_test:customers_have_ids"], + expected_run_nodes=[("sql_test:customers_have_ids", "node", "success")], + rewrites={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, + ), + AdhocVerbTestCase( + description="test of a model and its failing SQL test fails both", + prerequisites=[], + command=["test", "--select", "customers", "test:customers_have_ids"], + expected_selection=["model:customers", "sql_test:customers_have_ids"], + expected_run_nodes=[ + ("model:customers", "node", "failed"), + ("sql_test:customers_have_ids", "node", "failed"), + ], + files={"tests/unit/test_customers.sql": CUSTOMERS_TEST.format(expected=2)}, + expected_status="failed", + ), AdhocVerbTestCase( description="freshness observes the verified sources upstream of the selection", prerequisites=[], @@ -303,7 +323,8 @@ def test_given_an_ad_hoc_verb_when_the_checkout_changes_after_verification_then_ "seeds/seeds.yml": SEEDS_YML.format(region_type="VARCHAR"), "sources/raw.yml": SOURCES_YML.format(table="raw_orders"), "python/rewrite_checkout.py": REWRITE_CHECKOUT, - }, + } + | test_case.files, ) with scheduled_worker( tmp_path, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 0169dd24b0..4231bf9a7d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -851,6 +851,23 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re {"model:orders", "source:raw_shipments", "source:stage_shipments"} ), ), + ClaimedLoadTestCase( + "a job reaching a managed source through + runs the source's loader", + "", + "SELECT amount FROM orders", + "success", + [(9,)], + files=INTERMEDIATE_LOADER_FILES + | { + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/upstream_loaded_orders.py" + ).read_text() + }, + expected_node_count=3, + expected_write_set=frozenset( + {"model:orders", "source:raw_shipments", "source:stage_shipments"} + ), + ), ], ids=lambda case: case.description, ) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 76d37d2d1b..179999d185 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -34,6 +34,8 @@ ) from sqlbuild.orchestration_declarations.models.authoring import Job, JobGroup from tests.integration.src.sqlbuild.compiler.compiled_project._test_types import ( + PythonCheckJobRejectionTestCase, + PythonCheckJobTestCase, PythonJobGroupCycleTestCase, PythonJobGroupTestCase, PythonJobSelectionTestCase, @@ -1997,3 +1999,50 @@ def test_given_a_python_dependency_cycle_across_jobs_when_publishing_a_group_the with pytest.raises(ValueError, match=re.escape(test_case.expected_message)): python_pipeline_snapshot(tmp_path, [*jobs, group]) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonCheckJobTestCase( + description="a job matching a check and the model it checks publishes the model", + select="tag:order_checks orders", + expected_selection=["model:orders"], + ), + ], + ids=lambda case: case.description, +) +def test_given_a_job_matching_a_check_and_its_node_when_publishing_then_the_check_runs_with_it( + tmp_path: Path, test_case: PythonCheckJobTestCase +) -> None: + nightly: Job = job("nightly_orders", description="Orders.", select=test_case.select) + + snapshot: dict[str, Any] = json.loads(python_pipeline_snapshot(tmp_path, [nightly]).to_json()) + + assert snapshot["documents"]["jobs"]["nightly_orders"]["selection"] == ( + test_case.expected_selection + ) + + +@pytest.mark.parametrize( + "test_case", + [ + PythonCheckJobRejectionTestCase( + description="a job matching a check without the model it checks names both", + select="tag:order_checks", + expected_message=( + "job nightly_orders: Python check 'orders_present' checks model:orders, which " + "the selection does not include; select it too, or narrow the selector so it no " + "longer matches the check." + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_job_matching_a_check_without_its_node_when_publishing_then_it_is_rejected( + tmp_path: Path, test_case: PythonCheckJobRejectionTestCase +) -> None: + nightly: Job = job("nightly_orders", description="Orders.", select=test_case.select) + + with pytest.raises(ValueError, match=re.escape(test_case.expected_message)): + python_pipeline_snapshot(tmp_path, [nightly]) diff --git a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py index ae3937e7da..f42a9bed93 100644 --- a/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py +++ b/tests/unit/src/sqlbuild/compiler/python_nodes/_helpers/test_run_selectors.py @@ -42,6 +42,13 @@ expected_sql_names=frozenset(), expected_python_node_names=frozenset({"export_orders"}), ), + PythonSqlSelectorTestCase( + description="accepts a check whose checked node the selection also matches", + select=("tag:exports",), + exclude=(), + expected_sql_names=frozenset(), + expected_python_node_names=frozenset({"export_orders"}), + ), PythonSqlSelectorTestCase( description="selects expanded source terminal loader requirement for run", select=("+source:raw_orders",), @@ -73,18 +80,14 @@ def test_given_run_selectors_when_resolving_then_excludes_python_checks( "test_case", [ PythonSqlSelectorErrorTestCase( - description="rejects explicit check selector for run", + description="rejects a check whose checked node the selection leaves out", select=("check:check_orders_export",), exclude=(), expected_error_type=ValueError, - expected_error_fragment=("Python checks are not selectable here: check_orders_export"), - ), - PythonSqlSelectorErrorTestCase( - description="rejects tag selector that includes a check for run", - select=("tag:exports",), - exclude=(), - expected_error_type=ValueError, - expected_error_fragment=("Python checks are not selectable here: check_orders_export"), + expected_error_fragment=( + "Python check 'check_orders_export' checks asset:export_orders, which the " + "selection does not include" + ), ), ], ids=lambda case: case.description, diff --git a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx index dd93a67b34..c1acdf2f86 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx @@ -109,7 +109,11 @@ Checks do not support `columns`, `column_lineage`, or `retry`. ## Running checks -Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. To run checks on their own, use [`sqb check`](/docs/cli/check): +Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. + +A `sqb build` selector that matches a check must also select the nodes the check depends on. Otherwise the build fails with an error that names the check and the missing node; select that node too, or narrow the selector so it no longer matches the check. Jobs, roles and server runs resolve selectors the same way. + +To run checks on their own, use [`sqb check`](/docs/cli/check): ```bash # Run all checks diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index ddc395623b..9d215b702c 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -141,7 +141,7 @@ Python nodes are selected like SQL resources, by bare name or typed selector: sqb build --select export_orders # bare name sqb build --select task:export_orders # typed sqb build --select asset:orders_export -sqb build --select check:check_orders +sqb build --select export_orders check:check_orders_exported # a check with the node it checks sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` From 879f9316b6be7d9a9aae48d9e0ca2d9504fed148 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:31:15 +0000 Subject: [PATCH 06/19] fix: refuse selecting a Python check on a source no loader writes A build never loads such a source, so a matched check on it would be accepted and then never run. The resolver now names the check and the source and points to sqb check. --- .../src/python_selection/_helpers/unified.rs | 79 +++++++++++++------ .../src/python_selection/tests/helpers.rs | 14 ++++ .../tests/test_python_selection.rs | 12 +++ .../test_build_no_tests_no_audits.py | 22 ++++++ .../docs/concepts/python-nodes/checks.mdx | 2 +- 5 files changed, 104 insertions(+), 25 deletions(-) diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs index 06eaa10482..56bf52afce 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/unified.rs @@ -102,35 +102,66 @@ pub(crate) fn run_selection( Ok(selection(context, sql_keys(&runnable), &runnable)) } -/// A matched check runs with the nodes it checks, so each of them must be selected too. +/// The first matched check a build cannot run after the nodes it checks, as an error message. fn unselected_checked_nodes( context: &SelectionContext<'_>, selected: &Atoms, checks: &[&str], ) -> Option { - checks.iter().find_map(|check| { - let python = context - .nodes - .edges(check, EdgeDirection::Upstream) - .iter() - .filter(|name| !selected.contains(&Atom::Python((*name).clone()))) - .filter_map(|name| Some(format!("{}:{name}", context.nodes.node(name)?.kind))); - let sql = context - .nodes - .node(check) - .map_or(&[][..], |node| &node.sql_deps); - let sql = sql - .iter() - .filter(|key| !selected.contains(&Atom::Sql((*key).clone()))) - .map(|(kind, name)| format!("{kind}:{name}")); - let missing: Vec = python.chain(sql).collect(); - (!missing.is_empty()).then(|| { - format!( - "Python check '{check}' checks {}, which the selection does not include; \ - select it too, or narrow the selector so it no longer matches the check.", - missing.join(", ") - ) - }) + let loaded: HashSet<&str> = source_loaders(context.sql) + .into_iter() + .map(|(source, _)| source) + .collect(); + for check in checks { + if let Some(message) = unrunnable_check(context, selected, &loaded, check) { + return Some(message); + } + } + None +} + +/// Why a build cannot run `check`: it checks a source no loader writes, or an unselected node. +fn unrunnable_check( + context: &SelectionContext<'_>, + selected: &Atoms, + loaded: &HashSet<&str>, + check: &str, +) -> Option { + let sql: &[ObjectKey] = context + .nodes + .node(check) + .map_or(&[][..], |node| &node.sql_deps); + let unloaded: Vec = sql + .iter() + .filter(|(kind, name)| kind == SOURCE_RESOURCE && !loaded.contains(name.as_str())) + .map(|(kind, name)| format!("{kind}:{name}")) + .collect(); + if !unloaded.is_empty() { + return Some(format!( + "Python check '{check}' checks {}, which a build does not load, so the check \ + would not run; run it with sqb check, or narrow the selector so it no longer \ + matches the check.", + unloaded.join(", ") + )); + } + let missing: Vec = context + .nodes + .edges(check, EdgeDirection::Upstream) + .iter() + .filter(|name| !selected.contains(&Atom::Python((*name).clone()))) + .filter_map(|name| Some(format!("{}:{name}", context.nodes.node(name)?.kind))) + .chain( + sql.iter() + .filter(|key| !selected.contains(&Atom::Sql((*key).clone()))) + .map(|(kind, name)| format!("{kind}:{name}")), + ) + .collect(); + (!missing.is_empty()).then(|| { + format!( + "Python check '{check}' checks {}, which the selection does not include; \ + select it too, or narrow the selector so it no longer matches the check.", + missing.join(", ") + ) }) } diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs index 2ac7b7de0e..2e8c3134f8 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/helpers.rs @@ -67,6 +67,14 @@ const ORDERS_NODES: &[NodeSpec] = &[ &[], &[], ), + ( + "check_raw_customers", + "check", + "python/checks/customers.py", + &["customer_checks"], + &[], + &[("source", "raw_customers")], + ), ]; fn owned(values: &[&str]) -> Vec { @@ -128,6 +136,12 @@ fn orders_sql_graph() -> ProjectGraph { tags: Vec::new(), folder: None, }, + GraphResource { + key: key(&("source", "raw_customers")), + deps: Vec::new(), + tags: Vec::new(), + folder: None, + }, ]) } diff --git a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs index 6e89230f2c..fad17b2ca9 100644 --- a/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/tests/test_python_selection.rs @@ -113,6 +113,7 @@ fn given_run_selectors_when_resolving_then_selection_matches_python() { &[ ("model", "orders"), ("source", "load_events"), + ("source", "raw_customers"), ("source", "raw_orders"), ], &[ @@ -144,6 +145,17 @@ fn given_run_selectors_when_resolving_then_selection_matches_python() { exclude: &[], expected_selection: Ok((&[], &["export_orders"])), }, + RunSelectionTestCase { + description: "a check on a source a build does not load is refused even when selected", + select: &["tag:customer_checks", "source:raw_customers"], + exclude: &[], + expected_selection: Err(( + "S000", + "Python check 'check_raw_customers' checks source:raw_customers, which a build \ + does not load, so the check would not run; run it with sqb check, or narrow the \ + selector so it no longer matches the check.", + )), + }, RunSelectionTestCase { description: "a check whose checked node is unselected names both", select: &["check:check_orders_export"], diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index b3114af936..3af739ac24 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -2204,6 +2204,18 @@ def test_given_model_task_loader_chain_when_running_run_then_command_rejects_bou expected_output_fragments=("check check_orders_export START", "PASS=2"), select=("check:check_orders_export", "task:prepare_orders"), ), + RunE2ETestCase( + description="build rejects a selected check on a source no loader writes", + expected_exit_code=1, + expected_table_names=(), + expected_view_names=(), + expected_output_fragments=( + "Python check 'check_raw_events' checks source:raw_events, which a build does " + "not load, so the check would not run; run it with sqb check, or narrow the " + "selector so it no longer matches the check.", + ), + select=("check:check_raw_events", "source:raw_events"), + ), ], ids=lambda case: case.description, ) @@ -2234,6 +2246,16 @@ def test_given_check_selector_when_running_build_then_check_needs_its_checked_no "def prepare_orders(ctx):\n" " '''Test task prepare_orders.'''\n return ctx.result()\n" ), + "python/checks/events.py": ( + "from sqlbuild.checks import check\n" + "from sqlbuild.refs import source\n\n" + "@check(depends_on=source('raw_events'))\n" + "def check_raw_events(ctx):\n" + " '''Test check check_raw_events.'''\n return ctx.pass_()\n" + ), + "sources/raw.yml": ( + "sources:\n - name: raw_events\n description: Test source raw_events.\n" + ), }, ) diff --git a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx index c1acdf2f86..17a34a1947 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/checks.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/checks.mdx @@ -111,7 +111,7 @@ Checks do not support `columns`, `column_lineage`, or `retry`. Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. -A `sqb build` selector that matches a check must also select the nodes the check depends on. Otherwise the build fails with an error that names the check and the missing node; select that node too, or narrow the selector so it no longer matches the check. Jobs, roles and server runs resolve selectors the same way. +A `sqb build` selector that matches a check must also select the nodes the check depends on. Otherwise the build fails with an error that names the check and the missing node; select that node too, or narrow the selector so it no longer matches the check. A check on a source that no loader writes never runs in a build, so a selector that matches it fails too; run that check with `sqb check`. Jobs and server runs resolve selectors the same way. To run checks on their own, use [`sqb check`](/docs/cli/check): From 059f8cdb7380c82e8d9e3a4449b1dfa517f1c724 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 05:56:00 +0000 Subject: [PATCH 07/19] refactor: share upstream result reads between loader and node contexts Loader and Python node contexts read upstream results through one mixin, and the selection entry points build their context in one place. Regenerates the packaged docs skill for the selector and check pages. --- .../src/python_selection/_helpers/nodes.rs | 17 ++++- .../main/attach_python_jobs.rs | 12 ++-- .../main/resolve_python_sql_nodes.rs | 10 +-- .../main/resolve_python_sql_run_selection.rs | 10 +-- .../docs/concepts/python-nodes/checks.md | 6 +- .../docs/concepts/python-nodes/overview.md | 16 ++++- .../references/docs/concepts/selectors.md | 2 +- src/sqlbuild/executor/load/models.py | 63 +------------------ .../classes/dependency_result_reader.py | 61 ++++++++++++++++++ .../executor/node_results/constants.py | 7 +++ src/sqlbuild/executor/python_nodes/models.py | 63 +------------------ 11 files changed, 118 insertions(+), 149 deletions(-) create mode 100644 src/sqlbuild/executor/node_results/classes/dependency_result_reader.py diff --git a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs index e2390a571b..7fc0e9cd4a 100644 --- a/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs +++ b/crates/sqlbuild-analysis/src/python_selection/_helpers/nodes.rs @@ -17,7 +17,9 @@ use crate::python_selection::constants::{ UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, UNMAPPED_KIND_CODE, }; -use crate::python_selection::models::{NodeIndex, PythonSelectionGraph, PythonSelectionNode}; +use crate::python_selection::models::{ + NodeIndex, PythonSelectionGraph, PythonSelectionNode, SelectionContext, SqlSelectionGraph, +}; use sqlbuild_core::constants::PYTHON_NODE_ROOT as PYTHON_NODE_PATH_ROOT; pub(crate) type Names = BTreeSet; @@ -59,6 +61,19 @@ fn keyed(rows: &[(String, Vec)]) -> HashMap<&str, &[String]> { .collect() } +impl<'graph> SelectionContext<'graph> { + pub(crate) fn new( + sql: SqlSelectionGraph<'graph>, + python: &'graph PythonSelectionGraph, + ) -> Self { + Self { + sql, + nodes: NodeIndex::new(python), + sql_names: sql_names(sql.graph), + } + } +} + impl<'graph> NodeIndex<'graph> { pub(crate) fn new(graph: &'graph PythonSelectionGraph) -> Self { Self { diff --git a/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs index 6e3095f6f7..21edb1f7eb 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/attach_python_jobs.rs @@ -2,10 +2,9 @@ use std::collections::BTreeMap; use crate::graph::main::attach_jobs_with::attach_jobs_with; use crate::graph::models::{JobSelection, ProjectGraph}; -use crate::python_selection::_helpers::nodes::sql_names; use crate::python_selection::_helpers::unified::run_selection; use crate::python_selection::models::{ - NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, + PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, }; /// Attach jobs as publication resolves them, keeping each job's Python nodes as `(kind, name)` keys. @@ -20,14 +19,13 @@ pub fn attach_python_jobs( .map(|node| (node.name.as_str(), node.kind.as_str())) .collect(); attach_jobs_with(sql.graph.clone(), catalog, |graph, select, exclude| { - let context: SelectionContext<'_> = SelectionContext { - sql: SqlSelectionGraph { + let context: SelectionContext<'_> = SelectionContext::new( + SqlSelectionGraph { graph, sources: sql.sources, }, - nodes: NodeIndex::new(python), - sql_names: sql_names(graph), - }; + python, + ); let RunSelection { sql_keys, python_node_names, diff --git a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs index 6a2d99a3cb..b482df85e7 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_nodes.rs @@ -1,8 +1,7 @@ use crate::graph::errors::SelectorError; -use crate::python_selection::_helpers::nodes::sql_names; use crate::python_selection::_helpers::unified::node_selection; use crate::python_selection::models::{ - NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, + PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, }; /// The SQL resources and Python nodes, checks included, that selectors name, as grants and sensor conditions read them. @@ -12,10 +11,5 @@ pub fn resolve_python_sql_nodes( select: &[String], exclude: &[String], ) -> Result { - let context: SelectionContext<'_> = SelectionContext { - sql, - nodes: NodeIndex::new(python), - sql_names: sql_names(sql.graph), - }; - node_selection(&context, select, exclude) + node_selection(&SelectionContext::new(sql, python), select, exclude) } diff --git a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs index a820fd41d7..fef9983c7b 100644 --- a/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs +++ b/crates/sqlbuild-analysis/src/python_selection/main/resolve_python_sql_run_selection.rs @@ -1,8 +1,7 @@ use crate::graph::errors::SelectorError; -use crate::python_selection::_helpers::nodes::sql_names; use crate::python_selection::_helpers::unified::run_selection; use crate::python_selection::models::{ - NodeIndex, PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, + PythonSelectionGraph, RunSelection, SelectionContext, SqlSelectionGraph, }; /// Python's `resolve_python_sql_run_selectors`: run selectors over SQL resources and Python nodes. @@ -12,10 +11,5 @@ pub fn resolve_python_sql_run_selection( select: &[String], exclude: &[String], ) -> Result { - let context: SelectionContext<'_> = SelectionContext { - sql, - nodes: NodeIndex::new(python), - sql_names: sql_names(sql.graph), - }; - run_selection(&context, select, exclude) + run_selection(&SelectionContext::new(sql, python), select, exclude) } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md index 4dc3e69973..fe54d4880a 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/checks.md @@ -120,7 +120,11 @@ Checks do not support `columns`, `column_lineage`, or `retry`. ## Running checks -Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. To run checks on their own, use [`sqb check`](../../cli/check.md): +Checks run automatically at the end of `sqb build` when all their Python and SQL dependencies ran in that build. They are skipped when `--no-audits` is passed. + +A `sqb build` selector that matches a check must also select the nodes the check depends on. Otherwise the build fails with an error that names the check and the missing node; select that node too, or narrow the selector so it no longer matches the check. A check on a source that no loader writes never runs in a build, so a selector that matches it fails too; run that check with `sqb check`. Jobs and server runs resolve selectors the same way. + +To run checks on their own, use [`sqb check`](../../cli/check.md): ```bash # Run all checks diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index 17cdf8438d..c4c2db12e9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -143,7 +143,7 @@ def export_orders(ctx): - `ctx.skip(reason, mode=...)` skips the node. `mode` accepts `"soft"` (default, skips only this node) or `"hard"` (also blocks dependents), as a string or the `SkipMode` enum from `sqlbuild.tasks`/`sqlbuild.assets`. - Assets may pass `materialized=True`/`False` to record whether an artifact was produced. -Downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. +Within one run, downstream nodes run only if at least one upstream succeeded. If all upstreams are skipped, the downstream is skipped. A failed or hard-skipped upstream blocks its dependents. ## Result persistence @@ -157,7 +157,7 @@ Python nodes are selected like SQL resources, by bare name or typed selector: sqb build --select export_orders # bare name sqb build --select task:export_orders # typed sqb build --select asset:orders_export -sqb build --select check:check_orders +sqb build --select export_orders check:check_orders_exported # a check with the node it checks sqb build --select tag:exports # by tag sqb build --select +orders_export # with upstreams ``` @@ -177,6 +177,18 @@ dependency fails or is skipped. A dependency that is not part of the run (not se unmanaged source) is read as it currently exists, so `sqb build --select task:export_orders` runs the task against the relations already in the warehouse. +Python dependencies follow the same rule. A task or asset selected without its Python upstream +reads that upstream's latest successful result through `ctx.result_of`. If the upstream has never +succeeded for the target, the node fails with an error that names the upstream, for example: + +```text +Upstream task 'prepare_orders' has no successful result to read. Build it first with +'sqb build --select task:prepare_orders', or build both with 'sqb build --select +asset:export_orders'. +``` + +Jobs, sensors, roles and ad-hoc runs on the server resolve selectors the same way, so a job can +select one Python node and a job group orders it after the job that builds its upstream. + The commands differ in what they include by default: | Command | SQL | Loaders / tasks / assets | Checks | Audits | diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md index cf563c84a9..09e66562eb 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/selectors.md @@ -82,7 +82,7 @@ sqb build --select source:raw__orders ### Job -Select the nodes a job or job group declared under `orchestration/` builds. Job names are globally +Select the SQL and Python nodes a job or job group declared under `orchestration/` builds. Job names are globally unique, so a bare name works; `job:` is the explicit form and accepts glob patterns: ```bash diff --git a/src/sqlbuild/executor/load/models.py b/src/sqlbuild/executor/load/models.py index 39097300b7..f64d2156b0 100644 --- a/src/sqlbuild/executor/load/models.py +++ b/src/sqlbuild/executor/load/models.py @@ -17,13 +17,12 @@ from sqlbuild.adapter.relations.main.resolve_qualified_name_parts import ( resolve_qualified_name_parts, ) -from sqlbuild.compiler.python_nodes.types import PythonNodeKind, SkipMode +from sqlbuild.compiler.python_nodes.types import SkipMode from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.constants import LOADER_RELATION_QUALIFIER_SEPARATOR from sqlbuild.executor.load.types import LoadProgressCallback -from sqlbuild.executor.node_results.models import NodeResultEnvelope +from sqlbuild.executor.node_results.classes.dependency_result_reader import DependencyResultReader from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard -from sqlbuild.executor.python_nodes.constants import MISSING_DEFAULT from sqlbuild.executor.scheduling.types import ExecutionStatus from sqlbuild.provider.main.runtime import ProviderContainer, _empty_provider_container from sqlbuild.python_nodes.models import SqlResourceRef @@ -95,7 +94,7 @@ class LoaderResult: @dataclass(frozen=True) -class LoaderContext: +class LoaderContext(DependencyResultReader): """Runtime context passed to a source loader function.""" adapter: BaseAdapter @@ -164,44 +163,6 @@ def result( materialized=materialized, ) - def result_of( - self, - *, - node_function: Callable[..., object], - run_id: str | None = None, - default: object = MISSING_DEFAULT, - ) -> NodeResultEnvelope | object: - """Return the latest persisted upstream result by Python node function reference.""" - - if self.result_store is None: - if default is not MISSING_DEFAULT: - return default - raise ExecutorInputError("No Python node result store is available") - node_type, node_name = self._result_dependency_identity(node_function) - return self.result_store.result_of( - node_type=node_type, - node_name=node_name, - run_id=run_id, - default=default, - ) - - def results_of( - self, - *, - node_function: Callable[..., object], - limit: int, - ) -> tuple[NodeResultEnvelope, ...]: - """Return persisted successful upstream result history, newest first.""" - - if self.result_store is None: - return () - node_type, node_name = self._result_dependency_identity(node_function) - return self.result_store.results_of( - node_type=node_type, - node_name=node_name, - limit=limit, - ) - def qualify_name( self, *, @@ -251,24 +212,6 @@ def _record_resolved(self, relation_ref: LoaderRelationRef) -> None: SqlResourceRef(kind=SqlResourceRefKind.SOURCE, name=relation_ref.name) ) - def _result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: - task_definition: object = getattr(node_function, "__sqlbuild_task__", None) - if task_definition is not None: - return PythonNodeKind.TASK.value, self._definition_name(task_definition) - asset_definition: object = getattr(node_function, "__sqlbuild_asset__", None) - if asset_definition is not None: - return PythonNodeKind.ASSET.value, self._definition_name(asset_definition) - loader_definition: object = getattr(node_function, "__sqlbuild_loader__", None) - if loader_definition is not None: - return PythonNodeKind.LOADER.value, self._definition_name(loader_definition) - raise ExecutorInputError("Python node result dependency must be a task, asset, or loader") - - def _definition_name(self, definition: object) -> str: - name: object = getattr(definition, "name", None) - if isinstance(name, str): - return name - raise ExecutorInputError("Python node result dependency has no resolved name") - @dataclass(frozen=True) class LoadExecutionResult: diff --git a/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py b/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py new file mode 100644 index 0000000000..7e5a36393f --- /dev/null +++ b/src/sqlbuild/executor/node_results/classes/dependency_result_reader.py @@ -0,0 +1,61 @@ +"""Reads of a Python node's upstream results, looked up by the upstream's decorated function.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import Any + +from sqlbuild.errors.contracts.exceptions import ExecutorInputError +from sqlbuild.executor.node_results.constants import RESULT_DEPENDENCY_ATTRIBUTES +from sqlbuild.executor.node_results.models import NodeResultEnvelope +from sqlbuild.executor.python_nodes.constants import MISSING_DEFAULT + + +class DependencyResultReader: + """Mixin for node contexts that read their upstream results from a `result_store`.""" + + result_store: Any | None + + def result_of( + self, + *, + node_function: Callable[..., object], + run_id: str | None = None, + default: object = MISSING_DEFAULT, + ) -> NodeResultEnvelope | object: + """Return the latest persisted upstream result by Python node function reference.""" + + if self.result_store is None: + if default is not MISSING_DEFAULT: + return default + raise ExecutorInputError("No Python node result store is available") + node_type, node_name = self.result_dependency_identity(node_function) + return self.result_store.result_of( + node_type=node_type, node_name=node_name, run_id=run_id, default=default + ) + + def results_of( + self, + *, + node_function: Callable[..., object], + limit: int, + ) -> tuple[NodeResultEnvelope, ...]: + """Return persisted successful upstream result history, newest first.""" + + if self.result_store is None: + return () + node_type, node_name = self.result_dependency_identity(node_function) + return self.result_store.results_of(node_type=node_type, node_name=node_name, limit=limit) + + def result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: + """Return the `(kind, name)` a dependency's results are stored under.""" + + for attribute, kind in RESULT_DEPENDENCY_ATTRIBUTES: + definition: object = getattr(node_function, attribute, None) + if definition is None: + continue + name: object = getattr(definition, "name", None) + if not isinstance(name, str): + raise ExecutorInputError("Python node result dependency has no resolved name") + return kind, name + raise ExecutorInputError("Python node result dependency must be a task, asset, or loader") diff --git a/src/sqlbuild/executor/node_results/constants.py b/src/sqlbuild/executor/node_results/constants.py index b2e7444998..3b2d792021 100644 --- a/src/sqlbuild/executor/node_results/constants.py +++ b/src/sqlbuild/executor/node_results/constants.py @@ -1,6 +1,7 @@ """Runtime node result storage constants.""" from sqlbuild._native import StateSqlValueType +from sqlbuild.compiler.python_nodes.types import PythonNodeKind NODE_RESULT_MATERIALIZED_TRUE_VALUE: str = "true" @@ -36,3 +37,9 @@ **{column: StateSqlValueType.STRING for column in NODE_RESULT_COLUMNS}, COLUMN_TIMESTAMP: StateSqlValueType.TEXT_TIMESTAMP, } + +RESULT_DEPENDENCY_ATTRIBUTES: tuple[tuple[str, str], ...] = ( + ("__sqlbuild_task__", PythonNodeKind.TASK.value), + ("__sqlbuild_asset__", PythonNodeKind.ASSET.value), + ("__sqlbuild_loader__", PythonNodeKind.LOADER.value), +) diff --git a/src/sqlbuild/executor/python_nodes/models.py b/src/sqlbuild/executor/python_nodes/models.py index 245fab2a91..4d7ffae694 100644 --- a/src/sqlbuild/executor/python_nodes/models.py +++ b/src/sqlbuild/executor/python_nodes/models.py @@ -21,10 +21,9 @@ ) from sqlbuild.errors.contracts.exceptions import ExecutorInputError from sqlbuild.executor.load.models import LoadExecutionResult -from sqlbuild.executor.node_results.models import NodeResultEnvelope +from sqlbuild.executor.node_results.classes.dependency_result_reader import DependencyResultReader from sqlbuild.executor.python_nodes.classes.runtime_relation_guard import RuntimeRelationGuard from sqlbuild.executor.python_nodes.constants import ( - MISSING_DEFAULT, PYTHON_NODE_RELATION_QUALIFIER_SEPARATOR, ) from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder @@ -165,7 +164,7 @@ class IngressCallbacks: @dataclass(frozen=True, kw_only=True) -class BasePythonNodeContext: +class BasePythonNodeContext(DependencyResultReader): """Shared runtime helpers for framework-owned Python nodes.""" adapter: BaseAdapter @@ -254,64 +253,6 @@ def skip( metadata={} if metadata is None else metadata, ) - def result_of( - self, - *, - node_function: Callable[..., object], - run_id: str | None = None, - default: object = MISSING_DEFAULT, - ) -> NodeResultEnvelope | object: - """Return the latest persisted upstream result by Python node function reference.""" - - if self.result_store is None: - if default is not MISSING_DEFAULT: - return default - raise ExecutorInputError("No Python node result store is available") - node_type, node_name = self.result_dependency_identity(node_function) - return self.result_store.result_of( - node_type=node_type, - node_name=node_name, - run_id=run_id, - default=default, - ) - - def results_of( - self, - *, - node_function: Callable[..., object], - limit: int, - ) -> tuple[NodeResultEnvelope, ...]: - """Return persisted successful upstream result history, newest first.""" - - if self.result_store is None: - return () - node_type, node_name = self.result_dependency_identity(node_function) - return self.result_store.results_of( - node_type=node_type, - node_name=node_name, - limit=limit, - ) - - def result_dependency_identity(self, node_function: Callable[..., object]) -> tuple[str, str]: - """Return the `(kind, name)` a dependency's results are stored under.""" - - task_definition: object = getattr(node_function, "__sqlbuild_task__", None) - if task_definition is not None: - return PythonNodeKind.TASK.value, self._definition_name(task_definition) - asset_definition: object = getattr(node_function, "__sqlbuild_asset__", None) - if asset_definition is not None: - return PythonNodeKind.ASSET.value, self._definition_name(asset_definition) - loader_definition: object = getattr(node_function, "__sqlbuild_loader__", None) - if loader_definition is not None: - return PythonNodeKind.LOADER.value, self._definition_name(loader_definition) - raise ExecutorInputError("Python node result dependency must be a task, asset, or loader") - - def _definition_name(self, definition: object) -> str: - name: object = getattr(definition, "name", None) - if isinstance(name, str): - return name - raise ExecutorInputError("Python node result dependency must have a string name") - @dataclass(frozen=True, kw_only=True) class TaskContext(BasePythonNodeContext): From e0c01ba2ca20b28aa0e9c78247ed97b9f7f04853 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:15:28 +0000 Subject: [PATCH 08/19] feat!: order job groups by lineage only and skip checks under --no-audits JobGroup no longer takes after: a group's order comes only from the lineage between the nodes its jobs build, and declaring after fails with an error that names the group. build --no-audits now skips Python checks alongside audits, as documented. --- .../src/declarations/_helpers/group_waits.rs | 2 +- .../declarations/main/group_dependencies.rs | 26 +++++------------ .../declarations/main/snapshot_definitions.rs | 3 +- .../src/declarations/models.rs | 4 +-- .../_helpers/build_command/execution.rs | 5 +++- .../compiled_project/snapshot_declarations.rs | 12 +------- .../native_host/orchestration_declarations.rs | 15 ---------- .../models/authoring.py | 14 +++++++--- .../cli/commands/main/build/helpers.py | 12 ++++++++ .../build/test_no_tests_no_audits_flags.py | 6 ++-- .../test_orchestration_snapshot.py | 28 +++++-------------- ui/src/lib/api/mock/operate-store.ts | 5 +--- ui/src/lib/api/mock/operate.ts | 15 +--------- ui/src/lib/api/types/operate.ts | 4 +-- ui/src/lib/lineage/job-graph.test.ts | 12 ++++---- ui/src/lib/lineage/job-graph.ts | 15 +++++----- 16 files changed, 66 insertions(+), 112 deletions(-) diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs index 76a2b097aa..fcc0b6ece8 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; -/// Keep the first reason found for a wait: an explicit `after`, or the first lineage edge. +/// Keep the first reason found for a wait: the first lineage edge. pub(crate) fn record_wait( mut reasons: BTreeMap<(String, String), String>, (job, dependency): (&str, &str), diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs index d2103f2439..6e78d95cfb 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs @@ -1,4 +1,4 @@ -//! Freeze job boundaries from snapshot lineage plus explicit waits, rejecting cycles. +//! Freeze job boundaries from snapshot lineage, rejecting cycles. use std::collections::{BTreeMap, BTreeSet}; @@ -10,19 +10,12 @@ use crate::store::errors::ControlError; pub fn group_dependencies( (group, input): (&str, &SnapshotInput), jobs: &BTreeMap, - (members, after): (&BTreeSet, &BTreeMap>), + members: &BTreeSet, ) -> Result>, ControlError> { - if members.is_empty() - || members.iter().any(|name| !jobs.contains_key(name)) - || after - .iter() - .any(|(name, dependencies)| !members.contains(name) || !dependencies.is_subset(members)) - { - return Err(ControlError::Invalid( - "group needs known jobs and dependencies within its membership".into(), - )); + if members.is_empty() || members.iter().any(|name| !jobs.contains_key(name)) { + return Err(ControlError::Invalid("group needs known jobs".into())); } - let waits = lineage_dependencies(input, jobs, (members, after)); + let waits = lineage_dependencies(input, jobs, members); ensure_acyclic(group, &waits.edges, &waits.reasons)?; Ok(waits.edges) } @@ -31,7 +24,7 @@ pub fn group_dependencies( fn lineage_dependencies( input: &SnapshotInput, jobs: &BTreeMap, - (members, after): (&BTreeSet, &BTreeMap>), + members: &BTreeSet, ) -> GroupWaits { let mut owners: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); for name in members { @@ -42,12 +35,7 @@ fn lineage_dependencies( let mut edges: BTreeMap> = BTreeMap::new(); let mut reasons: BTreeMap<(String, String), String> = BTreeMap::new(); for name in members { - let mut dependencies = after.get(name).cloned().unwrap_or_default(); - for dependency in &dependencies { - reasons = record_wait(reasons, (name, dependency), || { - "declared in after".to_owned() - }); - } + let mut dependencies: BTreeSet = BTreeSet::new(); let mut seen: BTreeSet<&str> = BTreeSet::new(); let mut pending: Vec<&str> = jobs[name].selection.iter().map(String::as_str).collect(); while let Some(key) = pending.pop() { diff --git a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs index a415f62d13..4d42f2baa4 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs @@ -30,8 +30,7 @@ pub fn snapshot_definitions(input: &SnapshotInput) -> Result = group.jobs.keys().cloned().collect(); - let dependencies = - group_dependencies((name, input), &definitions.jobs, (&members, &group.jobs))?; + let dependencies = group_dependencies((name, input), &definitions.jobs, &members)?; let selection: BTreeSet = group .jobs .keys() diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index b0a475a46d..91ab538733 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -201,7 +201,7 @@ pub struct JobDetailRecord { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct JobMember { pub job_key: String, - /// Members this one waits for, from lineage and from the group's `after`. + /// Members this one waits for, from the lineage between their nodes. pub waits_for: BTreeSet, } @@ -288,7 +288,7 @@ pub(crate) struct PreviousTriggerState { pub(crate) status_override: Option, } -/// Each member job's waits, and why it waits: an explicit `after`, or the first lineage edge found. +/// Each member job's waits, and why it waits: the first lineage edge found. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub(crate) struct GroupWaits { pub(crate) edges: BTreeMap>, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 3e5a9123ba..66f70a70d5 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -668,7 +668,10 @@ pub(crate) fn run_post_build_python_checks<'py>( ) -> PyResult> { let py: Python<'py> = context.py(); let result = &outcome.result; - if result.status != BuildStatus::Success { + // `--no-audits` skips Python checks alongside audits. + if result.status != BuildStatus::Success + || !native_field(&context.request, "run_audits")?.extract::()? + { return tuple(py, &[]); } let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index ca9901107b..ee439627fc 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -162,17 +162,7 @@ pub(crate) fn compile_declarations( if members.len() != group.jobs.len() { return Err(PyValueError::new_err("group members must be distinct")); } - let mut after: BTreeMap> = BTreeMap::new(); - for (job, dependencies) in group.after { - after.insert( - job.name, - dependencies - .into_iter() - .map(|dependency| dependency.name) - .collect(), - ); - } - let dependencies = group_dependencies((&group.name, &input), &jobs, (&members, &after)) + let dependencies = group_dependencies((&group.name, &input), &jobs, &members) .map_err(|error| PyValueError::new_err(error.to_string()))?; let selection: BTreeSet = members .iter() diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 89f8f159be..82a61b3484 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -49,7 +49,6 @@ pub(crate) struct AuthoredGroup { pub(crate) name: String, pub(crate) description: String, pub(crate) jobs: Vec, - pub(crate) after: BTreeMap>, pub(crate) concurrency_groups: Vec, pub(crate) schedule: Option>, pub(crate) targets: Vec, @@ -328,20 +327,6 @@ pub(crate) fn collect_declarations<'py>( } pending.push(member); } - for (member, dependencies) in declaration.getattr("after")?.cast::()?.iter() { - if !member.is_instance(&job)? { - return Err(PyTypeError::new_err( - "group dependencies must be Job objects", - )); - } - for dependency in dependencies.try_iter()? { - if !dependency?.is_instance(&job)? { - return Err(PyTypeError::new_err( - "group dependencies must be Job objects", - )); - } - } - } AuthoredDeclaration::Group(declaration.extract()?) } else if !declaration.is_instance(&job)? { return Err(PyTypeError::new_err( diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py index dbc3f4e8c1..3300af9aae 100644 --- a/src/sqlbuild/orchestration_declarations/models/authoring.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Callable, Iterable, Sequence -from dataclasses import dataclass, field, replace +from dataclasses import InitVar, dataclass, field, replace from datetime import datetime from typing import Literal @@ -53,12 +53,12 @@ def __post_init__(self) -> None: @dataclass(frozen=True, eq=False) class JobGroup: - """Jobs ordered by compiled lineage and optional explicit dependencies.""" + """Jobs ordered by the lineage between the nodes they build.""" name: str description: str = field(kw_only=True) jobs: Sequence[Job] = field(kw_only=True) - after: dict[Job, Sequence[Job]] = field(default_factory=dict, kw_only=True) + after: InitVar[object] = field(default=None, kw_only=True) concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) schedule: Schedule | None = field(default=None, kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) @@ -67,7 +67,13 @@ class JobGroup: on_failure: OnFailure | None = field(default=None, kw_only=True) _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) - def __post_init__(self) -> None: + def __post_init__(self, after: object) -> None: + if after is not None: + raise ValueError( + f"job group '{self.name}' declares after, which is no longer supported: groups " + "order jobs only by lineage. Declare the dependency on the nodes instead, for " + "example depends_on on the task that must wait." + ) object.__setattr__(self, "_declared_at", declaration_site()) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 712fe48b83..8140e645d2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -318,6 +318,18 @@ def prepare_build_test_audit_flag_project(*, tmp_path: Path, project_name: str) """ ).strip() + "\n", + "python/checks/orders.py": dedent( + """ + from sqlbuild.checks import check + from sqlbuild.refs import model + + + @check(depends_on=model("orders")) + def check_orders_exist(ctx): + '''Test check check_orders_exist.''' + return ctx.pass_() + """ + ).lstrip(), }, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py index 467368d178..ee86817ce3 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_tests_no_audits_flags.py @@ -21,15 +21,15 @@ description="no tests skips SQL tests but runs audits", project_name="build_no_tests_project", command=("--no-color", "build", "--no-tests"), - expected_stdout_fragments=("audit not_null",), + expected_stdout_fragments=("audit not_null", "check check_orders_exist"), unexpected_stdout_fragments=("test test_orders",), ), BuildNoTestsNoAuditsFlagE2ETestCase( - description="no audits skips audits but runs SQL tests", + description="no audits skips audits and Python checks but runs SQL tests", project_name="build_no_audits_project", command=("--no-color", "build", "--no-audits"), expected_stdout_fragments=("test test_orders",), - unexpected_stdout_fragments=("audit not_null",), + unexpected_stdout_fragments=("audit not_null", "check_orders_exist"), ), ), ids=lambda case: case.description, diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 179999d185..70b2bbf397 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -1102,7 +1102,7 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject expected_jobs=("daily", "staging"), ), SnapshotDiscoveryTestCase( - description="group-only references retain lineage and explicit waits", + description="group-only references retain lineage waits", files={ "orchestration/jobs.py": ( "from sqlbuild.orchestration import job, job_group\n" @@ -1111,14 +1111,14 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject " marts = job('marts', description='Build orders.', select='orders')\n" " inventory = job('inventory', description='Repeat staging.', select='stg_orders')\n" " return job_group('daily', description='Ordered jobs.', " - "jobs=[marts, inventory, staging], after={inventory: [staging]})\n" + "jobs=[marts, inventory, staging])\n" "daily = factory()\n" ), }, expected_jobs=("inventory", "marts", "staging"), expected_groups={ "daily": { - "inventory": ["staging"], + "inventory": [], "marts": ["inventory", "staging"], "staging": [], } @@ -1505,36 +1505,22 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_message="callbacks with when are not supported yet", ), SnapshotDiscoveryRejectionTestCase( - description="explicit group cycles cannot publish", + description="a group declaring after is refused and names the group", files={ "orchestration/jobs.py": ( "from sqlbuild.orchestration import job, job_group\n" "staging = job('staging', description='Stage orders.', select='stg_orders')\n" "marts = job('marts', description='Build orders.', select='orders')\n" - "daily = job_group('daily', description='Cyclic jobs.', " + "daily = job_group('daily', description='Ordered jobs.', " "jobs=[marts, staging], after={staging: [marts]})\n" ), }, expected_error=ValueError, expected_message=re.escape( - "group dependencies contain a cycle in job group daily: marts waits for staging " - "(model:orders reads model:stg_orders); staging waits for marts (declared in after)" + "job group 'daily' declares after, which is no longer supported: groups order " + "jobs only by lineage. Declare the dependency on the nodes instead" ), ), - SnapshotDiscoveryRejectionTestCase( - description="group waits cannot refer to nonmember jobs", - files={ - "orchestration/jobs.py": ( - "from sqlbuild.orchestration import job, job_group\n" - "staging = job('staging', description='Stage orders.', select='stg_orders')\n" - "marts = job('marts', description='Build orders.', select='orders')\n" - "daily = job_group('daily', description='Unknown upstream.', " - "jobs=[marts], after={marts: [staging]})\n" - ), - }, - expected_error=ValueError, - expected_message="group needs known jobs and dependencies within its membership", - ), SnapshotDiscoveryRejectionTestCase( description="distinct objects cannot share a job name", files={ diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index ab0a7b05a2..aacde0be9f 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -1155,7 +1155,6 @@ export class MockOperate { if (a === b) continue; if ([...nodesOf.get(a)!].some((n) => needs.has(n))) ups.push({ jobKey: a, source: 'lineage' }); } - for (const a of job.after[b] ?? []) if (!ups.some((u) => u.jobKey === a)) ups.push({ jobKey: a, source: 'after' }); upstream.set(b, ups); } const stage = new Map(); @@ -1176,10 +1175,8 @@ export class MockOperate { const view = latest ? this.#view(latest, scope) : null; const holders = this.#records.filter((r) => HOLDING.has(r.run.status)); const resolved = job.kind === 'group' ? jobNodes(key) : resolveJobSelection(job).nodes; - const { after, ...rest } = job; - void after; return structuredClone({ - ...rest, + ...job, nodeCount: resolved.filter((n) => snapshotNode(n)?.kind !== 'source').length, resolvedNodes: resolved, lastRun: view ? { id: view.id, status: view.status, startedAt: view.startedAt, endedAt: view.endedAt } : null, diff --git a/ui/src/lib/api/mock/operate.ts b/ui/src/lib/api/mock/operate.ts index 28c0da7c9f..168130a8fb 100644 --- a/ui/src/lib/api/mock/operate.ts +++ b/ui/src/lib/api/mock/operate.ts @@ -37,9 +37,7 @@ type Known = Omit & { [P in K]-?: NonNullable type JobDef = Known< Omit, 'members' | 'overlap' | 'priority' | 'sourceFile' | 'declaration' | 'settings' -> & { - after: Record; -}; +>; export const CONCURRENCY_LIMITS: Record = { warehouse_heavy: 1, 'target:prod': 4 }; @@ -62,7 +60,6 @@ export const JOB_DEFS: JobDef[] = [ targets: null, owner: 'ingest', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/jobs.py', 14, @@ -91,7 +88,6 @@ export const JOB_DEFS: JobDef[] = [ targets: null, owner: 'data-platform', criticalNodes: ['fact_orders'], - after: {}, declaration: decl( 'orchestration/jobs.py', 24, @@ -124,7 +120,6 @@ export const JOB_DEFS: JobDef[] = [ targets: ['prod', 'staging'], owner: 'analytics', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/jobs.py', 37, @@ -153,7 +148,6 @@ export const JOB_DEFS: JobDef[] = [ targets: null, owner: 'analytics', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/activity.py', 8, @@ -184,7 +178,6 @@ export const JOB_DEFS: JobDef[] = [ targets: ['prod'], owner: 'finance-data', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/finance.py', 11, @@ -215,7 +208,6 @@ export const JOB_DEFS: JobDef[] = [ targets: null, owner: 'analytics', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/finance.py', 23, @@ -243,7 +235,6 @@ export const JOB_DEFS: JobDef[] = [ targets: ['prod'], owner: 'analytics', criticalNodes: [], - after: {}, declaration: decl( 'orchestration/reporting.py', 6, @@ -273,7 +264,6 @@ export const JOB_DEFS: JobDef[] = [ targets: ['prod'], owner: 'data-platform', criticalNodes: ['fact_orders'], - after: {}, declaration: decl( 'orchestration/groups.py', 5, @@ -303,7 +293,6 @@ export const JOB_DEFS: JobDef[] = [ targets: ['prod'], owner: 'finance-data', criticalNodes: ['daily_revenue'], - after: { customer_segments: ['revenue_refresh'] }, declaration: decl( 'orchestration/groups.py', 16, @@ -311,8 +300,6 @@ export const JOB_DEFS: JobDef[] = [ "close_of_business", description="Revenue first, then activity and customer segments", jobs=[revenue_refresh, activity_microbatch, customer_segments], - # Segments read closed revenue through an external lookup lineage can't see. - after={customer_segments: [revenue_refresh]}, on_failure=on_failure(retries=1, retry_delay_seconds=900), critical=["daily_revenue"], concurrency_groups=[warehouse_heavy], diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index c8f98f9ec2..0dc7087afa 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -688,8 +688,8 @@ export type ConcurrencyGroup = { export type JobGroupMember = { jobKey: string; - /** Jobs this member waits for: lineage-derived plus declared `after`; null where the cause isn't reported. */ - upstream: { jobKey: string; source: 'lineage' | 'after' | null }[]; + /** Jobs this member waits for, from lineage; null where the cause isn't reported. */ + upstream: { jobKey: string; source: 'lineage' | null }[]; /** Position in the derived order; members in the same stage run side by side. */ stage: number; }; diff --git a/ui/src/lib/lineage/job-graph.test.ts b/ui/src/lib/lineage/job-graph.test.ts index eac4067156..00f4749e9c 100644 --- a/ui/src/lib/lineage/job-graph.test.ts +++ b/ui/src/lib/lineage/job-graph.test.ts @@ -6,7 +6,7 @@ import { boxId, jobGraph, layoutJobGraph } from './job-graph'; const node = (key: string, upstream: string[] = []) => ({ key, kind: 'table', upstream, tags: [], stale: null, checks: null }) as unknown as CatalogNode; const measured = (nodes: Node[]) => nodes.map((n) => (n.type === 'job' ? n : { ...n, measured: { width: 236, height: 61 } })); -// staging builds raw → stg; orders reads stg; reports waits for orders only through `after`. +// staging builds raw → stg; orders reads stg; reports waits for orders through lineage outside the graph. const catalog = [node('model:raw'), node('model:stg', ['model:raw']), node('model:orders', ['model:stg']), node('model:report')]; const all = catalog.map((n) => n.key); const boxes = [ @@ -17,11 +17,11 @@ const boxes = [ const members = [ { jobKey: 'staging', upstream: [], stage: 0 }, { jobKey: 'orders', upstream: [{ jobKey: 'staging', source: 'lineage' as const }], stage: 1 }, - { jobKey: 'reports', upstream: [{ jobKey: 'orders', source: 'after' as const }, { jobKey: 'staging', source: 'lineage' as const }], stage: 2 } + { jobKey: 'reports', upstream: [{ jobKey: 'orders', source: null }, { jobKey: 'staging', source: 'lineage' as const }], stage: 2 } ]; describe('job graph', () => { - it('puts each node in the first box that builds it and draws `after` only where no lineage links two boxes', () => { + it('puts each node in the first box that builds it and draws a wait edge only where no drawn lineage links two boxes', () => { const { nodes, edges } = jobGraph(all, catalog, boxes, members); const parent = Object.fromEntries(nodes.filter((n) => n.type === 'asset').map((n) => [n.id, n.parentId])); expect(parent).toEqual({ 'model:raw': boxId('staging'), 'model:stg': boxId('staging'), 'model:orders': boxId('orders'), 'model:report': boxId('reports') }); @@ -64,7 +64,7 @@ describe('job graph', () => { expect(nodes.filter((n) => n.type === 'asset').map((n) => n.id).sort()).toEqual(['model:orders', 'model:report']); expect(nodes.find((n) => n.id === boxId('staging'))!.data.collapsed).toBe(true); // raw → stg sits inside the box and disappears; stg → orders now leaves the box. Staging and - // reports still have no lineage between them, so their `after` edge stays. + // reports still have no lineage between them, so their wait edge stays. expect(edges.map((e) => `${e.source}>${e.target}`).sort()).toEqual( [`${boxId('staging')}>model:orders`, `${boxId('orders')}>${boxId('reports')}`, `${boxId('staging')}>${boxId('reports')}`].sort() ); @@ -106,8 +106,8 @@ describe('job graph', () => { { jobKey: 'b', nodes: wide.slice(30).map((n) => n.key) }, { jobKey: 'empty', nodes: [] } ]; - const after = (jobKey: string) => ({ jobKey, upstream: [{ jobKey: 'a', source: 'after' as const }], stage: 1 }); - const { nodes, edges } = jobGraph(wide.map((n) => n.key), wide, groupBoxes, [{ jobKey: 'a', upstream: [], stage: 0 }, after('b'), after('empty')]); + const waiting = (jobKey: string) => ({ jobKey, upstream: [{ jobKey: 'a', source: null }], stage: 1 }); + const { nodes, edges } = jobGraph(wide.map((n) => n.key), wide, groupBoxes, [{ jobKey: 'a', upstream: [], stage: 0 }, waiting('b'), waiting('empty')]); const laid = layoutJobGraph(measured(nodes), edges); const box = (k: string) => laid.find((n) => n.id === boxId(k))!; expect(box('a').height).toBeLessThan(15 * (61 + 24) + 60); diff --git a/ui/src/lib/lineage/job-graph.ts b/ui/src/lib/lineage/job-graph.ts index 4917d59c7d..b4d9ae3d8c 100644 --- a/ui/src/lib/lineage/job-graph.ts +++ b/ui/src/lib/lineage/job-graph.ts @@ -26,7 +26,7 @@ const MARGIN = 24; export type JobBox = { jobKey: string; nodes: string[]; lastRun?: RunRef | null }; const edgeStyle = 'stroke: var(--sb-edge); stroke-width: 1.5;'; -const afterStyle = 'stroke: var(--sb-edge); stroke-width: 1.5; stroke-dasharray: 5 4;'; +const waitStyle = 'stroke: var(--sb-edge); stroke-width: 1.5; stroke-dasharray: 5 4;'; export function boxId(jobKey: string): string { return `job:${jobKey}`; @@ -35,8 +35,9 @@ export function boxId(jobKey: string): string { /** * Cards for `nodes` and the lineage between them. With `boxes`, a node built by several members sits * in the most specific box, the one building the fewest nodes (the earliest on a tie), so a small - * job inside a broad one keeps its nodes; a declared `after` between two members with no lineage - * between them is a dashed edge between their boxes. A box in `collapsed` shows only its header, + * job inside a broad one keeps its nodes; a member that waits for another with no drawn lineage + * between them, because the lineage runs through nodes outside the graph, gets a dashed edge between + * their boxes. A box in `collapsed` shows only its header, * and lineage to its cards meets the box instead. */ export function jobGraph( @@ -99,13 +100,13 @@ export function jobGraph( ); const jobOf = (id: string) => owner.get(id) ?? id.slice('job:'.length); const linked = new Set(lineage.map((e) => `${jobOf(e.source)}>${jobOf(e.target)}`)); - const after: Edge[] = members.flatMap((m) => + const waits: Edge[] = members.flatMap((m) => m.upstream .filter((u) => !linked.has(`${u.jobKey}>${m.jobKey}`) && boxes.some((b) => b.jobKey === u.jobKey)) - .map((u) => ({ id: `after:${u.jobKey}-${m.jobKey}`, source: boxId(u.jobKey), target: boxId(m.jobKey), type: 'bezier', style: afterStyle })) + .map((u) => ({ id: `wait:${u.jobKey}-${m.jobKey}`, source: boxId(u.jobKey), target: boxId(m.jobKey), type: 'bezier', style: waitStyle })) ); // Svelte Flow needs a parent before its children. - return { nodes: [...boxNodes, ...cards], edges: [...lineage, ...after] }; + return { nodes: [...boxNodes, ...cards], edges: [...lineage, ...waits] }; } /** The most cards a column holds before it wraps, growing with the graph so its shape stays near the canvas's. */ @@ -200,7 +201,7 @@ function layoutLayer(items: Item[], pairs: ReadonlyArray Date: Sat, 14 Nov 2026 06:18:48 +0000 Subject: [PATCH 09/19] docs: list orchestration/ in the Project Python table --- .../sqlbuild/references/docs/concepts/project-configuration.md | 1 + website/src/content/docs/docs/concepts/project-configuration.mdx | 1 + 2 files changed, 2 insertions(+) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md index 1d1b1add4a..1bfea05bee 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/project-configuration.md @@ -738,6 +738,7 @@ Code that extends SQLBuild itself keeps its own root: | `hooks/python/` | Python model lifecycle hooks | | `functions/python/` | Python UDFs | | `macros/python/`, `enums/`, `constants/` and scoped `_macros/python/`, `_enums/`, `_constants/` | Python macros and declarations | +| `orchestration/` | Jobs, job groups, sensors and calendar schedules. SQLBuild imports these modules only when a selector names a job or an unknown name, and when a location publishes. See [Job selectors](selectors.md#job) | Compilation rejects Python anywhere else with `D016`, so indirectly importable modules cannot create an unofficial project structure. diff --git a/website/src/content/docs/docs/concepts/project-configuration.mdx b/website/src/content/docs/docs/concepts/project-configuration.mdx index 33a7d51f30..c7ec219c13 100644 --- a/website/src/content/docs/docs/concepts/project-configuration.mdx +++ b/website/src/content/docs/docs/concepts/project-configuration.mdx @@ -713,6 +713,7 @@ Code that extends SQLBuild itself keeps its own root: | `hooks/python/` | Python model lifecycle hooks | | `functions/python/` | Python UDFs | | `macros/python/`, `enums/`, `constants/` and scoped `_macros/python/`, `_enums/`, `_constants/` | Python macros and declarations | +| `orchestration/` | Jobs, job groups, sensors and calendar schedules. SQLBuild imports these modules only when a selector names a job or an unknown name, and when a location publishes. See [Job selectors](/docs/concepts/selectors#job) | Compilation rejects Python anywhere else with `D016`, so indirectly importable modules cannot create an unofficial project structure. From c6d766132f5292209ce6713335d547d2de6dd401 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:34:44 +0000 Subject: [PATCH 10/19] feat: apply max_description_lines to Python and orchestration descriptions A Python task, asset, loader, check, hook, provider or UDF description that wraps past [format] max_description_lines is a P027 compile error at its declaration, and a job, job group, sensor or calendar schedule with one cannot publish. --- .../compiled_project/attachment_inputs.rs | 2 + .../compiled_project/snapshot_declarations.rs | 44 +++++++++- .../_helpers/input_loops/attachment.rs | 16 ++++ .../input_loops/resource_diagnostics.rs | 37 +++++++- .../main/description_line_count.rs | 10 +++ .../src/dsl_headers/main/mod.rs | 1 + .../sqlbuild/references/docs/cli/format.md | 8 +- .../docs/concepts/models/configuration.md | 5 ++ .../compile/test_required_descriptions.py | 88 +++++++++++++++++++ .../test_orchestration_snapshot.py | 13 +++ website/src/content/docs/docs/cli/format.mdx | 8 +- .../docs/concepts/models/configuration.mdx | 5 ++ 12 files changed, 232 insertions(+), 5 deletions(-) create mode 100644 crates/sqlbuild-rules/src/dsl_headers/main/description_line_count.rs diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs index 8f6eef3210..4339117ee9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/attachment_inputs.rs @@ -226,6 +226,8 @@ pub(crate) struct DiagnosticSettings { pub(crate) loaders: HashMap, /// Python-defined resources: kind, name, description, location and help, read by Python. pub(crate) python_resources: Vec, + /// The project's `max_description_lines` and `line_width`, which Python descriptions keep to. + pub(crate) description_limit: (usize, i64), } /// The orphaned audit factory warnings, then the missing descriptions, then the cursor errors. diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index ee439627fc..a95fae69cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use serde_json::Value; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; @@ -27,6 +27,9 @@ use crate::bindings::_helpers::native_host::orchestration_declarations::{ use crate::bindings::_helpers::native_host::orchestration_triggers::{ compile_calendars, compile_cron, compile_sensors, }; +use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_count; +use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; +use sqlbuild_rules::lint_text::models::FormatSettings; /// Dedicated declaration modules, imported by publication and by selectors that name a job. pub(crate) fn discover_declarations<'py>( @@ -87,6 +90,45 @@ pub(crate) fn compile_declarations( AuthoredDeclaration::Sensor(sensor) => authored_sensors.push(sensor), } } + let format: FormatSettings = + resolve_format_settings(project_dir).map_err(PyValueError::new_err)?; + let mut described: Vec<(&str, String, String)> = Vec::new(); + described.extend( + authored_jobs + .iter() + .map(|job| ("job", job.name.clone(), job.description.clone())), + ); + described.extend( + authored_groups + .iter() + .map(|group| ("job group", group.name.clone(), group.description.clone())), + ); + described.extend( + authored_sensors + .iter() + .map(|sensor| ("sensor", sensor.name.clone(), sensor.description.clone())), + ); + for calendar in &authored_calendars { + let calendar: &Bound<'_, PyAny> = calendar.bind(py); + described.push(( + "calendar schedule", + calendar + .getattr("function")? + .getattr("__name__")? + .extract()?, + calendar.getattr("description")?.extract()?, + )); + } + for (kind, name, description) in described { + let limit: i64 = format.max_description_lines; + let lines: usize = description_line_count(&description, format.line_width); + if i64::try_from(lines).unwrap_or(i64::MAX) > limit { + return Err(PyValueError::new_err(format!( + "{kind} '{name}' description exceeds {limit} lines; shorten it to {limit} lines \ + and move detail into documentation" + ))); + } + } let mut triggers: BTreeMap = BTreeMap::new(); for job in authored_jobs { if !job.targets.is_empty() && !job.targets.contains(&input.target) { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs index d34298388a..bef8722e89 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs @@ -1,13 +1,17 @@ //! Seeds, functions, sources, SQL tests, scenarios and audits attached from discovered inputs. use std::collections::HashMap; +use std::path::PathBuf; +use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyModule, PyModuleMethods, PyResult, Python}; use pyo3::types::{PyCFunction, PyDict, PyDictMethods, PyTuple}; use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_attachments::audits::constants::BUILTIN_AUDITS; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_host::entries::models::{ExecutorHost, FacadeHost, UserCodeHost}; +use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; +use sqlbuild_rules::lint_text::models::FormatSettings; use crate::bindings::_helpers::boundary::python_calls::{KeywordArguments, call_with, imported}; use crate::bindings::_helpers::compiled_project::attachment_inputs::AuditBlockRow; @@ -564,6 +568,7 @@ fn diagnostic_settings<'py>( .collect::>()?; settings.set_item("sql_hooks", hooks)?; settings.set_item("loaders", loader_rows(discovered)?)?; + settings.set_item("description_limit", description_limit(discovered)?)?; settings.set_item( "python_resources", call_with( @@ -577,6 +582,17 @@ fn diagnostic_settings<'py>( Ok(settings) } +/// The project's `max_description_lines` and `line_width`, as `sqb format` resolves them. +fn description_limit(discovered: &Bound<'_, PyAny>) -> PyResult<(usize, i64)> { + let directory: Option = native_field(discovered, "project_dir")?.extract()?; + let settings: FormatSettings = + resolve_format_settings(&directory.unwrap_or_default()).map_err(PyValueError::new_err)?; + Ok(( + usize::try_from(settings.max_description_lines).unwrap_or(0), + settings.line_width, + )) +} + /// Attach every SQL test case, scenario and audit, then report every resource diagnostic. #[pyfunction] #[pyo3(signature = (**options))] diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index cae1f4f929..4866d5c379 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -19,6 +19,7 @@ use sqlbuild_analysis::resource_diagnostics::main::yaml_help::yaml_help; use sqlbuild_analysis::resource_diagnostics::models::{CursorModel, HeaderHelp}; use sqlbuild_discovery::yaml_files::main::yaml_entry_lines::yaml_entry_lines; use sqlbuild_pysource::definition_locations::main::decorator_location::decorator_location; +use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_count; use sqlbuild_sqltext::compiler::main::model_header_entry_span::model_header_entry_span; use crate::bindings::_helpers::compile_context::delta::{ @@ -35,6 +36,7 @@ use crate::bindings::_helpers::records::fields::field as native_field; use sqlbuild_project::models::ReferenceRow; const MISSING_DESCRIPTION_CODE: &str = "P010"; +const DESCRIPTION_LENGTH_CODE: &str = "P027"; const CURSOR_WITHOUT_INPUTS_CODE: &str = "P011"; const ORPHAN_FACTORY_CODE: &str = "C216"; const AUDIT_FACTORIES_KEY: &str = "audit_factories"; @@ -153,6 +155,37 @@ impl<'py> Diagnostics<'_, 'py> { location()?, )?)) } + + /// P010 for a Python `kind` `name` without a description, or P027 when it wraps past the limit. + fn described_python( + &self, + resource: (&str, &str, Option<&str>), + description: Option<&str>, + location: impl FnOnce() -> PyResult>, + help: &str, + ) -> PyResult>> { + let (kind, name, resource_type) = resource; + let (limit, line_width) = self.settings.description_limit; + let Some(text) = + description.filter(|_| missing_description(kind, name, description).is_none()) + else { + return self.described(resource, description, location, help); + }; + if description_line_count(text, line_width) <= limit { + return Ok(None); + } + Ok(Some(self.diagnostic( + (DESCRIPTION_LENGTH_CODE, "error"), + ( + &format!("{kind} '{name}' description exceeds {limit} lines"), + Some(&format!( + "Shorten the description to {limit} lines; move detail into documentation." + )), + ), + (resource_type, name), + location()?, + )?)) + } } /// Orphaned factory warnings, missing descriptions and cursor errors of the owner's resources. @@ -367,7 +400,7 @@ fn missing_descriptions<'py>( let text = description .as_ref() .map(|description| docstring_text(description.bind(py))); - found.push(diagnostics.described( + found.push(diagnostics.described_python( (kind, name, None), text.as_deref(), || location.bind(py).call0(), @@ -456,7 +489,7 @@ fn function_description<'py>( ), &[format!("@udf(description=\"What {name} returns\", ...)")], ); - return diagnostics.described( + return diagnostics.described_python( ("function", name, Some(resource_type)), attached.description.as_deref(), || { diff --git a/crates/sqlbuild-rules/src/dsl_headers/main/description_line_count.rs b/crates/sqlbuild-rules/src/dsl_headers/main/description_line_count.rs new file mode 100644 index 0000000000..fb3ca72df0 --- /dev/null +++ b/crates/sqlbuild-rules/src/dsl_headers/main/description_line_count.rs @@ -0,0 +1,10 @@ +//! `description_line_count`: the lines a description takes once wrapped at the line width. + +use crate::dsl_headers::_helpers::text::line_count; +use crate::dsl_headers::_helpers::wrap::wrap_description; + +/// The lines `description` takes wrapped at `line_width`, as the description-length rule counts them. +pub fn description_line_count(description: &str, line_width: i64) -> usize { + let characters: Vec = description.chars().collect(); + line_count(&wrap_description(&characters, line_width, line_width)) +} diff --git a/crates/sqlbuild-rules/src/dsl_headers/main/mod.rs b/crates/sqlbuild-rules/src/dsl_headers/main/mod.rs index 862b8dc0f6..19633251d9 100644 --- a/crates/sqlbuild-rules/src/dsl_headers/main/mod.rs +++ b/crates/sqlbuild-rules/src/dsl_headers/main/mod.rs @@ -1,4 +1,5 @@ pub mod declares_table_returns; +pub mod description_line_count; pub mod fix_escaped_quotes; pub mod format_headers; pub mod inner_header_text; diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md index 01cf27cd3e..1c09acc5b3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/format.md @@ -53,13 +53,19 @@ formatted. `--fix` never skips files. Pass ## Line width and descriptions -Configure the maximum physical line width in `sqlbuild_project.toml` (the default is `100`): +Configure the maximum physical line width and the most lines a description may take in +`sqlbuild_project.toml` (the defaults are `100` and `10`): ```toml [format] line_width = 100 +max_description_lines = 10 ``` +A header description longer than `max_description_lines`, once wrapped at the line width, fails +`sqb format --check` with `description-length`. Python descriptions keep to the same limit at +compile; see [Descriptions](../concepts/models/configuration.md#descriptions). + `MODEL`, `SCENARIO`, `FUNCTION` and `HOOK` headers need a non-empty `description`. When one is missing, `sqb format --check` fails with `description-present`, so pre-commit catches it before compile does. A model described by its `model_schema` or its `[path_defaults]` entry passes, as it diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md index f1eaafc8d9..0155b13858 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/models/configuration.md @@ -87,6 +87,11 @@ error[P010]: model 'orders_summary' has no description | Loader | `@loader(description="...")` or the function docstring; a loader that feeds a declared source may instead rely on the source's YAML description | | Provider | The provider class docstring | +A description may take at most `[format] max_description_lines` lines (default `10`) once wrapped +at the line width, the same limit `sqb format --check` applies to headers. A longer Python +description, from a decorator or a docstring, is a `P027` compile error at its declaration. A job, +job group, sensor or calendar schedule with a longer description cannot publish. + SQL tests, audits, macros, enums, constants, materializations and event or output sinks do not need a description. Descriptions do not change fingerprints, so adding or editing one does not rebuild a model or redeploy a function (a Python UDF docstring is part of the deployed body, so editing it diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py index eab3e5a839..28bd546329 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py @@ -506,6 +506,94 @@ ) +def _paragraphs(count: int) -> str: + """A docstring of `count` one-line paragraphs, which wrap to 2 * count - 1 lines.""" + + body: str = "\n\n ".join(f"Orders paragraph {index}." for index in range(count)) + return f' """{body}"""\n' + + +# The default max_description_lines is 10: five paragraphs fit in 9 lines, six need 11. +_LENGTH_CASES: tuple[RequiredDescriptionCase, ...] = ( + RequiredDescriptionCase( + description="task docstring", + missing_files=( + ( + "python/tasks/refresh_exports.py", + python_node_source( + module="tasks", + decorator="task", + name="refresh_exports", + docstring_line=_paragraphs(6), + ), + ), + ), + described_files=( + ( + "python/tasks/refresh_exports.py", + python_node_source( + module="tasks", + decorator="task", + name="refresh_exports", + docstring_line=_paragraphs(5), + ), + ), + ), + expected_diagnostic=("P027", "python/tasks/refresh_exports.py", 4), + expected_message="task 'refresh_exports' description exceeds 10 lines", + expected_help_fragments=("Shorten the description to 10 lines",), + ), + RequiredDescriptionCase( + description="asset description argument", + missing_files=( + ( + "python/assets/order_report.py", + python_node_source( + module="assets", + decorator="asset", + name="order_report", + decorator_arguments=f"(description={'Order report. ' * 90!r})", + ), + ), + ), + described_files=( + ( + "python/assets/order_report.py", + python_node_source( + module="assets", + decorator="asset", + name="order_report", + decorator_arguments=f"(description={'Order report. ' * 60!r})", + ), + ), + ), + expected_diagnostic=("P027", "python/assets/order_report.py", 4), + expected_message="asset 'order_report' description exceeds 10 lines", + expected_help_fragments=("Shorten the description to 10 lines",), + ), +) + + +@pytest.mark.parametrize("test_case", _LENGTH_CASES, ids=lambda case: case.description) +def test_given_python_description_past_the_line_limit_when_compiling_then_reports_p027( + test_case: RequiredDescriptionCase, tmp_path: Path +) -> None: + too_long: subprocess.CompletedProcess[str] = compile_inline_files( + tmp_path=tmp_path / "too_long", files=test_case.missing_files + ) + at_limit: subprocess.CompletedProcess[str] = compile_inline_files( + tmp_path=tmp_path / "at_limit", files=test_case.described_files + ) + + diagnostics, text = require_sql_analysis_output(too_long) + assert too_long.returncode == 1, too_long.stdout + too_long.stderr + assert diagnostics == (test_case.expected_diagnostic,) + assert json.loads(too_long.stdout)["diagnostics"][0]["message"] == test_case.expected_message + assert all(fragment in text for fragment in test_case.expected_help_fragments), text + assert at_limit.returncode == 0, at_limit.stdout + at_limit.stderr + assert require_sql_analysis_output(at_limit)[0] == () + + @pytest.mark.parametrize( "test_case", [RequiredDescriptionCase(**asdict(case)) for case in _CASES], diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 70b2bbf397..5d7b7b4ab3 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -1504,6 +1504,19 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects expected_error=ValueError, expected_message="callbacks with when are not supported yet", ), + SnapshotDiscoveryRejectionTestCase( + description="a job description past max_description_lines cannot publish", + files={ + "orchestration/jobs.py": ( + "from sqlbuild.orchestration import job\n" + "daily = job('daily', description='Build orders. ' * 100, select='orders')\n" + ), + }, + expected_error=ValueError, + expected_message=re.escape( + "job 'daily' description exceeds 10 lines; shorten it to 10 lines" + ), + ), SnapshotDiscoveryRejectionTestCase( description="a group declaring after is refused and names the group", files={ diff --git a/website/src/content/docs/docs/cli/format.mdx b/website/src/content/docs/docs/cli/format.mdx index 145dbab35c..e46f501a44 100644 --- a/website/src/content/docs/docs/cli/format.mdx +++ b/website/src/content/docs/docs/cli/format.mdx @@ -44,13 +44,19 @@ formatted. `--fix` never skips files. Pass ## Line width and descriptions -Configure the maximum physical line width in `sqlbuild_project.toml` (the default is `100`): +Configure the maximum physical line width and the most lines a description may take in +`sqlbuild_project.toml` (the defaults are `100` and `10`): ```toml [format] line_width = 100 +max_description_lines = 10 ``` +A header description longer than `max_description_lines`, once wrapped at the line width, fails +`sqb format --check` with `description-length`. Python descriptions keep to the same limit at +compile; see [Descriptions](/docs/concepts/models/configuration#descriptions). + `MODEL`, `SCENARIO`, `FUNCTION` and `HOOK` headers need a non-empty `description`. When one is missing, `sqb format --check` fails with `description-present`, so pre-commit catches it before compile does. A model described by its `model_schema` or its `[path_defaults]` entry passes, as it diff --git a/website/src/content/docs/docs/concepts/models/configuration.mdx b/website/src/content/docs/docs/concepts/models/configuration.mdx index e4ca878c5b..88f2653214 100644 --- a/website/src/content/docs/docs/concepts/models/configuration.mdx +++ b/website/src/content/docs/docs/concepts/models/configuration.mdx @@ -71,6 +71,11 @@ error[P010]: model 'orders_summary' has no description | Loader | `@loader(description="...")` or the function docstring; a loader that feeds a declared source may instead rely on the source's YAML description | | Provider | The provider class docstring | +A description may take at most `[format] max_description_lines` lines (default `10`) once wrapped +at the line width, the same limit `sqb format --check` applies to headers. A longer Python +description, from a decorator or a docstring, is a `P027` compile error at its declaration. A job, +job group, sensor or calendar schedule with a longer description cannot publish. + SQL tests, audits, macros, enums, constants, materializations and event or output sinks do not need a description. Descriptions do not change fingerprints, so adding or editing one does not rebuild a model or redeploy a function (a Python UDF docstring is part of the deployed body, so editing it From 1e3e9e8089a9841c85a1e24f5c72eac2470b8c6b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:15:52 +0000 Subject: [PATCH 11/19] fix: keep orchestration fixtures and description limits consistent with lineage-only groups Rust orchestration fixtures declare the node lineage their group waits come from. A Python description limit falls back to the defaults without a project directory, and a calendar schedule that never decorated a function no longer breaks publication for other targets; DuckShop's calendar edge case now declares one. --- .../edge_cases/orchestration/calendar_orders.py | 10 ++++++++++ .../tests/runs/test_group_admission.rs | 6 ++++++ .../tests/runs/test_groups.rs | 5 +++++ .../tests/runs/test_job_detail.rs | 3 +-- .../tests/runs/test_job_subsets.rs | 6 ++++++ .../tests/runs/test_reruns.rs | 13 ++++++++++++- .../compiled_project/snapshot_declarations.rs | 10 ++++++---- .../src/bindings/_helpers/input_loops/attachment.rs | 10 +++++++++- crates/sqlbuild-rules/src/lint_text/constants.rs | 4 ++-- 9 files changed, 57 insertions(+), 10 deletions(-) diff --git a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py index 80d336fbbb..ce5a9b6723 100644 --- a/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py +++ b/benchmarks/duckshop/edge_cases/orchestration/calendar_orders.py @@ -13,3 +13,13 @@ ) def fulfillment_calendar(window, fulfillment): return fulfillment.upcoming(window) + + +# Declared for another target without decorating a function: development publication skips it. +production_calendar = calendar_schedule( + job=orders, + description="Production fulfillment times, not yet attached to a provider.", + horizon="8s", + timezone="UTC", + targets=("production",), +) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs index 6f90bc7572..ebe07fb987 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -491,12 +491,18 @@ fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_th for (index, parent) in test_case.parents.iter().enumerate() { let mut members = serde_json::Map::new(); let mut selection = BTreeSet::new(); + let mut previous: Option<&str> = None; for (stage, needs) in parent.children.iter().enumerate() { let name = format!("orders_job_{index}_{stage}"); let node = ["orders", "customers", "inventory", "products"][child_count % 4]; child_count += 1; selection.insert(node); + // Each stage reads the previous stage's node, so lineage orders the stages. + if let Some(upstream) = previous.replace(node) { + input.nodes.get_mut(node).expect("stage node").definition["deps"] = + json!([upstream]); + } jobs.insert(name.clone(), json!({"description": "Build a synthetic stage", "selection": [node], "concurrency_groups": needs})); let after: Vec = (0..stage) .map(|earlier| format!("orders_job_{index}_{earlier}")) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs index cef10c013d..82d54108a1 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_groups.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_groups.rs @@ -152,6 +152,11 @@ fn given_a_pinned_group_when_redeployed_and_cancelled_then_children_keep_scope_a input.generation = 2; input.documents.get_mut("jobs").expect("jobs")["marts"]["selection"] = json!(["inventory"]); + input + .nodes + .get_mut("inventory") + .expect("inventory") + .definition["deps"] = json!(["customers"]); input.documents.get_mut("groups").expect("groups")["daily_orders"]["selection"] = json!(["customers", "inventory"]); publish_snapshot(store, &input) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs index e8586b2850..f0cbec653a 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs @@ -87,8 +87,7 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r .nodes .get_mut("customers") .expect("customers node") - .definition = - json!({"materialization": "table", "config": {"cursor_type": "timestamp"}}); + .definition = json!({"materialization": "table", "deps": ["orders"], "config": {"cursor_type": "timestamp"}}); publish_snapshot(store, &input).await.expect("publish jobs"); let administrator = (&roles.context, credentials(&roles.administrator)); let location = roles.location_id; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs b/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs index 34c162e5b1..26cae64df5 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_job_subsets.rs @@ -102,6 +102,12 @@ fn given_a_published_job_when_a_run_narrows_it_then_only_the_jobs_own_selected_n ip_address: None, }; let mut input = snapshot(location_id); + // Customers read orders, so customer_sync waits for daily_orders. + input + .nodes + .get_mut("customers") + .expect("customers model") + .definition["deps"] = json!(["orders"]); input.documents.insert("jobs".into(), json!({ "daily_orders": {"description": "Orders and stock", "selection": ["customers", "inventory", "orders"], "concurrency_groups": ["orders-load"]}, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index 7fff917906..f3c5e1af1a 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -68,6 +68,17 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ .get_mut("orders") .expect("orders model") .definition = json!({"materialization": "table", "deps": ["source:raw_orders"]}); + // Lineage orders the group: products reads orders, and inventory reads products. + input + .nodes + .get_mut("products") + .expect("products model") + .definition["deps"] = json!(["orders"]); + input + .nodes + .get_mut("inventory") + .expect("inventory model") + .definition["deps"] = json!(["products"]); input.documents.insert("jobs".into(), json!({ "staging": {"description": "Stage orders", "selection": ["orders", "customers"], "concurrency_groups": ["warehouse"], "settings": {"vars": {"batch": 23}}}, "marts": {"description": "Build inventory", "selection": ["inventory"], "concurrency_groups": ["inventory_limit"]}, @@ -75,7 +86,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ })); input.documents.insert("groups".into(), json!({"daily_orders": { "description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products"], - "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["completed"], "completed": ["staging"]} + "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["completed", "staging"], "completed": ["staging"]} }})); let published = publish_snapshot(store, &input).await.expect("publish"); let source_id = Uuid::now_v7(); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 97466cf8a4..fddcbbf852 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -110,12 +110,14 @@ pub(crate) fn compile_declarations( ); for calendar in &authored_calendars { let calendar: &Bound<'_, PyAny> = calendar.bind(py); + // A calendar that never decorated a function has no name; `compile_calendars` handles it per target. + let function: Bound<'_, PyAny> = calendar.getattr("function")?; + if function.is_none() { + continue; + } described.push(( "calendar schedule", - calendar - .getattr("function")? - .getattr("__name__")? - .extract()?, + function.getattr("__name__")?.extract()?, calendar.getattr("description")?.extract()?, )); } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs index bef8722e89..c6e02c9a87 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs @@ -10,6 +10,7 @@ use pyo3::{IntoPyObjectExt, pyfunction, wrap_pyfunction}; use sqlbuild_attachments::audits::constants::BUILTIN_AUDITS; use sqlbuild_host::compile_context::main::record_compile_timing::record_compile_timing; use sqlbuild_host::entries::models::{ExecutorHost, FacadeHost, UserCodeHost}; +use sqlbuild_rules::lint_text::constants::{DEFAULT_LINE_WIDTH, DEFAULT_MAX_DESCRIPTION_LINES}; use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; use sqlbuild_rules::lint_text::models::FormatSettings; @@ -585,8 +586,15 @@ fn diagnostic_settings<'py>( /// The project's `max_description_lines` and `line_width`, as `sqb format` resolves them. fn description_limit(discovered: &Bound<'_, PyAny>) -> PyResult<(usize, i64)> { let directory: Option = native_field(discovered, "project_dir")?.extract()?; + // Without a project directory there is no configuration to read, not the current directory's. + let Some(directory) = directory else { + return Ok(( + usize::try_from(DEFAULT_MAX_DESCRIPTION_LINES).unwrap_or(0), + DEFAULT_LINE_WIDTH, + )); + }; let settings: FormatSettings = - resolve_format_settings(&directory.unwrap_or_default()).map_err(PyValueError::new_err)?; + resolve_format_settings(&directory).map_err(PyValueError::new_err)?; Ok(( usize::try_from(settings.max_description_lines).unwrap_or(0), settings.line_width, diff --git a/crates/sqlbuild-rules/src/lint_text/constants.rs b/crates/sqlbuild-rules/src/lint_text/constants.rs index 3ead104853..a629cf9be8 100644 --- a/crates/sqlbuild-rules/src/lint_text/constants.rs +++ b/crates/sqlbuild-rules/src/lint_text/constants.rs @@ -169,8 +169,8 @@ pub(crate) const NON_CONTIGUOUS_FIX: &str = pub(crate) const GENERATED_FIX: &str = "fix overlaps generated SQL"; pub(crate) const GENERATED_SQL_SUFFIX: &str = "(in generated SQL)"; pub(crate) const LOCAL_CONFIG_FILENAME: &str = "sqlbuild_local.toml"; -pub(crate) const DEFAULT_LINE_WIDTH: i64 = 100; -pub(crate) const DEFAULT_MAX_DESCRIPTION_LINES: i64 = 10; +pub const DEFAULT_LINE_WIDTH: i64 = 100; +pub const DEFAULT_MAX_DESCRIPTION_LINES: i64 = 10; pub(crate) const DEFAULT_MAX_LITERAL_LENGTH: i64 = 100; pub(crate) const DEFAULT_MAX_RANKING_ORDER_BY: i64 = 5; pub(crate) const DEFAULT_DIALECT: &str = "generic"; From 66f2d7d9338f01a1c21c8d7d0eda3bffccf4419a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:26:03 +0000 Subject: [PATCH 12/19] test: keep the rerun wait-boundary case transitive under lineage-only groups --- .../tests/runs/test_reruns.rs | 30 +++++++++++-------- .../compiled_project/snapshot_declarations.rs | 3 +- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index f3c5e1af1a..339285d199 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -68,12 +68,11 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ .get_mut("orders") .expect("orders model") .definition = json!({"materialization": "table", "deps": ["source:raw_orders"]}); - // Lineage orders the group: products reads orders, and inventory reads products. - input - .nodes - .get_mut("products") - .expect("products model") - .definition["deps"] = json!(["orders"]); + // Lineage orders the group: shipments reads orders and inventory reads products, so + // marts waits for staging only through completed, never directly. + let mut shipments = input.nodes["products"].clone(); + shipments.definition["deps"] = json!(["orders"]); + input.nodes.insert("shipments".into(), shipments); input .nodes .get_mut("inventory") @@ -82,11 +81,11 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ input.documents.insert("jobs".into(), json!({ "staging": {"description": "Stage orders", "selection": ["orders", "customers"], "concurrency_groups": ["warehouse"], "settings": {"vars": {"batch": 23}}}, "marts": {"description": "Build inventory", "selection": ["inventory"], "concurrency_groups": ["inventory_limit"]}, - "completed": {"description": "Build products", "selection": ["products"]} + "completed": {"description": "Build products and shipments", "selection": ["products", "shipments"]} })); input.documents.insert("groups".into(), json!({"daily_orders": { - "description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products"], - "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["completed", "staging"], "completed": ["staging"]} + "description": "Build daily orders", "selection": ["customers", "orders", "inventory", "products", "shipments"], + "concurrency_groups": ["warehouse"], "jobs": {"staging": [], "marts": ["completed"], "completed": ["staging"]} }})); let published = publish_snapshot(store, &input).await.expect("publish"); let source_id = Uuid::now_v7(); @@ -142,6 +141,7 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ .expect("admit source"); let node_outcomes = BTreeMap::from([ ("products", NodeOutcome::Success), + ("shipments", NodeOutcome::Success), ("orders", NodeOutcome::Failed), ("customers", NodeOutcome::Cancelled), ("inventory", NodeOutcome::Skipped), @@ -276,9 +276,15 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ ..retry.clone() }, expected_applied: true, - expected_selection: ["orders", "inventory", "customers", "products"] - .map(str::to_owned) - .into(), + expected_selection: [ + "orders", + "inventory", + "customers", + "products", + "shipments", + ] + .map(str::to_owned) + .into(), expected_jobs: &["staging", "completed", "marts"], }, ] { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index fddcbbf852..93c4d93c45 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -110,7 +110,8 @@ pub(crate) fn compile_declarations( ); for calendar in &authored_calendars { let calendar: &Bound<'_, PyAny> = calendar.bind(py); - // A calendar that never decorated a function has no name; `compile_calendars` handles it per target. + // A calendar that never decorated a function has no name to report; on its own target, + // `compile_calendars` still fails publication for it. let function: Bound<'_, PyAny> = calendar.getattr("function")?; if function.is_none() { continue; From c949c54d95c3c47ea187a363ad46e9df0637ff92 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:39:03 +0000 Subject: [PATCH 13/19] refactor: report P010 and P027 from one description check --- .../input_loops/resource_diagnostics.rs | 78 ++++++++----------- 1 file changed, 31 insertions(+), 47 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index 4866d5c379..245343b853 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -136,52 +136,36 @@ impl<'py> Diagnostics<'_, 'py> { .call((), Some(&fields)) } - /// P010 for `kind` `name` when its description is missing or blank. + /// P010 for `kind` `name` when its description is missing or blank; for a Python resource, + /// P027 when it wraps past the project's `max_description_lines`. fn described( &self, resource: (&str, &str, Option<&str>), - description: Option<&str>, + (description, python): (Option<&str>, bool), location: impl FnOnce() -> PyResult>, help: &str, ) -> PyResult>> { let (kind, name, resource_type) = resource; - let Some(message) = missing_description(kind, name, description) else { - return Ok(None); - }; - Ok(Some(self.diagnostic( - (MISSING_DESCRIPTION_CODE, "error"), - (&message, Some(help)), - (resource_type, name), - location()?, - )?)) - } - - /// P010 for a Python `kind` `name` without a description, or P027 when it wraps past the limit. - fn described_python( - &self, - resource: (&str, &str, Option<&str>), - description: Option<&str>, - location: impl FnOnce() -> PyResult>, - help: &str, - ) -> PyResult>> { - let (kind, name, resource_type) = resource; - let (limit, line_width) = self.settings.description_limit; - let Some(text) = - description.filter(|_| missing_description(kind, name, description).is_none()) - else { - return self.described(resource, description, location, help); + let (code, message, help): (&str, String, String) = if let Some(message) = + missing_description(kind, name, description) + { + (MISSING_DESCRIPTION_CODE, message, help.to_owned()) + } else { + let (limit, line_width) = self.settings.description_limit; + match description { + Some(text) if python && description_line_count(text, line_width) > limit => ( + DESCRIPTION_LENGTH_CODE, + format!("{kind} '{name}' description exceeds {limit} lines"), + format!( + "Shorten the description to {limit} lines; move detail into documentation." + ), + ), + _ => return Ok(None), + } }; - if description_line_count(text, line_width) <= limit { - return Ok(None); - } Ok(Some(self.diagnostic( - (DESCRIPTION_LENGTH_CODE, "error"), - ( - &format!("{kind} '{name}' description exceeds {limit} lines"), - Some(&format!( - "Shorten the description to {limit} lines; move detail into documentation." - )), - ), + (code, "error"), + (&message, Some(&help)), (resource_type, name), location()?, )?)) @@ -304,7 +288,7 @@ fn missing_descriptions<'py>( .and_then(|schema| schema.description.as_deref()); found.push(diagnostics.described( ("model", &name, Some("model")), - description, + (description, false), || diagnostics.header_location(&path, contents), &help, )?); @@ -333,7 +317,7 @@ fn missing_descriptions<'py>( }); found.push(diagnostics.described( ("scenario", name, Some("sql_scenario")), - description.as_deref(), + (description.as_deref(), false), || diagnostics.header_location(path, contents), &help, )?); @@ -355,7 +339,7 @@ fn missing_descriptions<'py>( let contents = contents_value.downcast::()?.to_str()?; found.push(diagnostics.described( ("seed", &name, Some("seed")), - description.as_deref(), + (description.as_deref(), false), || diagnostics.yaml_location(&path, contents, &name), &yaml_help("seed", &name, &posix(&path)), )?); @@ -384,7 +368,7 @@ fn missing_descriptions<'py>( }); found.push(diagnostics.described( ("hook", name, None), - description.as_deref(), + (description.as_deref(), false), || diagnostics.header_location(path, contents), &help, )?); @@ -400,9 +384,9 @@ fn missing_descriptions<'py>( let text = description .as_ref() .map(|description| docstring_text(description.bind(py))); - found.push(diagnostics.described_python( + found.push(diagnostics.described( (kind, name, None), - text.as_deref(), + (text.as_deref(), true), || location.bind(py).call0(), help, )?); @@ -456,7 +440,7 @@ fn source_description<'py>( } diagnostics.described( ("source", name, Some("source")), - description.as_deref(), + (description.as_deref(), false), || diagnostics.yaml_location(&path, contents, name), &help, ) @@ -489,9 +473,9 @@ fn function_description<'py>( ), &[format!("@udf(description=\"What {name} returns\", ...)")], ); - return diagnostics.described_python( + return diagnostics.described( ("function", name, Some(resource_type)), - attached.description.as_deref(), + (attached.description.as_deref(), true), || { let version = py.version_info(); let position: (usize, usize) = decorator_location( @@ -515,7 +499,7 @@ fn function_description<'py>( }); diagnostics.described( ("function", name, Some(resource_type)), - attached.description.as_deref(), + (attached.description.as_deref(), false), || diagnostics.header_location(&path, &contents), &help, ) From cc210da27fc84b1513fa3a659abbd2af79cc0761 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:44:56 +0000 Subject: [PATCH 14/19] chore: regenerate the control client for lineage-only group waits --- clients/typescript/src/schema.d.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 31884a29df..fd2e5a038d 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1854,7 +1854,7 @@ export interface components { }; JobMember: { job_key: string; - /** @description Members this one waits for, from lineage and from the group's `after`. */ + /** @description Members this one waits for, from the lineage between their nodes. */ waits_for: string[]; }; /** @description A job or group of the current published snapshot. Settings are left out of person reads. */ From eed4c5528b7b62645b50b20c4abad79c062f7ce9 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:56:19 +0000 Subject: [PATCH 15/19] refactor: route calendar descriptions through the native host and drop JobGroup after JobGroup no longer has an after parameter at all, so passing it fails at the call site. Calendar names and descriptions are read through the native host edge, the admission fixture builds stage lineage without branching, and the description-limit test keeps its cases beside the test. --- .../tests/runs/test_group_admission.rs | 11 +- .../_helpers/build_command/execution.rs | 1 - .../compiled_project/snapshot_declarations.rs | 21 +--- .../_helpers/input_loops/attachment.rs | 3 +- .../input_loops/resource_diagnostics.rs | 3 +- .../native_host/orchestration_triggers.rs | 19 +++ .../models/authoring.py | 11 +- .../cli/commands/main/compile/helpers.py | 7 ++ .../compile/test_required_descriptions.py | 109 +++++++++--------- 9 files changed, 93 insertions(+), 92 deletions(-) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs index ebe07fb987..643b151e5c 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_group_admission.rs @@ -491,18 +491,17 @@ fn given_capacity_and_multistage_cycles_when_replicas_admit_and_resume_groups_th for (index, parent) in test_case.parents.iter().enumerate() { let mut members = serde_json::Map::new(); let mut selection = BTreeSet::new(); - let mut previous: Option<&str> = None; + let mut earlier_nodes: Vec<&str> = Vec::new(); for (stage, needs) in parent.children.iter().enumerate() { let name = format!("orders_job_{index}_{stage}"); let node = ["orders", "customers", "inventory", "products"][child_count % 4]; child_count += 1; selection.insert(node); - // Each stage reads the previous stage's node, so lineage orders the stages. - if let Some(upstream) = previous.replace(node) { - input.nodes.get_mut(node).expect("stage node").definition["deps"] = - json!([upstream]); - } + // Each stage reads every earlier stage's node, so lineage orders the stages. + input.nodes.get_mut(node).expect("stage node").definition["deps"] = + json!(earlier_nodes); + earlier_nodes.push(node); jobs.insert(name.clone(), json!({"description": "Build a synthetic stage", "selection": [node], "concurrency_groups": needs})); let after: Vec = (0..stage) .map(|earlier| format!("orders_job_{index}_{earlier}")) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs index 66f70a70d5..34e0873df8 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/execution.rs @@ -668,7 +668,6 @@ pub(crate) fn run_post_build_python_checks<'py>( ) -> PyResult> { let py: Python<'py> = context.py(); let result = &outcome.result; - // `--no-audits` skips Python checks alongside audits. if result.status != BuildStatus::Success || !native_field(&context.request, "run_audits")?.extract::()? { diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 93c4d93c45..c9482f9b38 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -25,7 +25,7 @@ use crate::bindings::_helpers::native_host::orchestration_declarations::{ module_jobs, run_controls, }; use crate::bindings::_helpers::native_host::orchestration_triggers::{ - compile_calendars, compile_cron, compile_sensors, + calendar_descriptions, compile_calendars, compile_cron, compile_sensors, }; use sqlbuild_rules::dsl_headers::main::description_line_count::description_line_count; use sqlbuild_rules::lint_text::main::resolve_format_settings::resolve_format_settings; @@ -108,20 +108,11 @@ pub(crate) fn compile_declarations( .iter() .map(|sensor| ("sensor", sensor.name.clone(), sensor.description.clone())), ); - for calendar in &authored_calendars { - let calendar: &Bound<'_, PyAny> = calendar.bind(py); - // A calendar that never decorated a function has no name to report; on its own target, - // `compile_calendars` still fails publication for it. - let function: Bound<'_, PyAny> = calendar.getattr("function")?; - if function.is_none() { - continue; - } - described.push(( - "calendar schedule", - function.getattr("__name__")?.extract()?, - calendar.getattr("description")?.extract()?, - )); - } + described.extend( + calendar_descriptions(py, &authored_calendars)? + .into_iter() + .map(|(name, description)| ("calendar schedule", name, description)), + ); for (kind, name, description) in described { let limit: i64 = format.max_description_lines; let lines: usize = description_line_count(&description, format.line_width); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs index c6e02c9a87..bf3340298c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/attachment.rs @@ -583,10 +583,9 @@ fn diagnostic_settings<'py>( Ok(settings) } -/// The project's `max_description_lines` and `line_width`, as `sqb format` resolves them. +/// The project's `max_description_lines` and `line_width` as `sqb format` resolves them, else the defaults. fn description_limit(discovered: &Bound<'_, PyAny>) -> PyResult<(usize, i64)> { let directory: Option = native_field(discovered, "project_dir")?.extract()?; - // Without a project directory there is no configuration to read, not the current directory's. let Some(directory) = directory else { return Ok(( usize::try_from(DEFAULT_MAX_DESCRIPTION_LINES).unwrap_or(0), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs index 245343b853..577882795c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/input_loops/resource_diagnostics.rs @@ -136,8 +136,7 @@ impl<'py> Diagnostics<'_, 'py> { .call((), Some(&fields)) } - /// P010 for `kind` `name` when its description is missing or blank; for a Python resource, - /// P027 when it wraps past the project's `max_description_lines`. + /// P010 when `kind` `name` has no description; for Python, P027 when it wraps past the limit. fn described( &self, resource: (&str, &str, Option<&str>), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs index b8923950a0..9faf31f8e9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_triggers.rs @@ -184,6 +184,25 @@ fn calendar_times( }) } +/// The name and description of each calendar that decorates a function. +pub(crate) fn calendar_descriptions( + py: Python<'_>, + calendars: &[Py], +) -> PyResult> { + let mut described: Vec<(String, String)> = Vec::new(); + for calendar in calendars { + let calendar: &Bound<'_, PyAny> = calendar.bind(py); + let function: Bound<'_, PyAny> = calendar.getattr("function")?; + if !function.is_none() { + described.push(( + function.getattr("__name__")?.extract()?, + calendar.getattr("description")?.extract()?, + )); + } + } + Ok(described) +} + pub(crate) fn compile_calendars( py: Python<'_>, (project_dir, target): (&Path, &str), diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py index 6d934a749f..0bf9122d9f 100644 --- a/src/sqlbuild/orchestration_declarations/models/authoring.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Callable, Iterable, Sequence -from dataclasses import InitVar, dataclass, field, replace +from dataclasses import dataclass, field, replace from datetime import datetime from typing import Literal @@ -68,7 +68,6 @@ class JobGroup: name: str description: str = field(kw_only=True) jobs: Sequence[Job] = field(kw_only=True) - after: InitVar[object] = field(default=None, kw_only=True) concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) schedule: Schedule | None = field(default=None, kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) @@ -77,13 +76,7 @@ class JobGroup: on_failure: OnFailure | None = field(default=None, kw_only=True) _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) - def __post_init__(self, after: object) -> None: - if after is not None: - raise ValueError( - f"job group '{self.name}' declares after, which is no longer supported: groups " - "order jobs only by lineage. Declare the dependency on the nodes instead, for " - "example depends_on on the task that must wait." - ) + def __post_init__(self) -> None: object.__setattr__(self, "_declared_at", declaration_site()) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index e7e6b97035..d3cfbc3b53 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -3265,6 +3265,13 @@ def hooked_model_file(*, model_name: str, hook: str) -> tuple[str, str]: ) +def description_paragraphs(count: int) -> str: + """A docstring of `count` one-line paragraphs, which wrap to 2 * count - 1 lines.""" + + body: str = "\n\n ".join(f"Orders paragraph {index}." for index in range(count)) + return f' """{body}"""\n' + + def python_node_source( *, module: str, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py index 28bd546329..fedac2da3e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_required_descriptions.py @@ -22,6 +22,7 @@ from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( REQUIRED_DESCRIPTIONS_PROJECT, compile_inline_files, + description_paragraphs, hooked_model_file, python_node_source, require_sql_analysis_output, @@ -506,75 +507,69 @@ ) -def _paragraphs(count: int) -> str: - """A docstring of `count` one-line paragraphs, which wrap to 2 * count - 1 lines.""" - - body: str = "\n\n ".join(f"Orders paragraph {index}." for index in range(count)) - return f' """{body}"""\n' - - # The default max_description_lines is 10: five paragraphs fit in 9 lines, six need 11. -_LENGTH_CASES: tuple[RequiredDescriptionCase, ...] = ( - RequiredDescriptionCase( - description="task docstring", - missing_files=( - ( - "python/tasks/refresh_exports.py", - python_node_source( - module="tasks", - decorator="task", - name="refresh_exports", - docstring_line=_paragraphs(6), +@pytest.mark.parametrize( + "test_case", + [ + RequiredDescriptionCase( + description="task docstring", + missing_files=( + ( + "python/tasks/refresh_exports.py", + python_node_source( + module="tasks", + decorator="task", + name="refresh_exports", + docstring_line=description_paragraphs(6), + ), ), ), - ), - described_files=( - ( - "python/tasks/refresh_exports.py", - python_node_source( - module="tasks", - decorator="task", - name="refresh_exports", - docstring_line=_paragraphs(5), + described_files=( + ( + "python/tasks/refresh_exports.py", + python_node_source( + module="tasks", + decorator="task", + name="refresh_exports", + docstring_line=description_paragraphs(5), + ), ), ), + expected_diagnostic=("P027", "python/tasks/refresh_exports.py", 4), + expected_message="task 'refresh_exports' description exceeds 10 lines", + expected_help_fragments=("Shorten the description to 10 lines",), ), - expected_diagnostic=("P027", "python/tasks/refresh_exports.py", 4), - expected_message="task 'refresh_exports' description exceeds 10 lines", - expected_help_fragments=("Shorten the description to 10 lines",), - ), - RequiredDescriptionCase( - description="asset description argument", - missing_files=( - ( - "python/assets/order_report.py", - python_node_source( - module="assets", - decorator="asset", - name="order_report", - decorator_arguments=f"(description={'Order report. ' * 90!r})", + RequiredDescriptionCase( + description="asset description argument", + missing_files=( + ( + "python/assets/order_report.py", + python_node_source( + module="assets", + decorator="asset", + name="order_report", + decorator_arguments=f"(description={'Order report. ' * 90!r})", + ), ), ), - ), - described_files=( - ( - "python/assets/order_report.py", - python_node_source( - module="assets", - decorator="asset", - name="order_report", - decorator_arguments=f"(description={'Order report. ' * 60!r})", + described_files=( + ( + "python/assets/order_report.py", + python_node_source( + module="assets", + decorator="asset", + name="order_report", + decorator_arguments=f"(description={'Order report. ' * 60!r})", + ), ), ), + expected_diagnostic=("P027", "python/assets/order_report.py", 4), + expected_message="asset 'order_report' description exceeds 10 lines", + expected_help_fragments=("Shorten the description to 10 lines",), ), - expected_diagnostic=("P027", "python/assets/order_report.py", 4), - expected_message="asset 'order_report' description exceeds 10 lines", - expected_help_fragments=("Shorten the description to 10 lines",), - ), + ], + ids=lambda case: case.description, ) - - -@pytest.mark.parametrize("test_case", _LENGTH_CASES, ids=lambda case: case.description) def test_given_python_description_past_the_line_limit_when_compiling_then_reports_p027( test_case: RequiredDescriptionCase, tmp_path: Path ) -> None: From 3b40dcc9e44d43ca4534ed3f538d90a888fd4301 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:04:11 +0000 Subject: [PATCH 16/19] test: expect the unknown-argument error for JobGroup after --- .../_helpers/compiled_project/snapshot_declarations.rs | 2 +- .../compiled_project/test_orchestration_snapshot.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index c9482f9b38..47304d5c3a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -4,7 +4,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::path::Path; use pyo3::exceptions::PyValueError; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; +use pyo3::prelude::{Bound, Py, PyAny, PyResult, Python}; use serde_json::Value; use sqlbuild_analysis::graph::models::ProjectGraph; use sqlbuild_cli::argv::main::job_settings_arguments::job_settings_arguments; diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 5d7b7b4ab3..06e61dc873 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -1518,7 +1518,7 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects ), ), SnapshotDiscoveryRejectionTestCase( - description="a group declaring after is refused and names the group", + description="a group declaring after is refused at its module", files={ "orchestration/jobs.py": ( "from sqlbuild.orchestration import job, job_group\n" @@ -1530,8 +1530,8 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects }, expected_error=ValueError, expected_message=re.escape( - "job group 'daily' declares after, which is no longer supported: groups order " - "jobs only by lineage. Declare the dependency on the nodes instead" + "Failed to import orchestration file orchestration/jobs.py: JobGroup.__init__() " + "got an unexpected keyword argument 'after'" ), ), SnapshotDiscoveryRejectionTestCase( From 0cdaba6f8a7bc091f9691015ebaf571a563f4d86 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 09:28:43 +0000 Subject: [PATCH 17/19] test: cover P027 in the compiler failure corpus --- .../_helpers/corpus/failure_cases.py | 13 +++++++++++++ scripts/compiler_differential/constants.py | 1 + 2 files changed, 14 insertions(+) diff --git a/scripts/compiler_differential/_helpers/corpus/failure_cases.py b/scripts/compiler_differential/_helpers/corpus/failure_cases.py index 67cf616bf2..c0922abdc4 100644 --- a/scripts/compiler_differential/_helpers/corpus/failure_cases.py +++ b/scripts/compiler_differential/_helpers/corpus/failure_cases.py @@ -169,6 +169,19 @@ def _compile_failure_cases() -> tuple[FailureCase, ...]: ) }, ), + failure_case( + name="python-node-description-too-long", + expected_code="P027", + files={ + "python/tasks/exports.py": ( + "from sqlbuild.tasks import TaskContext, task\n\n\n" + "@task\ndef export_orders(ctx: TaskContext):\n" + ' """' + + "\n\n ".join(f"Orders paragraph {index}." for index in range(6)) + + '"""\n return None\n' + ) + }, + ), failure_case( name="enum-parse", expected_code="D013", diff --git a/scripts/compiler_differential/constants.py b/scripts/compiler_differential/constants.py index 4626809a05..351c64faf4 100644 --- a/scripts/compiler_differential/constants.py +++ b/scripts/compiler_differential/constants.py @@ -574,6 +574,7 @@ def short_model_names(*, model: Model, ctx: RuleContext) -> list[Finding]: "P023", "P024", "P025", + "P027", "S026", } ) From 84365df4487d57265c2e9ab4b372b1ada1ad3532 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:39:27 +0000 Subject: [PATCH 18/19] test: record the golden for the P027 failure corpus case --- .../python-node-description-too-long.json | 271 ++++++++++++++++++ 1 file changed, 271 insertions(+) create mode 100644 tests/goldens/compiler/failure/python-node-description-too-long.json diff --git a/tests/goldens/compiler/failure/python-node-description-too-long.json b/tests/goldens/compiler/failure/python-node-description-too-long.json new file mode 100644 index 0000000000..34bd8ae483 --- /dev/null +++ b/tests/goldens/compiler/failure/python-node-description-too-long.json @@ -0,0 +1,271 @@ +{ + "commands": [ + { + "label": "compile", + "exit_code": 1, + "diagnostics": [ + { + "phase": "compile", + "severity": "error", + "code": "P027", + "message": "task 'export_orders' description exceeds 10 lines", + "resource_name": "export_orders", + "path": "python/tasks/exports.py", + "line": 4, + "column": 1, + "location": { + "path": "python/tasks/exports.py", + "line": 4, + "column": 1 + }, + "help": "Shorten the description to 10 lines; move detail into documentation." + } + ] + } + ], + "compiled": { + "models/marts/customer_totals.sql": [ + "SELECT customer_id, SUM(amount) AS total_amount", + "FROM __ref(\"stg_orders\")", + "GROUP BY customer_id", + "" + ], + "models/staging/stg_orders.sql": [ + "SELECT order_id, customer_id, amount, status", + "FROM __source(\"raw_orders\")", + "" + ] + }, + "manifest": { + "nodes": { + "model.failure_corpus.customer_totals": { + "database": null, + "schema": "main", + "name": "customer_totals", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/marts/customer_totals.sql", + "original_file_path": "models/marts/customer_totals.sql", + "unique_id": "model.failure_corpus.customer_totals", + "fqn": [ + "failure_corpus", + "models", + "marts", + "customer_totals" + ], + "alias": "customer_totals", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Order totals per customer", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.customer_totals", + "depends_on": { + "macros": [], + "nodes": [ + "model.failure_corpus.stg_orders" + ] + } + }, + "model.failure_corpus.stg_orders": { + "database": null, + "schema": "main", + "name": "stg_orders", + "resource_type": "model", + "package_name": "failure_corpus", + "path": "models/staging/stg_orders.sql", + "original_file_path": "models/staging/stg_orders.sql", + "unique_id": "model.failure_corpus.stg_orders", + "fqn": [ + "failure_corpus", + "models", + "staging", + "stg_orders" + ], + "alias": "stg_orders", + "config": { + "enabled": true, + "alias": null, + "schema": null, + "database": null, + "tags": [], + "meta": {}, + "group": null, + "materialized": "view", + "incremental_strategy": null, + "merge_exclude_columns": null, + "persist_docs": {}, + "post-hook": [], + "pre-hook": [], + "quoting": {}, + "column_types": {}, + "full_refresh": null, + "unique_key": null, + "on_schema_change": "ignore", + "on_configuration_change": "apply", + "grants": {}, + "packages": [], + "docs": { + "show": true, + "node_color": null + }, + "contract": { + "enforced": false, + "alias_types": true + } + }, + "tags": [], + "description": "Staged orders", + "columns": {}, + "meta": {}, + "group": null, + "docs": { + "show": true, + "node_color": null + }, + "patch_path": null, + "build_path": null, + "unrendered_config": {}, + "config_call_dict": {}, + "unrendered_config_call_dict": {}, + "relation_name": "main.stg_orders", + "depends_on": { + "macros": [], + "nodes": [ + "source.failure_corpus.raw_orders" + ] + } + } + }, + "sources": { + "source.failure_corpus.raw_orders": { + "database": null, + "schema": "", + "name": "raw_orders", + "resource_type": "source", + "package_name": "failure_corpus", + "path": "sources/raw.yml", + "original_file_path": "sources/raw.yml", + "unique_id": "source.failure_corpus.raw_orders", + "fqn": [ + "failure_corpus", + "raw_orders" + ], + "source_name": "raw_orders", + "source_description": "", + "loader": "", + "identifier": "raw_orders", + "quoting": {}, + "loaded_at_field": null, + "freshness": null, + "external": null, + "description": "Orders feed.", + "columns": { + "order_id": { + "name": "order_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "customer_id": { + "name": "customer_id", + "description": "", + "meta": {}, + "data_type": "INTEGER", + "constraints": [], + "quote": null, + "tags": [] + }, + "amount": { + "name": "amount", + "description": "", + "meta": {}, + "data_type": "DOUBLE", + "constraints": [], + "quote": null, + "tags": [] + }, + "status": { + "name": "status", + "description": "", + "meta": {}, + "data_type": "VARCHAR", + "constraints": [], + "quote": null, + "tags": [] + } + }, + "meta": {}, + "source_meta": {}, + "tags": [], + "config": {}, + "patch_path": null, + "unrendered_config": {}, + "relation_name": null + } + }, + "macros": {}, + "exposures": {}, + "metrics": {}, + "groups": {}, + "selectors": {}, + "disabled": {}, + "parent_map": { + "model.failure_corpus.customer_totals": [ + "model.failure_corpus.stg_orders" + ], + "model.failure_corpus.stg_orders": [ + "source.failure_corpus.raw_orders" + ], + "source.failure_corpus.raw_orders": [] + }, + "group_map": {}, + "docs": {}, + "saved_queries": {}, + "semantic_models": {}, + "unit_tests": {} + } +} From ebe1b7acf214a7c98db03214f29b574ecfec395b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:51:11 +0000 Subject: [PATCH 19/19] test: expect build --no-audits to skip Python checks in the E2E suites --- .../test_build_no_tests_no_audits.py | 18 ++---------------- .../cli/commands/main/check/test_check.py | 9 +++------ 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py index 3af739ac24..d7fd693423 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/no_tests_no_audits/test_build_no_tests_no_audits.py @@ -257,14 +257,7 @@ def export_check(ctx): db_path: Path = project_dir / "factory_nodes_project.duckdb" run_result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "build", - "--no-tests", - "--no-audits", - "--select", - "+asset:orders_export", - ), + command=("--no-color", "build", "--no-tests", "--select", "+asset:orders_export"), project_dir=project_dir, ) bare_result: subprocess.CompletedProcess[str] = run_sqb( @@ -2260,14 +2253,7 @@ def test_given_check_selector_when_running_build_then_check_needs_its_checked_no ) result: subprocess.CompletedProcess[str] = run_sqb( - command=( - "--no-color", - "build", - "--no-tests", - "--no-audits", - "--select", - *test_case.select, - ), + command=("--no-color", "build", "--no-tests", "--select", *test_case.select), project_dir=project_dir, ) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/check/test_check.py b/tests/e2e/src/sqlbuild/cli/commands/main/check/test_check.py index fd91edd38b..3d560ca5d7 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/check/test_check.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/check/test_check.py @@ -178,7 +178,7 @@ project_kind="terminal_loader", ), CheckCommandTestCase( - description="build without tests or audits still executes Python checks", + description="build without tests or audits skips Python checks", command=( "--no-color", "build", @@ -188,11 +188,8 @@ "tag:failure", ), expected_returncode=0, - expected_stdout_fragments=( - "Execution sqb build", - "Python checks", - "check_orders_export", - ), + expected_stdout_fragments=("Execution sqb build",), + expected_absent_fragments=("Python checks", "check_orders_export"), ), CheckCommandTestCase( description="audit remains SQL audit only",