From 49e5d0725e0e70fb674489769edd975c9a1221be Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:42:39 +0000 Subject: [PATCH 1/2] fix(orchestration): claim sources loaded through hook reads and loader chains Snapshots record `loads` on models (managed sources their hooks or audit gates read beyond `deps`) and on managed sources (the declared sources in their loader chain), additive and absent when empty. A job's write set claims a selected model's `loads` and each loaded source's chain, so the worker no longer refuses those runs. A directly selected source claims only itself, as only its own loader runs. An intermediate loader that no source declares stays unclaimed, so runs that need one still fail in the worker. Models and sources that gain `loads` show as changed once, on the first publication after upgrading. --- .../intermediate_loader_orders.sql | 7 ++ .../intermediate_loader_sources.py | 17 ++++ .../intermediate_loader_sources.yml | 9 ++ .../edge_cases/orchestration/loaded_orders.py | 14 +++ .../orchestration/loaded_orders.sql | 8 ++ .../orchestration/loaded_sources.py | 29 ++++++ .../orchestration/loaded_sources.yml | 33 +++++++ .../src/planner_scope/main/mod.rs | 1 + .../src/planner_scope/main/node_loads.rs | 38 ++++++++ .../src/planner_scope/tests/test_loaders.rs | 91 ++++++++++++++++++- .../src/planner_scope/tests/test_types.rs | 9 ++ .../src/snapshots/_helpers/mod.rs | 1 + .../src/snapshots/_helpers/write_set.rs | 64 +++++++++++++ .../src/snapshots/constants.rs | 2 + .../src/snapshots/main/job_write_set.rs | 48 ++-------- .../tests/runs/helpers.rs | 8 +- .../tests/runs/test_adhoc.rs | 15 ++- .../_helpers/location_command/snapshot.rs | 38 +++++++- .../publication/_test_types.py | 11 +++ .../compiled_project/publication/helpers.py | 42 +++++++++ .../publication/test_worker.py | 91 +++++++++++++++++++ 21 files changed, 531 insertions(+), 45 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql create mode 100644 benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/loaded_orders.sql create mode 100644 benchmarks/duckshop/edge_cases/orchestration/loaded_sources.py create mode 100644 benchmarks/duckshop/edge_cases/orchestration/loaded_sources.yml create mode 100644 crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs create mode 100644 crates/sqlbuild-orchestration/src/snapshots/_helpers/write_set.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql new file mode 100644 index 0000000000..63c70f3f93 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql @@ -0,0 +1,7 @@ +MODEL ( + description 'Orders read from shipments whose loader needs an intermediate loader.', + materialized table, + tags [marts], +); + +SELECT CAST(order_id AS BIGINT) AS amount FROM __source("raw_shipments") diff --git a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py new file mode 100644 index 0000000000..9512d788c5 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py @@ -0,0 +1,17 @@ +"""A source loader that depends on an intermediate loader no source declares.""" + +from sqlbuild.loaders import loader + + +@loader +def stage_shipments(ctx): + """Stage synthetic shipments in the intermediate loader's own relation.""" + return [{"order_id": 9}] + + +@loader(depends_on=[stage_shipments]) +def raw_shipments(ctx): + """Load the staged shipments.""" + staged = ctx.loader(stage_shipments) + rows = ctx.query(f"SELECT order_id FROM {staged.destination}").fetchall() + return [{"order_id": row[0]} for row in rows] diff --git a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml new file mode 100644 index 0000000000..1369216f03 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml @@ -0,0 +1,9 @@ +sources: + - name: raw_shipments + description: Shipments loaded after an intermediate loader that no source declares. + managed: true + write_strategy: table + table: raw_shipments + columns: + - name: order_id + type: BIGINT diff --git a/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py b/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py new file mode 100644 index 0000000000..2941037d70 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py @@ -0,0 +1,14 @@ +"""A tag-selected job whose model loads a loader chain and a hook-read source. + +The build loads raw_orders and the raw_customers loader it depends on, and raw_returns, which only +the model's pre-hook reads. The job's write set claims those three and not raw_inventory. +""" + +from sqlbuild.orchestration import job, schedule + +annual_orders = job( + "annual_orders", + description="Build the marts layer and the sources it loads.", + select="tag:marts", + schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.sql b/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.sql new file mode 100644 index 0000000000..04ed5af737 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.sql @@ -0,0 +1,8 @@ +MODEL ( + description 'Orders that check returns before they build.', + materialized table, + tags [marts], + pre_hooks [inline_sql('SELECT count(*) FROM __source("raw_returns")')], +); + +SELECT CAST(order_id AS BIGINT) AS amount FROM __source("raw_orders") diff --git a/benchmarks/duckshop/edge_cases/orchestration/loaded_sources.py b/benchmarks/duckshop/edge_cases/orchestration/loaded_sources.py new file mode 100644 index 0000000000..1227f7d231 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/loaded_sources.py @@ -0,0 +1,29 @@ +"""Loaders for a two-loader chain, a hook-read source and a source no job reads.""" + +from sqlbuild.loaders import loader + + +@loader +def raw_customers(ctx): + """Load synthetic customers.""" + return [{"customer_id": 7}] + + +@loader(depends_on=[raw_customers]) +def raw_orders(ctx): + """Load one order per loaded customer.""" + customers = ctx.loader(raw_customers) + rows = ctx.query(f"SELECT customer_id FROM {customers.destination}").fetchall() + return [{"order_id": row[0]} for row in rows] + + +@loader +def raw_returns(ctx): + """Load synthetic returns.""" + return [{"order_id": 7}] + + +@loader +def raw_inventory(ctx): + """Load synthetic inventory.""" + return [{"product_id": 1}] diff --git a/benchmarks/duckshop/edge_cases/orchestration/loaded_sources.yml b/benchmarks/duckshop/edge_cases/orchestration/loaded_sources.yml new file mode 100644 index 0000000000..7fed310ff0 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/loaded_sources.yml @@ -0,0 +1,33 @@ +sources: + - name: raw_customers + description: Customers the orders loader reads first. + managed: true + write_strategy: table + table: raw_customers + columns: + - name: customer_id + type: BIGINT + - name: raw_orders + description: Orders loaded after the customers loader they depend on. + managed: true + write_strategy: table + table: raw_orders + columns: + - name: order_id + type: BIGINT + - name: raw_returns + description: Returns that only a model hook reads. + managed: true + write_strategy: table + table: raw_returns + columns: + - name: order_id + type: BIGINT + - name: raw_inventory + description: A managed source that nothing the job builds reads. + managed: true + write_strategy: table + table: raw_inventory + columns: + - name: product_id + type: BIGINT diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/mod.rs b/crates/sqlbuild-analysis/src/planner_scope/main/mod.rs index 3fbafe8e4c..d95d839471 100644 --- a/crates/sqlbuild-analysis/src/planner_scope/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_scope/main/mod.rs @@ -3,4 +3,5 @@ pub mod expand_loaders; pub mod intermediate_sources; pub mod managed_sources; +pub mod node_loads; pub mod source_selectors; diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs b/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs new file mode 100644 index 0000000000..4d2f33a411 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs @@ -0,0 +1,38 @@ +//! Source writers a loading build plans for one node besides the node itself. + +use crate::assembly::project::types::ObjectKey; +use crate::planner_scope::constants::{MODEL, SOURCE}; +use crate::planner_scope::models::LoaderGraph; +use std::collections::{BTreeMap, BTreeSet}; + +/// Managed sources each model's execution deps read and each managed source's declared loader chain, as `expand_loaders` keys them. +pub fn node_loads( + graph: &LoaderGraph, + upstream: &[(ObjectKey, Vec)], +) -> BTreeMap> { + let mut loads: BTreeMap> = BTreeMap::new(); + for (key, deps) in upstream { + let writers: Vec = if key.0 == MODEL { + deps.iter() + .filter(|dep| dep.0 == SOURCE && graph.source_loader(dep).is_some()) + .cloned() + .collect() + } else if let Some(chain) = graph + .source_loader(key) + .and_then(|loader| graph.loaders.get(loader)) + { + graph + .upstream(chain) + .iter() + .filter(|name| !graph.intermediate(name)) + .map(|name| graph.key(name)) + .collect() + } else { + continue; + }; + if !writers.is_empty() { + loads.entry(key.clone()).or_default().extend(writers); + } + } + loads +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs index 838ceeee51..b90277e273 100644 --- a/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs +++ b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs @@ -2,9 +2,12 @@ use crate::assembly::project::types::ObjectKey; use crate::planner_scope::main::expand_loaders::expand_loaders; +use crate::planner_scope::main::node_loads::node_loads; use crate::planner_scope::models::LoaderGraph; -use crate::planner_scope::tests::test_types::{DeepLoaderTestCase, LoaderTestCase}; -use std::collections::HashSet; +use crate::planner_scope::tests::test_types::{ + DeepLoaderTestCase, LoaderTestCase, NodeLoadsTestCase, +}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; #[test] fn given_shared_loader_dependencies_when_expanding_then_preserves_depth_first_edges_and_breadth_first_names() @@ -120,3 +123,87 @@ fn given_deep_loader_chain_when_expanding_then_uses_iterative_traversal() { ); } } + +#[test] +fn given_loader_chains_and_unmanaged_reads_when_listing_node_loads_then_only_declared_source_writers_appear() + { + let test_cases = [ + NodeLoadsTestCase { + description: "a hook-read managed source is listed and an unmanaged read is not", + loaders: vec![("raw_returns", vec![])], + sources: vec![("raw_returns", Some("raw_returns")), ("raw_products", None)], + upstream: vec![ + ( + "model:orders", + vec![ + "source:raw_returns", + "source:raw_products", + "model:stg_orders", + ], + ), + ("model:stg_orders", vec!["source:raw_products"]), + ("source:raw_products", vec![]), + ], + expected_loads: vec![("model:orders", vec!["source:raw_returns"])], + }, + NodeLoadsTestCase { + // stage_orders has no declaring source, so a run that loads it still fails in the worker. + description: "a chain keeps declared loaders behind an intermediate one and drops the intermediate", + loaders: vec![ + ("raw_orders", vec!["stage_orders"]), + ("stage_orders", vec!["raw_customers"]), + ("raw_customers", vec![]), + ], + sources: vec![ + ("raw_orders", Some("raw_orders")), + ("raw_customers", Some("raw_customers")), + ], + upstream: vec![ + ("source:raw_orders", vec![]), + ("source:raw_customers", vec![]), + ], + expected_loads: vec![("source:raw_orders", vec!["source:raw_customers"])], + }, + ]; + let key = |text: &str| { + let (kind, name) = text.split_once(':').expect("kind:name"); + (kind.to_owned(), name.to_owned()) + }; + for test_case in test_cases { + let graph = LoaderGraph { + loaders: test_case + .loaders + .iter() + .map(|(name, deps)| { + ( + (*name).to_owned(), + deps.iter().map(|dep| (*dep).to_owned()).collect(), + ) + }) + .collect(), + sources: test_case + .sources + .iter() + .map(|(name, loader)| ((*name).to_owned(), loader.map(str::to_owned))) + .collect(), + }; + let upstream: Vec<(ObjectKey, Vec)> = test_case + .upstream + .iter() + .map(|(node, deps)| (key(node), deps.iter().map(|dep| key(dep)).collect())) + .collect(); + assert_eq!( + node_loads(&graph, &upstream), + test_case + .expected_loads + .iter() + .map(|(node, writers)| ( + key(node), + writers.iter().map(|writer| key(writer)).collect() + )) + .collect::>>(), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_scope/tests/test_types.rs index f3f3d6c266..4df6c9eb40 100644 --- a/crates/sqlbuild-analysis/src/planner_scope/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_scope/tests/test_types.rs @@ -7,6 +7,15 @@ pub(crate) struct LoaderTestCase { pub expected_selected: Vec<&'static str>, } +#[derive(Debug)] +pub(crate) struct NodeLoadsTestCase { + pub description: &'static str, + pub loaders: Vec<(&'static str, Vec<&'static str>)>, + pub sources: Vec<(&'static str, Option<&'static str>)>, + pub upstream: Vec<(&'static str, Vec<&'static str>)>, + pub expected_loads: Vec<(&'static str, Vec<&'static str>)>, +} + #[derive(Debug)] pub(crate) struct DeepLoaderTestCase { pub description: &'static str, diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs index 8853fb96b6..d7f4615435 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod content; pub(crate) mod preparation; pub(crate) mod publication; pub(crate) mod reading; +pub(crate) mod write_set; diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/write_set.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/write_set.rs new file mode 100644 index 0000000000..955f075b2b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/write_set.rs @@ -0,0 +1,64 @@ +//! Write-set pieces read from published node definitions. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; + +use crate::snapshots::constants::{ + LOADER_POINTER, LOADS_FIELD, MODEL_NODE_KIND, SOURCE_KEY_PREFIX, SOURCE_NODE_KIND, + WRITER_NODE_KINDS, +}; +use crate::snapshots::models::SnapshotNode; + +/// Selected writer keys, candidate source keys, and the sources selected models read when loading is on. +pub(crate) fn selected_writers( + selected: &BTreeMap, + loads_model_sources: bool, +) -> (BTreeSet, BTreeSet, BTreeSet) { + let mut writers: BTreeSet = BTreeSet::new(); + let mut sources: BTreeSet = BTreeSet::new(); + let mut model_reads: BTreeSet = BTreeSet::new(); + for (key, node) in selected { + if WRITER_NODE_KINDS.contains(&node.kind.as_str()) { + writers.insert(key.clone()); + } + if node.kind == SOURCE_NODE_KIND { + sources.insert(key.clone()); + } else if node.kind == MODEL_NODE_KIND && loads_model_sources { + model_reads.extend(source_deps(&node.definition)); + model_reads.extend(loads(&node.definition)); + } + } + sources.extend(model_reads.iter().cloned()); + (writers, sources, model_reads) +} + +/// A source with a non-null loader, which a loading build writes. +pub(crate) fn managed(node: &SnapshotNode) -> bool { + node.kind == SOURCE_NODE_KIND + && node + .definition + .pointer(LOADER_POINTER) + .is_some_and(|loader| !loader.is_null()) +} + +/// The source keys among a node's compiled `deps`. +pub(crate) fn source_deps(definition: &Value) -> impl Iterator + '_ { + keys(definition, "deps").filter(|key| key.starts_with(SOURCE_KEY_PREFIX)) +} + +/// The keys a node's definition lists in `loads`; snapshots published before it have none. +pub(crate) fn loads(definition: &Value) -> impl Iterator + '_ { + keys(definition, LOADS_FIELD) +} + +/// The string keys a node's definition lists under `field`. +fn keys<'a>(definition: &'a Value, field: &str) -> impl Iterator + 'a { + definition + .get(field) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_owned) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/constants.rs b/crates/sqlbuild-orchestration/src/snapshots/constants.rs index cf3e62ba43..0072f17f66 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/constants.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/constants.rs @@ -55,5 +55,7 @@ pub(crate) const SOURCE_NODE_KIND: &str = "source"; pub(crate) const SOURCE_KEY_PREFIX: &str = "source:"; /// A source with a non-null loader here is managed, so automatic source loading writes it. pub(crate) const LOADER_POINTER: &str = "/entry/loader"; +/// Source writers a loading build of the node plans beyond its `deps`: hook and audit reads, loader chains. +pub(crate) const LOADS_FIELD: &str = "loads"; /// The frozen setting for `--no-load-sources`; `false` stops a build loading sources its models read. pub(crate) const LOAD_SOURCES_SETTING: &str = "load_sources"; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs b/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs index a8f24c6600..488e6cf351 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs @@ -5,15 +5,13 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::Value; use uuid::Uuid; -use crate::snapshots::constants::{ - LOAD_SOURCES_SETTING, LOADER_POINTER, MODEL_NODE_KIND, SOURCE_KEY_PREFIX, SOURCE_NODE_KIND, - WRITER_NODE_KINDS, -}; +use crate::snapshots::_helpers::write_set::{loads, managed, selected_writers}; +use crate::snapshots::constants::LOAD_SOURCES_SETTING; use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; use crate::store::errors::ControlError; use crate::store::models::ControlTransaction; -/// Selected writers plus the managed sources a build loads: selected ones, and parents of selected models unless loading is off. +/// Selected writers, the managed sources a build loads, and loader chains of sources selected models read; a selected source runs only its own loader. pub(crate) async fn job_write_set( transaction: ControlTransaction, snapshot_id: Uuid, @@ -22,40 +20,14 @@ pub(crate) async fn job_write_set( let (transaction, selected) = read_snapshot_nodes(transaction, snapshot_id, Some(selection)).await?; let loads_model_sources = settings.get(LOAD_SOURCES_SETTING) != Some(&Value::Bool(false)); - let mut writers: BTreeSet = BTreeSet::new(); - let mut sources: BTreeSet = BTreeSet::new(); - for (key, node) in &selected { - if WRITER_NODE_KINDS.contains(&node.kind.as_str()) { - writers.insert(key.clone()); - } - if node.kind == SOURCE_NODE_KIND { - sources.insert(key.clone()); - } else if node.kind == MODEL_NODE_KIND && loads_model_sources { - sources.extend( - node.definition - .get("deps") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(Value::as_str) - .filter(|key| key.starts_with(SOURCE_KEY_PREFIX)) - .map(str::to_owned), - ); + let (mut writers, candidates, model_reads) = selected_writers(&selected, loads_model_sources); + let (transaction, sources) = + read_snapshot_nodes(transaction, snapshot_id, Some(&candidates)).await?; + for (key, node) in sources.into_iter().filter(|(_, node)| managed(node)) { + if model_reads.contains(&key) { + writers.extend(loads(&node.definition)); } + writers.insert(key); } - let (transaction, sources) = - read_snapshot_nodes(transaction, snapshot_id, Some(&sources)).await?; - writers.extend( - sources - .into_iter() - .filter(|(_, node)| { - node.kind == SOURCE_NODE_KIND - && node - .definition - .pointer(LOADER_POINTER) - .is_some_and(|loader| !loader.is_null()) - }) - .map(|(key, _)| key), - ); Ok((transaction, writers)) } diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index 9fd636bed0..5dec2a4ee4 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -135,7 +135,7 @@ pub(crate) fn json_column(value: &Value) -> Value { }) } -/// Two models, a seed, a read-only source, a loaded customers source and one person per role under test. +/// Two models, a seed, a read-only source, a customers source loaded after accounts and one person per role under test. pub(crate) async fn adhoc_fixture( store: &ControlStore, context: &AuditContext, @@ -154,8 +154,12 @@ pub(crate) async fn adhoc_fixture( .map(published_node) .collect(); let (key, mut managed) = published_node("source:shop.customers"); - managed.definition = json!({"entry": {"loader": "load_customers"}}); + managed.definition = + json!({"entry": {"loader": "load_customers"}, "loads": ["source:shop.accounts"]}); input.nodes.insert(key, managed); + let (key, mut upstream) = published_node("source:shop.accounts"); + upstream.definition = json!({"entry": {"loader": "load_accounts"}}); + input.nodes.insert(key, upstream); input .nodes .get_mut("model:customers") diff --git a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs index 1662bd3364..fec1c805ae 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs @@ -33,12 +33,22 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ expected_launch: Ok(true), }, AdhocCommandTestCase { - description: "a build claims the managed source it loads for a selected model", + description: "a build claims the managed source it loads for a selected model and its loader chain", person: AdhocPerson::Developer, command: &["build", "model:customers"], priority: 0, reason: "Refresh customers", - expected_check: json!({"selection": ["model:customers"], "write_set": ["model:customers", "source:shop.customers"], + expected_check: json!({"selection": ["model:customers"], "write_set": ["model:customers", "source:shop.accounts", "source:shop.customers"], + "permissions": [["run.adhoc", true]], "error": null, "launchable": true}), + expected_launch: Ok(true), + }, + AdhocCommandTestCase { + description: "a build of a selected source claims it alone, as only its own loader runs", + person: AdhocPerson::Developer, + command: &["build", "source:shop.customers"], + priority: 0, + reason: "Reload customers", + expected_check: json!({"selection": ["source:shop.customers"], "write_set": ["source:shop.customers"], "permissions": [["run.adhoc", true]], "error": null, "launchable": true}), expected_launch: Ok(true), }, @@ -293,6 +303,7 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ vec![ json!(["build", {}, "Refresh orders"]), json!(["build", {}, "Refresh customers"]), + json!(["build", {}, "Reload customers"]), json!(["build", {"load_sources": false}, "Refresh customers only"]), json!(["build", {"full_refresh": true, "warehouse": "BUILD_XS"}, "Rebuild orders now"]), json!(["test", {}, "Check orders"]), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index bf15b867e1..e98184f7c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -7,7 +7,8 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{IntoPyObject, pyfunction}; -use serde_json::json; +use serde_json::{Value, json}; +use sqlbuild_analysis::planner_scope::main::node_loads::node_loads; use sqlbuild_core::models::DiagnosticSeverity; use sqlbuild_discovery::project_validation::main::validate_orchestration_names::validate_orchestration_names; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; @@ -28,8 +29,10 @@ use crate::bindings::_helpers::native_host::orchestration_declarations::{ collect_declarations, orchestration_names, }; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; +use crate::bindings::_helpers::planner_identity::graph::execution_deps; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; +use crate::bindings::_helpers::planner_scope::values::project_graph; use crate::bindings::_helpers::records::fields::field; /// Where declarations passed to the snapshot directly are reported as declared. @@ -140,6 +143,7 @@ pub(crate) fn orchestration_snapshot<'py>( input.documents.insert("lineage".into(), lineage); } input.nodes = identity_nodes(project.as_any(), input.nodes)?; + input.nodes = load_nodes(project.as_any(), input.nodes)?; Ok(NativeOrchestrationSnapshot { input }) } @@ -174,3 +178,35 @@ fn identity_nodes( } Ok(nodes) } + +/// `loads` lists the source writers a loading build of the node plans beyond its compiled `deps`, absent when none. +fn load_nodes( + project: &Bound<'_, PyAny>, + mut nodes: BTreeMap, +) -> PyResult> { + for ((kind, name), writers) in node_loads(&project_graph(project)?, &execution_deps(project)?) { + let Some(node) = nodes.get_mut(&format!("{kind}:{name}")) else { + continue; + }; + let deps: Vec = node + .definition + .get("deps") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + let loads: Vec = writers + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .filter(|key| !listed(&deps, key)) + .collect(); + if !loads.is_empty() { + node.definition["loads"] = json!(loads); + } + } + Ok(nodes) +} + +/// Whether `key` is already one of the node's compiled `deps`. +fn listed(deps: &[Value], key: &str) -> bool { + deps.iter().any(|dep| dep.as_str() == Some(key)) +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 29300f5896..884eaeaf23 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -1,5 +1,6 @@ """Publication cases and isolated server credentials.""" +import sqlite3 from collections.abc import Callable from contextlib import AbstractContextManager from dataclasses import dataclass, field @@ -118,11 +119,21 @@ class WorkerExecutionTestCase: files: dict[str, str] = field(default_factory=dict) files_after_publication: dict[str, str] = field(default_factory=dict) store_setup: str = "" + store_edit: Callable[[sqlite3.Connection], None] = lambda _: None concurrency_limits: dict[str, int] = field(default_factory=dict) launcher: Literal["native", "uncooperative", "docker"] = "native" warehouse_setup: Callable[[Path], None] = lambda _: None +@dataclass(frozen=True) +class UnclaimedLoadTestCase: + description: str + files: dict[str, str] + store_edit: Callable[[sqlite3.Connection], None] + refused_writer: str + expected_write_set: frozenset[str] + + @dataclass(frozen=True) class CheckExecutionTestCase(WorkerExecutionTestCase): description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 6e041ea16f..2e4d50dc5f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -3,6 +3,7 @@ from __future__ import annotations import base64 +import gzip import hashlib import json import os @@ -826,6 +827,7 @@ def scheduled_worker( control_store: Path = directory / "control/.sqlbuild/control.sqlite3" with sqlite3.connect(control_store) as control: _ = control.executescript(test_case.store_setup) + test_case.store_edit(control) monkeypatch.setenv("SQLBUILD_SCHEDULER_TEST_STORE", f"sqlite://{control_store}") monkeypatch.setenv( "SQLBUILD_PLATFORM_WEBHOOK_URL", @@ -1537,3 +1539,43 @@ def hold_write_lock(database: Path) -> duckdb.DuckDBPyConnection: except duckdb.IOException: assert time.monotonic() < deadline time.sleep(0.05) + + +LOADED_ORDER_FILES: dict[str, str] = { + "sources/loaded.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/loaded_sources.yml" + ).read_text(), + "python/loaders/loaded.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/loaded_sources.py" + ).read_text(), + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.sql" + ).read_text(), + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" + ).read_text(), +} + + +def without_loads(control: sqlite3.Connection, node_key: str) -> None: + """Drop a published node's `loads` in place; its content address and the worker's check stay.""" + for node_hash, body in control.execute( + "SELECT d.node_hash, d.body FROM sqb_node_definitions d JOIN sqb_snapshot_nodes n " + "ON n.node_hash = d.node_hash WHERE n.node_key = ?", + (node_key,), + ).fetchall(): + content: dict[str, Any] = json.loads(gzip.decompress(body)) + del content["definition"]["loads"] + _ = control.execute( + "UPDATE sqb_node_definitions SET body = ? WHERE node_hash = ?", + (gzip.compress(json.dumps(content).encode()), node_hash), + ) + + +def run_write_set(control_store: Path, run_id: str) -> frozenset[str]: + """The frozen write set a run's request recorded.""" + with closing(sqlite3.connect(control_store)) as control: + (request,) = control.execute( + "SELECT request_json FROM sqb_runs WHERE run_id=?", (run_id,) + ).fetchone() + return frozenset(json.loads(request)["write_set"]) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index dd84ed7816..ff3cfb341e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -33,16 +33,20 @@ ClaimedWorker, GroupExecutionTestCase, SnapshotWorkerTestCase, + UnclaimedLoadTestCase, WorkerExecutionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + LOADED_ORDER_FILES, cancel_running_statement, claimed_worker, + run_write_set, running_statement, scheduled_worker, snapshot_api, wait_for_log, wait_for_run, + without_loads, worker_result, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -598,6 +602,93 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re assert run["status"] == test_case.expected_status, worker.log.read_text() +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "a tag-selected job claims its loader chain and hook-read source, not unread ones", + "", + "SELECT amount FROM orders", + "success", + [(7,)], + files=LOADED_ORDER_FILES, + ) + ], + ids=lambda case: case.description, +) +def test_given_loads_when_a_worker_executes_then_the_run_claims_exactly_what_it_loads( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert run_write_set(worker.control_store, run["run_id"]) == frozenset( + {"model:orders", "source:raw_orders", "source:raw_customers", "source:raw_returns"} + ) + + +@pytest.mark.parametrize( + "test_case", + [ + UnclaimedLoadTestCase( + "the hook-read source is loaded only when the published model claims it", + files=LOADED_ORDER_FILES, + store_edit=lambda control: without_loads(control, "model:orders"), + refused_writer="source:raw_returns", + expected_write_set=frozenset( + {"model:orders", "source:raw_orders", "source:raw_customers"} + ), + ), + UnclaimedLoadTestCase( + # A known gap: no snapshot node exists for an intermediate loader, so `loads` leaves it out. + "an intermediate loader no source declares is never claimed, so the run still fails", + files={ + "sources/shipments.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml" + ).read_text(), + "python/loaders/shipments.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py" + ).read_text(), + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql" + ).read_text(), + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" + ).read_text(), + }, + store_edit=lambda _: None, + refused_writer="source:stage_shipments", + expected_write_set=frozenset({"model:orders", "source:raw_shipments"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_load_the_job_does_not_claim_when_a_worker_plans_then_the_run_is_refused( + test_case: UnclaimedLoadTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + worker_case: WorkerExecutionTestCase = WorkerExecutionTestCase( + test_case.description, + "", + "", + "failed", + [], + files=test_case.files, + store_edit=test_case.store_edit, + ) + with scheduled_worker(tmp_path, monkeypatch, worker_case) as worker: + run: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + log: str = worker.log.read_text() + assert (run["status"], run["node_counts_json"]) == ("failed", {"skipped": 1}), log + assert ( + f"planned writer {test_case.refused_writer} is outside the frozen run write set" in log + ) + assert run_write_set(worker.control_store, run["run_id"]) == test_case.expected_write_set + with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: + assert warehouse.execute( + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'" + ).fetchone() == (0,) + + @pytest.mark.parametrize( "test_case", [ From 1d3e0258db85d5f77e92033317f0d30dead54da0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:13:02 +0000 Subject: [PATCH 2/2] test(orchestration): read ad-hoc launch audit rows in launch order --- crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs index fec1c805ae..a3af046bbd 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs @@ -286,7 +286,8 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ ); let audit = rows( store, - "SELECT after_json FROM sqb_security_audit WHERE action='run.adhoc_requested'", + "SELECT after_json FROM sqb_security_audit WHERE action='run.adhoc_requested' \ + ORDER BY occurred_at, event_id", Vec::new(), ) .await;