diff --git a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql new file mode 100644 index 0000000000..63c70f3f93 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql @@ -0,0 +1,7 @@ +MODEL ( + description 'Orders read from shipments whose loader needs an intermediate loader.', + materialized table, + tags [marts], +); + +SELECT CAST(order_id AS BIGINT) AS amount FROM __source("raw_shipments") diff --git a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py new file mode 100644 index 0000000000..9512d788c5 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py @@ -0,0 +1,17 @@ +"""A source loader that depends on an intermediate loader no source declares.""" + +from sqlbuild.loaders import loader + + +@loader +def stage_shipments(ctx): + """Stage synthetic shipments in the intermediate loader's own relation.""" + return [{"order_id": 9}] + + +@loader(depends_on=[stage_shipments]) +def raw_shipments(ctx): + """Load the staged shipments.""" + staged = ctx.loader(stage_shipments) + rows = ctx.query(f"SELECT order_id FROM {staged.destination}").fetchall() + return [{"order_id": row[0]} for row in rows] diff --git a/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml new file mode 100644 index 0000000000..1369216f03 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml @@ -0,0 +1,9 @@ +sources: + - name: raw_shipments + description: Shipments loaded after an intermediate loader that no source declares. + managed: true + write_strategy: table + table: raw_shipments + columns: + - name: order_id + type: BIGINT diff --git a/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py b/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py new file mode 100644 index 0000000000..2941037d70 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py @@ -0,0 +1,14 @@ +"""A tag-selected job whose model loads a loader chain and a hook-read source. + +The build loads raw_orders and the raw_customers loader it depends on, and raw_returns, which only +the model's pre-hook reads. The job's write set claims those three and not raw_inventory. +""" + +from sqlbuild.orchestration import job, schedule + +annual_orders = job( + "annual_orders", + description="Build the marts layer and the sources it loads.", + select="tag:marts", + schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"), +) diff --git a/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.sql b/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.sql new file mode 100644 index 0000000000..04ed5af737 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/loaded_orders.sql @@ -0,0 +1,8 @@ +MODEL ( + description 'Orders that check returns before they build.', + materialized table, + tags [marts], + pre_hooks [inline_sql('SELECT count(*) FROM __source("raw_returns")')], +); + +SELECT CAST(order_id AS BIGINT) AS amount FROM __source("raw_orders") diff --git a/benchmarks/duckshop/edge_cases/orchestration/loaded_sources.py b/benchmarks/duckshop/edge_cases/orchestration/loaded_sources.py new file mode 100644 index 0000000000..1227f7d231 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/loaded_sources.py @@ -0,0 +1,29 @@ +"""Loaders for a two-loader chain, a hook-read source and a source no job reads.""" + +from sqlbuild.loaders import loader + + +@loader +def raw_customers(ctx): + """Load synthetic customers.""" + return [{"customer_id": 7}] + + +@loader(depends_on=[raw_customers]) +def raw_orders(ctx): + """Load one order per loaded customer.""" + customers = ctx.loader(raw_customers) + rows = ctx.query(f"SELECT customer_id FROM {customers.destination}").fetchall() + return [{"order_id": row[0]} for row in rows] + + +@loader +def raw_returns(ctx): + """Load synthetic returns.""" + return [{"order_id": 7}] + + +@loader +def raw_inventory(ctx): + """Load synthetic inventory.""" + return [{"product_id": 1}] diff --git a/benchmarks/duckshop/edge_cases/orchestration/loaded_sources.yml b/benchmarks/duckshop/edge_cases/orchestration/loaded_sources.yml new file mode 100644 index 0000000000..7fed310ff0 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/loaded_sources.yml @@ -0,0 +1,33 @@ +sources: + - name: raw_customers + description: Customers the orders loader reads first. + managed: true + write_strategy: table + table: raw_customers + columns: + - name: customer_id + type: BIGINT + - name: raw_orders + description: Orders loaded after the customers loader they depend on. + managed: true + write_strategy: table + table: raw_orders + columns: + - name: order_id + type: BIGINT + - name: raw_returns + description: Returns that only a model hook reads. + managed: true + write_strategy: table + table: raw_returns + columns: + - name: order_id + type: BIGINT + - name: raw_inventory + description: A managed source that nothing the job builds reads. + managed: true + write_strategy: table + table: raw_inventory + columns: + - name: product_id + type: BIGINT diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/mod.rs b/crates/sqlbuild-analysis/src/planner_scope/main/mod.rs index 3fbafe8e4c..d95d839471 100644 --- a/crates/sqlbuild-analysis/src/planner_scope/main/mod.rs +++ b/crates/sqlbuild-analysis/src/planner_scope/main/mod.rs @@ -3,4 +3,5 @@ pub mod expand_loaders; pub mod intermediate_sources; pub mod managed_sources; +pub mod node_loads; pub mod source_selectors; diff --git a/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs b/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs new file mode 100644 index 0000000000..4d2f33a411 --- /dev/null +++ b/crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs @@ -0,0 +1,38 @@ +//! Source writers a loading build plans for one node besides the node itself. + +use crate::assembly::project::types::ObjectKey; +use crate::planner_scope::constants::{MODEL, SOURCE}; +use crate::planner_scope::models::LoaderGraph; +use std::collections::{BTreeMap, BTreeSet}; + +/// Managed sources each model's execution deps read and each managed source's declared loader chain, as `expand_loaders` keys them. +pub fn node_loads( + graph: &LoaderGraph, + upstream: &[(ObjectKey, Vec)], +) -> BTreeMap> { + let mut loads: BTreeMap> = BTreeMap::new(); + for (key, deps) in upstream { + let writers: Vec = if key.0 == MODEL { + deps.iter() + .filter(|dep| dep.0 == SOURCE && graph.source_loader(dep).is_some()) + .cloned() + .collect() + } else if let Some(chain) = graph + .source_loader(key) + .and_then(|loader| graph.loaders.get(loader)) + { + graph + .upstream(chain) + .iter() + .filter(|name| !graph.intermediate(name)) + .map(|name| graph.key(name)) + .collect() + } else { + continue; + }; + if !writers.is_empty() { + loads.entry(key.clone()).or_default().extend(writers); + } + } + loads +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs index 838ceeee51..b90277e273 100644 --- a/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs +++ b/crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs @@ -2,9 +2,12 @@ use crate::assembly::project::types::ObjectKey; use crate::planner_scope::main::expand_loaders::expand_loaders; +use crate::planner_scope::main::node_loads::node_loads; use crate::planner_scope::models::LoaderGraph; -use crate::planner_scope::tests::test_types::{DeepLoaderTestCase, LoaderTestCase}; -use std::collections::HashSet; +use crate::planner_scope::tests::test_types::{ + DeepLoaderTestCase, LoaderTestCase, NodeLoadsTestCase, +}; +use std::collections::{BTreeMap, BTreeSet, HashSet}; #[test] fn given_shared_loader_dependencies_when_expanding_then_preserves_depth_first_edges_and_breadth_first_names() @@ -120,3 +123,87 @@ fn given_deep_loader_chain_when_expanding_then_uses_iterative_traversal() { ); } } + +#[test] +fn given_loader_chains_and_unmanaged_reads_when_listing_node_loads_then_only_declared_source_writers_appear() + { + let test_cases = [ + NodeLoadsTestCase { + description: "a hook-read managed source is listed and an unmanaged read is not", + loaders: vec![("raw_returns", vec![])], + sources: vec![("raw_returns", Some("raw_returns")), ("raw_products", None)], + upstream: vec![ + ( + "model:orders", + vec![ + "source:raw_returns", + "source:raw_products", + "model:stg_orders", + ], + ), + ("model:stg_orders", vec!["source:raw_products"]), + ("source:raw_products", vec![]), + ], + expected_loads: vec![("model:orders", vec!["source:raw_returns"])], + }, + NodeLoadsTestCase { + // stage_orders has no declaring source, so a run that loads it still fails in the worker. + description: "a chain keeps declared loaders behind an intermediate one and drops the intermediate", + loaders: vec![ + ("raw_orders", vec!["stage_orders"]), + ("stage_orders", vec!["raw_customers"]), + ("raw_customers", vec![]), + ], + sources: vec![ + ("raw_orders", Some("raw_orders")), + ("raw_customers", Some("raw_customers")), + ], + upstream: vec![ + ("source:raw_orders", vec![]), + ("source:raw_customers", vec![]), + ], + expected_loads: vec![("source:raw_orders", vec!["source:raw_customers"])], + }, + ]; + let key = |text: &str| { + let (kind, name) = text.split_once(':').expect("kind:name"); + (kind.to_owned(), name.to_owned()) + }; + for test_case in test_cases { + let graph = LoaderGraph { + loaders: test_case + .loaders + .iter() + .map(|(name, deps)| { + ( + (*name).to_owned(), + deps.iter().map(|dep| (*dep).to_owned()).collect(), + ) + }) + .collect(), + sources: test_case + .sources + .iter() + .map(|(name, loader)| ((*name).to_owned(), loader.map(str::to_owned))) + .collect(), + }; + let upstream: Vec<(ObjectKey, Vec)> = test_case + .upstream + .iter() + .map(|(node, deps)| (key(node), deps.iter().map(|dep| key(dep)).collect())) + .collect(); + assert_eq!( + node_loads(&graph, &upstream), + test_case + .expected_loads + .iter() + .map(|(node, writers)| ( + key(node), + writers.iter().map(|writer| key(writer)).collect() + )) + .collect::>>(), + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-analysis/src/planner_scope/tests/test_types.rs b/crates/sqlbuild-analysis/src/planner_scope/tests/test_types.rs index f3f3d6c266..4df6c9eb40 100644 --- a/crates/sqlbuild-analysis/src/planner_scope/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/planner_scope/tests/test_types.rs @@ -7,6 +7,15 @@ pub(crate) struct LoaderTestCase { pub expected_selected: Vec<&'static str>, } +#[derive(Debug)] +pub(crate) struct NodeLoadsTestCase { + pub description: &'static str, + pub loaders: Vec<(&'static str, Vec<&'static str>)>, + pub sources: Vec<(&'static str, Option<&'static str>)>, + pub upstream: Vec<(&'static str, Vec<&'static str>)>, + pub expected_loads: Vec<(&'static str, Vec<&'static str>)>, +} + #[derive(Debug)] pub(crate) struct DeepLoaderTestCase { pub description: &'static str, diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs index 8853fb96b6..d7f4615435 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/mod.rs @@ -2,3 +2,4 @@ pub(crate) mod content; pub(crate) mod preparation; pub(crate) mod publication; pub(crate) mod reading; +pub(crate) mod write_set; diff --git a/crates/sqlbuild-orchestration/src/snapshots/_helpers/write_set.rs b/crates/sqlbuild-orchestration/src/snapshots/_helpers/write_set.rs new file mode 100644 index 0000000000..955f075b2b --- /dev/null +++ b/crates/sqlbuild-orchestration/src/snapshots/_helpers/write_set.rs @@ -0,0 +1,64 @@ +//! Write-set pieces read from published node definitions. + +use std::collections::{BTreeMap, BTreeSet}; + +use serde_json::Value; + +use crate::snapshots::constants::{ + LOADER_POINTER, LOADS_FIELD, MODEL_NODE_KIND, SOURCE_KEY_PREFIX, SOURCE_NODE_KIND, + WRITER_NODE_KINDS, +}; +use crate::snapshots::models::SnapshotNode; + +/// Selected writer keys, candidate source keys, and the sources selected models read when loading is on. +pub(crate) fn selected_writers( + selected: &BTreeMap, + loads_model_sources: bool, +) -> (BTreeSet, BTreeSet, BTreeSet) { + let mut writers: BTreeSet = BTreeSet::new(); + let mut sources: BTreeSet = BTreeSet::new(); + let mut model_reads: BTreeSet = BTreeSet::new(); + for (key, node) in selected { + if WRITER_NODE_KINDS.contains(&node.kind.as_str()) { + writers.insert(key.clone()); + } + if node.kind == SOURCE_NODE_KIND { + sources.insert(key.clone()); + } else if node.kind == MODEL_NODE_KIND && loads_model_sources { + model_reads.extend(source_deps(&node.definition)); + model_reads.extend(loads(&node.definition)); + } + } + sources.extend(model_reads.iter().cloned()); + (writers, sources, model_reads) +} + +/// A source with a non-null loader, which a loading build writes. +pub(crate) fn managed(node: &SnapshotNode) -> bool { + node.kind == SOURCE_NODE_KIND + && node + .definition + .pointer(LOADER_POINTER) + .is_some_and(|loader| !loader.is_null()) +} + +/// The source keys among a node's compiled `deps`. +pub(crate) fn source_deps(definition: &Value) -> impl Iterator + '_ { + keys(definition, "deps").filter(|key| key.starts_with(SOURCE_KEY_PREFIX)) +} + +/// The keys a node's definition lists in `loads`; snapshots published before it have none. +pub(crate) fn loads(definition: &Value) -> impl Iterator + '_ { + keys(definition, LOADS_FIELD) +} + +/// The string keys a node's definition lists under `field`. +fn keys<'a>(definition: &'a Value, field: &str) -> impl Iterator + 'a { + definition + .get(field) + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_owned) +} diff --git a/crates/sqlbuild-orchestration/src/snapshots/constants.rs b/crates/sqlbuild-orchestration/src/snapshots/constants.rs index cf3e62ba43..0072f17f66 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/constants.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/constants.rs @@ -55,5 +55,7 @@ pub(crate) const SOURCE_NODE_KIND: &str = "source"; pub(crate) const SOURCE_KEY_PREFIX: &str = "source:"; /// A source with a non-null loader here is managed, so automatic source loading writes it. pub(crate) const LOADER_POINTER: &str = "/entry/loader"; +/// Source writers a loading build of the node plans beyond its `deps`: hook and audit reads, loader chains. +pub(crate) const LOADS_FIELD: &str = "loads"; /// The frozen setting for `--no-load-sources`; `false` stops a build loading sources its models read. pub(crate) const LOAD_SOURCES_SETTING: &str = "load_sources"; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs b/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs index a8f24c6600..488e6cf351 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs @@ -5,15 +5,13 @@ use std::collections::{BTreeMap, BTreeSet}; use serde_json::Value; use uuid::Uuid; -use crate::snapshots::constants::{ - LOAD_SOURCES_SETTING, LOADER_POINTER, MODEL_NODE_KIND, SOURCE_KEY_PREFIX, SOURCE_NODE_KIND, - WRITER_NODE_KINDS, -}; +use crate::snapshots::_helpers::write_set::{loads, managed, selected_writers}; +use crate::snapshots::constants::LOAD_SOURCES_SETTING; use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; use crate::store::errors::ControlError; use crate::store::models::ControlTransaction; -/// Selected writers plus the managed sources a build loads: selected ones, and parents of selected models unless loading is off. +/// Selected writers, the managed sources a build loads, and loader chains of sources selected models read; a selected source runs only its own loader. pub(crate) async fn job_write_set( transaction: ControlTransaction, snapshot_id: Uuid, @@ -22,40 +20,14 @@ pub(crate) async fn job_write_set( let (transaction, selected) = read_snapshot_nodes(transaction, snapshot_id, Some(selection)).await?; let loads_model_sources = settings.get(LOAD_SOURCES_SETTING) != Some(&Value::Bool(false)); - let mut writers: BTreeSet = BTreeSet::new(); - let mut sources: BTreeSet = BTreeSet::new(); - for (key, node) in &selected { - if WRITER_NODE_KINDS.contains(&node.kind.as_str()) { - writers.insert(key.clone()); - } - if node.kind == SOURCE_NODE_KIND { - sources.insert(key.clone()); - } else if node.kind == MODEL_NODE_KIND && loads_model_sources { - sources.extend( - node.definition - .get("deps") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(Value::as_str) - .filter(|key| key.starts_with(SOURCE_KEY_PREFIX)) - .map(str::to_owned), - ); + let (mut writers, candidates, model_reads) = selected_writers(&selected, loads_model_sources); + let (transaction, sources) = + read_snapshot_nodes(transaction, snapshot_id, Some(&candidates)).await?; + for (key, node) in sources.into_iter().filter(|(_, node)| managed(node)) { + if model_reads.contains(&key) { + writers.extend(loads(&node.definition)); } + writers.insert(key); } - let (transaction, sources) = - read_snapshot_nodes(transaction, snapshot_id, Some(&sources)).await?; - writers.extend( - sources - .into_iter() - .filter(|(_, node)| { - node.kind == SOURCE_NODE_KIND - && node - .definition - .pointer(LOADER_POINTER) - .is_some_and(|loader| !loader.is_null()) - }) - .map(|(key, _)| key), - ); Ok((transaction, writers)) } diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index 9fd636bed0..5dec2a4ee4 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -135,7 +135,7 @@ pub(crate) fn json_column(value: &Value) -> Value { }) } -/// Two models, a seed, a read-only source, a loaded customers source and one person per role under test. +/// Two models, a seed, a read-only source, a customers source loaded after accounts and one person per role under test. pub(crate) async fn adhoc_fixture( store: &ControlStore, context: &AuditContext, @@ -154,8 +154,12 @@ pub(crate) async fn adhoc_fixture( .map(published_node) .collect(); let (key, mut managed) = published_node("source:shop.customers"); - managed.definition = json!({"entry": {"loader": "load_customers"}}); + managed.definition = + json!({"entry": {"loader": "load_customers"}, "loads": ["source:shop.accounts"]}); input.nodes.insert(key, managed); + let (key, mut upstream) = published_node("source:shop.accounts"); + upstream.definition = json!({"entry": {"loader": "load_accounts"}}); + input.nodes.insert(key, upstream); input .nodes .get_mut("model:customers") diff --git a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs index 1662bd3364..a3af046bbd 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs @@ -33,12 +33,22 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ expected_launch: Ok(true), }, AdhocCommandTestCase { - description: "a build claims the managed source it loads for a selected model", + description: "a build claims the managed source it loads for a selected model and its loader chain", person: AdhocPerson::Developer, command: &["build", "model:customers"], priority: 0, reason: "Refresh customers", - expected_check: json!({"selection": ["model:customers"], "write_set": ["model:customers", "source:shop.customers"], + expected_check: json!({"selection": ["model:customers"], "write_set": ["model:customers", "source:shop.accounts", "source:shop.customers"], + "permissions": [["run.adhoc", true]], "error": null, "launchable": true}), + expected_launch: Ok(true), + }, + AdhocCommandTestCase { + description: "a build of a selected source claims it alone, as only its own loader runs", + person: AdhocPerson::Developer, + command: &["build", "source:shop.customers"], + priority: 0, + reason: "Reload customers", + expected_check: json!({"selection": ["source:shop.customers"], "write_set": ["source:shop.customers"], "permissions": [["run.adhoc", true]], "error": null, "launchable": true}), expected_launch: Ok(true), }, @@ -276,7 +286,8 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ ); let audit = rows( store, - "SELECT after_json FROM sqb_security_audit WHERE action='run.adhoc_requested'", + "SELECT after_json FROM sqb_security_audit WHERE action='run.adhoc_requested' \ + ORDER BY occurred_at, event_id", Vec::new(), ) .await; @@ -293,6 +304,7 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ vec![ json!(["build", {}, "Refresh orders"]), json!(["build", {}, "Refresh customers"]), + json!(["build", {}, "Reload customers"]), json!(["build", {"load_sources": false}, "Refresh customers only"]), json!(["build", {"full_refresh": true, "warehouse": "BUILD_XS"}, "Rebuild orders now"]), json!(["test", {}, "Check orders"]), diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index bf15b867e1..e98184f7c9 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -7,7 +7,8 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyRef, PyResult}; use pyo3::types::{PyDict, PyDictMethods}; use pyo3::{IntoPyObject, pyfunction}; -use serde_json::json; +use serde_json::{Value, json}; +use sqlbuild_analysis::planner_scope::main::node_loads::node_loads; use sqlbuild_core::models::DiagnosticSeverity; use sqlbuild_discovery::project_validation::main::validate_orchestration_names::validate_orchestration_names; use sqlbuild_orchestration::snapshots::models::{SnapshotInput, SnapshotNode}; @@ -28,8 +29,10 @@ use crate::bindings::_helpers::native_host::orchestration_declarations::{ collect_declarations, orchestration_names, }; use crate::bindings::_helpers::native_host::orchestration_environment::environment_hash; +use crate::bindings::_helpers::planner_identity::graph::execution_deps; use crate::bindings::_helpers::planner_identity::identities::direct_model_version_identities; use crate::bindings::_helpers::planner_scope::scope::build_planner_scope; +use crate::bindings::_helpers::planner_scope::values::project_graph; use crate::bindings::_helpers::records::fields::field; /// Where declarations passed to the snapshot directly are reported as declared. @@ -140,6 +143,7 @@ pub(crate) fn orchestration_snapshot<'py>( input.documents.insert("lineage".into(), lineage); } input.nodes = identity_nodes(project.as_any(), input.nodes)?; + input.nodes = load_nodes(project.as_any(), input.nodes)?; Ok(NativeOrchestrationSnapshot { input }) } @@ -174,3 +178,35 @@ fn identity_nodes( } Ok(nodes) } + +/// `loads` lists the source writers a loading build of the node plans beyond its compiled `deps`, absent when none. +fn load_nodes( + project: &Bound<'_, PyAny>, + mut nodes: BTreeMap, +) -> PyResult> { + for ((kind, name), writers) in node_loads(&project_graph(project)?, &execution_deps(project)?) { + let Some(node) = nodes.get_mut(&format!("{kind}:{name}")) else { + continue; + }; + let deps: Vec = node + .definition + .get("deps") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + let loads: Vec = writers + .into_iter() + .map(|(kind, name)| format!("{kind}:{name}")) + .filter(|key| !listed(&deps, key)) + .collect(); + if !loads.is_empty() { + node.definition["loads"] = json!(loads); + } + } + Ok(nodes) +} + +/// Whether `key` is already one of the node's compiled `deps`. +fn listed(deps: &[Value], key: &str) -> bool { + deps.iter().any(|dep| dep.as_str() == Some(key)) +} diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index 29300f5896..884eaeaf23 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -1,5 +1,6 @@ """Publication cases and isolated server credentials.""" +import sqlite3 from collections.abc import Callable from contextlib import AbstractContextManager from dataclasses import dataclass, field @@ -118,11 +119,21 @@ class WorkerExecutionTestCase: files: dict[str, str] = field(default_factory=dict) files_after_publication: dict[str, str] = field(default_factory=dict) store_setup: str = "" + store_edit: Callable[[sqlite3.Connection], None] = lambda _: None concurrency_limits: dict[str, int] = field(default_factory=dict) launcher: Literal["native", "uncooperative", "docker"] = "native" warehouse_setup: Callable[[Path], None] = lambda _: None +@dataclass(frozen=True) +class UnclaimedLoadTestCase: + description: str + files: dict[str, str] + store_edit: Callable[[sqlite3.Connection], None] + refused_writer: str + expected_write_set: frozenset[str] + + @dataclass(frozen=True) class CheckExecutionTestCase(WorkerExecutionTestCase): description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py index 6e041ea16f..2e4d50dc5f 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/helpers.py @@ -3,6 +3,7 @@ from __future__ import annotations import base64 +import gzip import hashlib import json import os @@ -826,6 +827,7 @@ def scheduled_worker( control_store: Path = directory / "control/.sqlbuild/control.sqlite3" with sqlite3.connect(control_store) as control: _ = control.executescript(test_case.store_setup) + test_case.store_edit(control) monkeypatch.setenv("SQLBUILD_SCHEDULER_TEST_STORE", f"sqlite://{control_store}") monkeypatch.setenv( "SQLBUILD_PLATFORM_WEBHOOK_URL", @@ -1537,3 +1539,43 @@ def hold_write_lock(database: Path) -> duckdb.DuckDBPyConnection: except duckdb.IOException: assert time.monotonic() < deadline time.sleep(0.05) + + +LOADED_ORDER_FILES: dict[str, str] = { + "sources/loaded.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/loaded_sources.yml" + ).read_text(), + "python/loaders/loaded.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/loaded_sources.py" + ).read_text(), + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.sql" + ).read_text(), + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" + ).read_text(), +} + + +def without_loads(control: sqlite3.Connection, node_key: str) -> None: + """Drop a published node's `loads` in place; its content address and the worker's check stay.""" + for node_hash, body in control.execute( + "SELECT d.node_hash, d.body FROM sqb_node_definitions d JOIN sqb_snapshot_nodes n " + "ON n.node_hash = d.node_hash WHERE n.node_key = ?", + (node_key,), + ).fetchall(): + content: dict[str, Any] = json.loads(gzip.decompress(body)) + del content["definition"]["loads"] + _ = control.execute( + "UPDATE sqb_node_definitions SET body = ? WHERE node_hash = ?", + (gzip.compress(json.dumps(content).encode()), node_hash), + ) + + +def run_write_set(control_store: Path, run_id: str) -> frozenset[str]: + """The frozen write set a run's request recorded.""" + with closing(sqlite3.connect(control_store)) as control: + (request,) = control.execute( + "SELECT request_json FROM sqb_runs WHERE run_id=?", (run_id,) + ).fetchone() + return frozenset(json.loads(request)["write_set"]) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index dd84ed7816..ff3cfb341e 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -33,16 +33,20 @@ ClaimedWorker, GroupExecutionTestCase, SnapshotWorkerTestCase, + UnclaimedLoadTestCase, WorkerExecutionTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( + LOADED_ORDER_FILES, cancel_running_statement, claimed_worker, + run_write_set, running_statement, scheduled_worker, snapshot_api, wait_for_log, wait_for_run, + without_loads, worker_result, ) from tests.integration.src.sqlbuild.compiler.graph.helpers import write_files @@ -598,6 +602,93 @@ def test_given_a_scheduled_job_when_published_then_an_isolated_worker_reports_re assert run["status"] == test_case.expected_status, worker.log.read_text() +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + "a tag-selected job claims its loader chain and hook-read source, not unread ones", + "", + "SELECT amount FROM orders", + "success", + [(7,)], + files=LOADED_ORDER_FILES, + ) + ], + ids=lambda case: case.description, +) +def test_given_loads_when_a_worker_executes_then_the_run_claims_exactly_what_it_loads( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = worker_result(worker, test_case) + assert run["status"] == test_case.expected_status, worker.log.read_text() + assert run_write_set(worker.control_store, run["run_id"]) == frozenset( + {"model:orders", "source:raw_orders", "source:raw_customers", "source:raw_returns"} + ) + + +@pytest.mark.parametrize( + "test_case", + [ + UnclaimedLoadTestCase( + "the hook-read source is loaded only when the published model claims it", + files=LOADED_ORDER_FILES, + store_edit=lambda control: without_loads(control, "model:orders"), + refused_writer="source:raw_returns", + expected_write_set=frozenset( + {"model:orders", "source:raw_orders", "source:raw_customers"} + ), + ), + UnclaimedLoadTestCase( + # A known gap: no snapshot node exists for an intermediate loader, so `loads` leaves it out. + "an intermediate loader no source declares is never claimed, so the run still fails", + files={ + "sources/shipments.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.yml" + ).read_text(), + "python/loaders/shipments.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_sources.py" + ).read_text(), + "models/orders.sql": Path( + "benchmarks/duckshop/edge_cases/orchestration/intermediate_loader_orders.sql" + ).read_text(), + "orchestration/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py" + ).read_text(), + }, + store_edit=lambda _: None, + refused_writer="source:stage_shipments", + expected_write_set=frozenset({"model:orders", "source:raw_shipments"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_load_the_job_does_not_claim_when_a_worker_plans_then_the_run_is_refused( + test_case: UnclaimedLoadTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + worker_case: WorkerExecutionTestCase = WorkerExecutionTestCase( + test_case.description, + "", + "", + "failed", + [], + files=test_case.files, + store_edit=test_case.store_edit, + ) + with scheduled_worker(tmp_path, monkeypatch, worker_case) as worker: + run: dict[str, Any] = wait_for_run(worker.api, worker.process, worker.log) + log: str = worker.log.read_text() + assert (run["status"], run["node_counts_json"]) == ("failed", {"skipped": 1}), log + assert ( + f"planned writer {test_case.refused_writer} is outside the frozen run write set" in log + ) + assert run_write_set(worker.control_store, run["run_id"]) == test_case.expected_write_set + with duckdb.connect(str(worker.project / "warehouse.duckdb")) as warehouse: + assert warehouse.execute( + "SELECT count(*) FROM duckdb_tables() WHERE table_name = 'orders'" + ).fetchone() == (0,) + + @pytest.mark.parametrize( "test_case", [