Skip to content

Commit 2b67a43

Browse files
authored
fix(orchestration): claim sources loaded through hook reads and loader chains (#1446)
* fix(orchestration): claim sources loaded through hook reads and loader chains Snapshots record `loads` on models (managed sources their hooks or audit gates read beyond `deps`) and on managed sources (the declared sources in their loader chain), additive and absent when empty. A job's write set claims a selected model's `loads` and each loaded source's chain, so the worker no longer refuses those runs. A directly selected source claims only itself, as only its own loader runs. An intermediate loader that no source declares stays unclaimed, so runs that need one still fail in the worker. Models and sources that gain `loads` show as changed once, on the first publication after upgrading. * test(orchestration): read ad-hoc launch audit rows in launch order
1 parent 6251280 commit 2b67a43

21 files changed

Lines changed: 533 additions & 46 deletions

File tree

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
MODEL (
2+
description 'Orders read from shipments whose loader needs an intermediate loader.',
3+
materialized table,
4+
tags [marts],
5+
);
6+
7+
SELECT CAST(order_id AS BIGINT) AS amount FROM __source("raw_shipments")
Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
"""A source loader that depends on an intermediate loader no source declares."""
2+
3+
from sqlbuild.loaders import loader
4+
5+
6+
@loader
7+
def stage_shipments(ctx):
8+
"""Stage synthetic shipments in the intermediate loader's own relation."""
9+
return [{"order_id": 9}]
10+
11+
12+
@loader(depends_on=[stage_shipments])
13+
def raw_shipments(ctx):
14+
"""Load the staged shipments."""
15+
staged = ctx.loader(stage_shipments)
16+
rows = ctx.query(f"SELECT order_id FROM {staged.destination}").fetchall()
17+
return [{"order_id": row[0]} for row in rows]
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
sources:
2+
- name: raw_shipments
3+
description: Shipments loaded after an intermediate loader that no source declares.
4+
managed: true
5+
write_strategy: table
6+
table: raw_shipments
7+
columns:
8+
- name: order_id
9+
type: BIGINT
Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
"""A tag-selected job whose model loads a loader chain and a hook-read source.
2+
3+
The build loads raw_orders and the raw_customers loader it depends on, and raw_returns, which only
4+
the model's pre-hook reads. The job's write set claims those three and not raw_inventory.
5+
"""
6+
7+
from sqlbuild.orchestration import job, schedule
8+
9+
annual_orders = job(
10+
"annual_orders",
11+
description="Build the marts layer and the sources it loads.",
12+
select="tag:marts",
13+
schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"),
14+
)
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
MODEL (
2+
description 'Orders that check returns before they build.',
3+
materialized table,
4+
tags [marts],
5+
pre_hooks [inline_sql('SELECT count(*) FROM __source("raw_returns")')],
6+
);
7+
8+
SELECT CAST(order_id AS BIGINT) AS amount FROM __source("raw_orders")
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
"""Loaders for a two-loader chain, a hook-read source and a source no job reads."""
2+
3+
from sqlbuild.loaders import loader
4+
5+
6+
@loader
7+
def raw_customers(ctx):
8+
"""Load synthetic customers."""
9+
return [{"customer_id": 7}]
10+
11+
12+
@loader(depends_on=[raw_customers])
13+
def raw_orders(ctx):
14+
"""Load one order per loaded customer."""
15+
customers = ctx.loader(raw_customers)
16+
rows = ctx.query(f"SELECT customer_id FROM {customers.destination}").fetchall()
17+
return [{"order_id": row[0]} for row in rows]
18+
19+
20+
@loader
21+
def raw_returns(ctx):
22+
"""Load synthetic returns."""
23+
return [{"order_id": 7}]
24+
25+
26+
@loader
27+
def raw_inventory(ctx):
28+
"""Load synthetic inventory."""
29+
return [{"product_id": 1}]
Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
sources:
2+
- name: raw_customers
3+
description: Customers the orders loader reads first.
4+
managed: true
5+
write_strategy: table
6+
table: raw_customers
7+
columns:
8+
- name: customer_id
9+
type: BIGINT
10+
- name: raw_orders
11+
description: Orders loaded after the customers loader they depend on.
12+
managed: true
13+
write_strategy: table
14+
table: raw_orders
15+
columns:
16+
- name: order_id
17+
type: BIGINT
18+
- name: raw_returns
19+
description: Returns that only a model hook reads.
20+
managed: true
21+
write_strategy: table
22+
table: raw_returns
23+
columns:
24+
- name: order_id
25+
type: BIGINT
26+
- name: raw_inventory
27+
description: A managed source that nothing the job builds reads.
28+
managed: true
29+
write_strategy: table
30+
table: raw_inventory
31+
columns:
32+
- name: product_id
33+
type: BIGINT

‎crates/sqlbuild-analysis/src/planner_scope/main/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@
33
pub mod expand_loaders;
44
pub mod intermediate_sources;
55
pub mod managed_sources;
6+
pub mod node_loads;
67
pub mod source_selectors;
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
//! Source writers a loading build plans for one node besides the node itself.
2+
3+
use crate::assembly::project::types::ObjectKey;
4+
use crate::planner_scope::constants::{MODEL, SOURCE};
5+
use crate::planner_scope::models::LoaderGraph;
6+
use std::collections::{BTreeMap, BTreeSet};
7+
8+
/// Managed sources each model's execution deps read and each managed source's declared loader chain, as `expand_loaders` keys them.
9+
pub fn node_loads(
10+
graph: &LoaderGraph,
11+
upstream: &[(ObjectKey, Vec<ObjectKey>)],
12+
) -> BTreeMap<ObjectKey, BTreeSet<ObjectKey>> {
13+
let mut loads: BTreeMap<ObjectKey, BTreeSet<ObjectKey>> = BTreeMap::new();
14+
for (key, deps) in upstream {
15+
let writers: Vec<ObjectKey> = if key.0 == MODEL {
16+
deps.iter()
17+
.filter(|dep| dep.0 == SOURCE && graph.source_loader(dep).is_some())
18+
.cloned()
19+
.collect()
20+
} else if let Some(chain) = graph
21+
.source_loader(key)
22+
.and_then(|loader| graph.loaders.get(loader))
23+
{
24+
graph
25+
.upstream(chain)
26+
.iter()
27+
.filter(|name| !graph.intermediate(name))
28+
.map(|name| graph.key(name))
29+
.collect()
30+
} else {
31+
continue;
32+
};
33+
if !writers.is_empty() {
34+
loads.entry(key.clone()).or_default().extend(writers);
35+
}
36+
}
37+
loads
38+
}

‎crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs‎

Lines changed: 89 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,12 @@
22
33
use crate::assembly::project::types::ObjectKey;
44
use crate::planner_scope::main::expand_loaders::expand_loaders;
5+
use crate::planner_scope::main::node_loads::node_loads;
56
use crate::planner_scope::models::LoaderGraph;
6-
use crate::planner_scope::tests::test_types::{DeepLoaderTestCase, LoaderTestCase};
7-
use std::collections::HashSet;
7+
use crate::planner_scope::tests::test_types::{
8+
DeepLoaderTestCase, LoaderTestCase, NodeLoadsTestCase,
9+
};
10+
use std::collections::{BTreeMap, BTreeSet, HashSet};
811

912
#[test]
1013
fn given_shared_loader_dependencies_when_expanding_then_preserves_depth_first_edges_and_breadth_first_names()
@@ -120,3 +123,87 @@ fn given_deep_loader_chain_when_expanding_then_uses_iterative_traversal() {
120123
);
121124
}
122125
}
126+
127+
#[test]
128+
fn given_loader_chains_and_unmanaged_reads_when_listing_node_loads_then_only_declared_source_writers_appear()
129+
{
130+
let test_cases = [
131+
NodeLoadsTestCase {
132+
description: "a hook-read managed source is listed and an unmanaged read is not",
133+
loaders: vec![("raw_returns", vec![])],
134+
sources: vec![("raw_returns", Some("raw_returns")), ("raw_products", None)],
135+
upstream: vec![
136+
(
137+
"model:orders",
138+
vec![
139+
"source:raw_returns",
140+
"source:raw_products",
141+
"model:stg_orders",
142+
],
143+
),
144+
("model:stg_orders", vec!["source:raw_products"]),
145+
("source:raw_products", vec![]),
146+
],
147+
expected_loads: vec![("model:orders", vec!["source:raw_returns"])],
148+
},
149+
NodeLoadsTestCase {
150+
// stage_orders has no declaring source, so a run that loads it still fails in the worker.
151+
description: "a chain keeps declared loaders behind an intermediate one and drops the intermediate",
152+
loaders: vec![
153+
("raw_orders", vec!["stage_orders"]),
154+
("stage_orders", vec!["raw_customers"]),
155+
("raw_customers", vec![]),
156+
],
157+
sources: vec![
158+
("raw_orders", Some("raw_orders")),
159+
("raw_customers", Some("raw_customers")),
160+
],
161+
upstream: vec![
162+
("source:raw_orders", vec![]),
163+
("source:raw_customers", vec![]),
164+
],
165+
expected_loads: vec![("source:raw_orders", vec!["source:raw_customers"])],
166+
},
167+
];
168+
let key = |text: &str| {
169+
let (kind, name) = text.split_once(':').expect("kind:name");
170+
(kind.to_owned(), name.to_owned())
171+
};
172+
for test_case in test_cases {
173+
let graph = LoaderGraph {
174+
loaders: test_case
175+
.loaders
176+
.iter()
177+
.map(|(name, deps)| {
178+
(
179+
(*name).to_owned(),
180+
deps.iter().map(|dep| (*dep).to_owned()).collect(),
181+
)
182+
})
183+
.collect(),
184+
sources: test_case
185+
.sources
186+
.iter()
187+
.map(|(name, loader)| ((*name).to_owned(), loader.map(str::to_owned)))
188+
.collect(),
189+
};
190+
let upstream: Vec<(ObjectKey, Vec<ObjectKey>)> = test_case
191+
.upstream
192+
.iter()
193+
.map(|(node, deps)| (key(node), deps.iter().map(|dep| key(dep)).collect()))
194+
.collect();
195+
assert_eq!(
196+
node_loads(&graph, &upstream),
197+
test_case
198+
.expected_loads
199+
.iter()
200+
.map(|(node, writers)| (
201+
key(node),
202+
writers.iter().map(|writer| key(writer)).collect()
203+
))
204+
.collect::<BTreeMap<ObjectKey, BTreeSet<ObjectKey>>>(),
205+
"{}",
206+
test_case.description
207+
);
208+
}
209+
}

0 commit comments

Comments
 (0)