Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
MODEL (
description 'Orders read from shipments whose loader needs an intermediate loader.',
materialized table,
tags [marts],
);

SELECT CAST(order_id AS BIGINT) AS amount FROM __source("raw_shipments")
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
"""A source loader that depends on an intermediate loader no source declares."""

from sqlbuild.loaders import loader


@loader
def stage_shipments(ctx):
"""Stage synthetic shipments in the intermediate loader's own relation."""
return [{"order_id": 9}]


@loader(depends_on=[stage_shipments])
def raw_shipments(ctx):
"""Load the staged shipments."""
staged = ctx.loader(stage_shipments)
rows = ctx.query(f"SELECT order_id FROM {staged.destination}").fetchall()
return [{"order_id": row[0]} for row in rows]
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
sources:
- name: raw_shipments
description: Shipments loaded after an intermediate loader that no source declares.
managed: true
write_strategy: table
table: raw_shipments
columns:
- name: order_id
type: BIGINT
14 changes: 14 additions & 0 deletions benchmarks/duckshop/edge_cases/orchestration/loaded_orders.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
"""A tag-selected job whose model loads a loader chain and a hook-read source.

The build loads raw_orders and the raw_customers loader it depends on, and raw_returns, which only
the model's pre-hook reads. The job's write set claims those three and not raw_inventory.
"""

from sqlbuild.orchestration import job, schedule

annual_orders = job(
"annual_orders",
description="Build the marts layer and the sources it loads.",
select="tag:marts",
schedule=schedule(cron="0 0 * * *", timezone="UTC", starts_at="2000-01-01T00:00:00Z"),
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
MODEL (
description 'Orders that check returns before they build.',
materialized table,
tags [marts],
pre_hooks [inline_sql('SELECT count(*) FROM __source("raw_returns")')],
);

SELECT CAST(order_id AS BIGINT) AS amount FROM __source("raw_orders")
29 changes: 29 additions & 0 deletions benchmarks/duckshop/edge_cases/orchestration/loaded_sources.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
"""Loaders for a two-loader chain, a hook-read source and a source no job reads."""

from sqlbuild.loaders import loader


@loader
def raw_customers(ctx):
"""Load synthetic customers."""
return [{"customer_id": 7}]


@loader(depends_on=[raw_customers])
def raw_orders(ctx):
"""Load one order per loaded customer."""
customers = ctx.loader(raw_customers)
rows = ctx.query(f"SELECT customer_id FROM {customers.destination}").fetchall()
return [{"order_id": row[0]} for row in rows]


@loader
def raw_returns(ctx):
"""Load synthetic returns."""
return [{"order_id": 7}]


@loader
def raw_inventory(ctx):
"""Load synthetic inventory."""
return [{"product_id": 1}]
33 changes: 33 additions & 0 deletions benchmarks/duckshop/edge_cases/orchestration/loaded_sources.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
sources:
- name: raw_customers
description: Customers the orders loader reads first.
managed: true
write_strategy: table
table: raw_customers
columns:
- name: customer_id
type: BIGINT
- name: raw_orders
description: Orders loaded after the customers loader they depend on.
managed: true
write_strategy: table
table: raw_orders
columns:
- name: order_id
type: BIGINT
- name: raw_returns
description: Returns that only a model hook reads.
managed: true
write_strategy: table
table: raw_returns
columns:
- name: order_id
type: BIGINT
- name: raw_inventory
description: A managed source that nothing the job builds reads.
managed: true
write_strategy: table
table: raw_inventory
columns:
- name: product_id
type: BIGINT
1 change: 1 addition & 0 deletions crates/sqlbuild-analysis/src/planner_scope/main/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,5 @@
pub mod expand_loaders;
pub mod intermediate_sources;
pub mod managed_sources;
pub mod node_loads;
pub mod source_selectors;
38 changes: 38 additions & 0 deletions crates/sqlbuild-analysis/src/planner_scope/main/node_loads.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
//! Source writers a loading build plans for one node besides the node itself.

use crate::assembly::project::types::ObjectKey;
use crate::planner_scope::constants::{MODEL, SOURCE};
use crate::planner_scope::models::LoaderGraph;
use std::collections::{BTreeMap, BTreeSet};

/// Managed sources each model's execution deps read and each managed source's declared loader chain, as `expand_loaders` keys them.
pub fn node_loads(
graph: &LoaderGraph,
upstream: &[(ObjectKey, Vec<ObjectKey>)],
) -> BTreeMap<ObjectKey, BTreeSet<ObjectKey>> {
let mut loads: BTreeMap<ObjectKey, BTreeSet<ObjectKey>> = BTreeMap::new();
for (key, deps) in upstream {
let writers: Vec<ObjectKey> = if key.0 == MODEL {
deps.iter()
.filter(|dep| dep.0 == SOURCE && graph.source_loader(dep).is_some())
.cloned()
.collect()
} else if let Some(chain) = graph
.source_loader(key)
.and_then(|loader| graph.loaders.get(loader))
{
graph
.upstream(chain)
.iter()
.filter(|name| !graph.intermediate(name))
.map(|name| graph.key(name))
.collect()
} else {
continue;
};
if !writers.is_empty() {
loads.entry(key.clone()).or_default().extend(writers);
}
}
loads
}
91 changes: 89 additions & 2 deletions crates/sqlbuild-analysis/src/planner_scope/tests/test_loaders.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,12 @@

use crate::assembly::project::types::ObjectKey;
use crate::planner_scope::main::expand_loaders::expand_loaders;
use crate::planner_scope::main::node_loads::node_loads;
use crate::planner_scope::models::LoaderGraph;
use crate::planner_scope::tests::test_types::{DeepLoaderTestCase, LoaderTestCase};
use std::collections::HashSet;
use crate::planner_scope::tests::test_types::{
DeepLoaderTestCase, LoaderTestCase, NodeLoadsTestCase,
};
use std::collections::{BTreeMap, BTreeSet, HashSet};

#[test]
fn given_shared_loader_dependencies_when_expanding_then_preserves_depth_first_edges_and_breadth_first_names()
Expand Down Expand Up @@ -120,3 +123,87 @@ fn given_deep_loader_chain_when_expanding_then_uses_iterative_traversal() {
);
}
}

#[test]
fn given_loader_chains_and_unmanaged_reads_when_listing_node_loads_then_only_declared_source_writers_appear()
{
let test_cases = [
NodeLoadsTestCase {
description: "a hook-read managed source is listed and an unmanaged read is not",
loaders: vec![("raw_returns", vec![])],
sources: vec![("raw_returns", Some("raw_returns")), ("raw_products", None)],
upstream: vec![
(
"model:orders",
vec![
"source:raw_returns",
"source:raw_products",
"model:stg_orders",
],
),
("model:stg_orders", vec!["source:raw_products"]),
("source:raw_products", vec![]),
],
expected_loads: vec![("model:orders", vec!["source:raw_returns"])],
},
NodeLoadsTestCase {
// stage_orders has no declaring source, so a run that loads it still fails in the worker.
description: "a chain keeps declared loaders behind an intermediate one and drops the intermediate",
loaders: vec![
("raw_orders", vec!["stage_orders"]),
("stage_orders", vec!["raw_customers"]),
("raw_customers", vec![]),
],
sources: vec![
("raw_orders", Some("raw_orders")),
("raw_customers", Some("raw_customers")),
],
upstream: vec![
("source:raw_orders", vec![]),
("source:raw_customers", vec![]),
],
expected_loads: vec![("source:raw_orders", vec!["source:raw_customers"])],
},
];
let key = |text: &str| {
let (kind, name) = text.split_once(':').expect("kind:name");
(kind.to_owned(), name.to_owned())
};
for test_case in test_cases {
let graph = LoaderGraph {
loaders: test_case
.loaders
.iter()
.map(|(name, deps)| {
(
(*name).to_owned(),
deps.iter().map(|dep| (*dep).to_owned()).collect(),
)
})
.collect(),
sources: test_case
.sources
.iter()
.map(|(name, loader)| ((*name).to_owned(), loader.map(str::to_owned)))
.collect(),
};
let upstream: Vec<(ObjectKey, Vec<ObjectKey>)> = test_case
.upstream
.iter()
.map(|(node, deps)| (key(node), deps.iter().map(|dep| key(dep)).collect()))
.collect();
assert_eq!(
node_loads(&graph, &upstream),
test_case
.expected_loads
.iter()
.map(|(node, writers)| (
key(node),
writers.iter().map(|writer| key(writer)).collect()
))
.collect::<BTreeMap<ObjectKey, BTreeSet<ObjectKey>>>(),
"{}",
test_case.description
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,15 @@ pub(crate) struct LoaderTestCase {
pub expected_selected: Vec<&'static str>,
}

#[derive(Debug)]
pub(crate) struct NodeLoadsTestCase {
pub description: &'static str,
pub loaders: Vec<(&'static str, Vec<&'static str>)>,
pub sources: Vec<(&'static str, Option<&'static str>)>,
pub upstream: Vec<(&'static str, Vec<&'static str>)>,
pub expected_loads: Vec<(&'static str, Vec<&'static str>)>,
}

#[derive(Debug)]
pub(crate) struct DeepLoaderTestCase {
pub description: &'static str,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@ pub(crate) mod content;
pub(crate) mod preparation;
pub(crate) mod publication;
pub(crate) mod reading;
pub(crate) mod write_set;
64 changes: 64 additions & 0 deletions crates/sqlbuild-orchestration/src/snapshots/_helpers/write_set.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
//! Write-set pieces read from published node definitions.

use std::collections::{BTreeMap, BTreeSet};

use serde_json::Value;

use crate::snapshots::constants::{
LOADER_POINTER, LOADS_FIELD, MODEL_NODE_KIND, SOURCE_KEY_PREFIX, SOURCE_NODE_KIND,
WRITER_NODE_KINDS,
};
use crate::snapshots::models::SnapshotNode;

/// Selected writer keys, candidate source keys, and the sources selected models read when loading is on.
pub(crate) fn selected_writers(
selected: &BTreeMap<String, SnapshotNode>,
loads_model_sources: bool,
) -> (BTreeSet<String>, BTreeSet<String>, BTreeSet<String>) {
let mut writers: BTreeSet<String> = BTreeSet::new();
let mut sources: BTreeSet<String> = BTreeSet::new();
let mut model_reads: BTreeSet<String> = BTreeSet::new();
for (key, node) in selected {
if WRITER_NODE_KINDS.contains(&node.kind.as_str()) {
writers.insert(key.clone());
}
if node.kind == SOURCE_NODE_KIND {
sources.insert(key.clone());
} else if node.kind == MODEL_NODE_KIND && loads_model_sources {
model_reads.extend(source_deps(&node.definition));
model_reads.extend(loads(&node.definition));
}
}
sources.extend(model_reads.iter().cloned());
(writers, sources, model_reads)
}

/// A source with a non-null loader, which a loading build writes.
pub(crate) fn managed(node: &SnapshotNode) -> bool {
node.kind == SOURCE_NODE_KIND
&& node
.definition
.pointer(LOADER_POINTER)
.is_some_and(|loader| !loader.is_null())
}

/// The source keys among a node's compiled `deps`.
pub(crate) fn source_deps(definition: &Value) -> impl Iterator<Item = String> + '_ {
keys(definition, "deps").filter(|key| key.starts_with(SOURCE_KEY_PREFIX))
}

/// The keys a node's definition lists in `loads`; snapshots published before it have none.
pub(crate) fn loads(definition: &Value) -> impl Iterator<Item = String> + '_ {
keys(definition, LOADS_FIELD)
}

/// The string keys a node's definition lists under `field`.
fn keys<'a>(definition: &'a Value, field: &str) -> impl Iterator<Item = String> + 'a {
definition
.get(field)
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(Value::as_str)
.map(str::to_owned)
}
2 changes: 2 additions & 0 deletions crates/sqlbuild-orchestration/src/snapshots/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,5 +55,7 @@ pub(crate) const SOURCE_NODE_KIND: &str = "source";
pub(crate) const SOURCE_KEY_PREFIX: &str = "source:";
/// A source with a non-null loader here is managed, so automatic source loading writes it.
pub(crate) const LOADER_POINTER: &str = "/entry/loader";
/// Source writers a loading build of the node plans beyond its `deps`: hook and audit reads, loader chains.
pub(crate) const LOADS_FIELD: &str = "loads";
/// The frozen setting for `--no-load-sources`; `false` stops a build loading sources its models read.
pub(crate) const LOAD_SOURCES_SETTING: &str = "load_sources";
Loading
Loading