Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
f9d6aa4
feat!: remove the dlt and ingestr integrations
kvlonge Nov 16, 2026
c091dc6
docs: remove the dlt and ingestr pages and mentions
kvlonge Nov 16, 2026
338d64c
fix: skip job group members not declared for the target
kvlonge Nov 15, 2026
a65e080
feat(config): gate warehouse round trips behind allow_warehouse_round…
kvlonge Nov 16, 2026
3363730
feat(build): run a round-trip loader between its task and the models …
kvlonge Nov 16, 2026
f573ca8
fix(dag): point Python edges into a declared loader at its source node
kvlonge Nov 16, 2026
1511a49
docs: describe warehouse round trips and loaders depending on tasks a…
kvlonge Nov 16, 2026
befbb6e
feat(selectors): select every node of one kind with kind:
kvlonge Nov 16, 2026
418deb4
perf(selectors): find the functions and seeds a selection needs in on…
kvlonge Nov 16, 2026
15ad6a4
fix(selectors): collect every graph key only when a selection needs them
kvlonge Nov 16, 2026
bcf4d6a
fix(compile): fail on every whole-graph refusal a plan makes, with th…
kvlonge Nov 17, 2026
ea5404d
test(build): expect the warehouse round-trip refusal for loaders afte…
kvlonge Nov 17, 2026
184ffa7
docs: regenerate the bundled selectors reference for kind:
kvlonge Nov 17, 2026
d22eeb3
refactor: read a Python node's SQL dependencies in one place
kvlonge Nov 17, 2026
1080aad
refactor: keep one public function per planner identity entry
kvlonge Nov 17, 2026
94b9962
test: expect compile to refuse cycles and loaders after SQL-reading t…
kvlonge Nov 17, 2026
9b3cdb2
Merge remote-tracking branch 'origin/main' into feat/warehouse-round-…
kvlonge Nov 17, 2026
ad644bf
test: list the round-trip setting in goldens and allow it in the Pyth…
kvlonge Nov 17, 2026
60be2eb
feat!: refuse warehouse round trips and graph cycles at compile time
kvlonge Nov 17, 2026
866c721
Merge remote-tracking branch 'origin/main' into feat/warehouse-round-…
kvlonge Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ E2E_DUCKDB_BUILD_CORE_PATHS := \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_template_expressions.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_terminal_failure.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_no_load_sources.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_duckdb_storage_key.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_source_end_audits.py \
tests/e2e/src/sqlbuild/cli/commands/main/build/test_python_check_outcomes.py \
Expand Down
26 changes: 17 additions & 9 deletions crates/sqlbuild-analysis/src/dag_artifact/_helpers/edges.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use crate::dag_artifact::_helpers::nodes::{
loader_or_source_id, node_id, python_node_id, sources_by_loader,
};
use crate::dag_artifact::_helpers::values::{object, text};
use crate::dag_artifact::constants::LOADER_KIND;
use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput};
use crate::dag_artifact::types::DagValue;
use crate::graph::models::ProjectGraph;
Expand All @@ -19,21 +20,28 @@ pub(crate) fn python_kinds(python: &DagPythonGraphInput) -> HashMap<&str, &str>
.collect()
}

fn python_edges(python: &DagPythonGraphInput) -> Vec<(String, String)> {
/// Python dependency and SQL read edges; a loader a source declares is drawn as that source.
fn python_edges(
python: &DagPythonGraphInput,
sources: &HashMap<&str, &str>,
) -> Vec<(String, String)> {
let kinds: HashMap<&str, &str> = python_kinds(python);
let id = |name: &str| {
let kind: &str = kinds[name];
if kind == LOADER_KIND {
loader_or_source_id(name, sources)
} else {
python_node_id(kind, name)
}
};
let mut edges: Vec<(String, String)> = python
.dependency_edges
.iter()
.map(|(upstream, downstream)| {
(
python_node_id(kinds[upstream.as_str()], upstream),
python_node_id(kinds[downstream.as_str()], downstream),
)
})
.map(|(upstream, downstream)| (id(upstream), id(downstream)))
.collect();
for node in &python.nodes {
for dependency in &node.sql_deps {
edges.push((node_id(dependency), python_node_id(&node.kind, &node.name)));
edges.push((node_id(dependency), id(&node.name)));
}
}
edges
Expand Down Expand Up @@ -61,7 +69,7 @@ pub(crate) fn edges(
}
}
if let Some(python) = python {
edges.extend(python_edges(python));
edges.extend(python_edges(python, &sources));
}
edges.sort();
DagValue::Array(
Expand Down
24 changes: 23 additions & 1 deletion crates/sqlbuild-analysis/src/dag_artifact/tests/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use crate::dag_artifact::main::build_dag_json::build_dag_json;
use crate::dag_artifact::models::{
DagColumnInput, DagLoaderInput, DagPythonNodeInput, DagSourceEntryInput, DagSourceInput,
};
use crate::dag_artifact::tests::test_types::BuildDagJsonTestCase;
use crate::dag_artifact::tests::test_types::{BuildDagJsonTestCase, DagEdgeTestCase};
use crate::dag_artifact::types::DagValue;
use crate::graph::models::ProjectGraph;
use crate::python_values::main::text_key::text_key;
Expand Down Expand Up @@ -89,6 +89,28 @@ pub(crate) fn built(test_case: &BuildDagJsonTestCase) -> Result<String, DagArtif
)
}

/// `(from_id, to_id)` of each edge of an edge test case's artifact, in artifact order.
pub(crate) fn dag_edges(test_case: &DagEdgeTestCase) -> Vec<(String, String)> {
let artifact: String = build_dag_json(
&ProjectGraph::default(),
&test_case.project,
Some(&test_case.python),
"orders_project",
)
.expect("artifact");
let lines: Vec<&str> = artifact.lines().map(str::trim).collect();
lines
.windows(2)
.filter_map(|pair| {
let from: &str = pair[0]
.strip_prefix("\"from_id\": \"")?
.strip_suffix("\",")?;
let to: &str = pair[1].strip_prefix("\"to_id\": \"")?.strip_suffix('"')?;
Some((from.to_owned(), to.to_owned()))
})
.collect()
}

/// Source meta with a `None` owner and the given team.
pub(crate) fn team_meta(value: DagValue) -> DagValue {
DagValue::Object(vec![
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
use crate::dag_artifact::errors::DagArtifactError;
use crate::dag_artifact::models::{DagProjectInput, DagPythonGraphInput};
use crate::dag_artifact::tests::helpers::{built, key, loader, python_node, source, team_meta};
use crate::dag_artifact::tests::test_types::BuildDagJsonTestCase;
use crate::dag_artifact::tests::helpers::{
built, dag_edges, key, loader, python_node, source, team_meta,
};
use crate::dag_artifact::tests::test_types::{BuildDagJsonTestCase, DagEdgeTestCase};
use crate::dag_artifact::types::DagValue;

const EMPTY_JSON: &str = "{
Expand Down Expand Up @@ -172,3 +174,46 @@ fn given_project_when_building_dag_json_then_python_artifact_agrees() {
);
}
}

#[test]
fn given_a_loader_depending_on_a_task_when_building_dag_json_then_the_edge_reaches_its_source() {
let test_cases = [DagEdgeTestCase {
description: "a task upstream of a declared loader points at the loader's source",
project: DagProjectInput {
sources: vec![source(
"partner_scores",
Some("partner_scores"),
DagValue::Object(Vec::new()),
)],
loaders: vec![loader("partner_scores", None, &[])],
..DagProjectInput::default()
},
python: DagPythonGraphInput {
nodes: vec![
python_node("task", "score_orders", vec![key("model", "orders")]),
python_node("loader", "partner_scores", Vec::new()),
],
dependency_edges: vec![("score_orders".to_owned(), "partner_scores".to_owned())],
kinds_by_name: vec![
("score_orders".to_owned(), "task".to_owned()),
("partner_scores".to_owned(), "loader".to_owned()),
],
},
expected_edges: &[
("model:orders", "task:score_orders"),
("task:score_orders", "source:partner_scores"),
],
}];
for test_case in test_cases {
assert_eq!(
dag_edges(&test_case),
test_case
.expected_edges
.iter()
.map(|(from, to)| ((*from).to_owned(), (*to).to_owned()))
.collect::<Vec<(String, String)>>(),
"{}",
test_case.description
);
}
}
8 changes: 8 additions & 0 deletions crates/sqlbuild-analysis/src/dag_artifact/tests/test_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,11 @@ pub(crate) struct SpecKindsTestCase {
pub(crate) materialization_type: &'static str,
pub(crate) expected_kinds: Vec<&'static str>,
}

pub(crate) struct DagEdgeTestCase {
pub(crate) description: &'static str,
pub(crate) project: DagProjectInput,
pub(crate) python: DagPythonGraphInput,
/// `(from_id, to_id)` of every edge, sorted.
pub(crate) expected_edges: &'static [(&'static str, &'static str)],
}
91 changes: 72 additions & 19 deletions crates/sqlbuild-analysis/src/graph/_helpers/selectors.rs
Original file line number Diff line number Diff line change
@@ -1,22 +1,24 @@
//! Python's planner selector grammar and resolution over the project graph.

use std::collections::BTreeSet;
use std::collections::{BTreeSet, HashSet};
use std::convert::Infallible;

use crate::assembly::project::types::ObjectKey;
use crate::graph::constants::{
EMPTY_SELECTOR_CODE, EMPTY_VALUE_CODE, EXPANSION_MARKER, FOLDER_SEPARATOR,
INTERSECTION_SEPARATOR, JOB_KIND, KIND_SEPARATOR, MISPLACED_MARKER_CODE, MISSING_NAME_CODE,
MODEL_RESOURCE, MODEL_ROOT, MODEL_ROOT_PREFIX, NAME_KIND, PATH_KIND, PATH_ROOT_CODE,
PATH_ROOT_ERROR, PATH_SELECTOR_CODE, PATH_SEPARATOR, PATTERN_CHARACTERS, PLANNER_DEFAULT_CODE,
SEED_RESOURCE, SOURCE_RESOURCE, SQL_FILE_SUFFIX, SUGGESTION_CUTOFF, SUGGESTION_LIMIT,
TABLE_FUNCTION_RESOURCE, TAG_KIND, TEST_KIND, UDF_RESOURCE, UNIT_TEST_CODE,
UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_KIND_CODE, UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE,
UNKNOWN_TAG_CODE, UNMAPPED_KIND_CODE, UNMAPPED_KINDS,
MODEL_RESOURCE, MODEL_ROOT, MODEL_ROOT_PREFIX, NAME_KIND, NODE_KIND_KIND, PATH_KIND,
PATH_ROOT_CODE, PATH_ROOT_ERROR, PATH_SELECTOR_CODE, PATH_SEPARATOR, PATTERN_CHARACTERS,
PLANNER_DEFAULT_CODE, SEED_RESOURCE, SELECTABLE_NODE_KINDS, SOURCE_RESOURCE, SQL_FILE_SUFFIX,
SUGGESTION_CUTOFF, SUGGESTION_LIMIT, TABLE_FUNCTION_RESOURCE, TAG_KIND, TEST_KIND,
UDF_RESOURCE, UNIT_TEST_CODE, UNIT_TEST_ONLY_TEST_AND_BUILD, UNKNOWN_KIND_CODE,
UNKNOWN_NAME_CODE, UNKNOWN_PATH_CODE, UNKNOWN_TAG_CODE, UNMAPPED_KIND_CODE, UNMAPPED_KINDS,
};
use crate::graph::errors::SelectorError;
use crate::graph::models::{
BuildResources, EdgeDirection as Direction, ParsedSelector, ProjectGraph,
};
use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach;
use sqlbuild_core::text::main::close_matches::close_matches;
use sqlbuild_core::text::main::glob_match::glob_match;

Expand All @@ -36,12 +38,12 @@ pub(crate) fn resolve(
select: &[String],
exclude: &[String],
) -> Result<Vec<ObjectKey>, SelectorError> {
let all: Keys = graph.names.iter().map(|(_, key)| key.clone()).collect();
let all = || -> Keys { graph.names.iter().map(|(_, key)| key.clone()).collect() };
if select.is_empty() && exclude.is_empty() {
return Ok(all.into_iter().collect());
return Ok(all().into_iter().collect());
}
let selected: Keys = if select.is_empty() {
all
all()
} else {
tokens(graph, select)?
};
Expand All @@ -52,21 +54,27 @@ pub(crate) fn resolve(
.collect())
}

/// Python's `expand_required_build_resources`: functions and seeds a selected scope needs.
/// Python's `expand_required_build_resources`: functions and seeds a selected scope needs; one walk from every selected key reaches the union of their upstream closures.
pub(crate) fn build_resources(
graph: &ProjectGraph,
selected: &Keys,
include: BuildResources,
) -> Keys {
let mut expanded: Keys = selected.clone();
for key in selected {
for upstream in graph.closure(key, Direction::Upstream) {
let wanted: bool = (include.upstream_functions && is_function(&upstream))
|| (include.upstream_seeds && upstream.0 == SEED_RESOURCE);
if wanted {
expanded.insert(upstream);
}
}
if include.upstream_functions || include.upstream_seeds {
let mut reached: HashSet<&ObjectKey> = HashSet::new();
let Ok(()) = breadth_first_reach(selected, None, &mut reached, |current| {
Ok::<_, Infallible>(graph.edges(current, Direction::Upstream))
});
expanded.extend(
reached
.into_iter()
.filter(|upstream| {
(include.upstream_functions && is_function(upstream))
|| (include.upstream_seeds && upstream.0 == SEED_RESOURCE)
})
.cloned(),
);
}
if include.downstream_functions {
for key in selected.iter().filter(|key| key.0 == MODEL_RESOURCE) {
Expand Down Expand Up @@ -273,6 +281,7 @@ fn is_selector_kind(prefix: &str) -> bool {
PATH_KIND,
TEST_KIND,
JOB_KIND,
NODE_KIND_KIND,
]
.contains(&prefix)
|| UNMAPPED_KINDS.contains(&prefix)
Expand All @@ -293,6 +302,7 @@ pub(crate) fn matched(graph: &ProjectGraph, kind: &str, value: &str) -> Resolved
}
PATH_KIND => folder(graph, value),
JOB_KIND => job(graph, value),
NODE_KIND_KIND => of_kind(graph, value),
TEST_KIND => Err(error(
UNIT_TEST_CODE,
format!(
Expand Down Expand Up @@ -373,6 +383,49 @@ fn lookup(graph: &ProjectGraph, kind: &str, value: &str) -> Resolved {
.collect())
}

/// The SQL resources of one node kind; Python node kinds need the Python node graph.
fn of_kind(graph: &ProjectGraph, value: &str) -> Resolved {
if !SELECTABLE_NODE_KINDS.contains(&value) {
return Err(unknown_node_kind(value));
}
if UNMAPPED_KINDS.contains(&value) {
return Err(error(
UNMAPPED_KIND_CODE,
format!(
"selector '{NODE_KIND_KIND}{KIND_SEPARATOR}{value}' selects Python nodes; \
they cannot be selected here"
),
));
}
let keys: Keys = graph
.names
.iter()
.filter(|(_, key)| key.0 == value)
.map(|(_, key)| key.clone())
.collect();
if keys.is_empty() {
return Err(error(
PLANNER_DEFAULT_CODE,
format!("no nodes found of kind '{value}'"),
));
}
Ok(keys)
}

/// `value` is not a node kind; suggest the closest ones.
pub(crate) fn unknown_node_kind(value: &str) -> SelectorError {
SelectorError {
code: UNKNOWN_NAME_CODE,
message: format!("unknown node kind '{value}'"),
help: suggestion(value, SELECTABLE_NODE_KINDS.to_vec()).or_else(|| {
Some(format!(
"node kinds are {}",
SELECTABLE_NODE_KINDS.join(", ")
))
}),
}
}

/// The nodes the jobs and job groups `value` names or matches select.
fn job(graph: &ProjectGraph, value: &str) -> Resolved {
let Some(jobs) = &graph.jobs else {
Expand Down
7 changes: 6 additions & 1 deletion crates/sqlbuild-analysis/src/graph/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@ pub(crate) const TAG_KIND: &str = "tag";
pub(crate) const PATH_KIND: &str = "path";
pub(crate) const TEST_KIND: &str = "test";
pub(crate) const JOB_KIND: &str = "job";
pub(crate) const NODE_KIND_KIND: &str = "kind";
/// The node kinds `kind:` selects, as the catalog names them: SQL resources, then Python nodes.
pub(crate) const SELECTABLE_NODE_KINDS: [&str; 9] = [
"model", "source", "seed", "udf", "table_fn", "task", "asset", "loader", "check",
];
/// Selector kinds Python parses but `sqb compile` cannot map to a compiled resource.
pub(crate) const UNMAPPED_KINDS: [&str; 4] = ["task", "asset", "loader", "check"];
pub(crate) const SUGGESTION_CUTOFF: f64 = 0.8;
Expand All @@ -35,7 +40,7 @@ pub(crate) const UNKNOWN_PATH_CODE: &str = "S009";
pub(crate) const UNMAPPED_KIND_CODE: &str = "S010";
pub(crate) const PATH_ROOT_CODE: &str = "S012";
pub(crate) const UNIT_TEST_CODE: &str = "S013";
pub(crate) const PLANNER_DEFAULT_CODE: &str = "S000";
pub const PLANNER_DEFAULT_CODE: &str = "S000";
pub(crate) const PATH_ROOT_ERROR: &str =
"path selectors require an explicit root: use 'models/' or 'python/'";
pub(crate) const UNIT_TEST_ONLY_TEST_AND_BUILD: &str =
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
use crate::graph::constants::{INTERSECTION_SEPARATOR, JOB_KIND, PATH_KIND, TAG_KIND};
use crate::graph::constants::{
INTERSECTION_SEPARATOR, JOB_KIND, NODE_KIND_KIND, PATH_KIND, TAG_KIND,
};
use crate::graph::models::ParsedSelector;

/// The atoms of `selection` that name one node: bare or typed names, without `+`, globs, tags, paths or jobs.
/// The atoms of `selection` that name one node: bare or typed names, without `+`, globs, tags, paths, jobs or kinds.
#[must_use]
pub fn named_selector_atoms(selection: &str) -> Vec<String> {
selection
Expand All @@ -20,7 +22,7 @@ fn names_one_node(atom: &str) -> bool {
upstream: false,
downstream: false,
}) => {
![TAG_KIND, PATH_KIND, JOB_KIND].contains(&kind.as_str())
![TAG_KIND, PATH_KIND, JOB_KIND, NODE_KIND_KIND].contains(&kind.as_str())
&& !crate::graph::_helpers::selectors::is_pattern(&value)
}
_ => false,
Expand Down
2 changes: 1 addition & 1 deletion crates/sqlbuild-analysis/src/graph/mod.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
//! The compiled project graph: dependency indexes and selector resolution.

pub(crate) mod _helpers;
pub(crate) mod constants;
pub mod constants;
pub mod errors;
pub mod main;
pub mod models;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ fn given_a_selection_when_listing_named_atoms_then_only_single_node_names_count(
selection: "tag:orders tag:finance",
expected_atoms: &[],
},
NamedAtomsTestCase {
description: "kinds do not count",
selection: "kind:model kind:task",
expected_atoms: &[],
},
NamedAtomsTestCase {
description: "folder paths do not count",
selection: "models/orders python/orders path:models/finance",
Expand Down
Loading
Loading