Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
f9d6aa4
feat!: remove the dlt and ingestr integrations
kvlonge Nov 16, 2026
c091dc6
docs: remove the dlt and ingestr pages and mentions
kvlonge Nov 16, 2026
338d64c
fix: skip job group members not declared for the target
kvlonge Nov 15, 2026
a65e080
feat(config): gate warehouse round trips behind allow_warehouse_round…
kvlonge Nov 16, 2026
3363730
feat(build): run a round-trip loader between its task and the models …
kvlonge Nov 16, 2026
f573ca8
fix(dag): point Python edges into a declared loader at its source node
kvlonge Nov 16, 2026
1511a49
docs: describe warehouse round trips and loaders depending on tasks a…
kvlonge Nov 16, 2026
befbb6e
feat(selectors): select every node of one kind with kind:
kvlonge Nov 16, 2026
418deb4
perf(selectors): find the functions and seeds a selection needs in on…
kvlonge Nov 16, 2026
15ad6a4
fix(selectors): collect every graph key only when a selection needs them
kvlonge Nov 16, 2026
bcf4d6a
fix(compile): fail on every whole-graph refusal a plan makes, with th…
kvlonge Nov 17, 2026
ea5404d
test(build): expect the warehouse round-trip refusal for loaders afte…
kvlonge Nov 17, 2026
184ffa7
docs: regenerate the bundled selectors reference for kind:
kvlonge Nov 17, 2026
d22eeb3
refactor: read a Python node's SQL dependencies in one place
kvlonge Nov 17, 2026
1080aad
refactor: keep one public function per planner identity entry
kvlonge Nov 17, 2026
94b9962
test: expect compile to refuse cycles and loaders after SQL-reading t…
kvlonge Nov 17, 2026
1032e58
test: add DuckShop cases for warehouse round trips and off-target gro…
kvlonge Nov 17, 2026
d08bff4
Merge origin/main into test/duckshop-round-trip-cases
kvlonge Nov 17, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
MODEL (description "Orders to score", materialized table);

SELECT 1 AS order_id UNION ALL SELECT 2 AS order_id
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
MODEL (description "Orders with their partner score", materialized table);

SELECT o.order_id, s.score
FROM __ref("orders") AS o
JOIN __source("partner_scores") AS s ON o.order_id = s.order_id
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
from python.tasks.scoring import score_orders

from sqlbuild.loaders import loader


@loader(depends_on=[score_orders])
def partner_scores(ctx):
"""Load the scores the partner returned."""
result = ctx.result_of(node_function=score_orders)
return [{"order_id": o, "score": s} for o, s in result.payload["scores"]]
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
from sqlbuild.refs import model
from sqlbuild.tasks import task


@task(depends_on=model("orders"))
def score_orders(ctx):
"""Send the orders to the partner and keep the scores it returns."""
relation = ctx.relation(model("orders"))
rows = ctx.query(f"SELECT order_id FROM {relation} ORDER BY order_id").fetchall()
return ctx.result(payload={"scores": [[row[0], row[0] * 10] for row in rows]})
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
sources:
- name: partner_scores
description: Scores the partner returned for each order
managed: true
write_strategy: table
columns:
- name: order_id
type: INTEGER
- name: score
type: INTEGER
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
from sqlbuild.refs import model
from sqlbuild.tasks import task


@task(depends_on=model("scored_orders"))
def score_orders(ctx):
"""Send the scored orders to the partner."""
return ctx.result(payload={})
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
MODEL (description "Ranks per order", materialized table);

SELECT order_id FROM __ref("order_totals")
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
MODEL (description "Totals per order", materialized table);

SELECT order_id FROM __ref("order_ranks")
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
from python.loaders.partner import partner_scores

from sqlbuild.tasks import task


@task(depends_on=[partner_scores])
def report_scores(ctx):
"""Report the scores the partner returned."""
return ctx.result(payload={})
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
from sqlbuild.tasks import task


@task()
def score_orders(ctx):
"""Fetch the partner scores."""
return ctx.result(payload={})
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
name = "daily_job_group"
description = "Orders everywhere; restocking only where it is declared."
jobs = ["orders_job", "restock_job"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
name = "orders_job"
description = "Build orders on every target."
select = "+orders"
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
name = "restock_job"
description = "Restock customers, declared only for production."
select = "customers"
targets = ["prod"]
61 changes: 22 additions & 39 deletions tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -1544,9 +1544,23 @@ def node_results_in(*, db_path: Path, schema: str, node_type: str) -> list[tuple
FAILING_SCORING_TASK: str = ' raise RuntimeError("partner service unavailable")\n'


ROUND_TRIP_FIXTURE: Path = Path("benchmarks/duckshop/edge_cases/loaders/round_trip")


def round_trip_project_files(*, project_settings: str, task_body: str) -> dict[str, str]:
"""Orders -> score task -> partner loader -> partner_scores source -> scored_orders model."""
"""DuckShop's round trip: orders -> score task -> partner loader -> source -> scored_orders."""

project: Path = ROUND_TRIP_FIXTURE / "project"
files: dict[str, str] = {
path.relative_to(project).as_posix(): path.read_text()
for path in sorted(
[*project.rglob("*.py"), *project.rglob("*.sql"), *project.rglob("*.yml")]
)
}
assert SCORING_TASK in files["python/tasks/scoring.py"]
files["python/tasks/scoring.py"] = files["python/tasks/scoring.py"].replace(
SCORING_TASK, task_body
)
return {
"sqlbuild_project.toml": (
'name = "partner_scoring"\n'
Expand All @@ -1558,47 +1572,16 @@ def round_trip_project_files(*, project_settings: str, task_body: str) -> dict[s
'connection = "local"\n'
'schema = "main"\n' + project_settings
),
"models/orders.sql": (
'MODEL (description "Orders to score", materialized table);\n\n'
"SELECT 1 AS order_id UNION ALL SELECT 2 AS order_id\n"
),
"sources/partner.yml": (
"sources:\n"
" - name: partner_scores\n"
" description: Scores the partner returned for each order\n"
" managed: true\n"
" write_strategy: table\n"
" columns:\n"
" - name: order_id\n"
" type: INTEGER\n"
" - name: score\n"
" type: INTEGER\n"
),
"models/scored_orders.sql": (
'MODEL (description "Orders with their partner score", materialized table);\n\n'
"SELECT o.order_id, s.score\n"
'FROM __ref("orders") AS o\n'
'JOIN __source("partner_scores") AS s ON o.order_id = s.order_id\n'
),
"python/tasks/scoring.py": (
"from sqlbuild.refs import model\n"
"from sqlbuild.tasks import task\n\n\n"
'@task(depends_on=model("orders"))\n'
"def score_orders(ctx):\n"
' """Send the orders to the partner and keep the scores it returns."""\n' + task_body
),
"python/loaders/partner.py": (
"from sqlbuild.loaders import loader\n\n"
"from python.tasks.scoring import score_orders\n\n\n"
"@loader(depends_on=[score_orders])\n"
"def partner_scores(ctx):\n"
' """Load the scores the partner returned."""\n'
" result = ctx.result_of(node_function=score_orders)\n"
' return [{"order_id": o, "score": s} for o, s in result.payload["scores"]]\n'
),
**files,
}


def round_trip_refusal(name: str) -> str:
"""One of DuckShop's round-trip refusal variants, such as `cycle_task.py`."""

return (ROUND_TRIP_FIXTURE / "refusals" / name).read_text()


def build_round_trip_project(
*, tmp_path: Path, test_case: WarehouseRoundTripE2ETestCase
) -> tuple[Path, str]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import json
import subprocess
from itertools import chain
from pathlib import Path

import pytest
Expand All @@ -13,12 +14,21 @@
DiamondDagE2ETestCase,
DiamondDagJsonE2ETestCase,
)
from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import (
ALLOW_ROUND_TRIPS,
SCORING_TASK,
round_trip_project_files,
)
from tests.e2e.src.sqlbuild.cli.commands.main.lineage.helpers import (
DIAMOND_EDGE_COUNT,
diamond_model_names,
prepare_diamond_lineage_project,
)
from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_waffle_shop, run_sqb
from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import (
prepare_inline_project,
prepare_waffle_shop,
run_sqb,
)


@pytest.mark.parametrize(
Expand Down Expand Up @@ -93,6 +103,60 @@ def test_given_waffle_shop_when_running_dag_json_then_it_reports_static_graph(
assert "action" not in result.stdout


@pytest.mark.parametrize(
"test_case",
[
DagJsonBuildE2ETestCase(
description="a task upstream of a declared loader points at the loader's source",
command=("dag", "--json"),
expected_exit_code=0,
expected_project_name="partner_scoring",
expected_node_ids=(
"model:orders",
"task:score_orders",
"source:partner_scores",
"model:scored_orders",
),
expected_edge_pairs=(
("model:orders", "task:score_orders"),
("task:score_orders", "source:partner_scores"),
("source:partner_scores", "model:scored_orders"),
),
expected_check_ids=(),
expected_absent_fragments=("loader:partner_scores",),
)
],
ids=lambda case: case.description,
)
def test_given_a_round_trip_when_running_dag_json_then_every_edge_ends_at_a_node(
test_case: DagJsonBuildE2ETestCase,
tmp_path: Path,
) -> None:
project_dir: Path = prepare_inline_project(
tmp_path=tmp_path,
project_name="partner_scoring",
repo_files=round_trip_project_files(
project_settings=ALLOW_ROUND_TRIPS, task_body=SCORING_TASK
),
)
result: subprocess.CompletedProcess[str] = run_sqb(
command=test_case.command, project_dir=project_dir
)

assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr
payload: dict[str, object] = json.loads(result.stdout)
node_ids: set[str] = {str(node["id"]) for node in payload["nodes"]}
edge_pairs: set[tuple[str, str]] = {
(str(edge["from_id"]), str(edge["to_id"])) for edge in payload["edges"]
}
assert payload["project_name"] == test_case.expected_project_name, test_case.description
assert set(test_case.expected_node_ids) <= node_ids, test_case.description
assert set(test_case.expected_edge_pairs) <= edge_pairs, test_case.description
assert set(chain.from_iterable(edge_pairs)) <= node_ids, edge_pairs
for fragment in test_case.expected_absent_fragments:
assert fragment not in result.stdout, test_case.description


@pytest.mark.parametrize(
"test_case",
[
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
ALLOW_ROUND_TRIPS,
SCORING_TASK,
round_trip_project_files,
round_trip_refusal,
)
from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import (
GraphBoundaryCompileTestCase,
Expand All @@ -20,30 +21,6 @@
run_sqb,
)

CYCLE_TASK: str = (
"from sqlbuild.refs import model\n"
"from sqlbuild.tasks import task\n\n\n"
'@task(depends_on=model("scored_orders"))\n'
"def score_orders(ctx):\n"
' """Send the scored orders to the partner."""\n'
" return ctx.result(payload={})\n"
)
UNREAD_TASK: str = (
"from sqlbuild.tasks import task\n\n\n"
"@task()\n"
"def score_orders(ctx):\n"
' """Fetch the partner scores."""\n'
" return ctx.result(payload={})\n"
)
TERMINAL_LOADER_READER: str = (
"from sqlbuild.tasks import task\n\n"
"from python.loaders.partner import partner_scores\n\n\n"
"@task(depends_on=[partner_scores])\n"
"def report_scores(ctx):\n"
' """Report the scores the partner returned."""\n'
" return ctx.result(payload={})\n"
)


@pytest.mark.parametrize(
"test_case",
Expand All @@ -58,7 +35,7 @@
GraphBoundaryCompileTestCase(
description="a cycle through a task, models, a source and its loader fails compile",
project_settings=ALLOW_ROUND_TRIPS,
edited_files={"python/tasks/scoring.py": CYCLE_TASK},
edited_files={"python/tasks/scoring.py": round_trip_refusal("cycle_task.py")},
expected_exit_code=1,
expected_output_fragments=(
"S000",
Expand All @@ -71,14 +48,8 @@
description="a cycle of model refs alone fails compile with the plan's message",
project_settings=ALLOW_ROUND_TRIPS,
edited_files={
"models/order_totals.sql": (
'MODEL (description "Totals per order", materialized table);\n\n'
'SELECT order_id FROM __ref("order_ranks")\n'
),
"models/order_ranks.sql": (
'MODEL (description "Ranks per order", materialized table);\n\n'
'SELECT order_id FROM __ref("order_totals")\n'
),
"models/order_totals.sql": round_trip_refusal("order_totals.sql"),
"models/order_ranks.sql": round_trip_refusal("order_ranks.sql"),
},
expected_exit_code=1,
expected_output_fragments=(
Expand All @@ -100,8 +71,8 @@
description="a task depending on a terminal loader instead of its source fails compile",
project_settings="",
edited_files={
"python/tasks/scoring.py": UNREAD_TASK,
"python/tasks/reporting.py": TERMINAL_LOADER_READER,
"python/tasks/scoring.py": round_trip_refusal("unread_task.py"),
"python/tasks/reporting.py": round_trip_refusal("terminal_loader_reader.py"),
},
expected_exit_code=1,
expected_output_fragments=(
Expand Down
Loading
Loading