diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/project/models/orders.sql b/benchmarks/duckshop/edge_cases/loaders/round_trip/project/models/orders.sql new file mode 100644 index 0000000000..c167db3316 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/project/models/orders.sql @@ -0,0 +1,3 @@ +MODEL (description "Orders to score", materialized table); + +SELECT 1 AS order_id UNION ALL SELECT 2 AS order_id diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/project/models/scored_orders.sql b/benchmarks/duckshop/edge_cases/loaders/round_trip/project/models/scored_orders.sql new file mode 100644 index 0000000000..acc221d1c1 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/project/models/scored_orders.sql @@ -0,0 +1,5 @@ +MODEL (description "Orders with their partner score", materialized table); + +SELECT o.order_id, s.score +FROM __ref("orders") AS o +JOIN __source("partner_scores") AS s ON o.order_id = s.order_id diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/project/python/loaders/partner.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/project/python/loaders/partner.py new file mode 100644 index 0000000000..ec9332e1bb --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/project/python/loaders/partner.py @@ -0,0 +1,10 @@ +from python.tasks.scoring import score_orders + +from sqlbuild.loaders import loader + + +@loader(depends_on=[score_orders]) +def partner_scores(ctx): + """Load the scores the partner returned.""" + result = ctx.result_of(node_function=score_orders) + return [{"order_id": o, "score": s} for o, s in result.payload["scores"]] diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/project/python/tasks/scoring.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/project/python/tasks/scoring.py new file mode 100644 index 0000000000..7d7f16b5d4 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/project/python/tasks/scoring.py @@ -0,0 +1,10 @@ +from sqlbuild.refs import model +from sqlbuild.tasks import task + + +@task(depends_on=model("orders")) +def score_orders(ctx): + """Send the orders to the partner and keep the scores it returns.""" + relation = ctx.relation(model("orders")) + rows = ctx.query(f"SELECT order_id FROM {relation} ORDER BY order_id").fetchall() + return ctx.result(payload={"scores": [[row[0], row[0] * 10] for row in rows]}) diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/project/sources/partner.yml b/benchmarks/duckshop/edge_cases/loaders/round_trip/project/sources/partner.yml new file mode 100644 index 0000000000..2cf533d75c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/project/sources/partner.yml @@ -0,0 +1,10 @@ +sources: + - name: partner_scores + description: Scores the partner returned for each order + managed: true + write_strategy: table + columns: + - name: order_id + type: INTEGER + - name: score + type: INTEGER diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/cycle_task.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/cycle_task.py new file mode 100644 index 0000000000..788f0bb748 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/cycle_task.py @@ -0,0 +1,8 @@ +from sqlbuild.refs import model +from sqlbuild.tasks import task + + +@task(depends_on=model("scored_orders")) +def score_orders(ctx): + """Send the scored orders to the partner.""" + return ctx.result(payload={}) diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/order_ranks.sql b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/order_ranks.sql new file mode 100644 index 0000000000..687ad16443 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/order_ranks.sql @@ -0,0 +1,3 @@ +MODEL (description "Ranks per order", materialized table); + +SELECT order_id FROM __ref("order_totals") diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/order_totals.sql b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/order_totals.sql new file mode 100644 index 0000000000..c7ddbcec75 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/order_totals.sql @@ -0,0 +1,3 @@ +MODEL (description "Totals per order", materialized table); + +SELECT order_id FROM __ref("order_ranks") diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/terminal_loader_reader.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/terminal_loader_reader.py new file mode 100644 index 0000000000..c6d5cf6de1 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/terminal_loader_reader.py @@ -0,0 +1,9 @@ +from python.loaders.partner import partner_scores + +from sqlbuild.tasks import task + + +@task(depends_on=[partner_scores]) +def report_scores(ctx): + """Report the scores the partner returned.""" + return ctx.result(payload={}) diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/unread_task.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/unread_task.py new file mode 100644 index 0000000000..44d7cc37db --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/refusals/unread_task.py @@ -0,0 +1,7 @@ +from sqlbuild.tasks import task + + +@task() +def score_orders(ctx): + """Fetch the partner scores.""" + return ctx.result(payload={}) diff --git a/benchmarks/duckshop/edge_cases/orchestration/off_target_members/job_groups/daily_job_group.toml b/benchmarks/duckshop/edge_cases/orchestration/off_target_members/job_groups/daily_job_group.toml new file mode 100644 index 0000000000..7db9a4484b --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/off_target_members/job_groups/daily_job_group.toml @@ -0,0 +1,3 @@ +name = "daily_job_group" +description = "Orders everywhere; restocking only where it is declared." +jobs = ["orders_job", "restock_job"] diff --git a/benchmarks/duckshop/edge_cases/orchestration/off_target_members/jobs/orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/off_target_members/jobs/orders_job.toml new file mode 100644 index 0000000000..4f46b4c813 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/off_target_members/jobs/orders_job.toml @@ -0,0 +1,3 @@ +name = "orders_job" +description = "Build orders on every target." +select = "+orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/off_target_members/jobs/restock_job.toml b/benchmarks/duckshop/edge_cases/orchestration/off_target_members/jobs/restock_job.toml new file mode 100644 index 0000000000..4eb644c23f --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/off_target_members/jobs/restock_job.toml @@ -0,0 +1,4 @@ +name = "restock_job" +description = "Restock customers, declared only for production." +select = "customers" +targets = ["prod"] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index 54df02477e..ed18c96b0b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1544,9 +1544,23 @@ def node_results_in(*, db_path: Path, schema: str, node_type: str) -> list[tuple FAILING_SCORING_TASK: str = ' raise RuntimeError("partner service unavailable")\n' +ROUND_TRIP_FIXTURE: Path = Path("benchmarks/duckshop/edge_cases/loaders/round_trip") + + def round_trip_project_files(*, project_settings: str, task_body: str) -> dict[str, str]: - """Orders -> score task -> partner loader -> partner_scores source -> scored_orders model.""" + """DuckShop's round trip: orders -> score task -> partner loader -> source -> scored_orders.""" + project: Path = ROUND_TRIP_FIXTURE / "project" + files: dict[str, str] = { + path.relative_to(project).as_posix(): path.read_text() + for path in sorted( + [*project.rglob("*.py"), *project.rglob("*.sql"), *project.rglob("*.yml")] + ) + } + assert SCORING_TASK in files["python/tasks/scoring.py"] + files["python/tasks/scoring.py"] = files["python/tasks/scoring.py"].replace( + SCORING_TASK, task_body + ) return { "sqlbuild_project.toml": ( 'name = "partner_scoring"\n' @@ -1558,47 +1572,16 @@ def round_trip_project_files(*, project_settings: str, task_body: str) -> dict[s 'connection = "local"\n' 'schema = "main"\n' + project_settings ), - "models/orders.sql": ( - 'MODEL (description "Orders to score", materialized table);\n\n' - "SELECT 1 AS order_id UNION ALL SELECT 2 AS order_id\n" - ), - "sources/partner.yml": ( - "sources:\n" - " - name: partner_scores\n" - " description: Scores the partner returned for each order\n" - " managed: true\n" - " write_strategy: table\n" - " columns:\n" - " - name: order_id\n" - " type: INTEGER\n" - " - name: score\n" - " type: INTEGER\n" - ), - "models/scored_orders.sql": ( - 'MODEL (description "Orders with their partner score", materialized table);\n\n' - "SELECT o.order_id, s.score\n" - 'FROM __ref("orders") AS o\n' - 'JOIN __source("partner_scores") AS s ON o.order_id = s.order_id\n' - ), - "python/tasks/scoring.py": ( - "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n\n" - '@task(depends_on=model("orders"))\n' - "def score_orders(ctx):\n" - ' """Send the orders to the partner and keep the scores it returns."""\n' + task_body - ), - "python/loaders/partner.py": ( - "from sqlbuild.loaders import loader\n\n" - "from python.tasks.scoring import score_orders\n\n\n" - "@loader(depends_on=[score_orders])\n" - "def partner_scores(ctx):\n" - ' """Load the scores the partner returned."""\n' - " result = ctx.result_of(node_function=score_orders)\n" - ' return [{"order_id": o, "score": s} for o, s in result.payload["scores"]]\n' - ), + **files, } +def round_trip_refusal(name: str) -> str: + """One of DuckShop's round-trip refusal variants, such as `cycle_task.py`.""" + + return (ROUND_TRIP_FIXTURE / "refusals" / name).read_text() + + def build_round_trip_project( *, tmp_path: Path, test_case: WarehouseRoundTripE2ETestCase ) -> tuple[Path, str]: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dag_json_behavior.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dag_json_behavior.py index dd883884a8..66c65da442 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dag_json_behavior.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_dag_json_behavior.py @@ -4,6 +4,7 @@ import json import subprocess +from itertools import chain from pathlib import Path import pytest @@ -13,12 +14,21 @@ DiamondDagE2ETestCase, DiamondDagJsonE2ETestCase, ) +from tests.e2e.src.sqlbuild.cli.commands.main.build.helpers import ( + ALLOW_ROUND_TRIPS, + SCORING_TASK, + round_trip_project_files, +) from tests.e2e.src.sqlbuild.cli.commands.main.lineage.helpers import ( DIAMOND_EDGE_COUNT, diamond_model_names, prepare_diamond_lineage_project, ) -from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_waffle_shop, run_sqb +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( + prepare_inline_project, + prepare_waffle_shop, + run_sqb, +) @pytest.mark.parametrize( @@ -93,6 +103,60 @@ def test_given_waffle_shop_when_running_dag_json_then_it_reports_static_graph( assert "action" not in result.stdout +@pytest.mark.parametrize( + "test_case", + [ + DagJsonBuildE2ETestCase( + description="a task upstream of a declared loader points at the loader's source", + command=("dag", "--json"), + expected_exit_code=0, + expected_project_name="partner_scoring", + expected_node_ids=( + "model:orders", + "task:score_orders", + "source:partner_scores", + "model:scored_orders", + ), + expected_edge_pairs=( + ("model:orders", "task:score_orders"), + ("task:score_orders", "source:partner_scores"), + ("source:partner_scores", "model:scored_orders"), + ), + expected_check_ids=(), + expected_absent_fragments=("loader:partner_scores",), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_round_trip_when_running_dag_json_then_every_edge_ends_at_a_node( + test_case: DagJsonBuildE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="partner_scoring", + repo_files=round_trip_project_files( + project_settings=ALLOW_ROUND_TRIPS, task_body=SCORING_TASK + ), + ) + result: subprocess.CompletedProcess[str] = run_sqb( + command=test_case.command, project_dir=project_dir + ) + + assert result.returncode == test_case.expected_exit_code, result.stdout + result.stderr + payload: dict[str, object] = json.loads(result.stdout) + node_ids: set[str] = {str(node["id"]) for node in payload["nodes"]} + edge_pairs: set[tuple[str, str]] = { + (str(edge["from_id"]), str(edge["to_id"])) for edge in payload["edges"] + } + assert payload["project_name"] == test_case.expected_project_name, test_case.description + assert set(test_case.expected_node_ids) <= node_ids, test_case.description + assert set(test_case.expected_edge_pairs) <= edge_pairs, test_case.description + assert set(chain.from_iterable(edge_pairs)) <= node_ids, edge_pairs + for fragment in test_case.expected_absent_fragments: + assert fragment not in result.stdout, test_case.description + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_sql_graph_boundaries.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_sql_graph_boundaries.py index c33810f739..b27cc1d738 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_sql_graph_boundaries.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_python_sql_graph_boundaries.py @@ -11,6 +11,7 @@ ALLOW_ROUND_TRIPS, SCORING_TASK, round_trip_project_files, + round_trip_refusal, ) from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import ( GraphBoundaryCompileTestCase, @@ -20,30 +21,6 @@ run_sqb, ) -CYCLE_TASK: str = ( - "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n\n" - '@task(depends_on=model("scored_orders"))\n' - "def score_orders(ctx):\n" - ' """Send the scored orders to the partner."""\n' - " return ctx.result(payload={})\n" -) -UNREAD_TASK: str = ( - "from sqlbuild.tasks import task\n\n\n" - "@task()\n" - "def score_orders(ctx):\n" - ' """Fetch the partner scores."""\n' - " return ctx.result(payload={})\n" -) -TERMINAL_LOADER_READER: str = ( - "from sqlbuild.tasks import task\n\n" - "from python.loaders.partner import partner_scores\n\n\n" - "@task(depends_on=[partner_scores])\n" - "def report_scores(ctx):\n" - ' """Report the scores the partner returned."""\n' - " return ctx.result(payload={})\n" -) - @pytest.mark.parametrize( "test_case", @@ -58,7 +35,7 @@ GraphBoundaryCompileTestCase( description="a cycle through a task, models, a source and its loader fails compile", project_settings=ALLOW_ROUND_TRIPS, - edited_files={"python/tasks/scoring.py": CYCLE_TASK}, + edited_files={"python/tasks/scoring.py": round_trip_refusal("cycle_task.py")}, expected_exit_code=1, expected_output_fragments=( "S000", @@ -71,14 +48,8 @@ description="a cycle of model refs alone fails compile with the plan's message", project_settings=ALLOW_ROUND_TRIPS, edited_files={ - "models/order_totals.sql": ( - 'MODEL (description "Totals per order", materialized table);\n\n' - 'SELECT order_id FROM __ref("order_ranks")\n' - ), - "models/order_ranks.sql": ( - 'MODEL (description "Ranks per order", materialized table);\n\n' - 'SELECT order_id FROM __ref("order_totals")\n' - ), + "models/order_totals.sql": round_trip_refusal("order_totals.sql"), + "models/order_ranks.sql": round_trip_refusal("order_ranks.sql"), }, expected_exit_code=1, expected_output_fragments=( @@ -100,8 +71,8 @@ description="a task depending on a terminal loader instead of its source fails compile", project_settings="", edited_files={ - "python/tasks/scoring.py": UNREAD_TASK, - "python/tasks/reporting.py": TERMINAL_LOADER_READER, + "python/tasks/scoring.py": round_trip_refusal("unread_task.py"), + "python/tasks/reporting.py": round_trip_refusal("terminal_loader_reader.py"), }, expected_exit_code=1, expected_output_fragments=( diff --git a/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py b/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py index 9a8aaf725e..9d340b007a 100644 --- a/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py +++ b/tests/integration/src/sqlbuild/cli/commands/main/test_job_selectors.py @@ -10,7 +10,7 @@ from sqlbuild._native import run_cli as main from tests.integration.src.sqlbuild.cli.commands.main._test_types import JobSelectorBuildTestCase -_PROJECT_FILES: dict[str, str] = { +_MODEL_FILES: dict[str, str] = { "sqlbuild_project.toml": ( 'name = "demo"\nadapter = "duckdb"\ndefault_target = "dev"\n\n' '[connection]\ndatabase = "demo.duckdb"\n\n[targets.dev]\n\n[targets.prod]\n' @@ -26,6 +26,11 @@ "models/customers.sql": ( "MODEL (description 'Customers.', materialized table);\nSELECT 7 AS customer_id\n" ), +} +_OFF_TARGET_FIXTURE: Path = Path("benchmarks/duckshop/edge_cases/orchestration/off_target_members") + +_PROJECT_FILES: dict[str, str] = { + **_MODEL_FILES, "orchestration/jobs/nightly_job.toml": ( 'name = "nightly_job"\ndescription = "Orders."\nselect = "+orders"\n' ), @@ -137,3 +142,59 @@ def test_given_declared_jobs_when_building_by_job_then_only_their_nodes_build( "WHERE NOT internal AND NOT starts_with(table_name, '_sqlbuild')" ).fetchall() assert {name for (name,) in rows} == test_case.expected_tables + + +@pytest.mark.parametrize( + "test_case", + [ + JobSelectorBuildTestCase( + description="a group skips a member declared only for another target", + build_args=("--select", "job:daily_job_group"), + expected_exit_code=0, + expected_tables=frozenset({"orders", "stg_orders"}), + ), + JobSelectorBuildTestCase( + description="a group runs that member under the target it is declared for", + build_args=("--target", "prod", "--select", "job:daily_job_group"), + expected_exit_code=0, + expected_tables=frozenset({"customers", "orders", "stg_orders"}), + ), + ], + ids=lambda case: case.description, +) +def test_given_a_group_member_declared_for_another_target_when_building_the_group_then_it_is_skipped( + test_case: JobSelectorBuildTestCase, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + files: dict[str, str] = { + **_MODEL_FILES, + **{ + f"orchestration/{path.relative_to(_OFF_TARGET_FIXTURE).as_posix()}": path.read_text() + for path in sorted(_OFF_TARGET_FIXTURE.rglob("*.toml")) + }, + } + for relative_path, contents in files.items(): + destination: Path = tmp_path / relative_path + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(contents, encoding="utf-8") + + exit_code: int = main( + [ + "--project-dir", + str(tmp_path), + "--no-color", + "build", + "--no-tests", + *test_case.build_args, + ] + ) + captured: str = "".join(capsys.readouterr()) + + assert exit_code == test_case.expected_exit_code, f"{test_case.description}: {captured}" + with duckdb.connect(str(tmp_path / "demo.duckdb")) as warehouse: + rows: list[tuple[str]] = warehouse.sql( + "SELECT table_name FROM duckdb_tables() " + "WHERE NOT internal AND NOT starts_with(table_name, '_sqlbuild')" + ).fetchall() + assert {name for (name,) in rows} == test_case.expected_tables, test_case.description