From a58ce8f9cc6cad0933701d9f74037ad4309fca03 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 14:44:57 +0000 Subject: [PATCH 1/4] fix: name --no-python as the reason a round-trip load is skipped With --no-python the task gating a round-trip loader never runs, and the load was soft-skipped with "the SQL it reads was not rebuilt", which is not why. The reason now says --no-python skips read-side tasks and assets. --- .../_helpers/build_command/lifecycle.rs | 10 +++++++++- .../cli/commands/main/build/_test_types.py | 3 ++- .../main/build/test_warehouse_round_trips.py | 18 ++++++++++++++++-- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 3a4c3a8872..9857ce9477 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -105,6 +105,8 @@ pub(crate) struct DirectPythonLifecycle { round_trip_gates: HashMap>, /// Each round-trip loader's source load key and the SQL keys it waits for. round_trip_waits: Vec<(ObjectKey, Vec)>, + /// Whether the build runs read-side tasks and assets; `--no-python` turns this off. + include_python: bool, } #[pymethods] @@ -469,7 +471,8 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( let connection_config: Bound<'py, PyAny> = context.get("connection_config")?; let project: Bound<'py, PyAny> = native_field(pipeline, "project")?; let plan_output: Bound<'py, PyAny> = native_field(pipeline, "plan_output")?; - let selected: Vec = if context.request_flag("include_python")? { + let include_python: bool = context.request_flag("include_python")?; + let selected: Vec = if include_python { task_asset_names(&discovered, pipeline)? } else { Vec::new() @@ -618,6 +621,7 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( ingress_loader_names: native_field(&lifecycle.plan, "ingress_loader_names")?.unbind(), round_trip_gates, round_trip_waits, + include_python, }, ) } @@ -690,6 +694,10 @@ impl SourceLoadGate { Some(SUCCESS_STATUS) => continue, Some(SKIPPED_STATUS) => ("was skipped", HARD_SKIP), Some(_) => ("failed", HARD_SKIP), + None if !lifecycle.include_python => ( + "did not run; --no-python skips read-side tasks and assets", + SOFT_SKIP, + ), None => ("did not run; the SQL it reads was not rebuilt", SOFT_SKIP), }; return Ok(Some((format!("Upstream {kind} '{name}' {outcome}"), mode))); diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 8728f33513..5ed761f0e8 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1715,9 +1715,10 @@ class WarehouseRoundTripE2ETestCase: @dataclass(frozen=True) class ChangedRoundTripE2ETestCase: - """A second `--changed` build of a round trip after an optional change to `orders`.""" + """A second build of a round trip, with `build_flags`, after an optional change to `orders`.""" description: str orders_sql: str expected_output_fragments: tuple[str, ...] expected_scored_rows: tuple[tuple[object, ...], ...] + build_flags: tuple[str, ...] = ("--changed",) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py index 308596fae2..a375c54f9b 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py @@ -136,10 +136,24 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ ), expected_scored_rows=((1, 10), (2, 20)), ), + ChangedRoundTripE2ETestCase( + description="--no-python leaves the task unrun and names the flag as the skip reason", + orders_sql=( + 'MODEL (description "Orders to score", materialized table);\n\n' + "SELECT 1 AS order_id UNION ALL SELECT 2 UNION ALL SELECT 3\n" + ), + expected_output_fragments=( + "soft skip: Upstream task 'score_orders' did not run; --no-python skips " + "read-side tasks and assets", + "Completed successfully", + ), + expected_scored_rows=((1, 10), (2, 20)), + build_flags=("--no-python",), + ), ], ids=lambda case: case.description, ) -def test_given_a_built_round_trip_when_building_changed_then_the_decision_follows_the_chain( +def test_given_a_built_round_trip_when_rebuilding_then_the_decision_follows_the_chain( tmp_path: Path, test_case: ChangedRoundTripE2ETestCase, ) -> None: @@ -159,7 +173,7 @@ def test_given_a_built_round_trip_when_building_changed_then_the_decision_follow (project_dir / "models" / "orders.sql").write_text(test_case.orders_sql, encoding="utf-8") result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--changed"), project_dir=project_dir + command=("--no-color", "build", *test_case.build_flags), project_dir=project_dir ) output: str = result.stdout + result.stderr From 358b43fe76a8e3f73feca20f4e6a67fdf8a4f5d0 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:00:51 +0000 Subject: [PATCH 2/4] fix: keep a gating task's soft skip soft for its round-trip load A round-trip load was hard-skipped whenever its gating task or asset skipped, so a task's default soft skip blocked every model reading the source. The load now takes the upstream's skip mode: a soft skip skips only the load, and models reading the source still build from its last loaded data. A hard skip or failure still blocks them. --- .../_helpers/build_command/lifecycle.rs | 23 +++++++++---- .../docs/concepts/python-nodes/loaders.md | 7 ++-- .../docs/concepts/python-nodes/overview.md | 5 +-- .../cli/commands/main/build/_test_types.py | 3 +- .../main/build/test_warehouse_round_trips.py | 34 +++++++++++++++++++ .../docs/concepts/python-nodes/loaders.mdx | 7 ++-- .../docs/concepts/python-nodes/overview.mdx | 5 +-- 7 files changed, 66 insertions(+), 18 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 9857ce9477..d46a559eea 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -669,7 +669,7 @@ pub(crate) struct SourceLoadGate { #[pymethods] impl SourceLoadGate { - /// Why a round-trip load is skipped: `hard` after a failed or skipped upstream, `soft` when it never ran. + /// Why a round-trip load is skipped: `hard` after a failed or hard-skipped upstream, `soft` after a soft skip or when it never ran. fn __call__( &self, py: Python<'_>, @@ -680,19 +680,28 @@ impl SourceLoadGate { let Some(gates) = loader.and_then(|loader| lifecycle.round_trip_gates.get(&loader)) else { return Ok(None); }; - let mut statuses: HashMap = HashMap::new(); + let mut statuses: HashMap)> = HashMap::new(); for result in lifecycle.python_results(py)?.try_iter()? { let result: Bound<'_, PyAny> = result?; + let skip_mode: Bound<'_, PyAny> = native_field(&result, "skip_mode")?; statuses.insert( native_field(&result, "node_name")?.extract()?, - native_field(&result, "status")?.str()?.extract()?, + ( + native_field(&result, "status")?.str()?.extract()?, + (!skip_mode.is_none()) + .then(|| skip_mode.str()?.extract()) + .transpose()?, + ), ); } for (kind, name) in gates { - let (outcome, mode): (&str, &'static str) = match statuses.get(name).map(String::as_str) - { - Some(SUCCESS_STATUS) => continue, - Some(SKIPPED_STATUS) => ("was skipped", HARD_SKIP), + let status = statuses + .get(name) + .map(|(status, mode)| (status.as_str(), mode.as_deref())); + let (outcome, mode): (&str, &'static str) = match status { + Some((SUCCESS_STATUS, _)) => continue, + Some((SKIPPED_STATUS, Some(SOFT_SKIP))) => ("was skipped", SOFT_SKIP), + Some((SKIPPED_STATUS, _)) => ("was skipped", HARD_SKIP), Some(_) => ("failed", HARD_SKIP), None if !lifecycle.include_python => ( "did not run; --no-python skips read-side tasks and assets", diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md index c439564e0f..ae176ffd50 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -324,9 +324,10 @@ Dependencies form a DAG. SQLBuild schedules loaders in topological order and exe ### Depending on tasks and assets A loader reads an upstream task's or asset's result with `ctx.result_of(...)`; `ctx.loader(...)` -takes loaders only. A failed or skipped upstream task or asset skips the loader and everything -downstream of the source it fills. A loader selected without its upstream task or asset reads that -upstream's latest successful result, and fails naming the upstream if it never succeeded. +takes loaders only. A failed or hard-skipped upstream task or asset skips the loader and everything +downstream of the source it fills. A soft-skipped one skips only the load, so the source keeps its +last loaded data and its readers still build. A loader selected without its upstream task or asset +reads that upstream's latest successful result, and fails naming the upstream if it never succeeded. ```python from sqlbuild.loaders import loader diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index 3c3ae4c95a..e282fedff3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -110,8 +110,9 @@ allow_warehouse_round_trips = true With the setting on: - One `sqb build` runs the SQL the task reads, then the task, then the loader, then the models that - read the loader's source. If the task fails or is skipped, the loader and everything downstream of - its source are skipped. + read the loader's source. If the task fails or skips with `mode="hard"`, the loader and everything + downstream of its source are skipped. If it skips softly, only the load is skipped: the source + keeps its last loaded data, and models reading it still build when another of their inputs ran. - Lineage and column lineage restart at the source: the source's columns come from the loader, not from the models the task read. - Job groups run a member that builds the task before the member that runs the loader, and that one diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 5ed761f0e8..addd329583 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1715,10 +1715,11 @@ class WarehouseRoundTripE2ETestCase: @dataclass(frozen=True) class ChangedRoundTripE2ETestCase: - """A second build of a round trip, with `build_flags`, after an optional change to `orders`.""" + """A second round-trip build with `build_flags`, after changing `orders` and maybe the task.""" description: str orders_sql: str expected_output_fragments: tuple[str, ...] expected_scored_rows: tuple[tuple[object, ...], ...] build_flags: tuple[str, ...] = ("--changed",) + task_body: str | None = None diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py index a375c54f9b..90ab14a260 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py @@ -150,6 +150,33 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ expected_scored_rows=((1, 10), (2, 20)), build_flags=("--no-python",), ), + ChangedRoundTripE2ETestCase( + description="a soft-skipped task soft-skips the load, and a model reading a changed model still runs", + orders_sql=( + 'MODEL (description "Orders to score", materialized table);\n\n' + "SELECT 1 AS order_id UNION ALL SELECT 2 UNION ALL SELECT 3\n" + ), + task_body=' return ctx.skip(reason="partner offline")\n', + expected_output_fragments=( + "soft skip: Upstream task 'score_orders' was skipped", + "table scored_orders OK", + "Completed successfully", + ), + expected_scored_rows=((1, 10), (2, 20)), + ), + ChangedRoundTripE2ETestCase( + description="a hard-skipped task hard-skips the load and blocks the models reading its source", + orders_sql=( + 'MODEL (description "Orders to score", materialized table);\n\n' + "SELECT 1 AS order_id UNION ALL SELECT 2 UNION ALL SELECT 3\n" + ), + task_body=' return ctx.skip(reason="partner offline", mode="hard")\n', + expected_output_fragments=( + "hard skip: Upstream task 'score_orders' was skipped", + "table scored_orders SKIP", + ), + expected_scored_rows=((1, 10), (2, 20)), + ), ], ids=lambda case: case.description, ) @@ -171,6 +198,13 @@ def test_given_a_built_round_trip_when_rebuilding_then_the_decision_follows_the_ ) project_dir: Path = database.parent (project_dir / "models" / "orders.sql").write_text(test_case.orders_sql, encoding="utf-8") + if test_case.task_body is not None: + task_file: Path = project_dir / "python" / "tasks" / "scoring.py" + task_source: str = task_file.read_text(encoding="utf-8") + task_file.write_text( + task_source[: task_source.index(" relation = ")] + test_case.task_body, + encoding="utf-8", + ) result: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "build", *test_case.build_flags), project_dir=project_dir diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx index 5d0a06bbc3..34a2219e68 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx @@ -304,9 +304,10 @@ Dependencies form a DAG. SQLBuild schedules loaders in topological order and exe ### Depending on tasks and assets A loader reads an upstream task's or asset's result with `ctx.result_of(...)`; `ctx.loader(...)` -takes loaders only. A failed or skipped upstream task or asset skips the loader and everything -downstream of the source it fills. A loader selected without its upstream task or asset reads that -upstream's latest successful result, and fails naming the upstream if it never succeeded. +takes loaders only. A failed or hard-skipped upstream task or asset skips the loader and everything +downstream of the source it fills. A soft-skipped one skips only the load, so the source keeps its +last loaded data and its readers still build. A loader selected without its upstream task or asset +reads that upstream's latest successful result, and fails naming the upstream if it never succeeded. ```python from sqlbuild.loaders import loader diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index ff8e0475bf..88d7ae58e6 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -94,8 +94,9 @@ allow_warehouse_round_trips = true With the setting on: - One `sqb build` runs the SQL the task reads, then the task, then the loader, then the models that - read the loader's source. If the task fails or is skipped, the loader and everything downstream of - its source are skipped. + read the loader's source. If the task fails or skips with `mode="hard"`, the loader and everything + downstream of its source are skipped. If it skips softly, only the load is skipped: the source + keeps its last loaded data, and models reading it still build when another of their inputs ran. - Lineage and column lineage restart at the source: the source's columns come from the loader, not from the models the task read. - Job groups run a member that builds the task before the member that runs the loader, and that one From 5644817fb722a8f14bdc4ac8f46245637c5541f4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:12:52 +0000 Subject: [PATCH 3/4] fix: apply the Python fan-in rule to a round-trip load's gating upstreams A round-trip load skipped at the first gating task or asset that did not succeed. It now follows the rule every other Python node follows: a failed or hard-skipped upstream skips it hard, any successful upstream lets it run, and it skips softly only when every upstream skipped softly or did not run. --- .../_helpers/build_command/lifecycle.rs | 46 +++++++++---- .../docs/concepts/python-nodes/loaders.md | 7 +- .../docs/concepts/python-nodes/overview.md | 8 ++- .../cli/commands/main/build/_test_types.py | 1 + .../main/build/test_warehouse_round_trips.py | 67 +++++++++++++++++++ .../docs/concepts/python-nodes/loaders.mdx | 7 +- .../docs/concepts/python-nodes/overview.mdx | 8 ++- 7 files changed, 119 insertions(+), 25 deletions(-) diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index d46a559eea..42b7baeea1 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -669,7 +669,10 @@ pub(crate) struct SourceLoadGate { #[pymethods] impl SourceLoadGate { - /// Why a round-trip load is skipped: `hard` after a failed or hard-skipped upstream, `soft` after a soft skip or when it never ran. + /// Why a round-trip load is skipped, by the Python fan-in rule over its gating upstreams. + /// + /// A failed or hard-skipped upstream skips it `hard`. Otherwise it runs when any upstream + /// succeeded, and skips `soft`, naming the first upstream, when every one soft-skipped or never ran. fn __call__( &self, py: Python<'_>, @@ -694,23 +697,40 @@ impl SourceLoadGate { ), ); } + let mut succeeded: bool = false; + let mut soft_reason: Option = None; for (kind, name) in gates { let status = statuses .get(name) .map(|(status, mode)| (status.as_str(), mode.as_deref())); - let (outcome, mode): (&str, &'static str) = match status { - Some((SUCCESS_STATUS, _)) => continue, - Some((SKIPPED_STATUS, Some(SOFT_SKIP))) => ("was skipped", SOFT_SKIP), - Some((SKIPPED_STATUS, _)) => ("was skipped", HARD_SKIP), - Some(_) => ("failed", HARD_SKIP), - None if !lifecycle.include_python => ( - "did not run; --no-python skips read-side tasks and assets", - SOFT_SKIP, - ), - None => ("did not run; the SQL it reads was not rebuilt", SOFT_SKIP), + let outcome: &str = match status { + Some((SUCCESS_STATUS, _)) => { + succeeded = true; + continue; + } + Some((SKIPPED_STATUS, Some(SOFT_SKIP))) => "was skipped", + Some((SKIPPED_STATUS, _)) => { + return Ok(Some(( + format!("Upstream {kind} '{name}' was skipped"), + HARD_SKIP, + ))); + } + Some(_) => { + return Ok(Some(( + format!("Upstream {kind} '{name}' failed"), + HARD_SKIP, + ))); + } + None if !lifecycle.include_python => { + "did not run; --no-python skips read-side tasks and assets" + } + None => "did not run; the SQL it reads was not rebuilt", }; - return Ok(Some((format!("Upstream {kind} '{name}' {outcome}"), mode))); + soft_reason.get_or_insert_with(|| format!("Upstream {kind} '{name}' {outcome}")); + } + if succeeded { + return Ok(None); } - Ok(None) + Ok(soft_reason.map(|reason| (reason, SOFT_SKIP))) } } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md index ae176ffd50..fe2e0615c9 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -325,9 +325,10 @@ Dependencies form a DAG. SQLBuild schedules loaders in topological order and exe A loader reads an upstream task's or asset's result with `ctx.result_of(...)`; `ctx.loader(...)` takes loaders only. A failed or hard-skipped upstream task or asset skips the loader and everything -downstream of the source it fills. A soft-skipped one skips only the load, so the source keeps its -last loaded data and its readers still build. A loader selected without its upstream task or asset -reads that upstream's latest successful result, and fails naming the upstream if it never succeeded. +downstream of the source it fills. Otherwise the loader runs when any upstream succeeded; when all +of them skip softly, only the load is skipped, so the source keeps its last loaded data and its +readers still build. A loader selected without its upstream task or asset reads that upstream's +latest successful result, and fails naming the upstream if it never succeeded. ```python from sqlbuild.loaders import loader diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index e282fedff3..d5991ed34e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -110,9 +110,11 @@ allow_warehouse_round_trips = true With the setting on: - One `sqb build` runs the SQL the task reads, then the task, then the loader, then the models that - read the loader's source. If the task fails or skips with `mode="hard"`, the loader and everything - downstream of its source are skipped. If it skips softly, only the load is skipped: the source - keeps its last loaded data, and models reading it still build when another of their inputs ran. + read the loader's source. The loader follows the usual rule for its upstream tasks and assets: if + one fails or skips with `mode="hard"`, the loader and everything downstream of its source are + skipped; otherwise it runs when any of them succeeded. When all of them skip softly, only the load + is skipped: the source keeps its last loaded data, and models reading it still build when another + of their inputs ran. - Lineage and column lineage restart at the source: the source's columns come from the loader, not from the models the task read. - Job groups run a member that builds the task before the member that runs the loader, and that one diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index addd329583..9a1c3b2046 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1723,3 +1723,4 @@ class ChangedRoundTripE2ETestCase: expected_scored_rows: tuple[tuple[object, ...], ...] build_flags: tuple[str, ...] = ("--changed",) task_body: str | None = None + extra_files: dict[str, str] = field(default_factory=dict) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py index 90ab14a260..c0cf6868e1 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py @@ -23,6 +23,36 @@ table_exists, ) +_ORDERS_WITH_A_THIRD: str = ( + 'MODEL (description "Orders to score", materialized table);\n\n' + "SELECT 1 AS order_id UNION ALL SELECT 2 UNION ALL SELECT 3\n" +) +_TWO_GATE_LOADER: str = ( + "from sqlbuild.loaders import loader\n\n" + "from python.tasks.ranking import rank_orders\n" + "from python.tasks.scoring import score_orders\n\n\n" + "@loader(depends_on=[score_orders, rank_orders])\n" + "def partner_scores(ctx):\n" + ' """Load the scores the partner returned."""\n' + " result = ctx.result_of(node_function=score_orders)\n" + ' return [{"order_id": o, "score": s} for o, s in result.payload["scores"]]\n' +) + + +def _ranking_task(body: str) -> dict[str, str]: + """A second task gating the partner loader, with `body`, and the loader depending on both.""" + + return { + "python/tasks/ranking.py": ( + "from sqlbuild.refs import model\n" + "from sqlbuild.tasks import task\n\n\n" + '@task(depends_on=model("orders"))\n' + "def rank_orders(ctx):\n" + ' """Rank the orders for the partner."""\n' + body + ), + "python/loaders/partner.py": _TWO_GATE_LOADER, + } + @pytest.mark.parametrize( "test_case", @@ -177,6 +207,41 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ ), expected_scored_rows=((1, 10), (2, 20)), ), + ChangedRoundTripE2ETestCase( + description="one gating task soft-skips and another succeeds, so the load runs", + orders_sql=_ORDERS_WITH_A_THIRD, + task_body=' return ctx.skip(reason="partner offline")\n', + extra_files=_ranking_task(' return ctx.result(payload={"ranked": True})\n'), + expected_output_fragments=( + "task rank_orders OK", + "source partner_scores OK", + "table scored_orders OK", + ), + expected_scored_rows=((1, 10), (2, 20)), + ), + ChangedRoundTripE2ETestCase( + description="every gating task soft-skips, so the load soft-skips", + orders_sql=_ORDERS_WITH_A_THIRD, + task_body=' return ctx.skip(reason="partner offline")\n', + extra_files=_ranking_task(' return ctx.skip(reason="nothing to rank")\n'), + expected_output_fragments=( + "partner_scores SKIP", + "soft skip: Upstream task '", + "table scored_orders OK", + ), + expected_scored_rows=((1, 10), (2, 20)), + ), + ChangedRoundTripE2ETestCase( + description="one gating task hard-skips while another succeeds, so the load hard-skips", + orders_sql=_ORDERS_WITH_A_THIRD, + task_body=' return ctx.skip(reason="partner offline", mode="hard")\n', + extra_files=_ranking_task(' return ctx.result(payload={"ranked": True})\n'), + expected_output_fragments=( + "hard skip: Upstream task 'score_orders' was skipped", + "table scored_orders SKIP", + ), + expected_scored_rows=((1, 10), (2, 20)), + ), ], ids=lambda case: case.description, ) @@ -198,6 +263,8 @@ def test_given_a_built_round_trip_when_rebuilding_then_the_decision_follows_the_ ) project_dir: Path = database.parent (project_dir / "models" / "orders.sql").write_text(test_case.orders_sql, encoding="utf-8") + for relative, text in test_case.extra_files.items(): + (project_dir / relative).write_text(text, encoding="utf-8") if test_case.task_body is not None: task_file: Path = project_dir / "python" / "tasks" / "scoring.py" task_source: str = task_file.read_text(encoding="utf-8") diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx index 34a2219e68..fed492df5c 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx @@ -305,9 +305,10 @@ Dependencies form a DAG. SQLBuild schedules loaders in topological order and exe A loader reads an upstream task's or asset's result with `ctx.result_of(...)`; `ctx.loader(...)` takes loaders only. A failed or hard-skipped upstream task or asset skips the loader and everything -downstream of the source it fills. A soft-skipped one skips only the load, so the source keeps its -last loaded data and its readers still build. A loader selected without its upstream task or asset -reads that upstream's latest successful result, and fails naming the upstream if it never succeeded. +downstream of the source it fills. Otherwise the loader runs when any upstream succeeded; when all +of them skip softly, only the load is skipped, so the source keeps its last loaded data and its +readers still build. A loader selected without its upstream task or asset reads that upstream's +latest successful result, and fails naming the upstream if it never succeeded. ```python from sqlbuild.loaders import loader diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index 88d7ae58e6..190d19d562 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -94,9 +94,11 @@ allow_warehouse_round_trips = true With the setting on: - One `sqb build` runs the SQL the task reads, then the task, then the loader, then the models that - read the loader's source. If the task fails or skips with `mode="hard"`, the loader and everything - downstream of its source are skipped. If it skips softly, only the load is skipped: the source - keeps its last loaded data, and models reading it still build when another of their inputs ran. + read the loader's source. The loader follows the usual rule for its upstream tasks and assets: if + one fails or skips with `mode="hard"`, the loader and everything downstream of its source are + skipped; otherwise it runs when any of them succeeded. When all of them skip softly, only the load + is skipped: the source keeps its last loaded data, and models reading it still build when another + of their inputs ran. - Lineage and column lineage restart at the source: the source's columns come from the loader, not from the models the task read. - Job groups run a member that builds the task before the member that runs the loader, and that one From 318d037bd72a72d020aa040ecd5b26ebc970b7fb Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:15:26 +0000 Subject: [PATCH 4/4] fix: gate a round-trip load only on the tasks and assets it reads directly A load ran when any task or asset anywhere upstream of it succeeded, so a succeeding task behind a soft-skipped one loaded that one's stale result. Only the tasks and assets a loader reads directly, or through other loaders, can now let it run; a failed or hard-skipped one anywhere still skips it hard. With --no-python, a read-side gate that did not run soft-skips the load even when an ingress gate succeeded. --- .../variants/ingress_gate/partner.py | 11 +++ .../variants/ingress_gate/rates.yml | 8 ++ .../variants/ingress_gate/rates_loader.py | 9 ++ .../variants/ingress_gate/rates_task.py | 7 ++ .../variants/transitive_gate/packaging.py | 9 ++ .../variants/transitive_gate/partner.py | 10 ++ .../_helpers/lifecycle/gate_frontier.rs | 37 +++++++ .../src/python_plan/_helpers/lifecycle/mod.rs | 1 + .../src/python_plan/_helpers/mod.rs | 1 + .../src/python_plan/main/run_lifecycle.rs | 5 +- .../sqlbuild-analysis/src/python_plan/mod.rs | 1 + .../src/python_plan/models.rs | 4 +- .../python_plan/tests/test_run_lifecycle.rs | 32 +++++- .../src/python_plan/tests/test_types.rs | 3 +- .../_helpers/build_command/lifecycle.rs | 30 +++--- .../docs/concepts/python-nodes/loaders.md | 7 +- .../cli/commands/main/build/_test_types.py | 3 +- .../cli/commands/main/build/helpers.py | 43 ++++++++ .../main/build/test_warehouse_round_trips.py | 98 +++++++++++-------- .../docs/concepts/python-nodes/loaders.mdx | 7 +- 20 files changed, 262 insertions(+), 64 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/partner.py create mode 100644 benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates.yml create mode 100644 benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_loader.py create mode 100644 benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_task.py create mode 100644 benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/packaging.py create mode 100644 benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/partner.py create mode 100644 crates/sqlbuild-analysis/src/python_plan/_helpers/lifecycle/gate_frontier.rs create mode 100644 crates/sqlbuild-analysis/src/python_plan/_helpers/lifecycle/mod.rs create mode 100644 crates/sqlbuild-analysis/src/python_plan/_helpers/mod.rs diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/partner.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/partner.py new file mode 100644 index 0000000000..c7d7520d10 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/partner.py @@ -0,0 +1,11 @@ +from python.tasks.rates import fetch_rates +from python.tasks.scoring import score_orders + +from sqlbuild.loaders import loader + + +@loader(depends_on=[score_orders, fetch_rates]) +def partner_scores(ctx): + """Load the scores, gated by a read-side task and an ingress task.""" + result = ctx.result_of(node_function=score_orders) + return [{"order_id": o, "score": s} for o, s in result.payload["scores"]] diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates.yml b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates.yml new file mode 100644 index 0000000000..86db69691c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates.yml @@ -0,0 +1,8 @@ +sources: + - name: partner_rates + description: Rates the partner publishes + managed: true + write_strategy: table + columns: + - name: rate + type: INTEGER diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_loader.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_loader.py new file mode 100644 index 0000000000..4cc3351265 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_loader.py @@ -0,0 +1,9 @@ +from python.tasks.rates import fetch_rates + +from sqlbuild.loaders import loader + + +@loader(depends_on=[fetch_rates]) +def partner_rates(ctx): + """Load the partner's rates before the build's SQL.""" + return [{"rate": ctx.result_of(node_function=fetch_rates).payload["rate"]}] diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_task.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_task.py new file mode 100644 index 0000000000..9fdb868e39 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_task.py @@ -0,0 +1,7 @@ +from sqlbuild.tasks import task + + +@task() +def fetch_rates(ctx): + """Fetch the partner's rates; reads no SQL, so it runs before the build's SQL.""" + return ctx.result(payload={"rate": 10}) diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/packaging.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/packaging.py new file mode 100644 index 0000000000..a1fe66883c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/packaging.py @@ -0,0 +1,9 @@ +from python.tasks.scoring import score_orders + +from sqlbuild.tasks import task + + +@task(depends_on=[score_orders]) +def package_scores(ctx): + """Package the partner's scores; skips softly when there is nothing new to send.""" + return ctx.skip(reason="nothing to package") diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/partner.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/partner.py new file mode 100644 index 0000000000..6054a3ee61 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/partner.py @@ -0,0 +1,10 @@ +from python.tasks.packaging import package_scores + +from sqlbuild.loaders import loader + + +@loader(depends_on=[package_scores]) +def partner_scores(ctx): + """Load the packaged scores; only its direct upstream decides whether it runs.""" + result = ctx.result_of(node_function=package_scores) + return [{"order_id": o, "score": s} for o, s in result.payload["scores"]] diff --git a/crates/sqlbuild-analysis/src/python_plan/_helpers/lifecycle/gate_frontier.rs b/crates/sqlbuild-analysis/src/python_plan/_helpers/lifecycle/gate_frontier.rs new file mode 100644 index 0000000000..52ea06f5f4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/_helpers/lifecycle/gate_frontier.rs @@ -0,0 +1,37 @@ +//! The gates a round-trip loader reads directly or through other loaders. + +use std::collections::{HashMap, HashSet}; +use std::convert::Infallible; + +use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; + +use crate::python_plan::models::LifecycleNode; +use crate::python_selection::constants::LOADER_KIND; + +/// The `gates` reached upstream of `loader` by walking only through loaders, in `gates` order. +pub(crate) fn gate_frontier( + loader: &str, + gates: &[String], + nodes: &HashMap, + upstream: &HashMap>, +) -> Vec { + let mut reached: HashSet<&str> = HashSet::new(); + let Ok(()) = breadth_first_reach([loader], None, &mut reached, |current| { + let through: bool = nodes + .get(*current) + .is_some_and(|node| node.kind == LOADER_KIND); + Ok::<_, Infallible>( + upstream + .get(*current) + .filter(|_| through) + .into_iter() + .flatten() + .map(String::as_str), + ) + }); + gates + .iter() + .filter(|gate| reached.contains(gate.as_str())) + .cloned() + .collect() +} diff --git a/crates/sqlbuild-analysis/src/python_plan/_helpers/lifecycle/mod.rs b/crates/sqlbuild-analysis/src/python_plan/_helpers/lifecycle/mod.rs new file mode 100644 index 0000000000..e05af83b30 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/_helpers/lifecycle/mod.rs @@ -0,0 +1 @@ +pub(crate) mod gate_frontier; diff --git a/crates/sqlbuild-analysis/src/python_plan/_helpers/mod.rs b/crates/sqlbuild-analysis/src/python_plan/_helpers/mod.rs new file mode 100644 index 0000000000..60d651af72 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod lifecycle; diff --git a/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs b/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs index 6d8da0ca03..0c66855de1 100644 --- a/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs +++ b/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeSet, HashMap}; use crate::assembly::project::types::ObjectKey; +use crate::python_plan::_helpers::lifecycle::gate_frontier::gate_frontier; use crate::python_plan::main::upstream_closure::upstream_closure; use crate::python_plan::models::{LifecycleNode, RoundTripLoader, RunLifecycle}; use crate::python_selection::constants::{ASSET_KIND, LOADER_KIND, TASK_KIND}; @@ -33,10 +34,12 @@ pub fn run_lifecycle( ingress_loaders.insert(loader.clone()); continue; } + let gates: Vec = closure.into_iter().filter(|name| runs(name)).collect(); round_trip_loaders.push(RoundTripLoader { name: loader.clone(), waits_for: waits_for.into_iter().collect(), - gates: closure.into_iter().filter(|name| runs(name)).collect(), + frontier: gate_frontier(loader, &gates, nodes, upstream), + gates, }); } let mut ingress: BTreeSet = ingress_loaders.clone(); diff --git a/crates/sqlbuild-analysis/src/python_plan/mod.rs b/crates/sqlbuild-analysis/src/python_plan/mod.rs index 1a70a48981..e6e5a52eb4 100644 --- a/crates/sqlbuild-analysis/src/python_plan/mod.rs +++ b/crates/sqlbuild-analysis/src/python_plan/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod _helpers; pub mod main; pub mod models; diff --git a/crates/sqlbuild-analysis/src/python_plan/models.rs b/crates/sqlbuild-analysis/src/python_plan/models.rs index b04f58cbd1..5c60f8844a 100644 --- a/crates/sqlbuild-analysis/src/python_plan/models.rs +++ b/crates/sqlbuild-analysis/src/python_plan/models.rs @@ -17,8 +17,10 @@ pub struct RoundTripLoader { pub name: String, /// The SQL its selected upstream tasks and assets read, which its load waits for; sorted. pub waits_for: Vec, - /// Its selected upstream tasks and assets, which must succeed for it to load; sorted. + /// Its selected upstream tasks and assets, none of which may fail or hard-skip for it to load; sorted. pub gates: Vec, + /// The gates it reads directly or through other loaders, one of which must succeed for it to load; sorted. + pub frontier: Vec, } /// Selected Python nodes split into ingress and read-side regions, each sorted. diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs index 69d0a9d4c9..b52018a631 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs @@ -55,6 +55,35 @@ fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { "partner_scores", &[("model", "orders")], &["package_scores", "score_orders"], + &["package_scores"], + )], + }, + LifecycleTestCase { + description: "only the tasks and assets a loader reads directly can let it run", + selected: &[ + "fetch_rates", + "score_orders", + "package_scores", + "partner_scores", + ], + nodes: &[ + ("fetch_rates", "task", &[]), + ("score_orders", "task", &[("model", "orders")]), + ("package_scores", "asset", &[]), + ("partner_scores", "loader", &[]), + ], + upstream: &[ + ("partner_scores", &["package_scores", "fetch_rates"]), + ("package_scores", &["score_orders"]), + ], + expected_ingress: &[], + expected_loaders: &[], + expected_read_side: &["fetch_rates", "package_scores", "score_orders"], + expected_round_trips: &[( + "partner_scores", + &[("model", "orders")], + &["fetch_rates", "package_scores", "score_orders"], + &["fetch_rates", "package_scores"], )], }, LifecycleTestCase { @@ -94,13 +123,14 @@ fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { round_trip_loaders: test_case .expected_round_trips .iter() - .map(|(name, waits_for, gates)| RoundTripLoader { + .map(|(name, waits_for, gates, frontier)| RoundTripLoader { name: (*name).to_owned(), waits_for: waits_for .iter() .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) .collect(), gates: names(gates), + frontier: names(frontier), }) .collect(), }, diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs index c4044b7ecc..d23e8c2540 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs @@ -20,7 +20,7 @@ pub(crate) struct LifecycleTestCase { pub(crate) expected_ingress: &'static [&'static str], pub(crate) expected_loaders: &'static [&'static str], pub(crate) expected_read_side: &'static [&'static str], - /// `(loader, SQL it waits for, tasks and assets that gate it)`. + /// `(loader, SQL it waits for, tasks and assets that gate it, the gates it reads directly)`. pub(crate) expected_round_trips: &'static [ExpectedRoundTrip], } @@ -36,4 +36,5 @@ pub(crate) type ExpectedRoundTrip = ( &'static str, &'static [(&'static str, &'static str)], &'static [&'static str], + &'static [&'static str], ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 42b7baeea1..f5e23ede68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -101,8 +101,8 @@ pub(crate) struct DirectPythonLifecycle { ingress_python_results: Py, ingress_load_results: Py, ingress_loader_names: Py, - /// Each round-trip loader's selected upstream tasks and assets, `(kind, name)`, which must succeed. - round_trip_gates: HashMap>, + /// Each round-trip loader's selected upstream tasks and assets, `(kind, name)`, and whether each is on its frontier. + round_trip_gates: RoundTripGates, /// Each round-trip loader's source load key and the SQL keys it waits for. round_trip_waits: Vec<(ObjectKey, Vec)>, /// Whether the build runs read-side tasks and assets; `--no-python` turns this off. @@ -626,7 +626,7 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( ) } -type RoundTripGates = HashMap>; +type RoundTripGates = HashMap>; type RoundTripWaits = Vec<(ObjectKey, Vec)>; /// Each round-trip loader's gating upstreams, and its source load key with the SQL it waits for. @@ -638,10 +638,10 @@ fn round_trip_ordering( let mut gates: RoundTripGates = HashMap::new(); let mut waits: RoundTripWaits = Vec::new(); for loader in &lifecycle.round_trips { - let mut kinds: Vec = Vec::new(); + let mut kinds: Vec<(ObjectKey, bool)> = Vec::new(); for name in &loader.gates { let kind: String = native_field(&nodes.get_item(name)?, "kind")?.extract()?; - kinds.push((kind, name.clone())); + kinds.push(((kind, name.clone()), loader.frontier.contains(name))); } gates.insert(loader.name.clone(), kinds); for entry in native_field(plan_output, "source_load_entries")?.try_iter()? { @@ -669,10 +669,7 @@ pub(crate) struct SourceLoadGate { #[pymethods] impl SourceLoadGate { - /// Why a round-trip load is skipped, by the Python fan-in rule over its gating upstreams. - /// - /// A failed or hard-skipped upstream skips it `hard`. Otherwise it runs when any upstream - /// succeeded, and skips `soft`, naming the first upstream, when every one soft-skipped or never ran. + /// Why a round-trip load is skipped: `hard` for any failed or hard-skipped gate, else `soft` unless a frontier gate succeeded. fn __call__( &self, py: Python<'_>, @@ -699,13 +696,14 @@ impl SourceLoadGate { } let mut succeeded: bool = false; let mut soft_reason: Option = None; - for (kind, name) in gates { + let mut not_run_reason: Option = None; + for ((kind, name), frontier) in gates { let status = statuses .get(name) .map(|(status, mode)| (status.as_str(), mode.as_deref())); let outcome: &str = match status { Some((SUCCESS_STATUS, _)) => { - succeeded = true; + succeeded |= *frontier; continue; } Some((SKIPPED_STATUS, Some(SOFT_SKIP))) => "was skipped", @@ -722,12 +720,20 @@ impl SourceLoadGate { ))); } None if !lifecycle.include_python => { - "did not run; --no-python skips read-side tasks and assets" + not_run_reason.get_or_insert_with(|| { + format!( + "Upstream {kind} '{name}' did not run; --no-python skips read-side tasks and assets" + ) + }); + continue; } None => "did not run; the SQL it reads was not rebuilt", }; soft_reason.get_or_insert_with(|| format!("Upstream {kind} '{name}' {outcome}")); } + if let Some(reason) = not_run_reason { + return Ok(Some((reason, SOFT_SKIP))); + } if succeeded { return Ok(None); } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md index fe2e0615c9..fb7deeb529 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -325,9 +325,10 @@ Dependencies form a DAG. SQLBuild schedules loaders in topological order and exe A loader reads an upstream task's or asset's result with `ctx.result_of(...)`; `ctx.loader(...)` takes loaders only. A failed or hard-skipped upstream task or asset skips the loader and everything -downstream of the source it fills. Otherwise the loader runs when any upstream succeeded; when all -of them skip softly, only the load is skipped, so the source keeps its last loaded data and its -readers still build. A loader selected without its upstream task or asset reads that upstream's +downstream of the source it fills. Otherwise the loader runs when a task or asset it depends on, +directly or through other loaders, succeeded; when all of them skip softly or did not run, only the +load is skipped, so the source keeps its last loaded data and its readers still build when another +of their inputs ran. A loader selected without its upstream task or asset reads that upstream's latest successful result, and fails naming the upstream if it never succeeded. ```python diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 9a1c3b2046..b6b4c624a2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1715,12 +1715,11 @@ class WarehouseRoundTripE2ETestCase: @dataclass(frozen=True) class ChangedRoundTripE2ETestCase: - """A second round-trip build with `build_flags`, after changing `orders` and maybe the task.""" + """A second round-trip build with `build_flags`, after changing `orders` and `extra_files`.""" description: str orders_sql: str expected_output_fragments: tuple[str, ...] expected_scored_rows: tuple[tuple[object, ...], ...] build_flags: tuple[str, ...] = ("--changed",) - task_body: str | None = None extra_files: dict[str, str] = field(default_factory=dict) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index ed18c96b0b..64a8fdd8ce 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1545,6 +1545,16 @@ def node_results_in(*, db_path: Path, schema: str, node_type: str) -> list[tuple ROUND_TRIP_FIXTURE: Path = Path("benchmarks/duckshop/edge_cases/loaders/round_trip") +_TWO_GATE_LOADER: str = ( + "from sqlbuild.loaders import loader\n\n" + "from python.tasks.ranking import rank_orders\n" + "from python.tasks.scoring import score_orders\n\n\n" + "@loader(depends_on=[score_orders, rank_orders])\n" + "def partner_scores(ctx):\n" + ' """Load the scores the partner returned."""\n' + " result = ctx.result_of(node_function=score_orders)\n" + ' return [{"order_id": o, "score": s} for o, s in result.payload["scores"]]\n' +) def round_trip_project_files(*, project_settings: str, task_body: str) -> dict[str, str]: @@ -1582,6 +1592,39 @@ def round_trip_refusal(name: str) -> str: return (ROUND_TRIP_FIXTURE / "refusals" / name).read_text() +def round_trip_variant(variant: str, files: dict[str, str]) -> dict[str, str]: + """DuckShop's round-trip `variant` files, keyed by where each goes in the project.""" + + return { + target: (ROUND_TRIP_FIXTURE / "variants" / variant / name).read_text() + for target, name in files.items() + } + + +def scoring_task_files(body: str) -> dict[str, str]: + """The round trip's score task with `body`, keyed by where it goes in the project.""" + + path: str = "python/tasks/scoring.py" + return { + path: round_trip_project_files(project_settings=ALLOW_ROUND_TRIPS, task_body=body)[path] + } + + +def ranking_task_files(body: str) -> dict[str, str]: + """A second task gating the partner loader, with `body`, and the loader depending on both.""" + + return { + "python/tasks/ranking.py": ( + "from sqlbuild.refs import model\n" + "from sqlbuild.tasks import task\n\n\n" + '@task(depends_on=model("orders"))\n' + "def rank_orders(ctx):\n" + ' """Rank the orders for the partner."""\n' + body + ), + "python/loaders/partner.py": _TWO_GATE_LOADER, + } + + def build_round_trip_project( *, tmp_path: Path, test_case: WarehouseRoundTripE2ETestCase ) -> tuple[Path, str]: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py index c0cf6868e1..3531ad691a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py @@ -16,6 +16,9 @@ FAILING_SCORING_TASK, SCORING_TASK, build_round_trip_project, + ranking_task_files, + round_trip_variant, + scoring_task_files, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( query_duckdb, @@ -27,31 +30,6 @@ 'MODEL (description "Orders to score", materialized table);\n\n' "SELECT 1 AS order_id UNION ALL SELECT 2 UNION ALL SELECT 3\n" ) -_TWO_GATE_LOADER: str = ( - "from sqlbuild.loaders import loader\n\n" - "from python.tasks.ranking import rank_orders\n" - "from python.tasks.scoring import score_orders\n\n\n" - "@loader(depends_on=[score_orders, rank_orders])\n" - "def partner_scores(ctx):\n" - ' """Load the scores the partner returned."""\n' - " result = ctx.result_of(node_function=score_orders)\n" - ' return [{"order_id": o, "score": s} for o, s in result.payload["scores"]]\n' -) - - -def _ranking_task(body: str) -> dict[str, str]: - """A second task gating the partner loader, with `body`, and the loader depending on both.""" - - return { - "python/tasks/ranking.py": ( - "from sqlbuild.refs import model\n" - "from sqlbuild.tasks import task\n\n\n" - '@task(depends_on=model("orders"))\n' - "def rank_orders(ctx):\n" - ' """Rank the orders for the partner."""\n' + body - ), - "python/loaders/partner.py": _TWO_GATE_LOADER, - } @pytest.mark.parametrize( @@ -186,7 +164,7 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ 'MODEL (description "Orders to score", materialized table);\n\n' "SELECT 1 AS order_id UNION ALL SELECT 2 UNION ALL SELECT 3\n" ), - task_body=' return ctx.skip(reason="partner offline")\n', + extra_files=scoring_task_files(' return ctx.skip(reason="partner offline")\n'), expected_output_fragments=( "soft skip: Upstream task 'score_orders' was skipped", "table scored_orders OK", @@ -200,7 +178,9 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ 'MODEL (description "Orders to score", materialized table);\n\n' "SELECT 1 AS order_id UNION ALL SELECT 2 UNION ALL SELECT 3\n" ), - task_body=' return ctx.skip(reason="partner offline", mode="hard")\n', + extra_files=scoring_task_files( + ' return ctx.skip(reason="partner offline", mode="hard")\n' + ), expected_output_fragments=( "hard skip: Upstream task 'score_orders' was skipped", "table scored_orders SKIP", @@ -210,8 +190,10 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ ChangedRoundTripE2ETestCase( description="one gating task soft-skips and another succeeds, so the load runs", orders_sql=_ORDERS_WITH_A_THIRD, - task_body=' return ctx.skip(reason="partner offline")\n', - extra_files=_ranking_task(' return ctx.result(payload={"ranked": True})\n'), + extra_files={ + **scoring_task_files(' return ctx.skip(reason="partner offline")\n'), + **ranking_task_files(' return ctx.result(payload={"ranked": True})\n'), + }, expected_output_fragments=( "task rank_orders OK", "source partner_scores OK", @@ -222,8 +204,10 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ ChangedRoundTripE2ETestCase( description="every gating task soft-skips, so the load soft-skips", orders_sql=_ORDERS_WITH_A_THIRD, - task_body=' return ctx.skip(reason="partner offline")\n', - extra_files=_ranking_task(' return ctx.skip(reason="nothing to rank")\n'), + extra_files={ + **scoring_task_files(' return ctx.skip(reason="partner offline")\n'), + **ranking_task_files(' return ctx.skip(reason="nothing to rank")\n'), + }, expected_output_fragments=( "partner_scores SKIP", "soft skip: Upstream task '", @@ -234,14 +218,55 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ ChangedRoundTripE2ETestCase( description="one gating task hard-skips while another succeeds, so the load hard-skips", orders_sql=_ORDERS_WITH_A_THIRD, - task_body=' return ctx.skip(reason="partner offline", mode="hard")\n', - extra_files=_ranking_task(' return ctx.result(payload={"ranked": True})\n'), + extra_files={ + **scoring_task_files( + ' return ctx.skip(reason="partner offline", mode="hard")\n' + ), + **ranking_task_files(' return ctx.result(payload={"ranked": True})\n'), + }, expected_output_fragments=( "hard skip: Upstream task 'score_orders' was skipped", "table scored_orders SKIP", ), expected_scored_rows=((1, 10), (2, 20)), ), + ChangedRoundTripE2ETestCase( + description="a task behind the load's soft-skipped upstream succeeding does not run it", + orders_sql=_ORDERS_WITH_A_THIRD, + extra_files=round_trip_variant( + "transitive_gate", + { + "python/tasks/packaging.py": "packaging.py", + "python/loaders/partner.py": "partner.py", + }, + ), + expected_output_fragments=( + "soft skip: Upstream task 'package_scores'", + "table scored_orders OK", + "Completed successfully", + ), + expected_scored_rows=((1, 10), (2, 20)), + ), + ChangedRoundTripE2ETestCase( + description="--no-python soft-skips the load even when an ingress gate succeeds", + orders_sql=_ORDERS_WITH_A_THIRD, + extra_files=round_trip_variant( + "ingress_gate", + { + "python/tasks/rates.py": "rates_task.py", + "python/loaders/rates.py": "rates_loader.py", + "sources/rates.yml": "rates.yml", + "python/loaders/partner.py": "partner.py", + }, + ), + expected_output_fragments=( + "soft skip: Upstream task 'score_orders' did not run; --no-python skips " + "read-side tasks and assets", + "Completed successfully", + ), + expected_scored_rows=((1, 10), (2, 20)), + build_flags=("--no-python",), + ), ], ids=lambda case: case.description, ) @@ -265,13 +290,6 @@ def test_given_a_built_round_trip_when_rebuilding_then_the_decision_follows_the_ (project_dir / "models" / "orders.sql").write_text(test_case.orders_sql, encoding="utf-8") for relative, text in test_case.extra_files.items(): (project_dir / relative).write_text(text, encoding="utf-8") - if test_case.task_body is not None: - task_file: Path = project_dir / "python" / "tasks" / "scoring.py" - task_source: str = task_file.read_text(encoding="utf-8") - task_file.write_text( - task_source[: task_source.index(" relation = ")] + test_case.task_body, - encoding="utf-8", - ) result: subprocess.CompletedProcess[str] = run_sqb( command=("--no-color", "build", *test_case.build_flags), project_dir=project_dir diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx index fed492df5c..4be22259f2 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx @@ -305,9 +305,10 @@ Dependencies form a DAG. SQLBuild schedules loaders in topological order and exe A loader reads an upstream task's or asset's result with `ctx.result_of(...)`; `ctx.loader(...)` takes loaders only. A failed or hard-skipped upstream task or asset skips the loader and everything -downstream of the source it fills. Otherwise the loader runs when any upstream succeeded; when all -of them skip softly, only the load is skipped, so the source keeps its last loaded data and its -readers still build. A loader selected without its upstream task or asset reads that upstream's +downstream of the source it fills. Otherwise the loader runs when a task or asset it depends on, +directly or through other loaders, succeeded; when all of them skip softly or did not run, only the +load is skipped, so the source keeps its last loaded data and its readers still build when another +of their inputs ran. A loader selected without its upstream task or asset reads that upstream's latest successful result, and fails naming the upstream if it never succeeded. ```python