diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/partner.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/partner.py new file mode 100644 index 0000000000..c7d7520d10 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/partner.py @@ -0,0 +1,11 @@ +from python.tasks.rates import fetch_rates +from python.tasks.scoring import score_orders + +from sqlbuild.loaders import loader + + +@loader(depends_on=[score_orders, fetch_rates]) +def partner_scores(ctx): + """Load the scores, gated by a read-side task and an ingress task.""" + result = ctx.result_of(node_function=score_orders) + return [{"order_id": o, "score": s} for o, s in result.payload["scores"]] diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates.yml b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates.yml new file mode 100644 index 0000000000..86db69691c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates.yml @@ -0,0 +1,8 @@ +sources: + - name: partner_rates + description: Rates the partner publishes + managed: true + write_strategy: table + columns: + - name: rate + type: INTEGER diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_loader.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_loader.py new file mode 100644 index 0000000000..4cc3351265 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_loader.py @@ -0,0 +1,9 @@ +from python.tasks.rates import fetch_rates + +from sqlbuild.loaders import loader + + +@loader(depends_on=[fetch_rates]) +def partner_rates(ctx): + """Load the partner's rates before the build's SQL.""" + return [{"rate": ctx.result_of(node_function=fetch_rates).payload["rate"]}] diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_task.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_task.py new file mode 100644 index 0000000000..9fdb868e39 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/ingress_gate/rates_task.py @@ -0,0 +1,7 @@ +from sqlbuild.tasks import task + + +@task() +def fetch_rates(ctx): + """Fetch the partner's rates; reads no SQL, so it runs before the build's SQL.""" + return ctx.result(payload={"rate": 10}) diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/packaging.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/packaging.py new file mode 100644 index 0000000000..a1fe66883c --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/packaging.py @@ -0,0 +1,9 @@ +from python.tasks.scoring import score_orders + +from sqlbuild.tasks import task + + +@task(depends_on=[score_orders]) +def package_scores(ctx): + """Package the partner's scores; skips softly when there is nothing new to send.""" + return ctx.skip(reason="nothing to package") diff --git a/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/partner.py b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/partner.py new file mode 100644 index 0000000000..6054a3ee61 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/loaders/round_trip/variants/transitive_gate/partner.py @@ -0,0 +1,10 @@ +from python.tasks.packaging import package_scores + +from sqlbuild.loaders import loader + + +@loader(depends_on=[package_scores]) +def partner_scores(ctx): + """Load the packaged scores; only its direct upstream decides whether it runs.""" + result = ctx.result_of(node_function=package_scores) + return [{"order_id": o, "score": s} for o, s in result.payload["scores"]] diff --git a/crates/sqlbuild-analysis/src/python_plan/_helpers/lifecycle/gate_frontier.rs b/crates/sqlbuild-analysis/src/python_plan/_helpers/lifecycle/gate_frontier.rs new file mode 100644 index 0000000000..52ea06f5f4 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/_helpers/lifecycle/gate_frontier.rs @@ -0,0 +1,37 @@ +//! The gates a round-trip loader reads directly or through other loaders. + +use std::collections::{HashMap, HashSet}; +use std::convert::Infallible; + +use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach; + +use crate::python_plan::models::LifecycleNode; +use crate::python_selection::constants::LOADER_KIND; + +/// The `gates` reached upstream of `loader` by walking only through loaders, in `gates` order. +pub(crate) fn gate_frontier( + loader: &str, + gates: &[String], + nodes: &HashMap, + upstream: &HashMap>, +) -> Vec { + let mut reached: HashSet<&str> = HashSet::new(); + let Ok(()) = breadth_first_reach([loader], None, &mut reached, |current| { + let through: bool = nodes + .get(*current) + .is_some_and(|node| node.kind == LOADER_KIND); + Ok::<_, Infallible>( + upstream + .get(*current) + .filter(|_| through) + .into_iter() + .flatten() + .map(String::as_str), + ) + }); + gates + .iter() + .filter(|gate| reached.contains(gate.as_str())) + .cloned() + .collect() +} diff --git a/crates/sqlbuild-analysis/src/python_plan/_helpers/lifecycle/mod.rs b/crates/sqlbuild-analysis/src/python_plan/_helpers/lifecycle/mod.rs new file mode 100644 index 0000000000..e05af83b30 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/_helpers/lifecycle/mod.rs @@ -0,0 +1 @@ +pub(crate) mod gate_frontier; diff --git a/crates/sqlbuild-analysis/src/python_plan/_helpers/mod.rs b/crates/sqlbuild-analysis/src/python_plan/_helpers/mod.rs new file mode 100644 index 0000000000..60d651af72 --- /dev/null +++ b/crates/sqlbuild-analysis/src/python_plan/_helpers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod lifecycle; diff --git a/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs b/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs index 6d8da0ca03..0c66855de1 100644 --- a/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs +++ b/crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs @@ -3,6 +3,7 @@ use std::collections::{BTreeSet, HashMap}; use crate::assembly::project::types::ObjectKey; +use crate::python_plan::_helpers::lifecycle::gate_frontier::gate_frontier; use crate::python_plan::main::upstream_closure::upstream_closure; use crate::python_plan::models::{LifecycleNode, RoundTripLoader, RunLifecycle}; use crate::python_selection::constants::{ASSET_KIND, LOADER_KIND, TASK_KIND}; @@ -33,10 +34,12 @@ pub fn run_lifecycle( ingress_loaders.insert(loader.clone()); continue; } + let gates: Vec = closure.into_iter().filter(|name| runs(name)).collect(); round_trip_loaders.push(RoundTripLoader { name: loader.clone(), waits_for: waits_for.into_iter().collect(), - gates: closure.into_iter().filter(|name| runs(name)).collect(), + frontier: gate_frontier(loader, &gates, nodes, upstream), + gates, }); } let mut ingress: BTreeSet = ingress_loaders.clone(); diff --git a/crates/sqlbuild-analysis/src/python_plan/mod.rs b/crates/sqlbuild-analysis/src/python_plan/mod.rs index 1a70a48981..e6e5a52eb4 100644 --- a/crates/sqlbuild-analysis/src/python_plan/mod.rs +++ b/crates/sqlbuild-analysis/src/python_plan/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod _helpers; pub mod main; pub mod models; diff --git a/crates/sqlbuild-analysis/src/python_plan/models.rs b/crates/sqlbuild-analysis/src/python_plan/models.rs index b04f58cbd1..5c60f8844a 100644 --- a/crates/sqlbuild-analysis/src/python_plan/models.rs +++ b/crates/sqlbuild-analysis/src/python_plan/models.rs @@ -17,8 +17,10 @@ pub struct RoundTripLoader { pub name: String, /// The SQL its selected upstream tasks and assets read, which its load waits for; sorted. pub waits_for: Vec, - /// Its selected upstream tasks and assets, which must succeed for it to load; sorted. + /// Its selected upstream tasks and assets, none of which may fail or hard-skip for it to load; sorted. pub gates: Vec, + /// The gates it reads directly or through other loaders, one of which must succeed for it to load; sorted. + pub frontier: Vec, } /// Selected Python nodes split into ingress and read-side regions, each sorted. diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs index 69d0a9d4c9..b52018a631 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_run_lifecycle.rs @@ -55,6 +55,35 @@ fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { "partner_scores", &[("model", "orders")], &["package_scores", "score_orders"], + &["package_scores"], + )], + }, + LifecycleTestCase { + description: "only the tasks and assets a loader reads directly can let it run", + selected: &[ + "fetch_rates", + "score_orders", + "package_scores", + "partner_scores", + ], + nodes: &[ + ("fetch_rates", "task", &[]), + ("score_orders", "task", &[("model", "orders")]), + ("package_scores", "asset", &[]), + ("partner_scores", "loader", &[]), + ], + upstream: &[ + ("partner_scores", &["package_scores", "fetch_rates"]), + ("package_scores", &["score_orders"]), + ], + expected_ingress: &[], + expected_loaders: &[], + expected_read_side: &["fetch_rates", "package_scores", "score_orders"], + expected_round_trips: &[( + "partner_scores", + &[("model", "orders")], + &["fetch_rates", "package_scores", "score_orders"], + &["fetch_rates", "package_scores"], )], }, LifecycleTestCase { @@ -94,13 +123,14 @@ fn given_selection_when_classifying_lifecycle_then_matches_python_regions() { round_trip_loaders: test_case .expected_round_trips .iter() - .map(|(name, waits_for, gates)| RoundTripLoader { + .map(|(name, waits_for, gates, frontier)| RoundTripLoader { name: (*name).to_owned(), waits_for: waits_for .iter() .map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned())) .collect(), gates: names(gates), + frontier: names(frontier), }) .collect(), }, diff --git a/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs index c4044b7ecc..d23e8c2540 100644 --- a/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs +++ b/crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs @@ -20,7 +20,7 @@ pub(crate) struct LifecycleTestCase { pub(crate) expected_ingress: &'static [&'static str], pub(crate) expected_loaders: &'static [&'static str], pub(crate) expected_read_side: &'static [&'static str], - /// `(loader, SQL it waits for, tasks and assets that gate it)`. + /// `(loader, SQL it waits for, tasks and assets that gate it, the gates it reads directly)`. pub(crate) expected_round_trips: &'static [ExpectedRoundTrip], } @@ -36,4 +36,5 @@ pub(crate) type ExpectedRoundTrip = ( &'static str, &'static [(&'static str, &'static str)], &'static [&'static str], + &'static [&'static str], ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs index 3a4c3a8872..f5e23ede68 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/build_command/lifecycle.rs @@ -101,10 +101,12 @@ pub(crate) struct DirectPythonLifecycle { ingress_python_results: Py, ingress_load_results: Py, ingress_loader_names: Py, - /// Each round-trip loader's selected upstream tasks and assets, `(kind, name)`, which must succeed. - round_trip_gates: HashMap>, + /// Each round-trip loader's selected upstream tasks and assets, `(kind, name)`, and whether each is on its frontier. + round_trip_gates: RoundTripGates, /// Each round-trip loader's source load key and the SQL keys it waits for. round_trip_waits: Vec<(ObjectKey, Vec)>, + /// Whether the build runs read-side tasks and assets; `--no-python` turns this off. + include_python: bool, } #[pymethods] @@ -469,7 +471,8 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( let connection_config: Bound<'py, PyAny> = context.get("connection_config")?; let project: Bound<'py, PyAny> = native_field(pipeline, "project")?; let plan_output: Bound<'py, PyAny> = native_field(pipeline, "plan_output")?; - let selected: Vec = if context.request_flag("include_python")? { + let include_python: bool = context.request_flag("include_python")?; + let selected: Vec = if include_python { task_asset_names(&discovered, pipeline)? } else { Vec::new() @@ -618,11 +621,12 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>( ingress_loader_names: native_field(&lifecycle.plan, "ingress_loader_names")?.unbind(), round_trip_gates, round_trip_waits, + include_python, }, ) } -type RoundTripGates = HashMap>; +type RoundTripGates = HashMap>; type RoundTripWaits = Vec<(ObjectKey, Vec)>; /// Each round-trip loader's gating upstreams, and its source load key with the SQL it waits for. @@ -634,10 +638,10 @@ fn round_trip_ordering( let mut gates: RoundTripGates = HashMap::new(); let mut waits: RoundTripWaits = Vec::new(); for loader in &lifecycle.round_trips { - let mut kinds: Vec = Vec::new(); + let mut kinds: Vec<(ObjectKey, bool)> = Vec::new(); for name in &loader.gates { let kind: String = native_field(&nodes.get_item(name)?, "kind")?.extract()?; - kinds.push((kind, name.clone())); + kinds.push(((kind, name.clone()), loader.frontier.contains(name))); } gates.insert(loader.name.clone(), kinds); for entry in native_field(plan_output, "source_load_entries")?.try_iter()? { @@ -665,7 +669,7 @@ pub(crate) struct SourceLoadGate { #[pymethods] impl SourceLoadGate { - /// Why a round-trip load is skipped: `hard` after a failed or skipped upstream, `soft` when it never ran. + /// Why a round-trip load is skipped: `hard` for any failed or hard-skipped gate, else `soft` unless a frontier gate succeeded. fn __call__( &self, py: Python<'_>, @@ -676,24 +680,63 @@ impl SourceLoadGate { let Some(gates) = loader.and_then(|loader| lifecycle.round_trip_gates.get(&loader)) else { return Ok(None); }; - let mut statuses: HashMap = HashMap::new(); + let mut statuses: HashMap)> = HashMap::new(); for result in lifecycle.python_results(py)?.try_iter()? { let result: Bound<'_, PyAny> = result?; + let skip_mode: Bound<'_, PyAny> = native_field(&result, "skip_mode")?; statuses.insert( native_field(&result, "node_name")?.extract()?, - native_field(&result, "status")?.str()?.extract()?, + ( + native_field(&result, "status")?.str()?.extract()?, + (!skip_mode.is_none()) + .then(|| skip_mode.str()?.extract()) + .transpose()?, + ), ); } - for (kind, name) in gates { - let (outcome, mode): (&str, &'static str) = match statuses.get(name).map(String::as_str) - { - Some(SUCCESS_STATUS) => continue, - Some(SKIPPED_STATUS) => ("was skipped", HARD_SKIP), - Some(_) => ("failed", HARD_SKIP), - None => ("did not run; the SQL it reads was not rebuilt", SOFT_SKIP), + let mut succeeded: bool = false; + let mut soft_reason: Option = None; + let mut not_run_reason: Option = None; + for ((kind, name), frontier) in gates { + let status = statuses + .get(name) + .map(|(status, mode)| (status.as_str(), mode.as_deref())); + let outcome: &str = match status { + Some((SUCCESS_STATUS, _)) => { + succeeded |= *frontier; + continue; + } + Some((SKIPPED_STATUS, Some(SOFT_SKIP))) => "was skipped", + Some((SKIPPED_STATUS, _)) => { + return Ok(Some(( + format!("Upstream {kind} '{name}' was skipped"), + HARD_SKIP, + ))); + } + Some(_) => { + return Ok(Some(( + format!("Upstream {kind} '{name}' failed"), + HARD_SKIP, + ))); + } + None if !lifecycle.include_python => { + not_run_reason.get_or_insert_with(|| { + format!( + "Upstream {kind} '{name}' did not run; --no-python skips read-side tasks and assets" + ) + }); + continue; + } + None => "did not run; the SQL it reads was not rebuilt", }; - return Ok(Some((format!("Upstream {kind} '{name}' {outcome}"), mode))); + soft_reason.get_or_insert_with(|| format!("Upstream {kind} '{name}' {outcome}")); + } + if let Some(reason) = not_run_reason { + return Ok(Some((reason, SOFT_SKIP))); + } + if succeeded { + return Ok(None); } - Ok(None) + Ok(soft_reason.map(|reason| (reason, SOFT_SKIP))) } } diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md index c439564e0f..fb7deeb529 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/loaders.md @@ -324,9 +324,12 @@ Dependencies form a DAG. SQLBuild schedules loaders in topological order and exe ### Depending on tasks and assets A loader reads an upstream task's or asset's result with `ctx.result_of(...)`; `ctx.loader(...)` -takes loaders only. A failed or skipped upstream task or asset skips the loader and everything -downstream of the source it fills. A loader selected without its upstream task or asset reads that -upstream's latest successful result, and fails naming the upstream if it never succeeded. +takes loaders only. A failed or hard-skipped upstream task or asset skips the loader and everything +downstream of the source it fills. Otherwise the loader runs when a task or asset it depends on, +directly or through other loaders, succeeded; when all of them skip softly or did not run, only the +load is skipped, so the source keeps its last loaded data and its readers still build when another +of their inputs ran. A loader selected without its upstream task or asset reads that upstream's +latest successful result, and fails naming the upstream if it never succeeded. ```python from sqlbuild.loaders import loader diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md index 3c3ae4c95a..d5991ed34e 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/python-nodes/overview.md @@ -110,8 +110,11 @@ allow_warehouse_round_trips = true With the setting on: - One `sqb build` runs the SQL the task reads, then the task, then the loader, then the models that - read the loader's source. If the task fails or is skipped, the loader and everything downstream of - its source are skipped. + read the loader's source. The loader follows the usual rule for its upstream tasks and assets: if + one fails or skips with `mode="hard"`, the loader and everything downstream of its source are + skipped; otherwise it runs when any of them succeeded. When all of them skip softly, only the load + is skipped: the source keeps its last loaded data, and models reading it still build when another + of their inputs ran. - Lineage and column lineage restart at the source: the source's columns come from the loader, not from the models the task read. - Job groups run a member that builds the task before the member that runs the loader, and that one diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index 8728f33513..b6b4c624a2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1715,9 +1715,11 @@ class WarehouseRoundTripE2ETestCase: @dataclass(frozen=True) class ChangedRoundTripE2ETestCase: - """A second `--changed` build of a round trip after an optional change to `orders`.""" + """A second round-trip build with `build_flags`, after changing `orders` and `extra_files`.""" description: str orders_sql: str expected_output_fragments: tuple[str, ...] expected_scored_rows: tuple[tuple[object, ...], ...] + build_flags: tuple[str, ...] = ("--changed",) + extra_files: dict[str, str] = field(default_factory=dict) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index ed18c96b0b..64a8fdd8ce 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -1545,6 +1545,16 @@ def node_results_in(*, db_path: Path, schema: str, node_type: str) -> list[tuple ROUND_TRIP_FIXTURE: Path = Path("benchmarks/duckshop/edge_cases/loaders/round_trip") +_TWO_GATE_LOADER: str = ( + "from sqlbuild.loaders import loader\n\n" + "from python.tasks.ranking import rank_orders\n" + "from python.tasks.scoring import score_orders\n\n\n" + "@loader(depends_on=[score_orders, rank_orders])\n" + "def partner_scores(ctx):\n" + ' """Load the scores the partner returned."""\n' + " result = ctx.result_of(node_function=score_orders)\n" + ' return [{"order_id": o, "score": s} for o, s in result.payload["scores"]]\n' +) def round_trip_project_files(*, project_settings: str, task_body: str) -> dict[str, str]: @@ -1582,6 +1592,39 @@ def round_trip_refusal(name: str) -> str: return (ROUND_TRIP_FIXTURE / "refusals" / name).read_text() +def round_trip_variant(variant: str, files: dict[str, str]) -> dict[str, str]: + """DuckShop's round-trip `variant` files, keyed by where each goes in the project.""" + + return { + target: (ROUND_TRIP_FIXTURE / "variants" / variant / name).read_text() + for target, name in files.items() + } + + +def scoring_task_files(body: str) -> dict[str, str]: + """The round trip's score task with `body`, keyed by where it goes in the project.""" + + path: str = "python/tasks/scoring.py" + return { + path: round_trip_project_files(project_settings=ALLOW_ROUND_TRIPS, task_body=body)[path] + } + + +def ranking_task_files(body: str) -> dict[str, str]: + """A second task gating the partner loader, with `body`, and the loader depending on both.""" + + return { + "python/tasks/ranking.py": ( + "from sqlbuild.refs import model\n" + "from sqlbuild.tasks import task\n\n\n" + '@task(depends_on=model("orders"))\n' + "def rank_orders(ctx):\n" + ' """Rank the orders for the partner."""\n' + body + ), + "python/loaders/partner.py": _TWO_GATE_LOADER, + } + + def build_round_trip_project( *, tmp_path: Path, test_case: WarehouseRoundTripE2ETestCase ) -> tuple[Path, str]: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py index 308596fae2..3531ad691a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_warehouse_round_trips.py @@ -16,6 +16,9 @@ FAILING_SCORING_TASK, SCORING_TASK, build_round_trip_project, + ranking_task_files, + round_trip_variant, + scoring_task_files, ) from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import ( query_duckdb, @@ -23,6 +26,11 @@ table_exists, ) +_ORDERS_WITH_A_THIRD: str = ( + 'MODEL (description "Orders to score", materialized table);\n\n' + "SELECT 1 AS order_id UNION ALL SELECT 2 UNION ALL SELECT 3\n" +) + @pytest.mark.parametrize( "test_case", @@ -136,10 +144,133 @@ def test_given_a_round_trip_that_cannot_complete_when_building_then_its_readers_ ), expected_scored_rows=((1, 10), (2, 20)), ), + ChangedRoundTripE2ETestCase( + description="--no-python leaves the task unrun and names the flag as the skip reason", + orders_sql=( + 'MODEL (description "Orders to score", materialized table);\n\n' + "SELECT 1 AS order_id UNION ALL SELECT 2 UNION ALL SELECT 3\n" + ), + expected_output_fragments=( + "soft skip: Upstream task 'score_orders' did not run; --no-python skips " + "read-side tasks and assets", + "Completed successfully", + ), + expected_scored_rows=((1, 10), (2, 20)), + build_flags=("--no-python",), + ), + ChangedRoundTripE2ETestCase( + description="a soft-skipped task soft-skips the load, and a model reading a changed model still runs", + orders_sql=( + 'MODEL (description "Orders to score", materialized table);\n\n' + "SELECT 1 AS order_id UNION ALL SELECT 2 UNION ALL SELECT 3\n" + ), + extra_files=scoring_task_files(' return ctx.skip(reason="partner offline")\n'), + expected_output_fragments=( + "soft skip: Upstream task 'score_orders' was skipped", + "table scored_orders OK", + "Completed successfully", + ), + expected_scored_rows=((1, 10), (2, 20)), + ), + ChangedRoundTripE2ETestCase( + description="a hard-skipped task hard-skips the load and blocks the models reading its source", + orders_sql=( + 'MODEL (description "Orders to score", materialized table);\n\n' + "SELECT 1 AS order_id UNION ALL SELECT 2 UNION ALL SELECT 3\n" + ), + extra_files=scoring_task_files( + ' return ctx.skip(reason="partner offline", mode="hard")\n' + ), + expected_output_fragments=( + "hard skip: Upstream task 'score_orders' was skipped", + "table scored_orders SKIP", + ), + expected_scored_rows=((1, 10), (2, 20)), + ), + ChangedRoundTripE2ETestCase( + description="one gating task soft-skips and another succeeds, so the load runs", + orders_sql=_ORDERS_WITH_A_THIRD, + extra_files={ + **scoring_task_files(' return ctx.skip(reason="partner offline")\n'), + **ranking_task_files(' return ctx.result(payload={"ranked": True})\n'), + }, + expected_output_fragments=( + "task rank_orders OK", + "source partner_scores OK", + "table scored_orders OK", + ), + expected_scored_rows=((1, 10), (2, 20)), + ), + ChangedRoundTripE2ETestCase( + description="every gating task soft-skips, so the load soft-skips", + orders_sql=_ORDERS_WITH_A_THIRD, + extra_files={ + **scoring_task_files(' return ctx.skip(reason="partner offline")\n'), + **ranking_task_files(' return ctx.skip(reason="nothing to rank")\n'), + }, + expected_output_fragments=( + "partner_scores SKIP", + "soft skip: Upstream task '", + "table scored_orders OK", + ), + expected_scored_rows=((1, 10), (2, 20)), + ), + ChangedRoundTripE2ETestCase( + description="one gating task hard-skips while another succeeds, so the load hard-skips", + orders_sql=_ORDERS_WITH_A_THIRD, + extra_files={ + **scoring_task_files( + ' return ctx.skip(reason="partner offline", mode="hard")\n' + ), + **ranking_task_files(' return ctx.result(payload={"ranked": True})\n'), + }, + expected_output_fragments=( + "hard skip: Upstream task 'score_orders' was skipped", + "table scored_orders SKIP", + ), + expected_scored_rows=((1, 10), (2, 20)), + ), + ChangedRoundTripE2ETestCase( + description="a task behind the load's soft-skipped upstream succeeding does not run it", + orders_sql=_ORDERS_WITH_A_THIRD, + extra_files=round_trip_variant( + "transitive_gate", + { + "python/tasks/packaging.py": "packaging.py", + "python/loaders/partner.py": "partner.py", + }, + ), + expected_output_fragments=( + "soft skip: Upstream task 'package_scores'", + "table scored_orders OK", + "Completed successfully", + ), + expected_scored_rows=((1, 10), (2, 20)), + ), + ChangedRoundTripE2ETestCase( + description="--no-python soft-skips the load even when an ingress gate succeeds", + orders_sql=_ORDERS_WITH_A_THIRD, + extra_files=round_trip_variant( + "ingress_gate", + { + "python/tasks/rates.py": "rates_task.py", + "python/loaders/rates.py": "rates_loader.py", + "sources/rates.yml": "rates.yml", + "python/loaders/partner.py": "partner.py", + }, + ), + expected_output_fragments=( + "soft skip: Upstream task 'score_orders' did not run; --no-python skips " + "read-side tasks and assets", + "Completed successfully", + ), + expected_scored_rows=((1, 10), (2, 20)), + build_flags=("--no-python",), + ), ], ids=lambda case: case.description, ) -def test_given_a_built_round_trip_when_building_changed_then_the_decision_follows_the_chain( +def test_given_a_built_round_trip_when_rebuilding_then_the_decision_follows_the_chain( tmp_path: Path, test_case: ChangedRoundTripE2ETestCase, ) -> None: @@ -157,9 +288,11 @@ def test_given_a_built_round_trip_when_building_changed_then_the_decision_follow ) project_dir: Path = database.parent (project_dir / "models" / "orders.sql").write_text(test_case.orders_sql, encoding="utf-8") + for relative, text in test_case.extra_files.items(): + (project_dir / relative).write_text(text, encoding="utf-8") result: subprocess.CompletedProcess[str] = run_sqb( - command=("--no-color", "build", "--changed"), project_dir=project_dir + command=("--no-color", "build", *test_case.build_flags), project_dir=project_dir ) output: str = result.stdout + result.stderr diff --git a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx index 5d0a06bbc3..4be22259f2 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/loaders.mdx @@ -304,9 +304,12 @@ Dependencies form a DAG. SQLBuild schedules loaders in topological order and exe ### Depending on tasks and assets A loader reads an upstream task's or asset's result with `ctx.result_of(...)`; `ctx.loader(...)` -takes loaders only. A failed or skipped upstream task or asset skips the loader and everything -downstream of the source it fills. A loader selected without its upstream task or asset reads that -upstream's latest successful result, and fails naming the upstream if it never succeeded. +takes loaders only. A failed or hard-skipped upstream task or asset skips the loader and everything +downstream of the source it fills. Otherwise the loader runs when a task or asset it depends on, +directly or through other loaders, succeeded; when all of them skip softly or did not run, only the +load is skipped, so the source keeps its last loaded data and its readers still build when another +of their inputs ran. A loader selected without its upstream task or asset reads that upstream's +latest successful result, and fails naming the upstream if it never succeeded. ```python from sqlbuild.loaders import loader diff --git a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx index ff8e0475bf..190d19d562 100644 --- a/website/src/content/docs/docs/concepts/python-nodes/overview.mdx +++ b/website/src/content/docs/docs/concepts/python-nodes/overview.mdx @@ -94,8 +94,11 @@ allow_warehouse_round_trips = true With the setting on: - One `sqb build` runs the SQL the task reads, then the task, then the loader, then the models that - read the loader's source. If the task fails or is skipped, the loader and everything downstream of - its source are skipped. + read the loader's source. The loader follows the usual rule for its upstream tasks and assets: if + one fails or skips with `mode="hard"`, the loader and everything downstream of its source are + skipped; otherwise it runs when any of them succeeded. When all of them skip softly, only the load + is skipped: the source keeps its last loaded data, and models reading it still build when another + of their inputs ran. - Lineage and column lineage restart at the source: the source's columns come from the loader, not from the models the task read. - Job groups run a member that builds the task before the member that runs the loader, and that one