Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
from python.tasks.rates import fetch_rates
from python.tasks.scoring import score_orders

from sqlbuild.loaders import loader


@loader(depends_on=[score_orders, fetch_rates])
def partner_scores(ctx):
"""Load the scores, gated by a read-side task and an ingress task."""
result = ctx.result_of(node_function=score_orders)
return [{"order_id": o, "score": s} for o, s in result.payload["scores"]]
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
sources:
- name: partner_rates
description: Rates the partner publishes
managed: true
write_strategy: table
columns:
- name: rate
type: INTEGER
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
from python.tasks.rates import fetch_rates

from sqlbuild.loaders import loader


@loader(depends_on=[fetch_rates])
def partner_rates(ctx):
"""Load the partner's rates before the build's SQL."""
return [{"rate": ctx.result_of(node_function=fetch_rates).payload["rate"]}]
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
from sqlbuild.tasks import task


@task()
def fetch_rates(ctx):
"""Fetch the partner's rates; reads no SQL, so it runs before the build's SQL."""
return ctx.result(payload={"rate": 10})
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
from python.tasks.scoring import score_orders

from sqlbuild.tasks import task


@task(depends_on=[score_orders])
def package_scores(ctx):
"""Package the partner's scores; skips softly when there is nothing new to send."""
return ctx.skip(reason="nothing to package")
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
from python.tasks.packaging import package_scores

from sqlbuild.loaders import loader


@loader(depends_on=[package_scores])
def partner_scores(ctx):
"""Load the packaged scores; only its direct upstream decides whether it runs."""
result = ctx.result_of(node_function=package_scores)
return [{"order_id": o, "score": s} for o, s in result.payload["scores"]]
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
//! The gates a round-trip loader reads directly or through other loaders.

use std::collections::{HashMap, HashSet};
use std::convert::Infallible;

use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach;

use crate::python_plan::models::LifecycleNode;
use crate::python_selection::constants::LOADER_KIND;

/// The `gates` reached upstream of `loader` by walking only through loaders, in `gates` order.
pub(crate) fn gate_frontier(
loader: &str,
gates: &[String],
nodes: &HashMap<String, LifecycleNode>,
upstream: &HashMap<String, Vec<String>>,
) -> Vec<String> {
let mut reached: HashSet<&str> = HashSet::new();
let Ok(()) = breadth_first_reach([loader], None, &mut reached, |current| {
let through: bool = nodes
.get(*current)
.is_some_and(|node| node.kind == LOADER_KIND);
Ok::<_, Infallible>(
upstream
.get(*current)
.filter(|_| through)
.into_iter()
.flatten()
.map(String::as_str),
)
});
gates
.iter()
.filter(|gate| reached.contains(gate.as_str()))
.cloned()
.collect()
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pub(crate) mod gate_frontier;
1 change: 1 addition & 0 deletions crates/sqlbuild-analysis/src/python_plan/_helpers/mod.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pub(crate) mod lifecycle;
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
use std::collections::{BTreeSet, HashMap};

use crate::assembly::project::types::ObjectKey;
use crate::python_plan::_helpers::lifecycle::gate_frontier::gate_frontier;
use crate::python_plan::main::upstream_closure::upstream_closure;
use crate::python_plan::models::{LifecycleNode, RoundTripLoader, RunLifecycle};
use crate::python_selection::constants::{ASSET_KIND, LOADER_KIND, TASK_KIND};
Expand Down Expand Up @@ -33,10 +34,12 @@ pub fn run_lifecycle(
ingress_loaders.insert(loader.clone());
continue;
}
let gates: Vec<String> = closure.into_iter().filter(|name| runs(name)).collect();
round_trip_loaders.push(RoundTripLoader {
name: loader.clone(),
waits_for: waits_for.into_iter().collect(),
gates: closure.into_iter().filter(|name| runs(name)).collect(),
frontier: gate_frontier(loader, &gates, nodes, upstream),
gates,
});
}
let mut ingress: BTreeSet<String> = ingress_loaders.clone();
Expand Down
1 change: 1 addition & 0 deletions crates/sqlbuild-analysis/src/python_plan/mod.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
pub(crate) mod _helpers;
pub mod main;
pub mod models;

Expand Down
4 changes: 3 additions & 1 deletion crates/sqlbuild-analysis/src/python_plan/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ pub struct RoundTripLoader {
pub name: String,
/// The SQL its selected upstream tasks and assets read, which its load waits for; sorted.
pub waits_for: Vec<ObjectKey>,
/// Its selected upstream tasks and assets, which must succeed for it to load; sorted.
/// Its selected upstream tasks and assets, none of which may fail or hard-skip for it to load; sorted.
pub gates: Vec<String>,
/// The gates it reads directly or through other loaders, one of which must succeed for it to load; sorted.
pub frontier: Vec<String>,
}

/// Selected Python nodes split into ingress and read-side regions, each sorted.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,35 @@ fn given_selection_when_classifying_lifecycle_then_matches_python_regions() {
"partner_scores",
&[("model", "orders")],
&["package_scores", "score_orders"],
&["package_scores"],
)],
},
LifecycleTestCase {
description: "only the tasks and assets a loader reads directly can let it run",
selected: &[
"fetch_rates",
"score_orders",
"package_scores",
"partner_scores",
],
nodes: &[
("fetch_rates", "task", &[]),
("score_orders", "task", &[("model", "orders")]),
("package_scores", "asset", &[]),
("partner_scores", "loader", &[]),
],
upstream: &[
("partner_scores", &["package_scores", "fetch_rates"]),
("package_scores", &["score_orders"]),
],
expected_ingress: &[],
expected_loaders: &[],
expected_read_side: &["fetch_rates", "package_scores", "score_orders"],
expected_round_trips: &[(
"partner_scores",
&[("model", "orders")],
&["fetch_rates", "package_scores", "score_orders"],
&["fetch_rates", "package_scores"],
)],
},
LifecycleTestCase {
Expand Down Expand Up @@ -94,13 +123,14 @@ fn given_selection_when_classifying_lifecycle_then_matches_python_regions() {
round_trip_loaders: test_case
.expected_round_trips
.iter()
.map(|(name, waits_for, gates)| RoundTripLoader {
.map(|(name, waits_for, gates, frontier)| RoundTripLoader {
name: (*name).to_owned(),
waits_for: waits_for
.iter()
.map(|(kind, name)| ((*kind).to_owned(), (*name).to_owned()))
.collect(),
gates: names(gates),
frontier: names(frontier),
})
.collect(),
},
Expand Down
3 changes: 2 additions & 1 deletion crates/sqlbuild-analysis/src/python_plan/tests/test_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ pub(crate) struct LifecycleTestCase {
pub(crate) expected_ingress: &'static [&'static str],
pub(crate) expected_loaders: &'static [&'static str],
pub(crate) expected_read_side: &'static [&'static str],
/// `(loader, SQL it waits for, tasks and assets that gate it)`.
/// `(loader, SQL it waits for, tasks and assets that gate it, the gates it reads directly)`.
pub(crate) expected_round_trips: &'static [ExpectedRoundTrip],
}

Expand All @@ -36,4 +36,5 @@ pub(crate) type ExpectedRoundTrip = (
&'static str,
&'static [(&'static str, &'static str)],
&'static [&'static str],
&'static [&'static str],
);
Original file line number Diff line number Diff line change
Expand Up @@ -101,10 +101,12 @@ pub(crate) struct DirectPythonLifecycle {
ingress_python_results: Py<PyAny>,
ingress_load_results: Py<PyAny>,
ingress_loader_names: Py<PyAny>,
/// Each round-trip loader's selected upstream tasks and assets, `(kind, name)`, which must succeed.
round_trip_gates: HashMap<String, Vec<ObjectKey>>,
/// Each round-trip loader's selected upstream tasks and assets, `(kind, name)`, and whether each is on its frontier.
round_trip_gates: RoundTripGates,
/// Each round-trip loader's source load key and the SQL keys it waits for.
round_trip_waits: Vec<(ObjectKey, Vec<ObjectKey>)>,
/// Whether the build runs read-side tasks and assets; `--no-python` turns this off.
include_python: bool,
}

#[pymethods]
Expand Down Expand Up @@ -469,7 +471,8 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>(
let connection_config: Bound<'py, PyAny> = context.get("connection_config")?;
let project: Bound<'py, PyAny> = native_field(pipeline, "project")?;
let plan_output: Bound<'py, PyAny> = native_field(pipeline, "plan_output")?;
let selected: Vec<String> = if context.request_flag("include_python")? {
let include_python: bool = context.request_flag("include_python")?;
let selected: Vec<String> = if include_python {
task_asset_names(&discovered, pipeline)?
} else {
Vec::new()
Expand Down Expand Up @@ -618,11 +621,12 @@ pub(crate) fn prepare_direct_python_lifecycle<'py>(
ingress_loader_names: native_field(&lifecycle.plan, "ingress_loader_names")?.unbind(),
round_trip_gates,
round_trip_waits,
include_python,
},
)
}

type RoundTripGates = HashMap<String, Vec<ObjectKey>>;
type RoundTripGates = HashMap<String, Vec<(ObjectKey, bool)>>;
type RoundTripWaits = Vec<(ObjectKey, Vec<ObjectKey>)>;

/// Each round-trip loader's gating upstreams, and its source load key with the SQL it waits for.
Expand All @@ -634,10 +638,10 @@ fn round_trip_ordering(
let mut gates: RoundTripGates = HashMap::new();
let mut waits: RoundTripWaits = Vec::new();
for loader in &lifecycle.round_trips {
let mut kinds: Vec<ObjectKey> = Vec::new();
let mut kinds: Vec<(ObjectKey, bool)> = Vec::new();
for name in &loader.gates {
let kind: String = native_field(&nodes.get_item(name)?, "kind")?.extract()?;
kinds.push((kind, name.clone()));
kinds.push(((kind, name.clone()), loader.frontier.contains(name)));
}
gates.insert(loader.name.clone(), kinds);
for entry in native_field(plan_output, "source_load_entries")?.try_iter()? {
Expand Down Expand Up @@ -665,7 +669,7 @@ pub(crate) struct SourceLoadGate {

#[pymethods]
impl SourceLoadGate {
/// Why a round-trip load is skipped: `hard` after a failed or skipped upstream, `soft` when it never ran.
/// Why a round-trip load is skipped: `hard` for any failed or hard-skipped gate, else `soft` unless a frontier gate succeeded.
fn __call__(
&self,
py: Python<'_>,
Expand All @@ -676,24 +680,63 @@ impl SourceLoadGate {
let Some(gates) = loader.and_then(|loader| lifecycle.round_trip_gates.get(&loader)) else {
return Ok(None);
};
let mut statuses: HashMap<String, String> = HashMap::new();
let mut statuses: HashMap<String, (String, Option<String>)> = HashMap::new();
for result in lifecycle.python_results(py)?.try_iter()? {
let result: Bound<'_, PyAny> = result?;
let skip_mode: Bound<'_, PyAny> = native_field(&result, "skip_mode")?;
statuses.insert(
native_field(&result, "node_name")?.extract()?,
native_field(&result, "status")?.str()?.extract()?,
(
native_field(&result, "status")?.str()?.extract()?,
(!skip_mode.is_none())
.then(|| skip_mode.str()?.extract())
.transpose()?,
),
);
}
for (kind, name) in gates {
let (outcome, mode): (&str, &'static str) = match statuses.get(name).map(String::as_str)
{
Some(SUCCESS_STATUS) => continue,
Some(SKIPPED_STATUS) => ("was skipped", HARD_SKIP),
Some(_) => ("failed", HARD_SKIP),
None => ("did not run; the SQL it reads was not rebuilt", SOFT_SKIP),
let mut succeeded: bool = false;
let mut soft_reason: Option<String> = None;
let mut not_run_reason: Option<String> = None;
for ((kind, name), frontier) in gates {
let status = statuses
.get(name)
.map(|(status, mode)| (status.as_str(), mode.as_deref()));
let outcome: &str = match status {
Some((SUCCESS_STATUS, _)) => {
succeeded |= *frontier;
continue;
}
Some((SKIPPED_STATUS, Some(SOFT_SKIP))) => "was skipped",
Some((SKIPPED_STATUS, _)) => {
return Ok(Some((
format!("Upstream {kind} '{name}' was skipped"),
HARD_SKIP,
)));
}
Some(_) => {
return Ok(Some((
format!("Upstream {kind} '{name}' failed"),
HARD_SKIP,
)));
}
None if !lifecycle.include_python => {
not_run_reason.get_or_insert_with(|| {
format!(
"Upstream {kind} '{name}' did not run; --no-python skips read-side tasks and assets"
)
});
continue;
}
None => "did not run; the SQL it reads was not rebuilt",
};
return Ok(Some((format!("Upstream {kind} '{name}' {outcome}"), mode)));
soft_reason.get_or_insert_with(|| format!("Upstream {kind} '{name}' {outcome}"));
}
if let Some(reason) = not_run_reason {
return Ok(Some((reason, SOFT_SKIP)));
}
if succeeded {
return Ok(None);
}
Ok(None)
Ok(soft_reason.map(|reason| (reason, SOFT_SKIP)))
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -324,9 +324,12 @@ Dependencies form a DAG. SQLBuild schedules loaders in topological order and exe
### Depending on tasks and assets

A loader reads an upstream task's or asset's result with `ctx.result_of(...)`; `ctx.loader(...)`
takes loaders only. A failed or skipped upstream task or asset skips the loader and everything
downstream of the source it fills. A loader selected without its upstream task or asset reads that
upstream's latest successful result, and fails naming the upstream if it never succeeded.
takes loaders only. A failed or hard-skipped upstream task or asset skips the loader and everything
downstream of the source it fills. Otherwise the loader runs when a task or asset it depends on,
directly or through other loaders, succeeded; when all of them skip softly or did not run, only the
load is skipped, so the source keeps its last loaded data and its readers still build when another
of their inputs ran. A loader selected without its upstream task or asset reads that upstream's
latest successful result, and fails naming the upstream if it never succeeded.

```python
from sqlbuild.loaders import loader
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,11 @@ allow_warehouse_round_trips = true
With the setting on:

- One `sqb build` runs the SQL the task reads, then the task, then the loader, then the models that
read the loader's source. If the task fails or is skipped, the loader and everything downstream of
its source are skipped.
read the loader's source. The loader follows the usual rule for its upstream tasks and assets: if
one fails or skips with `mode="hard"`, the loader and everything downstream of its source are
skipped; otherwise it runs when any of them succeeded. When all of them skip softly, only the load
is skipped: the source keeps its last loaded data, and models reading it still build when another
of their inputs ran.
- Lineage and column lineage restart at the source: the source's columns come from the loader, not
from the models the task read.
- Job groups run a member that builds the task before the member that runs the loader, and that one
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1715,9 +1715,11 @@ class WarehouseRoundTripE2ETestCase:

@dataclass(frozen=True)
class ChangedRoundTripE2ETestCase:
"""A second `--changed` build of a round trip after an optional change to `orders`."""
"""A second round-trip build with `build_flags`, after changing `orders` and `extra_files`."""

description: str
orders_sql: str
expected_output_fragments: tuple[str, ...]
expected_scored_rows: tuple[tuple[object, ...], ...]
build_flags: tuple[str, ...] = ("--changed",)
extra_files: dict[str, str] = field(default_factory=dict)
Loading
Loading