Skip to content

Commit 318d037

Browse files
committed
fix: gate a round-trip load only on the tasks and assets it reads directly
A load ran when any task or asset anywhere upstream of it succeeded, so a succeeding task behind a soft-skipped one loaded that one's stale result. Only the tasks and assets a loader reads directly, or through other loaders, can now let it run; a failed or hard-skipped one anywhere still skips it hard. With --no-python, a read-side gate that did not run soft-skips the load even when an ingress gate succeeded.
1 parent 5644817 commit 318d037

20 files changed

Lines changed: 262 additions & 64 deletions

File tree

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
from python.tasks.rates import fetch_rates
2+
from python.tasks.scoring import score_orders
3+
4+
from sqlbuild.loaders import loader
5+
6+
7+
@loader(depends_on=[score_orders, fetch_rates])
8+
def partner_scores(ctx):
9+
"""Load the scores, gated by a read-side task and an ingress task."""
10+
result = ctx.result_of(node_function=score_orders)
11+
return [{"order_id": o, "score": s} for o, s in result.payload["scores"]]
Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
sources:
2+
- name: partner_rates
3+
description: Rates the partner publishes
4+
managed: true
5+
write_strategy: table
6+
columns:
7+
- name: rate
8+
type: INTEGER
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
from python.tasks.rates import fetch_rates
2+
3+
from sqlbuild.loaders import loader
4+
5+
6+
@loader(depends_on=[fetch_rates])
7+
def partner_rates(ctx):
8+
"""Load the partner's rates before the build's SQL."""
9+
return [{"rate": ctx.result_of(node_function=fetch_rates).payload["rate"]}]
Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
from sqlbuild.tasks import task
2+
3+
4+
@task()
5+
def fetch_rates(ctx):
6+
"""Fetch the partner's rates; reads no SQL, so it runs before the build's SQL."""
7+
return ctx.result(payload={"rate": 10})
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
from python.tasks.scoring import score_orders
2+
3+
from sqlbuild.tasks import task
4+
5+
6+
@task(depends_on=[score_orders])
7+
def package_scores(ctx):
8+
"""Package the partner's scores; skips softly when there is nothing new to send."""
9+
return ctx.skip(reason="nothing to package")
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
from python.tasks.packaging import package_scores
2+
3+
from sqlbuild.loaders import loader
4+
5+
6+
@loader(depends_on=[package_scores])
7+
def partner_scores(ctx):
8+
"""Load the packaged scores; only its direct upstream decides whether it runs."""
9+
result = ctx.result_of(node_function=package_scores)
10+
return [{"order_id": o, "score": s} for o, s in result.payload["scores"]]
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
//! The gates a round-trip loader reads directly or through other loaders.
2+
3+
use std::collections::{HashMap, HashSet};
4+
use std::convert::Infallible;
5+
6+
use sqlbuild_core::graph_walk::main::breadth_first_reach::breadth_first_reach;
7+
8+
use crate::python_plan::models::LifecycleNode;
9+
use crate::python_selection::constants::LOADER_KIND;
10+
11+
/// The `gates` reached upstream of `loader` by walking only through loaders, in `gates` order.
12+
pub(crate) fn gate_frontier(
13+
loader: &str,
14+
gates: &[String],
15+
nodes: &HashMap<String, LifecycleNode>,
16+
upstream: &HashMap<String, Vec<String>>,
17+
) -> Vec<String> {
18+
let mut reached: HashSet<&str> = HashSet::new();
19+
let Ok(()) = breadth_first_reach([loader], None, &mut reached, |current| {
20+
let through: bool = nodes
21+
.get(*current)
22+
.is_some_and(|node| node.kind == LOADER_KIND);
23+
Ok::<_, Infallible>(
24+
upstream
25+
.get(*current)
26+
.filter(|_| through)
27+
.into_iter()
28+
.flatten()
29+
.map(String::as_str),
30+
)
31+
});
32+
gates
33+
.iter()
34+
.filter(|gate| reached.contains(gate.as_str()))
35+
.cloned()
36+
.collect()
37+
}
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
pub(crate) mod gate_frontier;
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
pub(crate) mod lifecycle;

‎crates/sqlbuild-analysis/src/python_plan/main/run_lifecycle.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
use std::collections::{BTreeSet, HashMap};
44

55
use crate::assembly::project::types::ObjectKey;
6+
use crate::python_plan::_helpers::lifecycle::gate_frontier::gate_frontier;
67
use crate::python_plan::main::upstream_closure::upstream_closure;
78
use crate::python_plan::models::{LifecycleNode, RoundTripLoader, RunLifecycle};
89
use crate::python_selection::constants::{ASSET_KIND, LOADER_KIND, TASK_KIND};
@@ -33,10 +34,12 @@ pub fn run_lifecycle(
3334
ingress_loaders.insert(loader.clone());
3435
continue;
3536
}
37+
let gates: Vec<String> = closure.into_iter().filter(|name| runs(name)).collect();
3638
round_trip_loaders.push(RoundTripLoader {
3739
name: loader.clone(),
3840
waits_for: waits_for.into_iter().collect(),
39-
gates: closure.into_iter().filter(|name| runs(name)).collect(),
41+
frontier: gate_frontier(loader, &gates, nodes, upstream),
42+
gates,
4043
});
4144
}
4245
let mut ingress: BTreeSet<String> = ingress_loaders.clone();

0 commit comments

Comments
 (0)