Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# Bundling nodes into jobs creates three wait cycles the node lineage does not have: two between
# model jobs, and one through a seed that a single job builds for everyone else.
name = "nightly_job_group"
description = "Build everything nightly."
jobs = ["customers_job", "orders_job", "inventory_job", "shipments_job", "reference_job", "region_report_job"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
name = "customers_job"
description = "Build customers."
select = "customers"
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
name = "inventory_job"
description = "Build inventory and its staging model."
select = "inventory stg_inventory"
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
name = "orders_job"
description = "Build orders and their staging model."
select = "orders stg_orders"
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
name = "reference_job"
description = "Build the regions seed and regional orders."
select = "regions regional_orders"
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
name = "region_report_job"
description = "Build the region report."
select = "region_report"
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
name = "shipments_job"
description = "Build shipments."
select = "shipments"
2 changes: 2 additions & 0 deletions crates/sqlbuild-core/src/event_export/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ pub const EVENT_EXPORT_KINDS: [&str; 8] = [
"run",
"statement",
];
/// The event that starts an invocation, delivered before that invocation's other queued events.
pub const INVOCATION_START_EVENT_TYPE: &str = "invocation_started";
/// Invocation terminals, delivered only after every other queued event.
pub const INVOCATION_TERMINAL_EVENT_TYPES: [&str; 2] =
["invocation_completed", "invocation_failed"];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

use crate::event_export::models::{EventQueue, QueuedEvent};

/// The queue without its next event: the oldest at the highest priority, invocation terminals last.
/// The queue without its next event: the oldest at the highest priority, invocation terminals last, and a queued start of the chosen event's invocation before it.
pub fn take_lifecycle_event<T>(
mut queue: EventQueue<T>,
) -> (EventQueue<T>, Option<QueuedEvent<T>>) {
Expand All @@ -21,6 +21,25 @@ pub fn take_lifecycle_event<T>(
.filter(|(_, queued)| selectable(queued) && Some(queued.priority) == highest)
.min_by_key(|(_, queued)| queued.sequence)
.map(|(position, _)| position);
let position: Option<usize> = position.map(|position| start_first(&queue, position));
let next: Option<QueuedEvent<T>> = position.map(|position| queue.items.remove(position));
(queue, next)
}

/// The position of a queued start of the event at `position`'s invocation, or `position` when none is queued.
fn start_first<T>(queue: &EventQueue<T>, position: usize) -> usize {
let chosen: &QueuedEvent<T> = &queue.items[position];
if chosen.invocation_start || chosen.invocation_id.is_none() {
return position;
}
let mut start: Option<(usize, u64)> = None;
for (index, queued) in queue.items.iter().enumerate() {
if queued.invocation_start
&& queued.invocation_id == chosen.invocation_id
&& start.is_none_or(|(_, sequence)| queued.sequence < sequence)
{
start = Some((index, queued.sequence));
}
}
start.map_or(position, |(index, _)| index)
}
4 changes: 4 additions & 0 deletions crates/sqlbuild-core/src/event_export/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ pub struct QueuedEvent<T> {
pub invocation_terminal: bool,
/// Whether the event reports a run or invocation outcome, which the queue always admits.
pub outcome: bool,
/// The invocation the event belongs to, when it names one.
pub invocation_id: Option<String>,
/// Whether the event starts its invocation and so goes before that invocation's other events.
pub invocation_start: bool,
pub eligible_exporters: Vec<usize>,
}

Expand Down
14 changes: 14 additions & 0 deletions crates/sqlbuild-core/src/event_export/tests/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,24 @@ pub(crate) fn queued(
priority,
invocation_terminal,
outcome: false,
invocation_id: None,
invocation_start: false,
eligible_exporters: vec![0],
}
}

/// A queued event with `sequence` and `(priority, invocation start, invocation)`.
pub(crate) fn invocation_event(
sequence: u64,
(priority, invocation_start, invocation): (u8, bool, &str),
) -> QueuedEvent<()> {
QueuedEvent {
invocation_id: Some(invocation.to_owned()),
invocation_start,
..queued(sequence, (priority, false))
}
}

/// A queued non-terminal event with `sequence`, `(priority, outcome)` and one eligible exporter.
pub(crate) fn event((priority, outcome): (u8, bool), sequence: u64) -> QueuedEvent<()> {
QueuedEvent {
Expand Down
49 changes: 47 additions & 2 deletions crates/sqlbuild-core/src/event_export/tests/test_queue.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@ use crate::event_export::main::queue_lifecycle_event::queue_lifecycle_event;
use crate::event_export::main::retain_outcome_events::retain_outcome_events;
use crate::event_export::main::take_lifecycle_event::take_lifecycle_event;
use crate::event_export::models::EventQueue;
use crate::event_export::tests::helpers::{event, queued, sequences};
use crate::event_export::tests::helpers::{event, invocation_event, queued, sequences};
use crate::event_export::tests::test_types::{
DeliveryOrderTestCase, DisplacementTestCase, OutcomeAdmissionTestCase, OutcomeReserveTestCase,
DeliveryOrderTestCase, DisplacementTestCase, InvocationStartOrderTestCase,
OutcomeAdmissionTestCase, OutcomeReserveTestCase,
};

#[test]
Expand Down Expand Up @@ -186,3 +187,47 @@ fn given_queued_events_when_taking_then_priority_then_fifo_with_invocation_termi
);
}
}

#[test]
fn given_a_queued_invocation_start_when_taking_then_it_precedes_only_its_own_invocation() {
let test_cases = [
InvocationStartOrderTestCase {
description: "a start goes before its invocation's higher-priority events",
events: &[(0, true, "a"), (2, false, "a"), (3, false, "a")],
expected_sequences: &[0, 2, 1],
},
InvocationStartOrderTestCase {
description: "a start that is not queued holds nothing back",
events: &[(2, false, "a"), (0, false, "a"), (3, false, "a")],
expected_sequences: &[2, 0, 1],
},
InvocationStartOrderTestCase {
description: "one invocation's start does not jump another's events",
events: &[
(2, false, "b"),
(0, true, "a"),
(2, false, "a"),
(0, false, "b"),
],
expected_sequences: &[0, 1, 2, 3],
},
];

for test_case in test_cases {
let mut queue: EventQueue<()> = EventQueue::new(16);
for (sequence, event) in (0_u64..).zip(test_case.events) {
queue = queue_lifecycle_event(queue, invocation_event(sequence, *event)).queue;
}
let mut order: Vec<u64> = Vec::new();
for _ in test_case.events {
let (rest, next) = take_lifecycle_event(queue);
queue = rest;
order.extend(next.map(|event| event.sequence));
}
assert_eq!(
order, test_case.expected_sequences,
"{}",
test_case.description
);
}
}
7 changes: 7 additions & 0 deletions crates/sqlbuild-core/src/event_export/tests/test_types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,13 @@ pub(crate) struct DeliveryOrderTestCase {
pub(crate) expected_sequences: &'static [u64],
}

/// Queued events as `(priority, invocation start, invocation)`, and the order they are taken in.
pub(crate) struct InvocationStartOrderTestCase {
pub(crate) description: &'static str,
pub(crate) events: &'static [(u8, bool, &'static str)],
pub(crate) expected_sequences: &'static [u64],
}

/// One event routed across sink filters, and the resulting accounting.
pub(crate) struct RoutingTestCase {
pub(crate) description: &'static str,
Expand Down
134 changes: 105 additions & 29 deletions crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs
Original file line number Diff line number Diff line change
@@ -1,45 +1,121 @@
//! Record why one group member waits for another, and name the waits that close a cycle.
//! Find the group members that wait on each other, name a cycle among them, and say why each waits.

use std::collections::{BTreeMap, BTreeSet};

/// Keep the first reason found for a wait: the first lineage edge.
pub(crate) fn record_wait(
mut reasons: BTreeMap<(String, String), String>,
(job, dependency): (&str, &str),
reason: impl FnOnce() -> String,
) -> BTreeMap<(String, String), String> {
reasons
.entry((job.to_owned(), dependency.to_owned()))
.or_insert_with(reason);
reasons
use crate::declarations::constants::SEED_KEY_PREFIX;
use crate::declarations::models::GroupCycle;

/// Each member `name` waits for, with the first lineage edge that makes it wait: the node and the upstream it reads.
pub(crate) fn member_waits(
(name, selection): (&str, &BTreeSet<String>),
deps: &BTreeMap<String, BTreeSet<String>>,
owners: &BTreeMap<&str, BTreeSet<&str>>,
) -> BTreeMap<String, (String, String)> {
let mut waits: BTreeMap<String, (String, String)> = BTreeMap::new();
let mut seen: BTreeSet<&str> = BTreeSet::new();
let mut pending: Vec<&str> = selection.iter().map(String::as_str).collect();
while let Some(key) = pending.pop() {
if !seen.insert(key) {
continue;
}
for upstream in deps.get(key).into_iter().flatten() {
let builders: Option<&BTreeSet<&str>> = owners.get(upstream.as_str());
for owner in builders
.filter(|builders| !builders.contains(name))
.into_iter()
.flatten()
{
waits
.entry((*owner).to_owned())
.or_insert_with(|| (key.to_owned(), upstream.clone()));
}
if !seen.contains(upstream.as_str()) {
pending.push(upstream);
}
}
}
waits
}

/// Every job left waits on another, so following first dependencies from any of them closes a cycle.
pub(crate) fn describe_cycle(
remaining: &BTreeMap<String, BTreeSet<String>>,
reasons: &BTreeMap<(String, String), String>,
) -> String {
/// Groups of two or more members that each reach every other through waits, in name order.
pub(crate) fn strongly_connected(
edges: &BTreeMap<String, BTreeSet<String>>,
) -> Vec<BTreeSet<String>> {
let reach = |start: &str| {
let mut seen: BTreeSet<String> = BTreeSet::new();
let mut pending: Vec<&str> = vec![start];
while let Some(name) = pending.pop() {
for next in edges.get(name).into_iter().flatten() {
if seen.insert(next.clone()) {
pending.push(next);
}
}
}
seen
};
let reaches: BTreeMap<&str, BTreeSet<String>> = edges
.keys()
.map(|name| (name.as_str(), reach(name)))
.collect();
let mut components: Vec<BTreeSet<String>> = Vec::new();
for (name, reached) in &reaches {
if !reached.contains(*name) || components.iter().any(|done| done.contains(*name)) {
continue;
}
components.push(
reached
.iter()
.filter(|other| reaches[other.as_str()].contains(*name))
.cloned()
.collect(),
);
}
components
}

/// Follow first waits inside `members`, all of which wait on each other, until a member repeats.
pub(crate) fn cycle_path<'a>(
members: &'a BTreeSet<String>,
edges: &'a BTreeMap<String, BTreeSet<String>>,
) -> Vec<&'a str> {
let mut path: Vec<&str> = Vec::new();
let mut current = remaining.keys().next().map(String::as_str);
let mut current = members.iter().next().map(String::as_str);
while let Some(name) = current {
if let Some(start) = path.iter().position(|seen| *seen == name) {
path.drain(..start);
break;
}
path.push(name);
current = remaining
current = edges
.get(name)
.and_then(|deps| deps.iter().next())
.and_then(|deps| deps.iter().find(|dep| members.contains(*dep)))
.map(String::as_str);
}
path.iter()
.zip(path.iter().cycle().skip(1))
.map(|(job, dependency)| {
let reason = reasons
.get(&((*job).to_owned(), (*dependency).to_owned()))
.map_or("", String::as_str);
format!("{job} waits for {dependency} ({reason})")
})
.collect::<Vec<_>>()
.join("; ")
path
}

/// The cycle through `path`, each wait with the node and upstream that cause it, and whether one reads a seed or shared node.
pub(crate) fn named_cycle(
path: &[&str],
reasons: &BTreeMap<(String, String), (String, String)>,
owners: &BTreeMap<&str, BTreeSet<&str>>,
) -> GroupCycle {
let mut waits: Vec<String> = Vec::new();
let mut shared_read: bool = false;
for (index, job) in path.iter().enumerate() {
let dependency: &str = path[(index + 1) % path.len()];
let (key, upstream) = &reasons[&((*job).to_owned(), dependency.to_owned())];
waits.push(format!(
"{job} waits for {dependency} ({key} reads {upstream})"
));
shared_read = shared_read
|| upstream.starts_with(SEED_KEY_PREFIX)
|| owners
.get(upstream.as_str())
.is_some_and(|builders| builders.len() > 1);
}
GroupCycle {
description: waits.join("; "),
shared_read,
}
}
8 changes: 8 additions & 0 deletions crates/sqlbuild-orchestration/src/declarations/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,11 @@ pub(crate) const DEFER_SETTING_PREFIX: &str = "defer_";

/// How many due times a person's trigger listing gives for each running schedule.
pub(crate) const UPCOMING_TIMES: usize = 5;

/// The key prefix of seed nodes, which a job that loads them builds for every reader.
pub(crate) const SEED_KEY_PREFIX: &str = "seed:";

/// Added under a job group's cycles when one of their waits reads a seed or a node several members build.
pub const SHARED_READ_HINT: &str = "hint: a wait in this cycle reads a seed or a node that more than one \
member builds, so every member that reads it waits for all of its builders; give seeds and shared nodes \
their own job that runs first";
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
//! One line per cycle in a job group, then the shared-read hint when a cycle passes through a seed or shared node.

use crate::declarations::constants::SHARED_READ_HINT;
use crate::declarations::models::GroupCycle;

pub fn group_cycle_message(group: &str, cycles: &[GroupCycle]) -> Option<String> {
if cycles.is_empty() {
return None;
}
let mut lines: Vec<String> = cycles
.iter()
.map(|cycle| {
format!(
"group dependencies contain a cycle in job group {group}: {}",
cycle.description
)
})
.collect();
if cycles.iter().any(|cycle| cycle.shared_read) {
lines.push(SHARED_READ_HINT.to_owned());
}
Some(lines.join("\n"))
}
Loading
Loading