diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_cycles/job_groups/nightly_job_group.toml b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/job_groups/nightly_job_group.toml new file mode 100644 index 0000000000..960ffca1f7 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/job_groups/nightly_job_group.toml @@ -0,0 +1,5 @@ +# Bundling nodes into jobs creates three wait cycles the node lineage does not have: two between +# model jobs, and one through a seed that a single job builds for everyone else. +name = "nightly_job_group" +description = "Build everything nightly." +jobs = ["customers_job", "orders_job", "inventory_job", "shipments_job", "reference_job", "region_report_job"] diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/customers_job.toml b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/customers_job.toml new file mode 100644 index 0000000000..fc2588ed4e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/customers_job.toml @@ -0,0 +1,3 @@ +name = "customers_job" +description = "Build customers." +select = "customers" diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/inventory_job.toml b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/inventory_job.toml new file mode 100644 index 0000000000..e99f2bd558 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/inventory_job.toml @@ -0,0 +1,3 @@ +name = "inventory_job" +description = "Build inventory and its staging model." +select = "inventory stg_inventory" diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/orders_job.toml new file mode 100644 index 0000000000..e27bd9e3f9 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/orders_job.toml @@ -0,0 +1,3 @@ +name = "orders_job" +description = "Build orders and their staging model." +select = "orders stg_orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/reference_job.toml b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/reference_job.toml new file mode 100644 index 0000000000..70e8d370c1 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/reference_job.toml @@ -0,0 +1,3 @@ +name = "reference_job" +description = "Build the regions seed and regional orders." +select = "regions regional_orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/region_report_job.toml b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/region_report_job.toml new file mode 100644 index 0000000000..25604aba03 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/region_report_job.toml @@ -0,0 +1,3 @@ +name = "region_report_job" +description = "Build the region report." +select = "region_report" diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/shipments_job.toml b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/shipments_job.toml new file mode 100644 index 0000000000..ae0dd6b3df --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/shipments_job.toml @@ -0,0 +1,3 @@ +name = "shipments_job" +description = "Build shipments." +select = "shipments" diff --git a/crates/sqlbuild-core/src/event_export/constants.rs b/crates/sqlbuild-core/src/event_export/constants.rs index 75feaf7546..53a9a7d0bf 100644 --- a/crates/sqlbuild-core/src/event_export/constants.rs +++ b/crates/sqlbuild-core/src/event_export/constants.rs @@ -16,6 +16,8 @@ pub const EVENT_EXPORT_KINDS: [&str; 8] = [ "run", "statement", ]; +/// The event that starts an invocation, delivered before that invocation's other queued events. +pub const INVOCATION_START_EVENT_TYPE: &str = "invocation_started"; /// Invocation terminals, delivered only after every other queued event. pub const INVOCATION_TERMINAL_EVENT_TYPES: [&str; 2] = ["invocation_completed", "invocation_failed"]; diff --git a/crates/sqlbuild-core/src/event_export/main/take_lifecycle_event.rs b/crates/sqlbuild-core/src/event_export/main/take_lifecycle_event.rs index aec19201b6..f59b4ec8b6 100644 --- a/crates/sqlbuild-core/src/event_export/main/take_lifecycle_event.rs +++ b/crates/sqlbuild-core/src/event_export/main/take_lifecycle_event.rs @@ -2,7 +2,7 @@ use crate::event_export::models::{EventQueue, QueuedEvent}; -/// The queue without its next event: the oldest at the highest priority, invocation terminals last. +/// The queue without its next event: the oldest at the highest priority, invocation terminals last, and a queued start of the chosen event's invocation before it. pub fn take_lifecycle_event( mut queue: EventQueue, ) -> (EventQueue, Option>) { @@ -21,6 +21,25 @@ pub fn take_lifecycle_event( .filter(|(_, queued)| selectable(queued) && Some(queued.priority) == highest) .min_by_key(|(_, queued)| queued.sequence) .map(|(position, _)| position); + let position: Option = position.map(|position| start_first(&queue, position)); let next: Option> = position.map(|position| queue.items.remove(position)); (queue, next) } + +/// The position of a queued start of the event at `position`'s invocation, or `position` when none is queued. +fn start_first(queue: &EventQueue, position: usize) -> usize { + let chosen: &QueuedEvent = &queue.items[position]; + if chosen.invocation_start || chosen.invocation_id.is_none() { + return position; + } + let mut start: Option<(usize, u64)> = None; + for (index, queued) in queue.items.iter().enumerate() { + if queued.invocation_start + && queued.invocation_id == chosen.invocation_id + && start.is_none_or(|(_, sequence)| queued.sequence < sequence) + { + start = Some((index, queued.sequence)); + } + } + start.map_or(position, |(index, _)| index) +} diff --git a/crates/sqlbuild-core/src/event_export/models.rs b/crates/sqlbuild-core/src/event_export/models.rs index 9ee6750807..00a6c41cb1 100644 --- a/crates/sqlbuild-core/src/event_export/models.rs +++ b/crates/sqlbuild-core/src/event_export/models.rs @@ -46,6 +46,10 @@ pub struct QueuedEvent { pub invocation_terminal: bool, /// Whether the event reports a run or invocation outcome, which the queue always admits. pub outcome: bool, + /// The invocation the event belongs to, when it names one. + pub invocation_id: Option, + /// Whether the event starts its invocation and so goes before that invocation's other events. + pub invocation_start: bool, pub eligible_exporters: Vec, } diff --git a/crates/sqlbuild-core/src/event_export/tests/helpers.rs b/crates/sqlbuild-core/src/event_export/tests/helpers.rs index 2817d06c0a..252cbc4a81 100644 --- a/crates/sqlbuild-core/src/event_export/tests/helpers.rs +++ b/crates/sqlbuild-core/src/event_export/tests/helpers.rs @@ -11,10 +11,24 @@ pub(crate) fn queued( priority, invocation_terminal, outcome: false, + invocation_id: None, + invocation_start: false, eligible_exporters: vec![0], } } +/// A queued event with `sequence` and `(priority, invocation start, invocation)`. +pub(crate) fn invocation_event( + sequence: u64, + (priority, invocation_start, invocation): (u8, bool, &str), +) -> QueuedEvent<()> { + QueuedEvent { + invocation_id: Some(invocation.to_owned()), + invocation_start, + ..queued(sequence, (priority, false)) + } +} + /// A queued non-terminal event with `sequence`, `(priority, outcome)` and one eligible exporter. pub(crate) fn event((priority, outcome): (u8, bool), sequence: u64) -> QueuedEvent<()> { QueuedEvent { diff --git a/crates/sqlbuild-core/src/event_export/tests/test_queue.rs b/crates/sqlbuild-core/src/event_export/tests/test_queue.rs index 2ae6361f1d..c8b8c8d0a4 100644 --- a/crates/sqlbuild-core/src/event_export/tests/test_queue.rs +++ b/crates/sqlbuild-core/src/event_export/tests/test_queue.rs @@ -2,9 +2,10 @@ use crate::event_export::main::queue_lifecycle_event::queue_lifecycle_event; use crate::event_export::main::retain_outcome_events::retain_outcome_events; use crate::event_export::main::take_lifecycle_event::take_lifecycle_event; use crate::event_export::models::EventQueue; -use crate::event_export::tests::helpers::{event, queued, sequences}; +use crate::event_export::tests::helpers::{event, invocation_event, queued, sequences}; use crate::event_export::tests::test_types::{ - DeliveryOrderTestCase, DisplacementTestCase, OutcomeAdmissionTestCase, OutcomeReserveTestCase, + DeliveryOrderTestCase, DisplacementTestCase, InvocationStartOrderTestCase, + OutcomeAdmissionTestCase, OutcomeReserveTestCase, }; #[test] @@ -186,3 +187,47 @@ fn given_queued_events_when_taking_then_priority_then_fifo_with_invocation_termi ); } } + +#[test] +fn given_a_queued_invocation_start_when_taking_then_it_precedes_only_its_own_invocation() { + let test_cases = [ + InvocationStartOrderTestCase { + description: "a start goes before its invocation's higher-priority events", + events: &[(0, true, "a"), (2, false, "a"), (3, false, "a")], + expected_sequences: &[0, 2, 1], + }, + InvocationStartOrderTestCase { + description: "a start that is not queued holds nothing back", + events: &[(2, false, "a"), (0, false, "a"), (3, false, "a")], + expected_sequences: &[2, 0, 1], + }, + InvocationStartOrderTestCase { + description: "one invocation's start does not jump another's events", + events: &[ + (2, false, "b"), + (0, true, "a"), + (2, false, "a"), + (0, false, "b"), + ], + expected_sequences: &[0, 1, 2, 3], + }, + ]; + + for test_case in test_cases { + let mut queue: EventQueue<()> = EventQueue::new(16); + for (sequence, event) in (0_u64..).zip(test_case.events) { + queue = queue_lifecycle_event(queue, invocation_event(sequence, *event)).queue; + } + let mut order: Vec = Vec::new(); + for _ in test_case.events { + let (rest, next) = take_lifecycle_event(queue); + queue = rest; + order.extend(next.map(|event| event.sequence)); + } + assert_eq!( + order, test_case.expected_sequences, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/event_export/tests/test_types.rs b/crates/sqlbuild-core/src/event_export/tests/test_types.rs index 0338b49a40..1130367650 100644 --- a/crates/sqlbuild-core/src/event_export/tests/test_types.rs +++ b/crates/sqlbuild-core/src/event_export/tests/test_types.rs @@ -40,6 +40,13 @@ pub(crate) struct DeliveryOrderTestCase { pub(crate) expected_sequences: &'static [u64], } +/// Queued events as `(priority, invocation start, invocation)`, and the order they are taken in. +pub(crate) struct InvocationStartOrderTestCase { + pub(crate) description: &'static str, + pub(crate) events: &'static [(u8, bool, &'static str)], + pub(crate) expected_sequences: &'static [u64], +} + /// One event routed across sink filters, and the resulting accounting. pub(crate) struct RoutingTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs index fcc0b6ece8..5f0380af4c 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs @@ -1,45 +1,121 @@ -//! Record why one group member waits for another, and name the waits that close a cycle. +//! Find the group members that wait on each other, name a cycle among them, and say why each waits. use std::collections::{BTreeMap, BTreeSet}; -/// Keep the first reason found for a wait: the first lineage edge. -pub(crate) fn record_wait( - mut reasons: BTreeMap<(String, String), String>, - (job, dependency): (&str, &str), - reason: impl FnOnce() -> String, -) -> BTreeMap<(String, String), String> { - reasons - .entry((job.to_owned(), dependency.to_owned())) - .or_insert_with(reason); - reasons +use crate::declarations::constants::SEED_KEY_PREFIX; +use crate::declarations::models::GroupCycle; + +/// Each member `name` waits for, with the first lineage edge that makes it wait: the node and the upstream it reads. +pub(crate) fn member_waits( + (name, selection): (&str, &BTreeSet), + deps: &BTreeMap>, + owners: &BTreeMap<&str, BTreeSet<&str>>, +) -> BTreeMap { + let mut waits: BTreeMap = BTreeMap::new(); + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let mut pending: Vec<&str> = selection.iter().map(String::as_str).collect(); + while let Some(key) = pending.pop() { + if !seen.insert(key) { + continue; + } + for upstream in deps.get(key).into_iter().flatten() { + let builders: Option<&BTreeSet<&str>> = owners.get(upstream.as_str()); + for owner in builders + .filter(|builders| !builders.contains(name)) + .into_iter() + .flatten() + { + waits + .entry((*owner).to_owned()) + .or_insert_with(|| (key.to_owned(), upstream.clone())); + } + if !seen.contains(upstream.as_str()) { + pending.push(upstream); + } + } + } + waits } -/// Every job left waits on another, so following first dependencies from any of them closes a cycle. -pub(crate) fn describe_cycle( - remaining: &BTreeMap>, - reasons: &BTreeMap<(String, String), String>, -) -> String { +/// Groups of two or more members that each reach every other through waits, in name order. +pub(crate) fn strongly_connected( + edges: &BTreeMap>, +) -> Vec> { + let reach = |start: &str| { + let mut seen: BTreeSet = BTreeSet::new(); + let mut pending: Vec<&str> = vec![start]; + while let Some(name) = pending.pop() { + for next in edges.get(name).into_iter().flatten() { + if seen.insert(next.clone()) { + pending.push(next); + } + } + } + seen + }; + let reaches: BTreeMap<&str, BTreeSet> = edges + .keys() + .map(|name| (name.as_str(), reach(name))) + .collect(); + let mut components: Vec> = Vec::new(); + for (name, reached) in &reaches { + if !reached.contains(*name) || components.iter().any(|done| done.contains(*name)) { + continue; + } + components.push( + reached + .iter() + .filter(|other| reaches[other.as_str()].contains(*name)) + .cloned() + .collect(), + ); + } + components +} + +/// Follow first waits inside `members`, all of which wait on each other, until a member repeats. +pub(crate) fn cycle_path<'a>( + members: &'a BTreeSet, + edges: &'a BTreeMap>, +) -> Vec<&'a str> { let mut path: Vec<&str> = Vec::new(); - let mut current = remaining.keys().next().map(String::as_str); + let mut current = members.iter().next().map(String::as_str); while let Some(name) = current { if let Some(start) = path.iter().position(|seen| *seen == name) { path.drain(..start); break; } path.push(name); - current = remaining + current = edges .get(name) - .and_then(|deps| deps.iter().next()) + .and_then(|deps| deps.iter().find(|dep| members.contains(*dep))) .map(String::as_str); } - path.iter() - .zip(path.iter().cycle().skip(1)) - .map(|(job, dependency)| { - let reason = reasons - .get(&((*job).to_owned(), (*dependency).to_owned())) - .map_or("", String::as_str); - format!("{job} waits for {dependency} ({reason})") - }) - .collect::>() - .join("; ") + path +} + +/// The cycle through `path`, each wait with the node and upstream that cause it, and whether one reads a seed or shared node. +pub(crate) fn named_cycle( + path: &[&str], + reasons: &BTreeMap<(String, String), (String, String)>, + owners: &BTreeMap<&str, BTreeSet<&str>>, +) -> GroupCycle { + let mut waits: Vec = Vec::new(); + let mut shared_read: bool = false; + for (index, job) in path.iter().enumerate() { + let dependency: &str = path[(index + 1) % path.len()]; + let (key, upstream) = &reasons[&((*job).to_owned(), dependency.to_owned())]; + waits.push(format!( + "{job} waits for {dependency} ({key} reads {upstream})" + )); + shared_read = shared_read + || upstream.starts_with(SEED_KEY_PREFIX) + || owners + .get(upstream.as_str()) + .is_some_and(|builders| builders.len() > 1); + } + GroupCycle { + description: waits.join("; "), + shared_read, + } } diff --git a/crates/sqlbuild-orchestration/src/declarations/constants.rs b/crates/sqlbuild-orchestration/src/declarations/constants.rs index 125a59e986..00d87fef80 100644 --- a/crates/sqlbuild-orchestration/src/declarations/constants.rs +++ b/crates/sqlbuild-orchestration/src/declarations/constants.rs @@ -9,3 +9,11 @@ pub(crate) const DEFER_SETTING_PREFIX: &str = "defer_"; /// How many due times a person's trigger listing gives for each running schedule. pub(crate) const UPCOMING_TIMES: usize = 5; + +/// The key prefix of seed nodes, which a job that loads them builds for every reader. +pub(crate) const SEED_KEY_PREFIX: &str = "seed:"; + +/// Added under a job group's cycles when one of their waits reads a seed or a node several members build. +pub const SHARED_READ_HINT: &str = "hint: a wait in this cycle reads a seed or a node that more than one \ +member builds, so every member that reads it waits for all of its builders; give seeds and shared nodes \ +their own job that runs first"; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_cycle_message.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_cycle_message.rs new file mode 100644 index 0000000000..550c541015 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_cycle_message.rs @@ -0,0 +1,23 @@ +//! One line per cycle in a job group, then the shared-read hint when a cycle passes through a seed or shared node. + +use crate::declarations::constants::SHARED_READ_HINT; +use crate::declarations::models::GroupCycle; + +pub fn group_cycle_message(group: &str, cycles: &[GroupCycle]) -> Option { + if cycles.is_empty() { + return None; + } + let mut lines: Vec = cycles + .iter() + .map(|cycle| { + format!( + "group dependencies contain a cycle in job group {group}: {}", + cycle.description + ) + }) + .collect(); + if cycles.iter().any(|cycle| cycle.shared_read) { + lines.push(SHARED_READ_HINT.to_owned()); + } + Some(lines.join("\n")) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs index 6e78d95cfb..1623bd329b 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs @@ -2,8 +2,9 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::declarations::_helpers::group_waits::{describe_cycle, record_wait}; -use crate::declarations::models::{GroupWaits, JobDefinition}; +use crate::declarations::main::group_cycle_message::group_cycle_message; +use crate::declarations::main::group_wait_graph::group_wait_graph; +use crate::declarations::models::JobDefinition; use crate::snapshots::models::SnapshotInput; use crate::store::errors::ControlError; @@ -15,81 +16,29 @@ pub fn group_dependencies( if members.is_empty() || members.iter().any(|name| !jobs.contains_key(name)) { return Err(ControlError::Invalid("group needs known jobs".into())); } - let waits = lineage_dependencies(input, jobs, members); - ensure_acyclic(group, &waits.edges, &waits.reasons)?; - Ok(waits.edges) -} - -/// A job that builds a node itself, such as a shared function, never waits for its other builders. -fn lineage_dependencies( - input: &SnapshotInput, - jobs: &BTreeMap, - members: &BTreeSet, -) -> GroupWaits { - let mut owners: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); - for name in members { - for key in &jobs[name].selection { - owners.entry(key).or_default().insert(name); - } + let selections: BTreeMap> = members + .iter() + .map(|name| (name.clone(), jobs[name].selection.clone())) + .collect(); + let deps: BTreeMap> = input + .nodes + .iter() + .map(|(key, node)| { + let upstream: BTreeSet = node + .definition + .get("deps") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(serde_json::Value::as_str) + .map(str::to_owned) + .collect(); + (key.clone(), upstream) + }) + .collect(); + let waits = group_wait_graph(&selections, &deps); + if let Some(message) = group_cycle_message(group, &waits.cycles) { + return Err(ControlError::Invalid(message)); } - let mut edges: BTreeMap> = BTreeMap::new(); - let mut reasons: BTreeMap<(String, String), String> = BTreeMap::new(); - for name in members { - let mut dependencies: BTreeSet = BTreeSet::new(); - let mut seen: BTreeSet<&str> = BTreeSet::new(); - let mut pending: Vec<&str> = jobs[name].selection.iter().map(String::as_str).collect(); - while let Some(key) = pending.pop() { - if !seen.insert(key) { - continue; - } - if let Some(node) = input.nodes.get(key) - && let Some(upstream) = node - .definition - .get("deps") - .and_then(serde_json::Value::as_array) - { - for upstream in upstream.iter().filter_map(serde_json::Value::as_str) { - if let Some(owners) = owners.get(upstream) - && !owners.contains(name.as_str()) - { - for owner in owners { - dependencies.insert((*owner).to_owned()); - reasons = record_wait(reasons, (name, owner), || { - format!("{key} reads {upstream}") - }); - } - } - pending.push(upstream); - } - } - } - edges.insert(name.clone(), dependencies); - } - GroupWaits { edges, reasons } -} - -fn ensure_acyclic( - group: &str, - edges: &BTreeMap>, - reasons: &BTreeMap<(String, String), String>, -) -> Result<(), ControlError> { - let mut remaining = edges.clone(); - while !remaining.is_empty() { - let ready: BTreeSet = remaining - .iter() - .filter(|(_, deps)| deps.is_empty()) - .map(|(name, _)| name.clone()) - .collect(); - if ready.is_empty() { - return Err(ControlError::Invalid(format!( - "group dependencies contain a cycle in job group {group}: {}", - describe_cycle(&remaining, reasons) - ))); - } - remaining.retain(|name, deps| { - deps.retain(|dependency| !ready.contains(dependency)); - !ready.contains(name) - }); - } - Ok(()) + Ok(waits.edges) } diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_wait_graph.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_wait_graph.rs new file mode 100644 index 0000000000..474ace6d6e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_wait_graph.rs @@ -0,0 +1,44 @@ +//! Derive group member waits from node lineage and report every set of members that wait on each other. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::declarations::_helpers::group_waits::{ + cycle_path, member_waits, named_cycle, strongly_connected, +}; +use crate::declarations::models::{GroupCycle, GroupWaits}; + +/// Each member waits for every other member that builds a node its nodes read, unless it builds that node too. +pub fn group_wait_graph( + selections: &BTreeMap>, + deps: &BTreeMap>, +) -> GroupWaits { + let mut owners: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for (name, selection) in selections { + for key in selection { + owners.entry(key).or_default().insert(name); + } + } + let mut edges: BTreeMap> = BTreeMap::new(); + let mut reasons: BTreeMap<(String, String), (String, String)> = BTreeMap::new(); + for (name, selection) in selections { + let waits: BTreeMap = + member_waits((name, selection), deps, &owners); + edges.insert(name.clone(), waits.keys().cloned().collect()); + for (dependency, reason) in waits { + reasons.insert((name.clone(), dependency), reason); + } + } + let mut cycles: Vec = Vec::new(); + for members in strongly_connected(&edges) { + cycles.push(named_cycle( + &cycle_path(&members, &edges), + &reasons, + &owners, + )); + } + GroupWaits { + edges, + reasons, + cycles, + } +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs index 085e543d5b..d6c7a60fe4 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -1,7 +1,9 @@ pub mod change_person_trigger; pub mod change_trigger; pub(crate) mod frozen_definitions; +pub mod group_cycle_message; pub mod group_dependencies; +pub mod group_wait_graph; pub mod job_resources; pub mod list_person_jobs; pub mod list_person_trigger_changes; diff --git a/crates/sqlbuild-orchestration/src/declarations/mod.rs b/crates/sqlbuild-orchestration/src/declarations/mod.rs index f147865646..2dfde8b679 100644 --- a/crates/sqlbuild-orchestration/src/declarations/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/mod.rs @@ -1,4 +1,8 @@ pub(crate) mod _helpers; -pub(crate) mod constants; +pub mod constants; pub mod main; pub mod models; + +#[cfg(test)] +#[path = "../../tests/declarations/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 56f9c12183..f4bf04c8db 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -343,11 +343,20 @@ pub(crate) struct PreviousTriggerState { pub(crate) status_override: Option, } -/// Each member job's waits, and why it waits: the first lineage edge found. +/// Each member job's waits, why it waits (the first lineage edge found, as node and upstream), and its cycles. #[derive(Clone, Debug, Default, PartialEq, Eq)] -pub(crate) struct GroupWaits { - pub(crate) edges: BTreeMap>, - pub(crate) reasons: BTreeMap<(String, String), String>, +pub struct GroupWaits { + pub edges: BTreeMap>, + pub reasons: BTreeMap<(String, String), (String, String)>, + pub cycles: Vec, +} + +/// Members that wait on each other, named as one cycle among them with a reason per wait. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GroupCycle { + pub description: String, + /// Whether a wait in the cycle reads a seed or a node that more than one member builds. + pub shared_read: bool, } /// Providers the current snapshot declares; empty before a publish or when the project has none. diff --git a/crates/sqlbuild-orchestration/tests/declarations/helpers.rs b/crates/sqlbuild-orchestration/tests/declarations/helpers.rs new file mode 100644 index 0000000000..10b270cdad --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/declarations/helpers.rs @@ -0,0 +1,47 @@ +//! Build group selections and node lineage for the wait graph tests. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::declarations::tests::test_types::Pairs; + +pub(crate) fn map(pairs: Pairs) -> BTreeMap> { + pairs + .iter() + .map(|(key, values)| { + ( + (*key).to_owned(), + values.iter().map(|value| (*value).to_owned()).collect(), + ) + }) + .collect() +} + +/// `members` jobs over one acyclic chain of nodes, each node reading the one before it, where the first job also builds a tail reading the last node. +pub(crate) fn chained_group( + members: usize, + nodes_per_member: usize, +) -> ( + BTreeMap>, + BTreeMap>, +) { + let node = |index: usize| format!("model:m{index}"); + let last: usize = members * nodes_per_member - 1; + let mut selections: BTreeMap> = BTreeMap::new(); + for member in 0..members { + let first: usize = member * nodes_per_member; + selections.insert( + format!("job_{member:03}"), + (first..first + nodes_per_member).map(node).collect(), + ); + } + selections + .entry("job_000".to_owned()) + .or_default() + .insert("model:tail".to_owned()); + let mut deps: BTreeMap> = BTreeMap::new(); + for index in 1..=last { + deps.insert(node(index), BTreeSet::from([node(index - 1)])); + } + deps.insert("model:tail".to_owned(), BTreeSet::from([node(last)])); + (selections, deps) +} diff --git a/crates/sqlbuild-orchestration/tests/declarations/test_contracts.rs b/crates/sqlbuild-orchestration/tests/declarations/test_contracts.rs new file mode 100644 index 0000000000..89785f7cea --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/declarations/test_contracts.rs @@ -0,0 +1,8 @@ +//! Job group wait contracts, derived from node lineage without a control store. + +#[path = "helpers.rs"] +pub(crate) mod helpers; +#[path = "test_group_wait_graph.rs"] +mod test_group_wait_graph; +#[path = "test_types.rs"] +pub(crate) mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs b/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs new file mode 100644 index 0000000000..f686a87a53 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs @@ -0,0 +1,140 @@ +//! Every set of members that wait on each other is reported in one pass, and reads of seeds or shared nodes add the hint. + +use std::collections::BTreeSet; + +use crate::declarations::constants::SHARED_READ_HINT; +use crate::declarations::main::group_cycle_message::group_cycle_message; +use crate::declarations::main::group_wait_graph::group_wait_graph; +use crate::declarations::tests::helpers::{chained_group, map}; +use crate::declarations::tests::test_types::{GroupWaitGraphTestCase, LargeGroupTestCase}; + +#[test] +fn given_group_members_and_node_lineage_when_deriving_waits_then_every_cycle_is_reported_with_its_reasons() + { + let two_cycles = "group dependencies contain a cycle in job group nightly: customers_job waits for \ + orders_job (model:customers reads model:stg_orders); orders_job waits for customers_job \ + (model:orders reads model:customers)\n\ + group dependencies contain a cycle in job group nightly: inventory_job waits for \ + shipments_job (model:inventory reads model:shipments); shipments_job waits for \ + inventory_job (model:shipments reads model:stg_inventory)"; + let seed_hub = format!( + "group dependencies contain a cycle in job group nightly: customers_job waits for \ + seeds_and_orders_job (model:customers reads seed:regions); seeds_and_orders_job waits for \ + customers_job (model:orders reads model:customers)\n{SHARED_READ_HINT}" + ); + let test_cases = [ + GroupWaitGraphTestCase { + description: "two independent cycles are both reported, and a member that only waits on one is not", + selections: &[ + ("customers_job", &["model:customers"]), + ("finance_job", &["model:revenue"]), + ("inventory_job", &["model:inventory", "model:stg_inventory"]), + ("orders_job", &["model:orders", "model:stg_orders"]), + ("shipments_job", &["model:shipments"]), + ], + deps: &[ + ("model:customers", &["model:stg_orders"]), + ("model:inventory", &["model:shipments"]), + ("model:orders", &["model:customers"]), + ("model:revenue", &["model:orders"]), + ("model:shipments", &["model:stg_inventory"]), + ], + expected_edges: &[ + ("customers_job", &["orders_job"]), + ("finance_job", &["customers_job", "orders_job"]), + ("inventory_job", &["shipments_job"]), + ("orders_job", &["customers_job"]), + ("shipments_job", &["inventory_job"]), + ], + expected_message: Some(two_cycles), + }, + GroupWaitGraphTestCase { + description: "a cycle through a seed one member builds for everyone adds the hint", + selections: &[ + ("customers_job", &["model:customers"]), + ("seeds_and_orders_job", &["seed:regions", "model:orders"]), + ], + deps: &[ + ("model:customers", &["seed:regions"]), + ("model:orders", &["model:customers"]), + ], + expected_edges: &[ + ("customers_job", &["seeds_and_orders_job"]), + ("seeds_and_orders_job", &["customers_job"]), + ], + expected_message: Some(seed_hub.leak()), + }, + GroupWaitGraphTestCase { + description: "members that both build a shared function never wait on each other for it", + selections: &[ + ( + "customers_job", + &["function:normalize_email", "model:customers"], + ), + ("orders_job", &["function:normalize_email", "model:orders"]), + ], + deps: &[ + ( + "model:customers", + &["function:normalize_email", "model:orders"], + ), + ("model:orders", &["function:normalize_email"]), + ], + expected_edges: &[("customers_job", &["orders_job"]), ("orders_job", &[])], + expected_message: None, + }, + ]; + for test_case in test_cases { + let waits = group_wait_graph(&map(test_case.selections), &map(test_case.deps)); + assert_eq!( + waits.edges, + map(test_case.expected_edges), + "{}", + test_case.description + ); + assert_eq!( + group_cycle_message("nightly", &waits.cycles).as_deref(), + test_case.expected_message, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_large_group_on_a_long_chain_when_deriving_waits_then_one_cycle_is_reported_within_the_time_limit() + { + let test_cases = [LargeGroupTestCase { + description: "50 members over a 10,000-node chain close into one cycle", + members: 50, + nodes_per_member: 200, + expected_cycles: 1, + expected_waits: 49 + (1..50).sum::(), + expected_limit: std::time::Duration::from_secs(2), + }]; + for test_case in test_cases { + let (selections, deps) = chained_group(test_case.members, test_case.nodes_per_member); + + let started = std::time::Instant::now(); + let waits = group_wait_graph(&selections, &deps); + let elapsed = started.elapsed(); + + assert_eq!( + waits.cycles.len(), + test_case.expected_cycles, + "{}", + test_case.description + ); + assert_eq!( + waits.edges.values().map(BTreeSet::len).sum::(), + test_case.expected_waits, + "{}", + test_case.description + ); + assert!( + elapsed < test_case.expected_limit, + "{}: took {elapsed:?}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/declarations/test_types.rs b/crates/sqlbuild-orchestration/tests/declarations/test_types.rs new file mode 100644 index 0000000000..d0063a71ab --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/declarations/test_types.rs @@ -0,0 +1,23 @@ +//! Group members, the nodes they build and read, and the waits and cycle report expected. + +pub(crate) type Pairs = &'static [(&'static str, &'static [&'static str])]; + +pub(crate) struct GroupWaitGraphTestCase { + pub(crate) description: &'static str, + /// Each member job and the node keys it builds. + pub(crate) selections: Pairs, + /// Each node key and the keys it reads. + pub(crate) deps: Pairs, + pub(crate) expected_edges: Pairs, + pub(crate) expected_message: Option<&'static str>, +} + +pub(crate) struct LargeGroupTestCase { + pub(crate) description: &'static str, + pub(crate) members: usize, + pub(crate) nodes_per_member: usize, + pub(crate) expected_cycles: usize, + /// Waits across all members: each waits for every member before it, and the first, which builds the tail, for all others. + pub(crate) expected_waits: usize, + pub(crate) expected_limit: std::time::Duration, +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs index 8ab1d28a76..40c1ae83e3 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs @@ -35,8 +35,10 @@ use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_project::contracts::main::evaluate_model_contracts::evaluate_model_contracts; +use sqlbuild_python_project::graph::models::CompiledProjectGraph; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; +use sqlbuild_python_project::project_snapshot::main::job_group_cycles::job_group_cycles; const RICH_LINEAGE: &str = "rich"; const FAST_LINEAGE: &str = "fast"; @@ -222,6 +224,13 @@ fn analyzed<'py>( early_rules.stop(py); } early_lint_wait_ms = early_lint.borrow().stop(py)?.into_pyobject(py)?.into_any(); + } else { + let compiled: &Bound<'py, CompiledProjectGraph> = graph.cast()?; + job_group_cycles( + &options.project_dir, + (&project, &discovered), + compiled.get().native(py).get().graph(), + )?; } let (lineage, lineage_ms): (Bound<'py, PyAny>, u64) = driver.lineage(&graph, &dialect)?; let contracts: ContractPhase<'py> = driver.contracts(&project, (&dialect, &adapter))?; diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/mod.rs index a23aea17c6..1788c6e61d 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/mod.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/mod.rs @@ -1,6 +1,7 @@ pub mod consumer_uses; pub mod invocation_allows_missing_secrets; pub mod orchestration_declarations; +pub mod orchestration_failure; pub mod provider_uses; pub mod register_declaration_files; pub mod register_yaml_entries; diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_failure.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_failure.rs new file mode 100644 index 0000000000..08f938b9e4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_failure.rs @@ -0,0 +1,17 @@ +//! An orchestration problem found after discovery, raised as `OrchestrationDeclarationError` (D017). + +use crate::discovery::_helpers::failures::raised::failure_error; +use pyo3::prelude::{PyErr, Python}; +use sqlbuild_discovery::models::{DiscoveryFailure, FailureKind}; + +/// `OrchestrationDeclarationError` (D017) with `message`, which names the file, line and column itself. +pub fn orchestration_failure(py: Python<'_>, message: String) -> PyErr { + failure_error( + py, + DiscoveryFailure { + kind: FailureKind::Orchestration, + message, + help: None, + }, + ) +} diff --git a/crates/sqlbuild-python-project/src/graph/constants.rs b/crates/sqlbuild-python-project/src/graph/constants.rs index 574ce12638..e82311d348 100644 --- a/crates/sqlbuild-python-project/src/graph/constants.rs +++ b/crates/sqlbuild-python-project/src/graph/constants.rs @@ -5,6 +5,9 @@ use crate::graph::models::CompiledProjectGraph; use sqlbuild_python_base::records::main::registry::class_of::class_of; use sqlbuild_python_base::records::models::GetterEntry; +/// The Python node kind whose functions load sources. +pub(crate) const LOADER_KIND: &str = "loader"; + /// The getter readers of this module's native pyclasses. pub const GETTERS: &[GetterEntry] = &[GetterEntry { class: class_of::, diff --git a/crates/sqlbuild-python-project/src/graph/models.rs b/crates/sqlbuild-python-project/src/graph/models.rs index c55cc47781..d47fccc589 100644 --- a/crates/sqlbuild-python-project/src/graph/models.rs +++ b/crates/sqlbuild-python-project/src/graph/models.rs @@ -8,6 +8,7 @@ use crate::graph::_helpers::project::project_graph::outcome; use crate::graph::_helpers::project::views::lineage_graph_views; use crate::graph::_helpers::python_nodes::python_selection::project_sources; use crate::graph::_helpers::python_nodes::python_selection::selection_graph; +use crate::graph::constants::LOADER_KIND; use crate::graph::main::project::project_graph::project_graph; use crate::graph::main::python_nodes::python_node_graph::python_node_graph; use pyo3::prelude::Bound; @@ -47,6 +48,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use std::cell::RefCell; +use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; @@ -439,6 +441,32 @@ impl ProjectSelection { .map(|node| format!("{}:{name}", node.kind)) } + /// Each Python node's published key and the keys it reads: its upstream Python nodes and its SQL deps. + pub fn node_deps(&self) -> Vec<(String, BTreeSet)> { + let upstream: BTreeMap<&str, &[String]> = self + .nodes + .upstream + .iter() + .map(|(name, names)| (name.as_str(), names.as_slice())) + .collect(); + let mut nodes: Vec<(String, BTreeSet)> = Vec::new(); + for node in &self.nodes.nodes { + let mut deps: BTreeSet = BTreeSet::new(); + for name in upstream + .get(node.name.as_str()) + .copied() + .unwrap_or_default() + { + deps.extend(self.node_key(name)); + } + for (kind, name) in &node.sql_deps { + deps.insert(format!("{kind}:{name}")); + } + nodes.push((format!("{}:{}", node.kind, node.name), deps)); + } + nodes + } + /// The key a selected Python node's run reports under; a loader reports as the source it writes. pub(crate) fn run_key(&self, name: &str) -> Option { let node = self @@ -450,6 +478,11 @@ impl ProjectSelection { Some(python_run_key(&self.sources, (&node.kind, name))) } + /// The `source:` key a loader node publishes when it writes no declared source, so its chain's other loaders can read it. + pub fn intermediate_source(&self, kind: &str, name: &str) -> Option { + (kind == LOADER_KIND && self.intermediate_loader(name)).then(|| format!("source:{name}")) + } + /// Whether a loader writes no declared source and so runs only as part of another loader's chain. pub fn intermediate_loader(&self, name: &str) -> bool { !self diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs index 2f10cb6b41..35f65138a1 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod snapshot_boundary; pub(crate) mod snapshot_declarations; pub(crate) mod snapshot_execution; +pub(crate) mod snapshot_group_cycles; pub(crate) mod snapshot_models; pub(crate) mod snapshot_project; pub(crate) mod snapshot_resources; diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs index a628edd529..f510f466b7 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs @@ -40,7 +40,7 @@ fn file_site(path: &str, position: FilePosition) -> Option { } /// Whether a declaration with `targets` applies to `target`. -fn declared_for(targets: &[String], target: &str) -> bool { +pub(crate) fn declared_for(targets: &[String], target: &str) -> bool { targets.is_empty() || targets.iter().any(|name| name == target) } diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_group_cycles.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_group_cycles.rs new file mode 100644 index 0000000000..051b13903a --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_group_cycles.rs @@ -0,0 +1,93 @@ +//! Find job group members that wait on each other at compile, from the same waits publication derives. + +use std::collections::{BTreeMap, BTreeSet}; + +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_discovery::orchestration_files::models::{FilePosition, OrchestrationInventory}; +use sqlbuild_orchestration::declarations::main::group_cycle_message::group_cycle_message; +use sqlbuild_orchestration::declarations::main::group_wait_graph::group_wait_graph; + +use crate::graph::models::ProjectSelection; +use crate::project_snapshot::_helpers::snapshot::snapshot_declarations::declared_for; + +/// Whether any group declared for `target` lists two or more jobs, so its members could wait on each other. +pub(crate) fn has_groups(inventory: &OrchestrationInventory, target: &str) -> bool { + inventory + .groups + .iter() + .any(|(_, group)| group.jobs.len() > 1 && declared_for(&group.run.targets, target)) +} + +/// Each node key and the keys it reads, as publication records them in node `deps`. +fn node_deps( + graph: &ProjectGraph, + selected: &ProjectSelection, +) -> BTreeMap> { + let mut deps: BTreeMap> = BTreeMap::new(); + for ((kind, name), upstream) in &graph.upstream { + let reads: &mut BTreeSet = deps.entry(format!("{kind}:{name}")).or_default(); + for (upstream_kind, upstream_name) in upstream { + reads.insert(format!("{upstream_kind}:{upstream_name}")); + } + } + for (node, upstream) in selected.node_deps() { + deps.entry(node).or_default().extend(upstream); + } + for node in &selected.nodes.nodes { + if let Some(source) = selected.intermediate_source(&node.kind, &node.name) { + deps.entry(source) + .or_default() + .insert(format!("{}:{}", node.kind, node.name)); + } + } + deps +} + +/// One `path:line:column: message` per group declared for `target` whose members wait on each other; jobs whose selector fails are left out. +pub(crate) fn group_cycle_failures( + inventory: &OrchestrationInventory, + target: &str, + (graph, selected): (&ProjectGraph, &ProjectSelection), +) -> Vec { + let tokens = |value: &str| { + value + .split_whitespace() + .map(str::to_owned) + .collect::>() + }; + let mut jobs: BTreeMap<&str, BTreeSet> = BTreeMap::new(); + for (_, job) in &inventory.jobs { + if !declared_for(&job.run.targets, target) { + continue; + } + let exclude: &str = job.exclude.as_ref().map_or("", |exclude| &exclude.value); + if let Ok(selection) = + selected.run_keys(graph, &tokens(&job.select.value), &tokens(exclude)) + { + jobs.insert(&job.run.name.value, selection); + } + } + let deps: BTreeMap> = node_deps(graph, selected); + let mut failures: Vec = Vec::new(); + for (path, group) in &inventory.groups { + if !declared_for(&group.run.targets, target) { + continue; + } + let selections: BTreeMap> = group + .jobs + .iter() + .filter_map(|member| { + let selection = jobs.get(member.value.as_str())?; + Some((member.value.clone(), selection.clone())) + }) + .collect(); + let name: &str = &group.run.name.value; + if let Some(message) = + group_cycle_message(name, &group_wait_graph(&selections, &deps).cycles) + { + let FilePosition { line, column } = group.run.name.position; + failures.push(format!("{path}:{line}:{column}: {message}")); + } + } + failures +} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/job_group_cycles.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/job_group_cycles.rs new file mode 100644 index 0000000000..653977b71e --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/job_group_cycles.rs @@ -0,0 +1,34 @@ +//! Refuse a compile whose job groups have members that wait on each other, as `OrchestrationDeclarationError` (D017). + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; +use sqlbuild_python_front::discovery::main::sources::orchestration_failure::orchestration_failure; + +use crate::graph::models::ProjectSelection; +use crate::project_snapshot::_helpers::snapshot::snapshot_group_cycles::{ + group_cycle_failures, has_groups, +}; + +/// Every cycle in every job group declared for the project's target, in one error; reads Python nodes only when a group has two or more jobs. +pub fn job_group_cycles( + project_dir: &Bound<'_, PyAny>, + (project, discovered_inputs): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + graph: &ProjectGraph, +) -> PyResult<()> { + let py = project_dir.py(); + let inventory = orchestration_declarations(project_dir)?; + let target: String = field(project, "effective_target_name")? + .extract::>()? + .unwrap_or_default(); + if !has_groups(&inventory, &target) { + return Ok(()); + } + let selected = ProjectSelection::new(project, discovered_inputs)?; + let failures: Vec = group_cycle_failures(&inventory, &target, (graph, &selected)); + if failures.is_empty() { + return Ok(()); + } + Err(orchestration_failure(py, failures.join("\n"))) +} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/mod.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/mod.rs index ad14c41e07..ac1b2d7e50 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/main/mod.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/mod.rs @@ -1,5 +1,6 @@ pub mod compile_declarations; pub mod compile_project_roles; +pub mod job_group_cycles; pub mod model_nodes; pub mod python_declarations; pub mod register_snapshot; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs index 34ee4b45f7..39a80d892f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs @@ -13,8 +13,8 @@ use sqlbuild_core::command_output::constants::COMMAND_OUTPUT_RECORD_TYPE; use sqlbuild_core::event_export::constants::{ DEFAULT_EVENT_EXPORT_HEALTH_INTERVAL_SECONDS, DEFAULT_EVENT_EXPORT_INVOCATION_TIMEOUT_SECONDS, DEFAULT_EVENT_EXPORT_NOTIFICATION_QUEUE_CAPACITY, DEFAULT_EVENT_EXPORT_QUEUE_CAPACITY, - DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, INVOCATION_TERMINAL_EVENT_TYPES, - OUTCOME_EVENT_TYPES, OUTCOME_RESERVE_FRACTION, + DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, INVOCATION_START_EVENT_TYPE, + INVOCATION_TERMINAL_EVENT_TYPES, OUTCOME_EVENT_TYPES, OUTCOME_RESERVE_FRACTION, }; use sqlbuild_core::event_export::main::drop_lifecycle_attempts::drop_lifecycle_attempts; use sqlbuild_core::event_export::main::export_snapshot::export_snapshot; @@ -797,6 +797,10 @@ impl EventExporterDispatcher { if self.withheld_kinds.contains(&policy.kind) { return Ok(()); } + let invocation_id: Option = match native_field(event, "invocation_id") { + Ok(value) => value.extract()?, + Err(_) => None, + }; let shared = &self.shared; let _held = shared.hold(py)?; let mut state = shared.locked()?; @@ -832,6 +836,8 @@ impl EventExporterDispatcher { priority: policy.priority, invocation_terminal: INVOCATION_TERMINAL_EVENT_TYPES.contains(&event_type.as_str()), outcome: OUTCOME_EVENT_TYPES.contains(&event_type.as_str()), + invocation_id, + invocation_start: event_type == INVOCATION_START_EVENT_TYPE, eligible_exporters: eligible, }, ); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 2f165dfbf8..7b5b0c5c4a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -1,6 +1,6 @@ //! Location publication assembles catalog data and planner-owned local identities. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::path::PathBuf; use pyo3::exceptions::PyValueError; @@ -36,9 +36,6 @@ use sqlbuild_python_project::project_snapshot::main::python_declarations::python use sqlbuild_python_project::project_snapshot::main::resource_nodes::resource_nodes; use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationSnapshot; -/// The Python node kind whose functions load sources. -const LOADER_KIND: &str = "loader"; - /// Compile catalog nodes and identities in the location, never in the server. #[pyfunction] #[pyo3(signature = (project, metadata_json, project_dir, **options))] @@ -156,34 +153,17 @@ fn python_nodes( tags.entry(name.as_str()).or_default().push(tag.as_str()); } } - let upstream: BTreeMap<&str, &[String]> = graph - .upstream - .iter() - .map(|(name, names)| (name.as_str(), names.as_slice())) - .collect(); let mut outputs: Vec<(String, SnapshotNode)> = Vec::new(); - for node in &graph.nodes { - let key = format!("{}:{}", node.kind, node.name); + for (node, (key, deps)) in graph.nodes.iter().zip(selected.node_deps()) { let Some(published) = nodes.get_mut(&key) else { return Err(PyValueError::new_err(format!( "Python node {key} is missing from the snapshot" ))); }; - let deps: BTreeSet = upstream - .get(node.name.as_str()) - .into_iter() - .flat_map(|names| names.iter()) - .filter_map(|name| selected.node_key(name)) - .chain( - node.sql_deps - .iter() - .map(|(kind, name)| format!("{kind}:{name}")), - ) - .collect(); published.definition["deps"] = json!(deps); published.definition["tags"] = json!(tags.get(node.name.as_str()).cloned().unwrap_or_default()); - if node.kind == LOADER_KIND && selected.intermediate_loader(&node.name) { + if let Some(source) = selected.intermediate_source(&node.kind, &node.name) { let mut definition = json!({ "name": node.name, "entry": {"loader": node.name}, @@ -194,7 +174,7 @@ fn python_nodes( definition["path"] = path.clone(); } outputs.push(( - format!("source:{}", node.name), + source, SnapshotNode { kind: "source".into(), definition, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md index 6f96975b4c..9cb2e94a83 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md @@ -106,9 +106,11 @@ durability. SQLBuild core does not provide a Kafka or ClickHouse implementation. - `sqb compile` sends lifecycle sinks only its invocation events (`invocation_started` and its outcome), never its phase `operation` events, so a sink never lengthens a compile with phase deliveries. Other commands send every subscribed kind. -- Each sink receives events in order on its own long-lived delivery thread. -- Lifecycle dispatch prioritizes failures and terminal facts, remaining FIFO within equal priority; - `invocation_completed` and `invocation_failed` come after every other queued event. +- Each sink receives events on its own long-lived delivery thread. +- An invocation's `invocation_started` arrives before its other events, and `invocation_completed` + or `invocation_failed` after every other queued event. Events between them are prioritized: + failures and terminal facts first, FIFO within equal priority. For exact chronology, sort by + `invocation_sequence`. - Queue overflow can displace or drop lower-priority records and is reflected in sink accounting. Run and invocation outcomes (`run_started`, `run_completed`, `run_failed`, `invocation_started`, `invocation_completed`, `invocation_failed`) are always admitted and never displaced. diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md index 0f38b9505c..2dd7a15b20 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -105,7 +105,8 @@ reports a job or sensor `select` that names more than three nodes one by one; se ## Group jobs A job group runs its jobs as one run. A job waits for another job in the group when it reads a node -that job builds. +that job builds. Jobs that wait on each other, directly or through other members, are a cycle, and +`sqb compile` reports every cycle in the group as one `D017` error. ```toml # orchestration/job_groups/daily_orders_job_group.toml @@ -119,6 +120,9 @@ A group takes `name`, `description` and `jobs`, and the same `targets`, `owner`, `concurrency_groups`, `schedule` and `[on_failure]` as a job. Its members are jobs, never other groups. +A job that reads a seed or a node that several members build waits for all of those members. Give +seeds and shared nodes their own job, so the jobs that read them wait only for that job. + ## Schedule a job `schedule` takes a cron expression or the name of a calendar, and always a timezone. @@ -226,6 +230,7 @@ Every problem with an orchestration file is a `D017` error at its file, line and | `schedule cron: invalid cron expression: Pattern must have 5 fields when seconds are disallowed.` | Write a five-field cron, such as `0 2 * * *` | | `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | | `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | +| `group dependencies contain a cycle in job group daily_orders_job_group: orders_marts_job waits for staging_orders_job (model:orders reads model:stg_orders); staging_orders_job waits for orders_marts_job (model:stg_orders reads seed:regions)` | Move each node to one job, and give seeds and nodes that several members read their own job that runs first | Details: diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index ece7bf2c0c..8548909338 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1443,3 +1443,10 @@ class ManagedSourceLoaderLocationTestCase: source_schema: str command: tuple[str, ...] expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class JobGroupCyclesCase: + description: str + fixed_group_toml: str + expected_messages: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_job_group_cycles.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_job_group_cycles.py new file mode 100644 index 0000000000..5f88190d1f --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_job_group_cycles.py @@ -0,0 +1,62 @@ +"""E2E coverage for job group cycles: compile reports every cycle, then passes once they are gone.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import JobGroupCyclesCase +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + GROUP_CYCLE_MESSAGE, + GROUP_CYCLE_NODES, + duckshop_orchestration, +) + +_GROUP_PATH: str = "orchestration/job_groups/nightly_job_group.toml" + + +@pytest.mark.parametrize( + "test_case", + [ + JobGroupCyclesCase( + description="three cycles fail compile together, and one job per pair that never wait passes", + fixed_group_toml=( + 'name = "nightly_job_group"\ndescription = "Build everything nightly."\n' + 'jobs = ["orders_job", "shipments_job", "region_report_job"]\n' + ), + expected_messages=tuple(GROUP_CYCLE_MESSAGE.splitlines()), + ) + ], + ids=lambda case: case.description, +) +def test_given_a_job_group_with_cycles_when_compiling_then_every_cycle_fails_until_fixed( + tmp_path: Path, + test_case: JobGroupCyclesCase, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="group_cycles", + repo_files={ + "sqlbuild_project.toml": 'name = "group_cycles"\nadapter = "duckdb"\n', + **GROUP_CYCLE_NODES, + **duckshop_orchestration("group_cycles"), + }, + ) + + failed: subprocess.CompletedProcess[str] = run_sqb( + command=("compile",), project_dir=project_dir + ) + + assert failed.returncode == 1, failed.stdout + failed.stderr + assert "error[D017]" in failed.stderr + for message in test_case.expected_messages: + assert message in failed.stderr, failed.stderr + + (project_dir / _GROUP_PATH).write_text(test_case.fixed_group_toml, encoding="utf-8") + fixed: subprocess.CompletedProcess[str] = run_sqb(command=("compile",), project_dir=project_dir) + + assert fixed.returncode == 0, fixed.stdout + fixed.stderr + assert "D017" not in fixed.stderr diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 8c8165381e..c41b849efa 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -437,6 +437,58 @@ def duckshop_orchestration(fixture: str) -> dict[str, str]: } +def _group_cycle_model(name: str, reads: str) -> str: + """A one-column synthetic model reading `reads`.""" + return ( + f"MODEL (description 'Synthetic {name}.', columns (id (type INTEGER)));\n\n" + f'SELECT CAST(1 AS INTEGER) AS id FROM __ref("{reads}")\n' + ) + + +# Models and a seed for the DuckShop `group_cycles` jobs, grouped into three cycles. +GROUP_CYCLE_NODES: dict[str, str] = { + "seeds/regions.csv": "id\n1\n", + "seeds/regions.yml": ( + "seeds:\n - name: regions\n description: Regions\n columns:\n" + " - name: id\n type: INTEGER\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Synthetic stg_orders.', columns (id (type INTEGER)));\n\n" + "SELECT CAST(1 AS INTEGER) AS id\n" + ), + "models/stg_inventory.sql": ( + "MODEL (description 'Synthetic stg_inventory.', columns (id (type INTEGER)));\n\n" + "SELECT CAST(1 AS INTEGER) AS id\n" + ), + "models/customers.sql": _group_cycle_model("customers", "stg_orders"), + "models/orders.sql": _group_cycle_model("orders", "customers"), + "models/shipments.sql": _group_cycle_model("shipments", "stg_inventory"), + "models/inventory.sql": _group_cycle_model("inventory", "shipments"), + "models/region_report.sql": ( + "MODEL (description 'Synthetic region_report.', columns (id (type INTEGER)));\n\n" + 'SELECT CAST(1 AS INTEGER) AS id FROM __seed("regions")\n' + ), + "models/regional_orders.sql": _group_cycle_model("regional_orders", "region_report"), +} +# What compile and publish both report for that group: every cycle, then the shared-read hint. +GROUP_CYCLE_MESSAGE: str = "\n".join( + f"group dependencies contain a cycle in job group nightly_job_group: {cycle}" + for cycle in ( + "customers_job waits for orders_job (model:customers reads model:stg_orders); " + "orders_job waits for customers_job (model:orders reads model:customers)", + "inventory_job waits for shipments_job (model:inventory reads model:shipments); " + "shipments_job waits for inventory_job (model:shipments reads model:stg_inventory)", + "reference_job waits for region_report_job (model:regional_orders reads " + "model:region_report); region_report_job waits for reference_job " + "(model:region_report reads seed:regions)", + ) +) + ( + "\nhint: a wait in this cycle reads a seed or a node that more than one member builds, " + "so every member that reads it waits for all of its builders; give seeds and shared nodes " + "their own job that runs first" +) + + def edited_duckshop_orchestration(fixture: str, path: str, old: str, new: str) -> dict[str, str]: """A DuckShop orchestration fixture with `old` replaced by `new` in the file at `path`.""" files: dict[str, str] = duckshop_orchestration(fixture) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index d21ca16616..4a60108d7c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -43,6 +43,8 @@ ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( FUNCTION_PROJECT_FILES, + GROUP_CYCLE_MESSAGE, + GROUP_CYCLE_NODES, METADATA, PARITY_FILES, TYPE_PROOF_FILES, @@ -1502,6 +1504,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects @pytest.mark.parametrize( "test_case", [ + SnapshotDiscoveryRejectionTestCase( + description="every job group cycle is reported at once, with the seed hint", + files={**duckshop_orchestration("group_cycles"), **GROUP_CYCLE_NODES}, + expected_error=ValueError, + expected_message=re.escape(GROUP_CYCLE_MESSAGE), + ), SnapshotDiscoveryRejectionTestCase( description="group members are jobs, not nested groups", files=duckshop_orchestration("nested_groups"), diff --git a/website/src/content/docs/docs/concepts/observability/sinks.mdx b/website/src/content/docs/docs/concepts/observability/sinks.mdx index c1925f9773..d7c6ea84f0 100644 --- a/website/src/content/docs/docs/concepts/observability/sinks.mdx +++ b/website/src/content/docs/docs/concepts/observability/sinks.mdx @@ -98,9 +98,11 @@ durability. SQLBuild core does not provide a Kafka or ClickHouse implementation. - `sqb compile` sends lifecycle sinks only its invocation events (`invocation_started` and its outcome), never its phase `operation` events, so a sink never lengthens a compile with phase deliveries. Other commands send every subscribed kind. -- Each sink receives events in order on its own long-lived delivery thread. -- Lifecycle dispatch prioritizes failures and terminal facts, remaining FIFO within equal priority; - `invocation_completed` and `invocation_failed` come after every other queued event. +- Each sink receives events on its own long-lived delivery thread. +- An invocation's `invocation_started` arrives before its other events, and `invocation_completed` + or `invocation_failed` after every other queued event. Events between them are prioritized: + failures and terminal facts first, FIFO within equal priority. For exact chronology, sort by + `invocation_sequence`. - Queue overflow can displace or drop lower-priority records and is reflected in sink accounting. Run and invocation outcomes (`run_started`, `run_completed`, `run_failed`, `invocation_started`, `invocation_completed`, `invocation_failed`) are always admitted and never displaced. diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx index 30acbc8147..47c6891d3d 100644 --- a/website/src/content/docs/docs/concepts/orchestration.mdx +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -91,7 +91,8 @@ reports a job or sensor `select` that names more than three nodes one by one; se ## Group jobs A job group runs its jobs as one run. A job waits for another job in the group when it reads a node -that job builds. +that job builds. Jobs that wait on each other, directly or through other members, are a cycle, and +`sqb compile` reports every cycle in the group as one `D017` error. ```toml # orchestration/job_groups/daily_orders_job_group.toml @@ -105,6 +106,9 @@ A group takes `name`, `description` and `jobs`, and the same `targets`, `owner`, `concurrency_groups`, `schedule` and `[on_failure]` as a job. Its members are jobs, never other groups. +A job that reads a seed or a node that several members build waits for all of those members. Give +seeds and shared nodes their own job, so the jobs that read them wait only for that job. + ## Schedule a job `schedule` takes a cron expression or the name of a calendar, and always a timezone. @@ -213,6 +217,7 @@ Every problem with an orchestration file is a `D017` error at its file, line and | `schedule cron: invalid cron expression: Pattern must have 5 fields when seconds are disallowed.` | Write a five-field cron, such as `0 2 * * *` | | `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | | `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | +| `group dependencies contain a cycle in job group daily_orders_job_group: orders_marts_job waits for staging_orders_job (model:orders reads model:stg_orders); staging_orders_job waits for orders_marts_job (model:stg_orders reads seed:regions)` | Move each node to one job, and give seeds and nodes that several members read their own job that runs first | Details: