From d6efc03318a2394765a6eeb4739e5125c0e2815e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 03:22:16 +0000 Subject: [PATCH 1/9] fix: deliver each invocation's start before its other lifecycle events The lifecycle queue delivers by priority, so a terminal fact such as operation_completed could reach a sink before invocation_started. When the event the queue would pick next has a queued invocation_started of the same invocation, the queue now picks that start first. Events of other invocations, and events whose start is not queued, keep their priorities. --- .../src/event_export/constants.rs | 2 + .../event_export/main/take_lifecycle_event.rs | 17 ++++++- .../sqlbuild-core/src/event_export/models.rs | 4 ++ .../src/event_export/tests/helpers.rs | 14 ++++++ .../src/event_export/tests/test_queue.rs | 49 ++++++++++++++++++- .../src/event_export/tests/test_types.rs | 7 +++ .../_helpers/event_exporting/dispatcher.rs | 10 +++- .../docs/concepts/observability/sinks.md | 8 +-- .../docs/concepts/observability/sinks.mdx | 8 +-- 9 files changed, 108 insertions(+), 11 deletions(-) diff --git a/crates/sqlbuild-core/src/event_export/constants.rs b/crates/sqlbuild-core/src/event_export/constants.rs index 75feaf7546..53a9a7d0bf 100644 --- a/crates/sqlbuild-core/src/event_export/constants.rs +++ b/crates/sqlbuild-core/src/event_export/constants.rs @@ -16,6 +16,8 @@ pub const EVENT_EXPORT_KINDS: [&str; 8] = [ "run", "statement", ]; +/// The event that starts an invocation, delivered before that invocation's other queued events. +pub const INVOCATION_START_EVENT_TYPE: &str = "invocation_started"; /// Invocation terminals, delivered only after every other queued event. pub const INVOCATION_TERMINAL_EVENT_TYPES: [&str; 2] = ["invocation_completed", "invocation_failed"]; diff --git a/crates/sqlbuild-core/src/event_export/main/take_lifecycle_event.rs b/crates/sqlbuild-core/src/event_export/main/take_lifecycle_event.rs index aec19201b6..1684c2421b 100644 --- a/crates/sqlbuild-core/src/event_export/main/take_lifecycle_event.rs +++ b/crates/sqlbuild-core/src/event_export/main/take_lifecycle_event.rs @@ -2,7 +2,7 @@ use crate::event_export::models::{EventQueue, QueuedEvent}; -/// The queue without its next event: the oldest at the highest priority, invocation terminals last. +/// The queue without its next event: the oldest at the highest priority, invocation terminals last, and a queued start of the chosen event's invocation before it. pub fn take_lifecycle_event( mut queue: EventQueue, ) -> (EventQueue, Option>) { @@ -21,6 +21,21 @@ pub fn take_lifecycle_event( .filter(|(_, queued)| selectable(queued) && Some(queued.priority) == highest) .min_by_key(|(_, queued)| queued.sequence) .map(|(position, _)| position); + let position: Option = position.map(|position| { + let chosen = &queue.items[position]; + queue + .items + .iter() + .enumerate() + .filter(|(_, queued)| { + queued.invocation_start + && !chosen.invocation_start + && chosen.invocation_id.is_some() + && queued.invocation_id == chosen.invocation_id + }) + .min_by_key(|(_, queued)| queued.sequence) + .map_or(position, |(start, _)| start) + }); let next: Option> = position.map(|position| queue.items.remove(position)); (queue, next) } diff --git a/crates/sqlbuild-core/src/event_export/models.rs b/crates/sqlbuild-core/src/event_export/models.rs index 9ee6750807..00a6c41cb1 100644 --- a/crates/sqlbuild-core/src/event_export/models.rs +++ b/crates/sqlbuild-core/src/event_export/models.rs @@ -46,6 +46,10 @@ pub struct QueuedEvent { pub invocation_terminal: bool, /// Whether the event reports a run or invocation outcome, which the queue always admits. pub outcome: bool, + /// The invocation the event belongs to, when it names one. + pub invocation_id: Option, + /// Whether the event starts its invocation and so goes before that invocation's other events. + pub invocation_start: bool, pub eligible_exporters: Vec, } diff --git a/crates/sqlbuild-core/src/event_export/tests/helpers.rs b/crates/sqlbuild-core/src/event_export/tests/helpers.rs index 2817d06c0a..252cbc4a81 100644 --- a/crates/sqlbuild-core/src/event_export/tests/helpers.rs +++ b/crates/sqlbuild-core/src/event_export/tests/helpers.rs @@ -11,10 +11,24 @@ pub(crate) fn queued( priority, invocation_terminal, outcome: false, + invocation_id: None, + invocation_start: false, eligible_exporters: vec![0], } } +/// A queued event with `sequence` and `(priority, invocation start, invocation)`. +pub(crate) fn invocation_event( + sequence: u64, + (priority, invocation_start, invocation): (u8, bool, &str), +) -> QueuedEvent<()> { + QueuedEvent { + invocation_id: Some(invocation.to_owned()), + invocation_start, + ..queued(sequence, (priority, false)) + } +} + /// A queued non-terminal event with `sequence`, `(priority, outcome)` and one eligible exporter. pub(crate) fn event((priority, outcome): (u8, bool), sequence: u64) -> QueuedEvent<()> { QueuedEvent { diff --git a/crates/sqlbuild-core/src/event_export/tests/test_queue.rs b/crates/sqlbuild-core/src/event_export/tests/test_queue.rs index 2ae6361f1d..c8b8c8d0a4 100644 --- a/crates/sqlbuild-core/src/event_export/tests/test_queue.rs +++ b/crates/sqlbuild-core/src/event_export/tests/test_queue.rs @@ -2,9 +2,10 @@ use crate::event_export::main::queue_lifecycle_event::queue_lifecycle_event; use crate::event_export::main::retain_outcome_events::retain_outcome_events; use crate::event_export::main::take_lifecycle_event::take_lifecycle_event; use crate::event_export::models::EventQueue; -use crate::event_export::tests::helpers::{event, queued, sequences}; +use crate::event_export::tests::helpers::{event, invocation_event, queued, sequences}; use crate::event_export::tests::test_types::{ - DeliveryOrderTestCase, DisplacementTestCase, OutcomeAdmissionTestCase, OutcomeReserveTestCase, + DeliveryOrderTestCase, DisplacementTestCase, InvocationStartOrderTestCase, + OutcomeAdmissionTestCase, OutcomeReserveTestCase, }; #[test] @@ -186,3 +187,47 @@ fn given_queued_events_when_taking_then_priority_then_fifo_with_invocation_termi ); } } + +#[test] +fn given_a_queued_invocation_start_when_taking_then_it_precedes_only_its_own_invocation() { + let test_cases = [ + InvocationStartOrderTestCase { + description: "a start goes before its invocation's higher-priority events", + events: &[(0, true, "a"), (2, false, "a"), (3, false, "a")], + expected_sequences: &[0, 2, 1], + }, + InvocationStartOrderTestCase { + description: "a start that is not queued holds nothing back", + events: &[(2, false, "a"), (0, false, "a"), (3, false, "a")], + expected_sequences: &[2, 0, 1], + }, + InvocationStartOrderTestCase { + description: "one invocation's start does not jump another's events", + events: &[ + (2, false, "b"), + (0, true, "a"), + (2, false, "a"), + (0, false, "b"), + ], + expected_sequences: &[0, 1, 2, 3], + }, + ]; + + for test_case in test_cases { + let mut queue: EventQueue<()> = EventQueue::new(16); + for (sequence, event) in (0_u64..).zip(test_case.events) { + queue = queue_lifecycle_event(queue, invocation_event(sequence, *event)).queue; + } + let mut order: Vec = Vec::new(); + for _ in test_case.events { + let (rest, next) = take_lifecycle_event(queue); + queue = rest; + order.extend(next.map(|event| event.sequence)); + } + assert_eq!( + order, test_case.expected_sequences, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-core/src/event_export/tests/test_types.rs b/crates/sqlbuild-core/src/event_export/tests/test_types.rs index 0338b49a40..1130367650 100644 --- a/crates/sqlbuild-core/src/event_export/tests/test_types.rs +++ b/crates/sqlbuild-core/src/event_export/tests/test_types.rs @@ -40,6 +40,13 @@ pub(crate) struct DeliveryOrderTestCase { pub(crate) expected_sequences: &'static [u64], } +/// Queued events as `(priority, invocation start, invocation)`, and the order they are taken in. +pub(crate) struct InvocationStartOrderTestCase { + pub(crate) description: &'static str, + pub(crate) events: &'static [(u8, bool, &'static str)], + pub(crate) expected_sequences: &'static [u64], +} + /// One event routed across sink filters, and the resulting accounting. pub(crate) struct RoutingTestCase { pub(crate) description: &'static str, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs index 34ee4b45f7..3c0ef9b2cf 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs @@ -13,8 +13,8 @@ use sqlbuild_core::command_output::constants::COMMAND_OUTPUT_RECORD_TYPE; use sqlbuild_core::event_export::constants::{ DEFAULT_EVENT_EXPORT_HEALTH_INTERVAL_SECONDS, DEFAULT_EVENT_EXPORT_INVOCATION_TIMEOUT_SECONDS, DEFAULT_EVENT_EXPORT_NOTIFICATION_QUEUE_CAPACITY, DEFAULT_EVENT_EXPORT_QUEUE_CAPACITY, - DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, INVOCATION_TERMINAL_EVENT_TYPES, - OUTCOME_EVENT_TYPES, OUTCOME_RESERVE_FRACTION, + DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, INVOCATION_START_EVENT_TYPE, + INVOCATION_TERMINAL_EVENT_TYPES, OUTCOME_EVENT_TYPES, OUTCOME_RESERVE_FRACTION, }; use sqlbuild_core::event_export::main::drop_lifecycle_attempts::drop_lifecycle_attempts; use sqlbuild_core::event_export::main::export_snapshot::export_snapshot; @@ -797,6 +797,10 @@ impl EventExporterDispatcher { if self.withheld_kinds.contains(&policy.kind) { return Ok(()); } + let invocation_id: Option = match event.getattr("invocation_id") { + Ok(value) => value.extract()?, + Err(_) => None, + }; let shared = &self.shared; let _held = shared.hold(py)?; let mut state = shared.locked()?; @@ -832,6 +836,8 @@ impl EventExporterDispatcher { priority: policy.priority, invocation_terminal: INVOCATION_TERMINAL_EVENT_TYPES.contains(&event_type.as_str()), outcome: OUTCOME_EVENT_TYPES.contains(&event_type.as_str()), + invocation_id, + invocation_start: event_type == INVOCATION_START_EVENT_TYPE, eligible_exporters: eligible, }, ); diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md index 6f96975b4c..9cb2e94a83 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/observability/sinks.md @@ -106,9 +106,11 @@ durability. SQLBuild core does not provide a Kafka or ClickHouse implementation. - `sqb compile` sends lifecycle sinks only its invocation events (`invocation_started` and its outcome), never its phase `operation` events, so a sink never lengthens a compile with phase deliveries. Other commands send every subscribed kind. -- Each sink receives events in order on its own long-lived delivery thread. -- Lifecycle dispatch prioritizes failures and terminal facts, remaining FIFO within equal priority; - `invocation_completed` and `invocation_failed` come after every other queued event. +- Each sink receives events on its own long-lived delivery thread. +- An invocation's `invocation_started` arrives before its other events, and `invocation_completed` + or `invocation_failed` after every other queued event. Events between them are prioritized: + failures and terminal facts first, FIFO within equal priority. For exact chronology, sort by + `invocation_sequence`. - Queue overflow can displace or drop lower-priority records and is reflected in sink accounting. Run and invocation outcomes (`run_started`, `run_completed`, `run_failed`, `invocation_started`, `invocation_completed`, `invocation_failed`) are always admitted and never displaced. diff --git a/website/src/content/docs/docs/concepts/observability/sinks.mdx b/website/src/content/docs/docs/concepts/observability/sinks.mdx index c1925f9773..d7c6ea84f0 100644 --- a/website/src/content/docs/docs/concepts/observability/sinks.mdx +++ b/website/src/content/docs/docs/concepts/observability/sinks.mdx @@ -98,9 +98,11 @@ durability. SQLBuild core does not provide a Kafka or ClickHouse implementation. - `sqb compile` sends lifecycle sinks only its invocation events (`invocation_started` and its outcome), never its phase `operation` events, so a sink never lengthens a compile with phase deliveries. Other commands send every subscribed kind. -- Each sink receives events in order on its own long-lived delivery thread. -- Lifecycle dispatch prioritizes failures and terminal facts, remaining FIFO within equal priority; - `invocation_completed` and `invocation_failed` come after every other queued event. +- Each sink receives events on its own long-lived delivery thread. +- An invocation's `invocation_started` arrives before its other events, and `invocation_completed` + or `invocation_failed` after every other queued event. Events between them are prioritized: + failures and terminal facts first, FIFO within equal priority. For exact chronology, sort by + `invocation_sequence`. - Queue overflow can displace or drop lower-priority records and is reflected in sink accounting. Run and invocation outcomes (`run_started`, `run_completed`, `run_failed`, `invocation_started`, `invocation_completed`, `invocation_failed`) are always admitted and never displaced. From ac3dedd03e15513c83f1cb0232fa076e65aa4245 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 04:36:40 +0000 Subject: [PATCH 2/9] refactor: derive job group waits in one shared function that reports every cycle group_wait_graph takes member selections and node dependencies, so publish and compile can share it. It finds every set of members that wait on each other, names one cycle per set with a reason per wait, and flags cycles whose waits read a seed or a node several members build. Publication keeps its message, one line per cycle, plus a hint for shared reads. --- .../src/declarations/_helpers/group_waits.rs | 73 +++++++---- .../src/declarations/constants.rs | 5 + .../declarations/main/group_cycle_message.rs | 23 ++++ .../declarations/main/group_dependencies.rs | 105 +++++----------- .../src/declarations/main/group_wait_graph.rs | 84 +++++++++++++ .../src/declarations/main/mod.rs | 2 + .../src/declarations/mod.rs | 6 +- .../src/declarations/models.rs | 17 ++- .../tests/declarations/test_contracts.rs | 6 + .../declarations/test_group_wait_graph.rs | 113 ++++++++++++++++++ .../tests/declarations/test_types.rs | 13 ++ 11 files changed, 341 insertions(+), 106 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/group_cycle_message.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/main/group_wait_graph.rs create mode 100644 crates/sqlbuild-orchestration/tests/declarations/test_contracts.rs create mode 100644 crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs create mode 100644 crates/sqlbuild-orchestration/tests/declarations/test_types.rs diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs index fcc0b6ece8..61abc2d175 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs @@ -1,45 +1,72 @@ -//! Record why one group member waits for another, and name the waits that close a cycle. +//! Record why one group member waits for another, find the members that wait on each other, and name a cycle among them. use std::collections::{BTreeMap, BTreeSet}; -/// Keep the first reason found for a wait: the first lineage edge. +/// Keep the first reason found for a wait: the first lineage edge, as the node and the upstream it reads. pub(crate) fn record_wait( - mut reasons: BTreeMap<(String, String), String>, + mut reasons: BTreeMap<(String, String), (String, String)>, (job, dependency): (&str, &str), - reason: impl FnOnce() -> String, -) -> BTreeMap<(String, String), String> { + reason: impl FnOnce() -> (String, String), +) -> BTreeMap<(String, String), (String, String)> { reasons .entry((job.to_owned(), dependency.to_owned())) .or_insert_with(reason); reasons } -/// Every job left waits on another, so following first dependencies from any of them closes a cycle. -pub(crate) fn describe_cycle( - remaining: &BTreeMap>, - reasons: &BTreeMap<(String, String), String>, -) -> String { +/// Groups of two or more members that each reach every other through waits, in name order. +pub(crate) fn strongly_connected( + edges: &BTreeMap>, +) -> Vec> { + let reach = |start: &str| { + let mut seen: BTreeSet = BTreeSet::new(); + let mut pending: Vec<&str> = vec![start]; + while let Some(name) = pending.pop() { + for next in edges.get(name).into_iter().flatten() { + if seen.insert(next.clone()) { + pending.push(next); + } + } + } + seen + }; + let reaches: BTreeMap<&str, BTreeSet> = edges + .keys() + .map(|name| (name.as_str(), reach(name))) + .collect(); + let mut components: Vec> = Vec::new(); + for (name, reached) in &reaches { + if !reached.contains(*name) || components.iter().any(|done| done.contains(*name)) { + continue; + } + components.push( + reached + .iter() + .filter(|other| reaches[other.as_str()].contains(*name)) + .cloned() + .collect(), + ); + } + components +} + +/// Follow first waits inside `members`, all of which wait on each other, until a member repeats. +pub(crate) fn cycle_path<'a>( + members: &'a BTreeSet, + edges: &'a BTreeMap>, +) -> Vec<&'a str> { let mut path: Vec<&str> = Vec::new(); - let mut current = remaining.keys().next().map(String::as_str); + let mut current = members.iter().next().map(String::as_str); while let Some(name) = current { if let Some(start) = path.iter().position(|seen| *seen == name) { path.drain(..start); break; } path.push(name); - current = remaining + current = edges .get(name) - .and_then(|deps| deps.iter().next()) + .and_then(|deps| deps.iter().find(|dep| members.contains(*dep))) .map(String::as_str); } - path.iter() - .zip(path.iter().cycle().skip(1)) - .map(|(job, dependency)| { - let reason = reasons - .get(&((*job).to_owned(), (*dependency).to_owned())) - .map_or("", String::as_str); - format!("{job} waits for {dependency} ({reason})") - }) - .collect::>() - .join("; ") + path } diff --git a/crates/sqlbuild-orchestration/src/declarations/constants.rs b/crates/sqlbuild-orchestration/src/declarations/constants.rs index 125a59e986..1879ce5a37 100644 --- a/crates/sqlbuild-orchestration/src/declarations/constants.rs +++ b/crates/sqlbuild-orchestration/src/declarations/constants.rs @@ -9,3 +9,8 @@ pub(crate) const DEFER_SETTING_PREFIX: &str = "defer_"; /// How many due times a person's trigger listing gives for each running schedule. pub(crate) const UPCOMING_TIMES: usize = 5; + +/// Added under a job group's cycles when one of their waits reads a seed or a node several members build. +pub const SHARED_READ_HINT: &str = "hint: a wait in this cycle reads a seed or a node that more than one \ +member builds, so every member that reads it waits for all of its builders; give seeds and shared nodes \ +their own job that runs first"; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_cycle_message.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_cycle_message.rs new file mode 100644 index 0000000000..550c541015 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_cycle_message.rs @@ -0,0 +1,23 @@ +//! One line per cycle in a job group, then the shared-read hint when a cycle passes through a seed or shared node. + +use crate::declarations::constants::SHARED_READ_HINT; +use crate::declarations::models::GroupCycle; + +pub fn group_cycle_message(group: &str, cycles: &[GroupCycle]) -> Option { + if cycles.is_empty() { + return None; + } + let mut lines: Vec = cycles + .iter() + .map(|cycle| { + format!( + "group dependencies contain a cycle in job group {group}: {}", + cycle.description + ) + }) + .collect(); + if cycles.iter().any(|cycle| cycle.shared_read) { + lines.push(SHARED_READ_HINT.to_owned()); + } + Some(lines.join("\n")) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs index 6e78d95cfb..faff196a7b 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs @@ -2,8 +2,9 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::declarations::_helpers::group_waits::{describe_cycle, record_wait}; -use crate::declarations::models::{GroupWaits, JobDefinition}; +use crate::declarations::main::group_cycle_message::group_cycle_message; +use crate::declarations::main::group_wait_graph::group_wait_graph; +use crate::declarations::models::JobDefinition; use crate::snapshots::models::SnapshotInput; use crate::store::errors::ControlError; @@ -15,81 +16,29 @@ pub fn group_dependencies( if members.is_empty() || members.iter().any(|name| !jobs.contains_key(name)) { return Err(ControlError::Invalid("group needs known jobs".into())); } - let waits = lineage_dependencies(input, jobs, members); - ensure_acyclic(group, &waits.edges, &waits.reasons)?; - Ok(waits.edges) -} - -/// A job that builds a node itself, such as a shared function, never waits for its other builders. -fn lineage_dependencies( - input: &SnapshotInput, - jobs: &BTreeMap, - members: &BTreeSet, -) -> GroupWaits { - let mut owners: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); - for name in members { - for key in &jobs[name].selection { - owners.entry(key).or_default().insert(name); - } + let selections: BTreeMap> = members + .iter() + .map(|name| (name.clone(), jobs[name].selection.clone())) + .collect(); + let deps: BTreeMap> = input + .nodes + .iter() + .map(|(key, node)| { + let upstream = node + .definition + .get("deps") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .filter_map(serde_json::Value::as_str) + .map(str::to_owned) + .collect(); + (key.clone(), upstream) + }) + .collect(); + let waits = group_wait_graph(&selections, &deps); + if let Some(message) = group_cycle_message(group, &waits.cycles) { + return Err(ControlError::Invalid(message)); } - let mut edges: BTreeMap> = BTreeMap::new(); - let mut reasons: BTreeMap<(String, String), String> = BTreeMap::new(); - for name in members { - let mut dependencies: BTreeSet = BTreeSet::new(); - let mut seen: BTreeSet<&str> = BTreeSet::new(); - let mut pending: Vec<&str> = jobs[name].selection.iter().map(String::as_str).collect(); - while let Some(key) = pending.pop() { - if !seen.insert(key) { - continue; - } - if let Some(node) = input.nodes.get(key) - && let Some(upstream) = node - .definition - .get("deps") - .and_then(serde_json::Value::as_array) - { - for upstream in upstream.iter().filter_map(serde_json::Value::as_str) { - if let Some(owners) = owners.get(upstream) - && !owners.contains(name.as_str()) - { - for owner in owners { - dependencies.insert((*owner).to_owned()); - reasons = record_wait(reasons, (name, owner), || { - format!("{key} reads {upstream}") - }); - } - } - pending.push(upstream); - } - } - } - edges.insert(name.clone(), dependencies); - } - GroupWaits { edges, reasons } -} - -fn ensure_acyclic( - group: &str, - edges: &BTreeMap>, - reasons: &BTreeMap<(String, String), String>, -) -> Result<(), ControlError> { - let mut remaining = edges.clone(); - while !remaining.is_empty() { - let ready: BTreeSet = remaining - .iter() - .filter(|(_, deps)| deps.is_empty()) - .map(|(name, _)| name.clone()) - .collect(); - if ready.is_empty() { - return Err(ControlError::Invalid(format!( - "group dependencies contain a cycle in job group {group}: {}", - describe_cycle(&remaining, reasons) - ))); - } - remaining.retain(|name, deps| { - deps.retain(|dependency| !ready.contains(dependency)); - !ready.contains(name) - }); - } - Ok(()) + Ok(waits.edges) } diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_wait_graph.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_wait_graph.rs new file mode 100644 index 0000000000..bb417b5aa7 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_wait_graph.rs @@ -0,0 +1,84 @@ +//! Derive group member waits from node lineage and report every set of members that wait on each other. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::declarations::_helpers::group_waits::{cycle_path, record_wait, strongly_connected}; +use crate::declarations::models::{GroupCycle, GroupWaits}; + +/// `selections` maps each member job to the node keys it builds; `deps` maps a node key to the keys it reads. +/// A member waits for every other member that builds a node its nodes read, through any number of +/// nodes; a member that builds a node itself, such as a shared function, never waits for its other builders. +pub fn group_wait_graph( + selections: &BTreeMap>, + deps: &BTreeMap>, +) -> GroupWaits { + let mut owners: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); + for (name, selection) in selections { + for key in selection { + owners.entry(key).or_default().insert(name); + } + } + let mut edges: BTreeMap> = BTreeMap::new(); + let mut reasons: BTreeMap<(String, String), (String, String)> = BTreeMap::new(); + for (name, selection) in selections { + let mut dependencies: BTreeSet = BTreeSet::new(); + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let mut pending: Vec<&str> = selection.iter().map(String::as_str).collect(); + while let Some(key) = pending.pop() { + if !seen.insert(key) { + continue; + } + for upstream in deps.get(key).into_iter().flatten() { + if let Some(owners) = owners.get(upstream.as_str()) + && !owners.contains(name.as_str()) + { + for owner in owners { + dependencies.insert((*owner).to_owned()); + reasons = record_wait(reasons, (name, owner), || { + (key.to_owned(), upstream.clone()) + }); + } + } + pending.push(upstream); + } + } + edges.insert(name.clone(), dependencies); + } + let cycles: Vec = strongly_connected(&edges) + .iter() + .map(|members| { + let path = cycle_path(members, &edges); + let waits: Vec<(&str, &str, &(String, String))> = path + .iter() + .zip(path.iter().cycle().skip(1)) + .map(|(job, dependency)| { + ( + *job, + *dependency, + &reasons[&((*job).to_owned(), (*dependency).to_owned())], + ) + }) + .collect(); + GroupCycle { + description: waits + .iter() + .map(|(job, dependency, (key, upstream))| { + format!("{job} waits for {dependency} ({key} reads {upstream})") + }) + .collect::>() + .join("; "), + shared_read: waits.iter().any(|(_, _, (_, upstream))| { + upstream.starts_with("seed:") + || owners + .get(upstream.as_str()) + .is_some_and(|owners| owners.len() > 1) + }), + } + }) + .collect(); + GroupWaits { + edges, + reasons, + cycles, + } +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs index 085e543d5b..d6c7a60fe4 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -1,7 +1,9 @@ pub mod change_person_trigger; pub mod change_trigger; pub(crate) mod frozen_definitions; +pub mod group_cycle_message; pub mod group_dependencies; +pub mod group_wait_graph; pub mod job_resources; pub mod list_person_jobs; pub mod list_person_trigger_changes; diff --git a/crates/sqlbuild-orchestration/src/declarations/mod.rs b/crates/sqlbuild-orchestration/src/declarations/mod.rs index f147865646..2dfde8b679 100644 --- a/crates/sqlbuild-orchestration/src/declarations/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/mod.rs @@ -1,4 +1,8 @@ pub(crate) mod _helpers; -pub(crate) mod constants; +pub mod constants; pub mod main; pub mod models; + +#[cfg(test)] +#[path = "../../tests/declarations/test_contracts.rs"] +pub(crate) mod tests; diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index 56f9c12183..f4bf04c8db 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -343,11 +343,20 @@ pub(crate) struct PreviousTriggerState { pub(crate) status_override: Option, } -/// Each member job's waits, and why it waits: the first lineage edge found. +/// Each member job's waits, why it waits (the first lineage edge found, as node and upstream), and its cycles. #[derive(Clone, Debug, Default, PartialEq, Eq)] -pub(crate) struct GroupWaits { - pub(crate) edges: BTreeMap>, - pub(crate) reasons: BTreeMap<(String, String), String>, +pub struct GroupWaits { + pub edges: BTreeMap>, + pub reasons: BTreeMap<(String, String), (String, String)>, + pub cycles: Vec, +} + +/// Members that wait on each other, named as one cycle among them with a reason per wait. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct GroupCycle { + pub description: String, + /// Whether a wait in the cycle reads a seed or a node that more than one member builds. + pub shared_read: bool, } /// Providers the current snapshot declares; empty before a publish or when the project has none. diff --git a/crates/sqlbuild-orchestration/tests/declarations/test_contracts.rs b/crates/sqlbuild-orchestration/tests/declarations/test_contracts.rs new file mode 100644 index 0000000000..b44ec92466 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/declarations/test_contracts.rs @@ -0,0 +1,6 @@ +//! Job group wait contracts, derived from node lineage without a control store. + +#[path = "test_group_wait_graph.rs"] +mod test_group_wait_graph; +#[path = "test_types.rs"] +pub(crate) mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs b/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs new file mode 100644 index 0000000000..f0b2b78749 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs @@ -0,0 +1,113 @@ +//! Every set of members that wait on each other is reported in one pass, and reads of seeds or shared nodes add the hint. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::declarations::constants::SHARED_READ_HINT; +use crate::declarations::main::group_cycle_message::group_cycle_message; +use crate::declarations::main::group_wait_graph::group_wait_graph; +use crate::declarations::tests::test_types::{GroupWaitGraphTestCase, Pairs}; + +fn map(pairs: Pairs) -> BTreeMap> { + pairs + .iter() + .map(|(key, values)| { + ( + (*key).to_owned(), + values.iter().map(|value| (*value).to_owned()).collect(), + ) + }) + .collect() +} + +#[test] +fn given_group_members_and_node_lineage_when_deriving_waits_then_every_cycle_is_reported_with_its_reasons() + { + let two_cycles = "group dependencies contain a cycle in job group nightly: customers_job waits for \ + orders_job (model:customers reads model:stg_orders); orders_job waits for customers_job \ + (model:orders reads model:customers)\n\ + group dependencies contain a cycle in job group nightly: inventory_job waits for \ + shipments_job (model:inventory reads model:shipments); shipments_job waits for \ + inventory_job (model:shipments reads model:stg_inventory)"; + let seed_hub = format!( + "group dependencies contain a cycle in job group nightly: customers_job waits for \ + seeds_and_orders_job (model:customers reads seed:regions); seeds_and_orders_job waits for \ + customers_job (model:orders reads model:customers)\n{SHARED_READ_HINT}" + ); + let test_cases = [ + GroupWaitGraphTestCase { + description: "two independent cycles are both reported, and a member that only waits on one is not", + selections: &[ + ("customers_job", &["model:customers"]), + ("finance_job", &["model:revenue"]), + ("inventory_job", &["model:inventory", "model:stg_inventory"]), + ("orders_job", &["model:orders", "model:stg_orders"]), + ("shipments_job", &["model:shipments"]), + ], + deps: &[ + ("model:customers", &["model:stg_orders"]), + ("model:inventory", &["model:shipments"]), + ("model:orders", &["model:customers"]), + ("model:revenue", &["model:orders"]), + ("model:shipments", &["model:stg_inventory"]), + ], + expected_edges: &[ + ("customers_job", &["orders_job"]), + ("finance_job", &["customers_job", "orders_job"]), + ("inventory_job", &["shipments_job"]), + ("orders_job", &["customers_job"]), + ("shipments_job", &["inventory_job"]), + ], + expected_message: Some(two_cycles), + }, + GroupWaitGraphTestCase { + description: "a cycle through a seed one member builds for everyone adds the hint", + selections: &[ + ("customers_job", &["model:customers"]), + ("seeds_and_orders_job", &["seed:regions", "model:orders"]), + ], + deps: &[ + ("model:customers", &["seed:regions"]), + ("model:orders", &["model:customers"]), + ], + expected_edges: &[ + ("customers_job", &["seeds_and_orders_job"]), + ("seeds_and_orders_job", &["customers_job"]), + ], + expected_message: Some(seed_hub.leak()), + }, + GroupWaitGraphTestCase { + description: "members that both build a shared function never wait on each other for it", + selections: &[ + ( + "customers_job", + &["function:normalize_email", "model:customers"], + ), + ("orders_job", &["function:normalize_email", "model:orders"]), + ], + deps: &[ + ( + "model:customers", + &["function:normalize_email", "model:orders"], + ), + ("model:orders", &["function:normalize_email"]), + ], + expected_edges: &[("customers_job", &["orders_job"]), ("orders_job", &[])], + expected_message: None, + }, + ]; + for test_case in test_cases { + let waits = group_wait_graph(&map(test_case.selections), &map(test_case.deps)); + assert_eq!( + waits.edges, + map(test_case.expected_edges), + "{}", + test_case.description + ); + assert_eq!( + group_cycle_message("nightly", &waits.cycles).as_deref(), + test_case.expected_message, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-orchestration/tests/declarations/test_types.rs b/crates/sqlbuild-orchestration/tests/declarations/test_types.rs new file mode 100644 index 0000000000..e41811fd50 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/declarations/test_types.rs @@ -0,0 +1,13 @@ +//! Group members, the nodes they build and read, and the waits and cycle report expected. + +pub(crate) type Pairs = &'static [(&'static str, &'static [&'static str])]; + +pub(crate) struct GroupWaitGraphTestCase { + pub(crate) description: &'static str, + /// Each member job and the node keys it builds. + pub(crate) selections: Pairs, + /// Each node key and the keys it reads. + pub(crate) deps: Pairs, + pub(crate) expected_edges: Pairs, + pub(crate) expected_message: Option<&'static str>, +} From 7fc2b664fc73c48547f7ebaa294df3c4c1519e4c Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 04:42:34 +0000 Subject: [PATCH 3/9] test: a DuckShop job group with three cycles, reported together on publish and compile WIP: the publish test passes. The compile E2E fails until compile checks job group cycles, which is the next commit. --- .../job_groups/nightly_job_group.toml | 5 ++ .../group_cycles/jobs/customers_job.toml | 3 ++ .../group_cycles/jobs/inventory_job.toml | 3 ++ .../group_cycles/jobs/orders_job.toml | 3 ++ .../group_cycles/jobs/reference_job.toml | 3 ++ .../group_cycles/jobs/region_report_job.toml | 3 ++ .../group_cycles/jobs/shipments_job.toml | 3 ++ .../main/compile/test_job_group_cycles.py | 46 ++++++++++++++++ .../compiler/compiled_project/helpers.py | 52 +++++++++++++++++++ .../test_orchestration_snapshot.py | 8 +++ 10 files changed, 129 insertions(+) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/group_cycles/job_groups/nightly_job_group.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/customers_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/inventory_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/orders_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/reference_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/region_report_job.toml create mode 100644 benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/shipments_job.toml create mode 100644 tests/e2e/src/sqlbuild/cli/commands/main/compile/test_job_group_cycles.py diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_cycles/job_groups/nightly_job_group.toml b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/job_groups/nightly_job_group.toml new file mode 100644 index 0000000000..960ffca1f7 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/job_groups/nightly_job_group.toml @@ -0,0 +1,5 @@ +# Bundling nodes into jobs creates three wait cycles the node lineage does not have: two between +# model jobs, and one through a seed that a single job builds for everyone else. +name = "nightly_job_group" +description = "Build everything nightly." +jobs = ["customers_job", "orders_job", "inventory_job", "shipments_job", "reference_job", "region_report_job"] diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/customers_job.toml b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/customers_job.toml new file mode 100644 index 0000000000..fc2588ed4e --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/customers_job.toml @@ -0,0 +1,3 @@ +name = "customers_job" +description = "Build customers." +select = "customers" diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/inventory_job.toml b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/inventory_job.toml new file mode 100644 index 0000000000..e99f2bd558 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/inventory_job.toml @@ -0,0 +1,3 @@ +name = "inventory_job" +description = "Build inventory and its staging model." +select = "inventory stg_inventory" diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/orders_job.toml b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/orders_job.toml new file mode 100644 index 0000000000..e27bd9e3f9 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/orders_job.toml @@ -0,0 +1,3 @@ +name = "orders_job" +description = "Build orders and their staging model." +select = "orders stg_orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/reference_job.toml b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/reference_job.toml new file mode 100644 index 0000000000..70e8d370c1 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/reference_job.toml @@ -0,0 +1,3 @@ +name = "reference_job" +description = "Build the regions seed and regional orders." +select = "regions regional_orders" diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/region_report_job.toml b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/region_report_job.toml new file mode 100644 index 0000000000..25604aba03 --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/region_report_job.toml @@ -0,0 +1,3 @@ +name = "region_report_job" +description = "Build the region report." +select = "region_report" diff --git a/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/shipments_job.toml b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/shipments_job.toml new file mode 100644 index 0000000000..ae0dd6b3df --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/group_cycles/jobs/shipments_job.toml @@ -0,0 +1,3 @@ +name = "shipments_job" +description = "Build shipments." +select = "shipments" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_job_group_cycles.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_job_group_cycles.py new file mode 100644 index 0000000000..78e0fd14e2 --- /dev/null +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_job_group_cycles.py @@ -0,0 +1,46 @@ +"""E2E coverage for job group cycles: compile reports every cycle, then passes once they are gone.""" + +from __future__ import annotations + +from pathlib import Path + +from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb +from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( + GROUP_CYCLE_MESSAGE, + GROUP_CYCLE_NODES, + duckshop_orchestration, +) + +_GROUP_PATH: str = "orchestration/job_groups/nightly_job_group.toml" + + +def test_given_a_job_group_with_cycles_when_compiling_then_every_cycle_fails_until_fixed( + tmp_path: Path, +) -> None: + project_dir: Path = prepare_inline_project( + tmp_path=tmp_path, + project_name="group_cycles", + repo_files={ + "sqlbuild_project.toml": 'name = "group_cycles"\nadapter = "duckdb"\n', + **GROUP_CYCLE_NODES, + **duckshop_orchestration("group_cycles"), + }, + ) + + failed = run_sqb(command=("compile",), project_dir=project_dir) + + assert failed.returncode == 1, failed.stdout + failed.stderr + assert "error[D017]" in failed.stderr + for line in GROUP_CYCLE_MESSAGE.splitlines(): + assert line in failed.stderr, failed.stderr + + # One job per pair of members that never wait on each other: no cycle is left. + (project_dir / _GROUP_PATH).write_text( + 'name = "nightly_job_group"\ndescription = "Build everything nightly."\n' + 'jobs = ["orders_job", "shipments_job", "region_report_job"]\n', + encoding="utf-8", + ) + fixed = run_sqb(command=("compile",), project_dir=project_dir) + + assert fixed.returncode == 0, fixed.stdout + fixed.stderr + assert "D017" not in fixed.stderr diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 8c8165381e..0e0ff9dc12 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -437,6 +437,58 @@ def duckshop_orchestration(fixture: str) -> dict[str, str]: } +def _group_cycle_model(name: str, reads: str) -> str: + """A one-column synthetic model reading `reads`.""" + return ( + f"MODEL (description 'Synthetic {name}.', columns (id (type INTEGER)));\n\n" + f'SELECT CAST(1 AS INTEGER) AS id FROM __ref("{reads}")\n' + ) + + +# Models and a seed for the DuckShop `group_cycles` jobs, whose group bundles them into three cycles. +GROUP_CYCLE_NODES: dict[str, str] = { + "seeds/regions.csv": "id\n1\n", + "seeds/regions.yml": ( + "seeds:\n - name: regions\n description: Regions\n columns:\n" + " - name: id\n type: INTEGER\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Synthetic stg_orders.', columns (id (type INTEGER)));\n\n" + "SELECT CAST(1 AS INTEGER) AS id\n" + ), + "models/stg_inventory.sql": ( + "MODEL (description 'Synthetic stg_inventory.', columns (id (type INTEGER)));\n\n" + "SELECT CAST(1 AS INTEGER) AS id\n" + ), + "models/customers.sql": _group_cycle_model("customers", "stg_orders"), + "models/orders.sql": _group_cycle_model("orders", "customers"), + "models/shipments.sql": _group_cycle_model("shipments", "stg_inventory"), + "models/inventory.sql": _group_cycle_model("inventory", "shipments"), + "models/region_report.sql": ( + "MODEL (description 'Synthetic region_report.', columns (id (type INTEGER)));\n\n" + 'SELECT CAST(1 AS INTEGER) AS id FROM __seed("regions")\n' + ), + "models/regional_orders.sql": _group_cycle_model("regional_orders", "region_report"), +} +# What compile and publish both report for that group: every cycle, then the shared-read hint. +GROUP_CYCLE_MESSAGE: str = "\n".join( + f"group dependencies contain a cycle in job group nightly_job_group: {cycle}" + for cycle in ( + "customers_job waits for orders_job (model:customers reads model:stg_orders); " + "orders_job waits for customers_job (model:orders reads model:customers)", + "inventory_job waits for shipments_job (model:inventory reads model:shipments); " + "shipments_job waits for inventory_job (model:shipments reads model:stg_inventory)", + "reference_job waits for region_report_job (model:regional_orders reads " + "model:region_report); region_report_job waits for reference_job " + "(model:region_report reads seed:regions)", + ) +) + ( + "\nhint: a wait in this cycle reads a seed or a node that more than one member builds, " + "so every member that reads it waits for all of its builders; give seeds and shared nodes " + "their own job that runs first" +) + + def edited_duckshop_orchestration(fixture: str, path: str, old: str, new: str) -> dict[str, str]: """A DuckShop orchestration fixture with `old` replaced by `new` in the file at `path`.""" files: dict[str, str] = duckshop_orchestration(fixture) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index d21ca16616..4a60108d7c 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -43,6 +43,8 @@ ) from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( FUNCTION_PROJECT_FILES, + GROUP_CYCLE_MESSAGE, + GROUP_CYCLE_NODES, METADATA, PARITY_FILES, TYPE_PROOF_FILES, @@ -1502,6 +1504,12 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects @pytest.mark.parametrize( "test_case", [ + SnapshotDiscoveryRejectionTestCase( + description="every job group cycle is reported at once, with the seed hint", + files={**duckshop_orchestration("group_cycles"), **GROUP_CYCLE_NODES}, + expected_error=ValueError, + expected_message=re.escape(GROUP_CYCLE_MESSAGE), + ), SnapshotDiscoveryRejectionTestCase( description="group members are jobs, not nested groups", files=duckshop_orchestration("nested_groups"), From 015adc8da708c5f26b4c7817486d73a84e6fa19e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 05:58:41 +0000 Subject: [PATCH 4/9] feat: report job group cycles at compile Compile derives each job group's member waits from the same lineage publication uses and reports every cycle as one D017 error at the group's name line. The Python node deps publication records come from one shared selection method, and the wait traversal pushes each node once so large groups stay linear. --- .../src/declarations/_helpers/group_waits.rs | 14 +-- .../src/declarations/main/group_wait_graph.rs | 24 +++-- .../declarations/test_group_wait_graph.rs | 41 +++++++++ .../_helpers/stages/analysis.rs | 9 ++ .../src/graph/models.rs | 29 ++++++ .../project_snapshot/_helpers/snapshot/mod.rs | 1 + .../snapshot/snapshot_declarations.rs | 2 +- .../snapshot/snapshot_group_cycles.rs | 92 +++++++++++++++++++ .../project_snapshot/main/job_group_cycles.rs | 37 ++++++++ .../src/project_snapshot/main/mod.rs | 1 + .../_helpers/location_command/snapshot.rs | 21 +---- .../docs/docs/concepts/orchestration.mdx | 7 +- 12 files changed, 236 insertions(+), 42 deletions(-) create mode 100644 crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_group_cycles.rs create mode 100644 crates/sqlbuild-python-project/src/project_snapshot/main/job_group_cycles.rs diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs index 61abc2d175..a3dccc6a0c 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs @@ -1,19 +1,7 @@ -//! Record why one group member waits for another, find the members that wait on each other, and name a cycle among them. +//! Find the group members that wait on each other, and name a cycle among them. use std::collections::{BTreeMap, BTreeSet}; -/// Keep the first reason found for a wait: the first lineage edge, as the node and the upstream it reads. -pub(crate) fn record_wait( - mut reasons: BTreeMap<(String, String), (String, String)>, - (job, dependency): (&str, &str), - reason: impl FnOnce() -> (String, String), -) -> BTreeMap<(String, String), (String, String)> { - reasons - .entry((job.to_owned(), dependency.to_owned())) - .or_insert_with(reason); - reasons -} - /// Groups of two or more members that each reach every other through waits, in name order. pub(crate) fn strongly_connected( edges: &BTreeMap>, diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_wait_graph.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_wait_graph.rs index bb417b5aa7..bbd490e77b 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/group_wait_graph.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_wait_graph.rs @@ -2,7 +2,7 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::declarations::_helpers::group_waits::{cycle_path, record_wait, strongly_connected}; +use crate::declarations::_helpers::group_waits::{cycle_path, strongly_connected}; use crate::declarations::models::{GroupCycle, GroupWaits}; /// `selections` maps each member job to the node keys it builds; `deps` maps a node key to the keys it reads. @@ -21,7 +21,7 @@ pub fn group_wait_graph( let mut edges: BTreeMap> = BTreeMap::new(); let mut reasons: BTreeMap<(String, String), (String, String)> = BTreeMap::new(); for (name, selection) in selections { - let mut dependencies: BTreeSet = BTreeSet::new(); + let mut dependencies: BTreeSet<&str> = BTreeSet::new(); let mut seen: BTreeSet<&str> = BTreeSet::new(); let mut pending: Vec<&str> = selection.iter().map(String::as_str).collect(); while let Some(key) = pending.pop() { @@ -32,17 +32,25 @@ pub fn group_wait_graph( if let Some(owners) = owners.get(upstream.as_str()) && !owners.contains(name.as_str()) { + // The first lineage edge that makes a wait is its reason. for owner in owners { - dependencies.insert((*owner).to_owned()); - reasons = record_wait(reasons, (name, owner), || { - (key.to_owned(), upstream.clone()) - }); + if dependencies.insert(owner) { + reasons.insert( + (name.clone(), (*owner).to_owned()), + (key.to_owned(), upstream.clone()), + ); + } } } - pending.push(upstream); + if !seen.contains(upstream.as_str()) { + pending.push(upstream); + } } } - edges.insert(name.clone(), dependencies); + edges.insert( + name.clone(), + dependencies.into_iter().map(str::to_owned).collect(), + ); } let cycles: Vec = strongly_connected(&edges) .iter() diff --git a/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs b/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs index f0b2b78749..7aeef4a721 100644 --- a/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs +++ b/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs @@ -111,3 +111,44 @@ fn given_group_members_and_node_lineage_when_deriving_waits_then_every_cycle_is_ ); } } + +#[test] +fn given_a_large_group_on_a_long_chain_when_deriving_waits_then_one_cycle_is_reported_within_the_time_limit() + { + const MEMBERS: usize = 50; + const NODES_PER_MEMBER: usize = 200; + let node = |index: usize| format!("model:m{index}"); + let last = MEMBERS * NODES_PER_MEMBER - 1; + let mut selections: BTreeMap> = (0..MEMBERS) + .map(|member| { + let first = member * NODES_PER_MEMBER; + ( + format!("job_{member:03}"), + (first..first + NODES_PER_MEMBER).map(node).collect(), + ) + }) + .collect(); + // An acyclic chain of 10,000 nodes: each reads the one before it, and the first job also builds + // a tail that reads the last node, so every member reaches every other through lineage. + selections + .get_mut("job_000") + .expect("first member") + .insert("model:tail".to_owned()); + let mut deps: BTreeMap> = (1..=last) + .map(|index| (node(index), BTreeSet::from([node(index - 1)]))) + .collect(); + deps.insert("model:tail".to_owned(), BTreeSet::from([node(last)])); + + let started = std::time::Instant::now(); + let waits = group_wait_graph(&selections, &deps); + let elapsed = started.elapsed(); + + assert_eq!(waits.cycles.len(), 1); + assert_eq!(waits.edges["job_000"].len(), MEMBERS - 1); + assert_eq!(waits.edges[&format!("job_{:03}", MEMBERS - 1)].len(), MEMBERS - 1); + assert!( + elapsed < std::time::Duration::from_secs(2), + "deriving waits for {MEMBERS} members over {} nodes took {elapsed:?}", + last + 2 + ); +} diff --git a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs index 8ab1d28a76..40c1ae83e3 100644 --- a/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs +++ b/crates/sqlbuild-python-compile/src/compile_driver/_helpers/stages/analysis.rs @@ -35,8 +35,10 @@ use sqlbuild_python_base::records::main::fields::replaced::replaced; use sqlbuild_python_base::records::types::RecordId; use sqlbuild_python_front::contracts::main::resolve_effective_adapter_name::resolve_effective_adapter_name; use sqlbuild_python_project::contracts::main::evaluate_model_contracts::evaluate_model_contracts; +use sqlbuild_python_project::graph::models::CompiledProjectGraph; use sqlbuild_python_project::planner_selection::main::reject_unit_test_selectors::reject_unit_test_selectors; use sqlbuild_python_project::planner_selection::main::select_project_graph::select_project_graph; +use sqlbuild_python_project::project_snapshot::main::job_group_cycles::job_group_cycles; const RICH_LINEAGE: &str = "rich"; const FAST_LINEAGE: &str = "fast"; @@ -222,6 +224,13 @@ fn analyzed<'py>( early_rules.stop(py); } early_lint_wait_ms = early_lint.borrow().stop(py)?.into_pyobject(py)?.into_any(); + } else { + let compiled: &Bound<'py, CompiledProjectGraph> = graph.cast()?; + job_group_cycles( + &options.project_dir, + (&project, &discovered), + compiled.get().native(py).get().graph(), + )?; } let (lineage, lineage_ms): (Bound<'py, PyAny>, u64) = driver.lineage(&graph, &dialect)?; let contracts: ContractPhase<'py> = driver.contracts(&project, (&dialect, &adapter))?; diff --git a/crates/sqlbuild-python-project/src/graph/models.rs b/crates/sqlbuild-python-project/src/graph/models.rs index c55cc47781..b56b90101f 100644 --- a/crates/sqlbuild-python-project/src/graph/models.rs +++ b/crates/sqlbuild-python-project/src/graph/models.rs @@ -47,6 +47,7 @@ use sqlbuild_python_base::records::main::construct::record_class::record_class; use sqlbuild_python_base::records::main::fields::field::field as native_field; use sqlbuild_python_base::records::types::RecordId; use std::cell::RefCell; +use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; @@ -439,6 +440,34 @@ impl ProjectSelection { .map(|node| format!("{}:{name}", node.kind)) } + /// Each Python node's published key and the keys it reads: its upstream Python nodes and its SQL deps. + pub fn node_deps(&self) -> Vec<(String, BTreeSet)> { + let upstream: BTreeMap<&str, &[String]> = self + .nodes + .upstream + .iter() + .map(|(name, names)| (name.as_str(), names.as_slice())) + .collect(); + self.nodes + .nodes + .iter() + .map(|node| { + let deps: BTreeSet = upstream + .get(node.name.as_str()) + .into_iter() + .flat_map(|names| names.iter()) + .filter_map(|name| self.node_key(name)) + .chain( + node.sql_deps + .iter() + .map(|(kind, name)| format!("{kind}:{name}")), + ) + .collect(); + (format!("{}:{}", node.kind, node.name), deps) + }) + .collect() + } + /// The key a selected Python node's run reports under; a loader reports as the source it writes. pub(crate) fn run_key(&self, name: &str) -> Option { let node = self diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs index 2f10cb6b41..35f65138a1 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod snapshot_boundary; pub(crate) mod snapshot_declarations; pub(crate) mod snapshot_execution; +pub(crate) mod snapshot_group_cycles; pub(crate) mod snapshot_models; pub(crate) mod snapshot_project; pub(crate) mod snapshot_resources; diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs index a628edd529..f510f466b7 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_declarations.rs @@ -40,7 +40,7 @@ fn file_site(path: &str, position: FilePosition) -> Option { } /// Whether a declaration with `targets` applies to `target`. -fn declared_for(targets: &[String], target: &str) -> bool { +pub(crate) fn declared_for(targets: &[String], target: &str) -> bool { targets.is_empty() || targets.iter().any(|name| name == target) } diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_group_cycles.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_group_cycles.rs new file mode 100644 index 0000000000..c2e892cea1 --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_group_cycles.rs @@ -0,0 +1,92 @@ +//! Find job group members that wait on each other at compile, from the same waits publication derives. + +use std::collections::{BTreeMap, BTreeSet}; + +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_discovery::orchestration_files::models::{FilePosition, OrchestrationInventory}; +use sqlbuild_orchestration::declarations::main::group_cycle_message::group_cycle_message; +use sqlbuild_orchestration::declarations::main::group_wait_graph::group_wait_graph; + +use crate::graph::models::ProjectSelection; +use crate::project_snapshot::_helpers::snapshot::snapshot_declarations::declared_for; + +/// Whether any group declared for `target` lists two or more jobs, so its members could wait on each other. +pub(crate) fn has_groups(inventory: &OrchestrationInventory, target: &str) -> bool { + inventory + .groups + .iter() + .any(|(_, group)| group.jobs.len() > 1 && declared_for(&group.run.targets, target)) +} + +/// Each node key and the keys it reads, as publication records them in node `deps`. +fn node_deps( + graph: &ProjectGraph, + selected: &ProjectSelection, +) -> BTreeMap> { + let key = |(kind, name): &(String, String)| format!("{kind}:{name}"); + let mut deps: BTreeMap> = graph + .upstream + .iter() + .map(|(node, upstream)| (key(node), upstream.iter().map(key).collect())) + .collect(); + for (node, upstream) in selected.node_deps() { + deps.entry(node).or_default().extend(upstream); + } + for node in &selected.nodes.nodes { + if node.kind == "loader" && selected.intermediate_loader(&node.name) { + deps.entry(format!("source:{}", node.name)) + .or_default() + .insert(format!("loader:{}", node.name)); + } + } + deps +} + +/// One `path:line:column: message` per group declared for `target` whose members wait on each other. +/// +/// A job whose selector fails is left out, since compile reports the selector itself. +pub(crate) fn group_cycle_failures( + inventory: &OrchestrationInventory, + target: &str, + (graph, selected): (&ProjectGraph, &ProjectSelection), +) -> Vec { + let tokens = |value: &str| { + value + .split_whitespace() + .map(str::to_owned) + .collect::>() + }; + let mut jobs: BTreeMap<&str, BTreeSet> = BTreeMap::new(); + for (_, job) in &inventory.jobs { + if !declared_for(&job.run.targets, target) { + continue; + } + let exclude: &str = job.exclude.as_ref().map_or("", |exclude| &exclude.value); + if let Ok(selection) = selected.run_keys(graph, &tokens(&job.select.value), &tokens(exclude)) + { + jobs.insert(&job.run.name.value, selection); + } + } + let deps: BTreeMap> = node_deps(graph, selected); + let mut failures: Vec = Vec::new(); + for (path, group) in &inventory.groups { + if !declared_for(&group.run.targets, target) { + continue; + } + let selections: BTreeMap> = group + .jobs + .iter() + .filter_map(|member| { + let selection = jobs.get(member.value.as_str())?; + Some((member.value.clone(), selection.clone())) + }) + .collect(); + let name: &str = &group.run.name.value; + if let Some(message) = group_cycle_message(name, &group_wait_graph(&selections, &deps).cycles) + { + let FilePosition { line, column } = group.run.name.position; + failures.push(format!("{path}:{line}:{column}: {message}")); + } + } + failures +} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/job_group_cycles.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/job_group_cycles.rs new file mode 100644 index 0000000000..13671770fe --- /dev/null +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/job_group_cycles.rs @@ -0,0 +1,37 @@ +//! Refuse a compile whose job groups have members that wait on each other, as `OrchestrationDeclarationError` (D017). + +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; +use sqlbuild_analysis::graph::models::ProjectGraph; +use sqlbuild_python_base::records::main::construct::error_class::error_class; +use sqlbuild_python_base::records::main::fields::field::field; +use sqlbuild_python_base::records::types::ErrorId; +use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; + +use crate::graph::models::ProjectSelection; +use crate::project_snapshot::_helpers::snapshot::snapshot_group_cycles::{ + group_cycle_failures, has_groups, +}; + +/// Every cycle in every job group declared for the project's target, in one error; reads Python nodes only when a group has two or more jobs. +pub fn job_group_cycles( + project_dir: &Bound<'_, PyAny>, + (project, discovered_inputs): (&Bound<'_, PyAny>, &Bound<'_, PyAny>), + graph: &ProjectGraph, +) -> PyResult<()> { + let py = project_dir.py(); + let inventory = orchestration_declarations(project_dir)?; + let target: String = field(project, "effective_target_name")? + .extract::>()? + .unwrap_or_default(); + if !has_groups(&inventory, &target) { + return Ok(()); + } + let selected = ProjectSelection::new(project, discovered_inputs)?; + let failures: Vec = group_cycle_failures(&inventory, &target, (graph, &selected)); + if failures.is_empty() { + return Ok(()); + } + Err(PyErr::from_value( + error_class(py, ErrorId::OrchestrationDeclaration)?.call1((failures.join("\n"),))?, + )) +} diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/mod.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/mod.rs index ad14c41e07..ac1b2d7e50 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/main/mod.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/mod.rs @@ -1,5 +1,6 @@ pub mod compile_declarations; pub mod compile_project_roles; +pub mod job_group_cycles; pub mod model_nodes; pub mod python_declarations; pub mod register_snapshot; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index 2f165dfbf8..cf1819630c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -1,6 +1,6 @@ //! Location publication assembles catalog data and planner-owned local identities. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; use std::path::PathBuf; use pyo3::exceptions::PyValueError; @@ -156,30 +156,13 @@ fn python_nodes( tags.entry(name.as_str()).or_default().push(tag.as_str()); } } - let upstream: BTreeMap<&str, &[String]> = graph - .upstream - .iter() - .map(|(name, names)| (name.as_str(), names.as_slice())) - .collect(); let mut outputs: Vec<(String, SnapshotNode)> = Vec::new(); - for node in &graph.nodes { - let key = format!("{}:{}", node.kind, node.name); + for (node, (key, deps)) in graph.nodes.iter().zip(selected.node_deps()) { let Some(published) = nodes.get_mut(&key) else { return Err(PyValueError::new_err(format!( "Python node {key} is missing from the snapshot" ))); }; - let deps: BTreeSet = upstream - .get(node.name.as_str()) - .into_iter() - .flat_map(|names| names.iter()) - .filter_map(|name| selected.node_key(name)) - .chain( - node.sql_deps - .iter() - .map(|(kind, name)| format!("{kind}:{name}")), - ) - .collect(); published.definition["deps"] = json!(deps); published.definition["tags"] = json!(tags.get(node.name.as_str()).cloned().unwrap_or_default()); diff --git a/website/src/content/docs/docs/concepts/orchestration.mdx b/website/src/content/docs/docs/concepts/orchestration.mdx index 30acbc8147..47c6891d3d 100644 --- a/website/src/content/docs/docs/concepts/orchestration.mdx +++ b/website/src/content/docs/docs/concepts/orchestration.mdx @@ -91,7 +91,8 @@ reports a job or sensor `select` that names more than three nodes one by one; se ## Group jobs A job group runs its jobs as one run. A job waits for another job in the group when it reads a node -that job builds. +that job builds. Jobs that wait on each other, directly or through other members, are a cycle, and +`sqb compile` reports every cycle in the group as one `D017` error. ```toml # orchestration/job_groups/daily_orders_job_group.toml @@ -105,6 +106,9 @@ A group takes `name`, `description` and `jobs`, and the same `targets`, `owner`, `concurrency_groups`, `schedule` and `[on_failure]` as a job. Its members are jobs, never other groups. +A job that reads a seed or a node that several members build waits for all of those members. Give +seeds and shared nodes their own job, so the jobs that read them wait only for that job. + ## Schedule a job `schedule` takes a cron expression or the name of a calendar, and always a timezone. @@ -213,6 +217,7 @@ Every problem with an orchestration file is a `D017` error at its file, line and | `schedule cron: invalid cron expression: Pattern must have 5 fields when seconds are disallowed.` | Write a five-field cron, such as `0 2 * * *` | | `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | | `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | +| `group dependencies contain a cycle in job group daily_orders_job_group: orders_marts_job waits for staging_orders_job (model:orders reads model:stg_orders); staging_orders_job waits for orders_marts_job (model:stg_orders reads seed:regions)` | Move each node to one job, and give seeds and nodes that several members read their own job that runs first | Details: From 2c7d1bcf7f851a858692fa3a0b6703b62c2ac7b5 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 06:11:31 +0000 Subject: [PATCH 5/9] refactor: keep job group cycle checks within the architecture rules Compile raises the cycle error through discovery's existing failure path, the dispatcher reads an event's invocation through the native record reader, and the wait traversal, cycle naming and invocation-start lookup are named helpers. The large-group test states its expected waits as data. --- .../event_export/main/take_lifecycle_event.rs | 34 ++++---- .../src/declarations/_helpers/group_waits.rs | 63 +++++++++++++- .../src/declarations/constants.rs | 3 + .../declarations/main/group_dependencies.rs | 2 +- .../src/declarations/main/group_wait_graph.rs | 82 ++++--------------- .../tests/declarations/helpers.rs | 47 +++++++++++ .../tests/declarations/test_contracts.rs | 2 + .../declarations/test_group_wait_graph.rs | 82 ++++++++----------- .../tests/declarations/test_types.rs | 10 +++ .../src/discovery/main/sources/mod.rs | 1 + .../main/sources/orchestration_failure.rs | 17 ++++ .../src/graph/constants.rs | 3 + .../src/graph/models.rs | 34 ++++---- .../snapshot/snapshot_group_cycles.rs | 26 +++--- .../project_snapshot/main/job_group_cycles.rs | 9 +- .../_helpers/event_exporting/dispatcher.rs | 2 +- .../cli/commands/main/compile/_test_types.py | 7 ++ .../main/compile/test_job_group_cycles.py | 36 +++++--- 18 files changed, 282 insertions(+), 178 deletions(-) create mode 100644 crates/sqlbuild-orchestration/tests/declarations/helpers.rs create mode 100644 crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_failure.rs diff --git a/crates/sqlbuild-core/src/event_export/main/take_lifecycle_event.rs b/crates/sqlbuild-core/src/event_export/main/take_lifecycle_event.rs index 1684c2421b..f59b4ec8b6 100644 --- a/crates/sqlbuild-core/src/event_export/main/take_lifecycle_event.rs +++ b/crates/sqlbuild-core/src/event_export/main/take_lifecycle_event.rs @@ -21,21 +21,25 @@ pub fn take_lifecycle_event( .filter(|(_, queued)| selectable(queued) && Some(queued.priority) == highest) .min_by_key(|(_, queued)| queued.sequence) .map(|(position, _)| position); - let position: Option = position.map(|position| { - let chosen = &queue.items[position]; - queue - .items - .iter() - .enumerate() - .filter(|(_, queued)| { - queued.invocation_start - && !chosen.invocation_start - && chosen.invocation_id.is_some() - && queued.invocation_id == chosen.invocation_id - }) - .min_by_key(|(_, queued)| queued.sequence) - .map_or(position, |(start, _)| start) - }); + let position: Option = position.map(|position| start_first(&queue, position)); let next: Option> = position.map(|position| queue.items.remove(position)); (queue, next) } + +/// The position of a queued start of the event at `position`'s invocation, or `position` when none is queued. +fn start_first(queue: &EventQueue, position: usize) -> usize { + let chosen: &QueuedEvent = &queue.items[position]; + if chosen.invocation_start || chosen.invocation_id.is_none() { + return position; + } + let mut start: Option<(usize, u64)> = None; + for (index, queued) in queue.items.iter().enumerate() { + if queued.invocation_start + && queued.invocation_id == chosen.invocation_id + && start.is_none_or(|(_, sequence)| queued.sequence < sequence) + { + start = Some((index, queued.sequence)); + } + } + start.map_or(position, |(index, _)| index) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs index a3dccc6a0c..5f0380af4c 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs @@ -1,7 +1,42 @@ -//! Find the group members that wait on each other, and name a cycle among them. +//! Find the group members that wait on each other, name a cycle among them, and say why each waits. use std::collections::{BTreeMap, BTreeSet}; +use crate::declarations::constants::SEED_KEY_PREFIX; +use crate::declarations::models::GroupCycle; + +/// Each member `name` waits for, with the first lineage edge that makes it wait: the node and the upstream it reads. +pub(crate) fn member_waits( + (name, selection): (&str, &BTreeSet), + deps: &BTreeMap>, + owners: &BTreeMap<&str, BTreeSet<&str>>, +) -> BTreeMap { + let mut waits: BTreeMap = BTreeMap::new(); + let mut seen: BTreeSet<&str> = BTreeSet::new(); + let mut pending: Vec<&str> = selection.iter().map(String::as_str).collect(); + while let Some(key) = pending.pop() { + if !seen.insert(key) { + continue; + } + for upstream in deps.get(key).into_iter().flatten() { + let builders: Option<&BTreeSet<&str>> = owners.get(upstream.as_str()); + for owner in builders + .filter(|builders| !builders.contains(name)) + .into_iter() + .flatten() + { + waits + .entry((*owner).to_owned()) + .or_insert_with(|| (key.to_owned(), upstream.clone())); + } + if !seen.contains(upstream.as_str()) { + pending.push(upstream); + } + } + } + waits +} + /// Groups of two or more members that each reach every other through waits, in name order. pub(crate) fn strongly_connected( edges: &BTreeMap>, @@ -58,3 +93,29 @@ pub(crate) fn cycle_path<'a>( } path } + +/// The cycle through `path`, each wait with the node and upstream that cause it, and whether one reads a seed or shared node. +pub(crate) fn named_cycle( + path: &[&str], + reasons: &BTreeMap<(String, String), (String, String)>, + owners: &BTreeMap<&str, BTreeSet<&str>>, +) -> GroupCycle { + let mut waits: Vec = Vec::new(); + let mut shared_read: bool = false; + for (index, job) in path.iter().enumerate() { + let dependency: &str = path[(index + 1) % path.len()]; + let (key, upstream) = &reasons[&((*job).to_owned(), dependency.to_owned())]; + waits.push(format!( + "{job} waits for {dependency} ({key} reads {upstream})" + )); + shared_read = shared_read + || upstream.starts_with(SEED_KEY_PREFIX) + || owners + .get(upstream.as_str()) + .is_some_and(|builders| builders.len() > 1); + } + GroupCycle { + description: waits.join("; "), + shared_read, + } +} diff --git a/crates/sqlbuild-orchestration/src/declarations/constants.rs b/crates/sqlbuild-orchestration/src/declarations/constants.rs index 1879ce5a37..00d87fef80 100644 --- a/crates/sqlbuild-orchestration/src/declarations/constants.rs +++ b/crates/sqlbuild-orchestration/src/declarations/constants.rs @@ -10,6 +10,9 @@ pub(crate) const DEFER_SETTING_PREFIX: &str = "defer_"; /// How many due times a person's trigger listing gives for each running schedule. pub(crate) const UPCOMING_TIMES: usize = 5; +/// The key prefix of seed nodes, which a job that loads them builds for every reader. +pub(crate) const SEED_KEY_PREFIX: &str = "seed:"; + /// Added under a job group's cycles when one of their waits reads a seed or a node several members build. pub const SHARED_READ_HINT: &str = "hint: a wait in this cycle reads a seed or a node that more than one \ member builds, so every member that reads it waits for all of its builders; give seeds and shared nodes \ diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs index faff196a7b..1623bd329b 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs @@ -24,7 +24,7 @@ pub fn group_dependencies( .nodes .iter() .map(|(key, node)| { - let upstream = node + let upstream: BTreeSet = node .definition .get("deps") .and_then(serde_json::Value::as_array) diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_wait_graph.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_wait_graph.rs index bbd490e77b..474ace6d6e 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/group_wait_graph.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_wait_graph.rs @@ -2,12 +2,12 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::declarations::_helpers::group_waits::{cycle_path, strongly_connected}; +use crate::declarations::_helpers::group_waits::{ + cycle_path, member_waits, named_cycle, strongly_connected, +}; use crate::declarations::models::{GroupCycle, GroupWaits}; -/// `selections` maps each member job to the node keys it builds; `deps` maps a node key to the keys it reads. -/// A member waits for every other member that builds a node its nodes read, through any number of -/// nodes; a member that builds a node itself, such as a shared function, never waits for its other builders. +/// Each member waits for every other member that builds a node its nodes read, unless it builds that node too. pub fn group_wait_graph( selections: &BTreeMap>, deps: &BTreeMap>, @@ -21,69 +21,21 @@ pub fn group_wait_graph( let mut edges: BTreeMap> = BTreeMap::new(); let mut reasons: BTreeMap<(String, String), (String, String)> = BTreeMap::new(); for (name, selection) in selections { - let mut dependencies: BTreeSet<&str> = BTreeSet::new(); - let mut seen: BTreeSet<&str> = BTreeSet::new(); - let mut pending: Vec<&str> = selection.iter().map(String::as_str).collect(); - while let Some(key) = pending.pop() { - if !seen.insert(key) { - continue; - } - for upstream in deps.get(key).into_iter().flatten() { - if let Some(owners) = owners.get(upstream.as_str()) - && !owners.contains(name.as_str()) - { - // The first lineage edge that makes a wait is its reason. - for owner in owners { - if dependencies.insert(owner) { - reasons.insert( - (name.clone(), (*owner).to_owned()), - (key.to_owned(), upstream.clone()), - ); - } - } - } - if !seen.contains(upstream.as_str()) { - pending.push(upstream); - } - } + let waits: BTreeMap = + member_waits((name, selection), deps, &owners); + edges.insert(name.clone(), waits.keys().cloned().collect()); + for (dependency, reason) in waits { + reasons.insert((name.clone(), dependency), reason); } - edges.insert( - name.clone(), - dependencies.into_iter().map(str::to_owned).collect(), - ); } - let cycles: Vec = strongly_connected(&edges) - .iter() - .map(|members| { - let path = cycle_path(members, &edges); - let waits: Vec<(&str, &str, &(String, String))> = path - .iter() - .zip(path.iter().cycle().skip(1)) - .map(|(job, dependency)| { - ( - *job, - *dependency, - &reasons[&((*job).to_owned(), (*dependency).to_owned())], - ) - }) - .collect(); - GroupCycle { - description: waits - .iter() - .map(|(job, dependency, (key, upstream))| { - format!("{job} waits for {dependency} ({key} reads {upstream})") - }) - .collect::>() - .join("; "), - shared_read: waits.iter().any(|(_, _, (_, upstream))| { - upstream.starts_with("seed:") - || owners - .get(upstream.as_str()) - .is_some_and(|owners| owners.len() > 1) - }), - } - }) - .collect(); + let mut cycles: Vec = Vec::new(); + for members in strongly_connected(&edges) { + cycles.push(named_cycle( + &cycle_path(&members, &edges), + &reasons, + &owners, + )); + } GroupWaits { edges, reasons, diff --git a/crates/sqlbuild-orchestration/tests/declarations/helpers.rs b/crates/sqlbuild-orchestration/tests/declarations/helpers.rs new file mode 100644 index 0000000000..10b270cdad --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/declarations/helpers.rs @@ -0,0 +1,47 @@ +//! Build group selections and node lineage for the wait graph tests. + +use std::collections::{BTreeMap, BTreeSet}; + +use crate::declarations::tests::test_types::Pairs; + +pub(crate) fn map(pairs: Pairs) -> BTreeMap> { + pairs + .iter() + .map(|(key, values)| { + ( + (*key).to_owned(), + values.iter().map(|value| (*value).to_owned()).collect(), + ) + }) + .collect() +} + +/// `members` jobs over one acyclic chain of nodes, each node reading the one before it, where the first job also builds a tail reading the last node. +pub(crate) fn chained_group( + members: usize, + nodes_per_member: usize, +) -> ( + BTreeMap>, + BTreeMap>, +) { + let node = |index: usize| format!("model:m{index}"); + let last: usize = members * nodes_per_member - 1; + let mut selections: BTreeMap> = BTreeMap::new(); + for member in 0..members { + let first: usize = member * nodes_per_member; + selections.insert( + format!("job_{member:03}"), + (first..first + nodes_per_member).map(node).collect(), + ); + } + selections + .entry("job_000".to_owned()) + .or_default() + .insert("model:tail".to_owned()); + let mut deps: BTreeMap> = BTreeMap::new(); + for index in 1..=last { + deps.insert(node(index), BTreeSet::from([node(index - 1)])); + } + deps.insert("model:tail".to_owned(), BTreeSet::from([node(last)])); + (selections, deps) +} diff --git a/crates/sqlbuild-orchestration/tests/declarations/test_contracts.rs b/crates/sqlbuild-orchestration/tests/declarations/test_contracts.rs index b44ec92466..89785f7cea 100644 --- a/crates/sqlbuild-orchestration/tests/declarations/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/declarations/test_contracts.rs @@ -1,5 +1,7 @@ //! Job group wait contracts, derived from node lineage without a control store. +#[path = "helpers.rs"] +pub(crate) mod helpers; #[path = "test_group_wait_graph.rs"] mod test_group_wait_graph; #[path = "test_types.rs"] diff --git a/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs b/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs index 7aeef4a721..f686a87a53 100644 --- a/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs +++ b/crates/sqlbuild-orchestration/tests/declarations/test_group_wait_graph.rs @@ -1,23 +1,12 @@ //! Every set of members that wait on each other is reported in one pass, and reads of seeds or shared nodes add the hint. -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeSet; use crate::declarations::constants::SHARED_READ_HINT; use crate::declarations::main::group_cycle_message::group_cycle_message; use crate::declarations::main::group_wait_graph::group_wait_graph; -use crate::declarations::tests::test_types::{GroupWaitGraphTestCase, Pairs}; - -fn map(pairs: Pairs) -> BTreeMap> { - pairs - .iter() - .map(|(key, values)| { - ( - (*key).to_owned(), - values.iter().map(|value| (*value).to_owned()).collect(), - ) - }) - .collect() -} +use crate::declarations::tests::helpers::{chained_group, map}; +use crate::declarations::tests::test_types::{GroupWaitGraphTestCase, LargeGroupTestCase}; #[test] fn given_group_members_and_node_lineage_when_deriving_waits_then_every_cycle_is_reported_with_its_reasons() @@ -115,40 +104,37 @@ fn given_group_members_and_node_lineage_when_deriving_waits_then_every_cycle_is_ #[test] fn given_a_large_group_on_a_long_chain_when_deriving_waits_then_one_cycle_is_reported_within_the_time_limit() { - const MEMBERS: usize = 50; - const NODES_PER_MEMBER: usize = 200; - let node = |index: usize| format!("model:m{index}"); - let last = MEMBERS * NODES_PER_MEMBER - 1; - let mut selections: BTreeMap> = (0..MEMBERS) - .map(|member| { - let first = member * NODES_PER_MEMBER; - ( - format!("job_{member:03}"), - (first..first + NODES_PER_MEMBER).map(node).collect(), - ) - }) - .collect(); - // An acyclic chain of 10,000 nodes: each reads the one before it, and the first job also builds - // a tail that reads the last node, so every member reaches every other through lineage. - selections - .get_mut("job_000") - .expect("first member") - .insert("model:tail".to_owned()); - let mut deps: BTreeMap> = (1..=last) - .map(|index| (node(index), BTreeSet::from([node(index - 1)]))) - .collect(); - deps.insert("model:tail".to_owned(), BTreeSet::from([node(last)])); + let test_cases = [LargeGroupTestCase { + description: "50 members over a 10,000-node chain close into one cycle", + members: 50, + nodes_per_member: 200, + expected_cycles: 1, + expected_waits: 49 + (1..50).sum::(), + expected_limit: std::time::Duration::from_secs(2), + }]; + for test_case in test_cases { + let (selections, deps) = chained_group(test_case.members, test_case.nodes_per_member); - let started = std::time::Instant::now(); - let waits = group_wait_graph(&selections, &deps); - let elapsed = started.elapsed(); + let started = std::time::Instant::now(); + let waits = group_wait_graph(&selections, &deps); + let elapsed = started.elapsed(); - assert_eq!(waits.cycles.len(), 1); - assert_eq!(waits.edges["job_000"].len(), MEMBERS - 1); - assert_eq!(waits.edges[&format!("job_{:03}", MEMBERS - 1)].len(), MEMBERS - 1); - assert!( - elapsed < std::time::Duration::from_secs(2), - "deriving waits for {MEMBERS} members over {} nodes took {elapsed:?}", - last + 2 - ); + assert_eq!( + waits.cycles.len(), + test_case.expected_cycles, + "{}", + test_case.description + ); + assert_eq!( + waits.edges.values().map(BTreeSet::len).sum::(), + test_case.expected_waits, + "{}", + test_case.description + ); + assert!( + elapsed < test_case.expected_limit, + "{}: took {elapsed:?}", + test_case.description + ); + } } diff --git a/crates/sqlbuild-orchestration/tests/declarations/test_types.rs b/crates/sqlbuild-orchestration/tests/declarations/test_types.rs index e41811fd50..d0063a71ab 100644 --- a/crates/sqlbuild-orchestration/tests/declarations/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/declarations/test_types.rs @@ -11,3 +11,13 @@ pub(crate) struct GroupWaitGraphTestCase { pub(crate) expected_edges: Pairs, pub(crate) expected_message: Option<&'static str>, } + +pub(crate) struct LargeGroupTestCase { + pub(crate) description: &'static str, + pub(crate) members: usize, + pub(crate) nodes_per_member: usize, + pub(crate) expected_cycles: usize, + /// Waits across all members: each waits for every member before it, and the first, which builds the tail, for all others. + pub(crate) expected_waits: usize, + pub(crate) expected_limit: std::time::Duration, +} diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/mod.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/mod.rs index a23aea17c6..1788c6e61d 100644 --- a/crates/sqlbuild-python-front/src/discovery/main/sources/mod.rs +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/mod.rs @@ -1,6 +1,7 @@ pub mod consumer_uses; pub mod invocation_allows_missing_secrets; pub mod orchestration_declarations; +pub mod orchestration_failure; pub mod provider_uses; pub mod register_declaration_files; pub mod register_yaml_entries; diff --git a/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_failure.rs b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_failure.rs new file mode 100644 index 0000000000..08f938b9e4 --- /dev/null +++ b/crates/sqlbuild-python-front/src/discovery/main/sources/orchestration_failure.rs @@ -0,0 +1,17 @@ +//! An orchestration problem found after discovery, raised as `OrchestrationDeclarationError` (D017). + +use crate::discovery::_helpers::failures::raised::failure_error; +use pyo3::prelude::{PyErr, Python}; +use sqlbuild_discovery::models::{DiscoveryFailure, FailureKind}; + +/// `OrchestrationDeclarationError` (D017) with `message`, which names the file, line and column itself. +pub fn orchestration_failure(py: Python<'_>, message: String) -> PyErr { + failure_error( + py, + DiscoveryFailure { + kind: FailureKind::Orchestration, + message, + help: None, + }, + ) +} diff --git a/crates/sqlbuild-python-project/src/graph/constants.rs b/crates/sqlbuild-python-project/src/graph/constants.rs index 574ce12638..e82311d348 100644 --- a/crates/sqlbuild-python-project/src/graph/constants.rs +++ b/crates/sqlbuild-python-project/src/graph/constants.rs @@ -5,6 +5,9 @@ use crate::graph::models::CompiledProjectGraph; use sqlbuild_python_base::records::main::registry::class_of::class_of; use sqlbuild_python_base::records::models::GetterEntry; +/// The Python node kind whose functions load sources. +pub(crate) const LOADER_KIND: &str = "loader"; + /// The getter readers of this module's native pyclasses. pub const GETTERS: &[GetterEntry] = &[GetterEntry { class: class_of::, diff --git a/crates/sqlbuild-python-project/src/graph/models.rs b/crates/sqlbuild-python-project/src/graph/models.rs index b56b90101f..7a7c07e65f 100644 --- a/crates/sqlbuild-python-project/src/graph/models.rs +++ b/crates/sqlbuild-python-project/src/graph/models.rs @@ -448,24 +448,22 @@ impl ProjectSelection { .iter() .map(|(name, names)| (name.as_str(), names.as_slice())) .collect(); - self.nodes - .nodes - .iter() - .map(|node| { - let deps: BTreeSet = upstream - .get(node.name.as_str()) - .into_iter() - .flat_map(|names| names.iter()) - .filter_map(|name| self.node_key(name)) - .chain( - node.sql_deps - .iter() - .map(|(kind, name)| format!("{kind}:{name}")), - ) - .collect(); - (format!("{}:{}", node.kind, node.name), deps) - }) - .collect() + let mut nodes: Vec<(String, BTreeSet)> = Vec::new(); + for node in &self.nodes.nodes { + let mut deps: BTreeSet = BTreeSet::new(); + for name in upstream + .get(node.name.as_str()) + .copied() + .unwrap_or_default() + { + deps.extend(self.node_key(name)); + } + for (kind, name) in &node.sql_deps { + deps.insert(format!("{kind}:{name}")); + } + nodes.push((format!("{}:{}", node.kind, node.name), deps)); + } + nodes } /// The key a selected Python node's run reports under; a loader reports as the source it writes. diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_group_cycles.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_group_cycles.rs index c2e892cea1..d5c964528d 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_group_cycles.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_group_cycles.rs @@ -7,6 +7,7 @@ use sqlbuild_discovery::orchestration_files::models::{FilePosition, Orchestratio use sqlbuild_orchestration::declarations::main::group_cycle_message::group_cycle_message; use sqlbuild_orchestration::declarations::main::group_wait_graph::group_wait_graph; +use crate::graph::constants::LOADER_KIND; use crate::graph::models::ProjectSelection; use crate::project_snapshot::_helpers::snapshot::snapshot_declarations::declared_for; @@ -23,17 +24,18 @@ fn node_deps( graph: &ProjectGraph, selected: &ProjectSelection, ) -> BTreeMap> { - let key = |(kind, name): &(String, String)| format!("{kind}:{name}"); - let mut deps: BTreeMap> = graph - .upstream - .iter() - .map(|(node, upstream)| (key(node), upstream.iter().map(key).collect())) - .collect(); + let mut deps: BTreeMap> = BTreeMap::new(); + for ((kind, name), upstream) in &graph.upstream { + let reads: &mut BTreeSet = deps.entry(format!("{kind}:{name}")).or_default(); + for (upstream_kind, upstream_name) in upstream { + reads.insert(format!("{upstream_kind}:{upstream_name}")); + } + } for (node, upstream) in selected.node_deps() { deps.entry(node).or_default().extend(upstream); } for node in &selected.nodes.nodes { - if node.kind == "loader" && selected.intermediate_loader(&node.name) { + if node.kind == LOADER_KIND && selected.intermediate_loader(&node.name) { deps.entry(format!("source:{}", node.name)) .or_default() .insert(format!("loader:{}", node.name)); @@ -42,9 +44,7 @@ fn node_deps( deps } -/// One `path:line:column: message` per group declared for `target` whose members wait on each other. -/// -/// A job whose selector fails is left out, since compile reports the selector itself. +/// One `path:line:column: message` per group declared for `target` whose members wait on each other; jobs whose selector fails are left out. pub(crate) fn group_cycle_failures( inventory: &OrchestrationInventory, target: &str, @@ -62,7 +62,8 @@ pub(crate) fn group_cycle_failures( continue; } let exclude: &str = job.exclude.as_ref().map_or("", |exclude| &exclude.value); - if let Ok(selection) = selected.run_keys(graph, &tokens(&job.select.value), &tokens(exclude)) + if let Ok(selection) = + selected.run_keys(graph, &tokens(&job.select.value), &tokens(exclude)) { jobs.insert(&job.run.name.value, selection); } @@ -82,7 +83,8 @@ pub(crate) fn group_cycle_failures( }) .collect(); let name: &str = &group.run.name.value; - if let Some(message) = group_cycle_message(name, &group_wait_graph(&selections, &deps).cycles) + if let Some(message) = + group_cycle_message(name, &group_wait_graph(&selections, &deps).cycles) { let FilePosition { line, column } = group.run.name.position; failures.push(format!("{path}:{line}:{column}: {message}")); diff --git a/crates/sqlbuild-python-project/src/project_snapshot/main/job_group_cycles.rs b/crates/sqlbuild-python-project/src/project_snapshot/main/job_group_cycles.rs index 13671770fe..653977b71e 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/main/job_group_cycles.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/main/job_group_cycles.rs @@ -1,11 +1,10 @@ //! Refuse a compile whose job groups have members that wait on each other, as `OrchestrationDeclarationError` (D017). -use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult}; +use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult}; use sqlbuild_analysis::graph::models::ProjectGraph; -use sqlbuild_python_base::records::main::construct::error_class::error_class; use sqlbuild_python_base::records::main::fields::field::field; -use sqlbuild_python_base::records::types::ErrorId; use sqlbuild_python_front::discovery::main::sources::orchestration_declarations::orchestration_declarations; +use sqlbuild_python_front::discovery::main::sources::orchestration_failure::orchestration_failure; use crate::graph::models::ProjectSelection; use crate::project_snapshot::_helpers::snapshot::snapshot_group_cycles::{ @@ -31,7 +30,5 @@ pub fn job_group_cycles( if failures.is_empty() { return Ok(()); } - Err(PyErr::from_value( - error_class(py, ErrorId::OrchestrationDeclaration)?.call1((failures.join("\n"),))?, - )) + Err(orchestration_failure(py, failures.join("\n"))) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs index 3c0ef9b2cf..39a80d892f 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs @@ -797,7 +797,7 @@ impl EventExporterDispatcher { if self.withheld_kinds.contains(&policy.kind) { return Ok(()); } - let invocation_id: Option = match event.getattr("invocation_id") { + let invocation_id: Option = match native_field(event, "invocation_id") { Ok(value) => value.extract()?, Err(_) => None, }; diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py index ece7bf2c0c..8548909338 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py @@ -1443,3 +1443,10 @@ class ManagedSourceLoaderLocationTestCase: source_schema: str command: tuple[str, ...] expected_rows: tuple[tuple[object, ...], ...] + + +@dataclass(frozen=True) +class JobGroupCyclesCase: + description: str + fixed_group_toml: str + expected_messages: tuple[str, ...] diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_job_group_cycles.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_job_group_cycles.py index 78e0fd14e2..5841d01e76 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_job_group_cycles.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_job_group_cycles.py @@ -2,8 +2,12 @@ from __future__ import annotations +import subprocess from pathlib import Path +import pytest + +from tests.e2e.src.sqlbuild.cli.commands.main.compile._test_types import JobGroupCyclesCase from tests.e2e.src.sqlbuild.cli.commands.shared.helpers import prepare_inline_project, run_sqb from tests.integration.src.sqlbuild.compiler.compiled_project.helpers import ( GROUP_CYCLE_MESSAGE, @@ -14,8 +18,23 @@ _GROUP_PATH: str = "orchestration/job_groups/nightly_job_group.toml" +@pytest.mark.parametrize( + "test_case", + [ + JobGroupCyclesCase( + description="three cycles fail compile together, and one job per pair that never wait passes", + fixed_group_toml=( + 'name = "nightly_job_group"\ndescription = "Build everything nightly."\n' + 'jobs = ["orders_job", "shipments_job", "region_report_job"]\n' + ), + expected_messages=tuple(GROUP_CYCLE_MESSAGE.splitlines()), + ) + ], + ids=lambda case: case.description, +) def test_given_a_job_group_with_cycles_when_compiling_then_every_cycle_fails_until_fixed( tmp_path: Path, + test_case: JobGroupCyclesCase, ) -> None: project_dir: Path = prepare_inline_project( tmp_path=tmp_path, @@ -27,20 +46,15 @@ def test_given_a_job_group_with_cycles_when_compiling_then_every_cycle_fails_unt }, ) - failed = run_sqb(command=("compile",), project_dir=project_dir) + failed: subprocess.CompletedProcess[str] = run_sqb(command=("compile",), project_dir=project_dir) assert failed.returncode == 1, failed.stdout + failed.stderr assert "error[D017]" in failed.stderr - for line in GROUP_CYCLE_MESSAGE.splitlines(): - assert line in failed.stderr, failed.stderr - - # One job per pair of members that never wait on each other: no cycle is left. - (project_dir / _GROUP_PATH).write_text( - 'name = "nightly_job_group"\ndescription = "Build everything nightly."\n' - 'jobs = ["orders_job", "shipments_job", "region_report_job"]\n', - encoding="utf-8", - ) - fixed = run_sqb(command=("compile",), project_dir=project_dir) + for message in test_case.expected_messages: + assert message in failed.stderr, failed.stderr + + (project_dir / _GROUP_PATH).write_text(test_case.fixed_group_toml, encoding="utf-8") + fixed: subprocess.CompletedProcess[str] = run_sqb(command=("compile",), project_dir=project_dir) assert fixed.returncode == 0, fixed.stdout + fixed.stderr assert "D017" not in fixed.stderr From c66088706b750b2927ed9a9df0e53617b152192e Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 06:12:19 +0000 Subject: [PATCH 6/9] docs: regenerate the orchestration skill reference for job group cycles --- .../sqlbuild/references/docs/concepts/orchestration.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md index 0f38b9505c..2dd7a15b20 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/concepts/orchestration.md @@ -105,7 +105,8 @@ reports a job or sensor `select` that names more than three nodes one by one; se ## Group jobs A job group runs its jobs as one run. A job waits for another job in the group when it reads a node -that job builds. +that job builds. Jobs that wait on each other, directly or through other members, are a cycle, and +`sqb compile` reports every cycle in the group as one `D017` error. ```toml # orchestration/job_groups/daily_orders_job_group.toml @@ -119,6 +120,9 @@ A group takes `name`, `description` and `jobs`, and the same `targets`, `owner`, `concurrency_groups`, `schedule` and `[on_failure]` as a job. Its members are jobs, never other groups. +A job that reads a seed or a node that several members build waits for all of those members. Give +seeds and shared nodes their own job, so the jobs that read them wait only for that job. + ## Schedule a job `schedule` takes a cron expression or the name of a calendar, and always a timezone. @@ -226,6 +230,7 @@ Every problem with an orchestration file is a `D017` error at its file, line and | `schedule cron: invalid cron expression: Pattern must have 5 fields when seconds are disallowed.` | Write a five-field cron, such as `0 2 * * *` | | `job nightly_orders_job resources: cpus must be a positive number` | Set `cpus` above zero | | `sensor 'orders_sensor' is declared here, but importing the module does not define it` | Assign the sensor to a module-level name or list, not a dictionary or function body | +| `group dependencies contain a cycle in job group daily_orders_job_group: orders_marts_job waits for staging_orders_job (model:orders reads model:stg_orders); staging_orders_job waits for orders_marts_job (model:stg_orders reads seed:regions)` | Move each node to one job, and give seeds and nodes that several members read their own job that runs first | Details: From 05a9264dc6b6d5b0984ba79bba79c4ccf14f0c61 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 06:26:41 +0000 Subject: [PATCH 7/9] refactor: decide in one place which loaders publish an intermediate source --- crates/sqlbuild-python-project/src/graph/models.rs | 6 ++++++ .../_helpers/snapshot/snapshot_group_cycles.rs | 7 +++---- .../src/bindings/_helpers/location_command/snapshot.rs | 7 ++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/crates/sqlbuild-python-project/src/graph/models.rs b/crates/sqlbuild-python-project/src/graph/models.rs index 7a7c07e65f..d47fccc589 100644 --- a/crates/sqlbuild-python-project/src/graph/models.rs +++ b/crates/sqlbuild-python-project/src/graph/models.rs @@ -8,6 +8,7 @@ use crate::graph::_helpers::project::project_graph::outcome; use crate::graph::_helpers::project::views::lineage_graph_views; use crate::graph::_helpers::python_nodes::python_selection::project_sources; use crate::graph::_helpers::python_nodes::python_selection::selection_graph; +use crate::graph::constants::LOADER_KIND; use crate::graph::main::project::project_graph::project_graph; use crate::graph::main::python_nodes::python_node_graph::python_node_graph; use pyo3::prelude::Bound; @@ -477,6 +478,11 @@ impl ProjectSelection { Some(python_run_key(&self.sources, (&node.kind, name))) } + /// The `source:` key a loader node publishes when it writes no declared source, so its chain's other loaders can read it. + pub fn intermediate_source(&self, kind: &str, name: &str) -> Option { + (kind == LOADER_KIND && self.intermediate_loader(name)).then(|| format!("source:{name}")) + } + /// Whether a loader writes no declared source and so runs only as part of another loader's chain. pub fn intermediate_loader(&self, name: &str) -> bool { !self diff --git a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_group_cycles.rs b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_group_cycles.rs index d5c964528d..051b13903a 100644 --- a/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_group_cycles.rs +++ b/crates/sqlbuild-python-project/src/project_snapshot/_helpers/snapshot/snapshot_group_cycles.rs @@ -7,7 +7,6 @@ use sqlbuild_discovery::orchestration_files::models::{FilePosition, Orchestratio use sqlbuild_orchestration::declarations::main::group_cycle_message::group_cycle_message; use sqlbuild_orchestration::declarations::main::group_wait_graph::group_wait_graph; -use crate::graph::constants::LOADER_KIND; use crate::graph::models::ProjectSelection; use crate::project_snapshot::_helpers::snapshot::snapshot_declarations::declared_for; @@ -35,10 +34,10 @@ fn node_deps( deps.entry(node).or_default().extend(upstream); } for node in &selected.nodes.nodes { - if node.kind == LOADER_KIND && selected.intermediate_loader(&node.name) { - deps.entry(format!("source:{}", node.name)) + if let Some(source) = selected.intermediate_source(&node.kind, &node.name) { + deps.entry(source) .or_default() - .insert(format!("loader:{}", node.name)); + .insert(format!("{}:{}", node.kind, node.name)); } } deps diff --git a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs index cf1819630c..7b5b0c5c4a 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/location_command/snapshot.rs @@ -36,9 +36,6 @@ use sqlbuild_python_project::project_snapshot::main::python_declarations::python use sqlbuild_python_project::project_snapshot::main::resource_nodes::resource_nodes; use sqlbuild_python_project::project_snapshot::models::NativeOrchestrationSnapshot; -/// The Python node kind whose functions load sources. -const LOADER_KIND: &str = "loader"; - /// Compile catalog nodes and identities in the location, never in the server. #[pyfunction] #[pyo3(signature = (project, metadata_json, project_dir, **options))] @@ -166,7 +163,7 @@ fn python_nodes( published.definition["deps"] = json!(deps); published.definition["tags"] = json!(tags.get(node.name.as_str()).cloned().unwrap_or_default()); - if node.kind == LOADER_KIND && selected.intermediate_loader(&node.name) { + if let Some(source) = selected.intermediate_source(&node.kind, &node.name) { let mut definition = json!({ "name": node.name, "entry": {"loader": node.name}, @@ -177,7 +174,7 @@ fn python_nodes( definition["path"] = path.clone(); } outputs.push(( - format!("source:{}", node.name), + source, SnapshotNode { kind: "source".into(), definition, From fbb63f9c7712eab4cc4d76ff180d3a17f3ef419b Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 06:31:07 +0000 Subject: [PATCH 8/9] style: format the job group cycles E2E test --- .../cli/commands/main/compile/test_job_group_cycles.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_job_group_cycles.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_job_group_cycles.py index 5841d01e76..5f88190d1f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_job_group_cycles.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_job_group_cycles.py @@ -46,7 +46,9 @@ def test_given_a_job_group_with_cycles_when_compiling_then_every_cycle_fails_unt }, ) - failed: subprocess.CompletedProcess[str] = run_sqb(command=("compile",), project_dir=project_dir) + failed: subprocess.CompletedProcess[str] = run_sqb( + command=("compile",), project_dir=project_dir + ) assert failed.returncode == 1, failed.stdout + failed.stderr assert "error[D017]" in failed.stderr From ba6c62a3dff0424e28a75c71d5569ae8dfd3835d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 17 Nov 2026 06:34:49 +0000 Subject: [PATCH 9/9] style: keep the group cycle fixture comment within the line limit --- .../src/sqlbuild/compiler/compiled_project/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py index 0e0ff9dc12..c41b849efa 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/helpers.py @@ -445,7 +445,7 @@ def _group_cycle_model(name: str, reads: str) -> str: ) -# Models and a seed for the DuckShop `group_cycles` jobs, whose group bundles them into three cycles. +# Models and a seed for the DuckShop `group_cycles` jobs, grouped into three cycles. GROUP_CYCLE_NODES: dict[str, str] = { "seeds/regions.csv": "id\n1\n", "seeds/regions.yml": (