From 1a66a075da166b75aad1dc0c4358372b239b3aca Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 07:24:23 +0000 Subject: [PATCH 1/3] fix(orchestration): claim loaded sources and order layer jobs by lineage A build that loads a managed source for a selected model now claims that source in its frozen write set, so the worker no longer refuses the run. Group children compute their own write sets, and reruns recompute writers for their narrowed selection. A job that builds a node itself, such as a function its models share with another layer's job, no longer waits for that node's other builders, so tag-selected layer jobs stop reporting a false group cycle. A real cycle now names the group and the edges that close it. Locations and workers warn once per process when a bearer token would travel over plain HTTP to a non-loopback server. --- .../orchestration/shared_function_layers.py | 14 ++++ .../src/api/_helpers/resources/mod.rs | 1 + .../src/api/_helpers/resources/plain_http.rs | 24 +++++++ .../src/api/constants.rs | 2 + .../src/api/main/connect_client.rs | 4 +- .../src/declarations/_helpers/group_waits.rs | 45 +++++++++++++ .../src/declarations/_helpers/mod.rs | 1 + .../declarations/_helpers/trigger_scope.rs | 8 ++- .../declarations/main/group_dependencies.rs | 51 ++++++++++----- .../declarations/main/snapshot_definitions.rs | 3 +- .../src/declarations/models.rs | 7 ++ .../src/runs/_helpers/requests/adhoc.rs | 8 ++- .../src/runs/_helpers/requests/job.rs | 8 ++- .../src/runs/_helpers/requests/permissions.rs | 6 +- .../src/runs/_helpers/requests/publication.rs | 8 ++- .../src/runs/_helpers/requests/rerun.rs | 25 +++++-- .../src/runs/main/frozen_job_request.rs | 8 ++- .../src/scheduler/_helpers/persistence.rs | 2 +- .../src/snapshots/constants.rs | 9 +++ .../src/snapshots/main/job_write_set.rs | 65 ++++++++++++++----- .../tests/runs/helpers.rs | 10 ++- .../tests/runs/test_adhoc.rs | 22 +++++++ .../tests/runs/test_reruns.rs | 9 +++ .../tests/runs/test_types.rs | 7 +- .../compiled_project/snapshot_declarations.rs | 2 +- fensu.toml | 6 ++ .../publication/_test_types.py | 7 ++ .../publication/test_publication.py | 64 ++++++++++++++++++ .../publication/test_worker.py | 15 +++++ .../test_orchestration_snapshot.py | 34 +++++++++- website/src/content/docs/docs/cli/server.mdx | 13 ++++ 31 files changed, 427 insertions(+), 61 deletions(-) create mode 100644 benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py create mode 100644 crates/sqlbuild-orchestration/src/api/_helpers/resources/plain_http.rs create mode 100644 crates/sqlbuild-orchestration/src/declarations/_helpers/group_waits.rs diff --git a/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py new file mode 100644 index 0000000000..77a937120d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py @@ -0,0 +1,14 @@ +"""Layer jobs selected by tag share the function their models call, and still order by lineage. + +A tag selection brings in the functions its models call, so both jobs build the shared function. A +job that builds a node itself does not wait for that node's other builders, so the group runs the +marts layer after the staging layer instead of reporting a cycle. +""" + +from sqlbuild.orchestration import job, job_group + +staging = job("staging_layer", description="Build the staging layer.", select="tag:staging") +marts = job("marts_layer", description="Build the marts layer.", select="tag:marts") +layers = job_group( + "layered_orders", description="Build staging, then marts.", jobs=[marts, staging] +) diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs index 376e0414f6..fc208658a0 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs @@ -14,6 +14,7 @@ pub(crate) mod person_node_sql; pub(crate) mod person_operations; pub(crate) mod person_plans; pub(crate) mod person_reads; +pub(crate) mod plain_http; pub(crate) mod request_json; pub(crate) mod router; pub(crate) mod run_reads; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/plain_http.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/plain_http.rs new file mode 100644 index 0000000000..60f535063e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/plain_http.rs @@ -0,0 +1,24 @@ +//! Warn once per process before a bearer token crosses the network over plain HTTP. + +use std::net::IpAddr; +use std::sync::Once; + +use crate::api::constants::{HTTP_SCHEME, LOOPBACK_HOST}; + +/// Loopback origins stay silent; any other `http://` origin warns on the first connection only. +pub(crate) fn warn_plain_http(url: &reqwest::Url) { + static WARNED: Once = Once::new(); + let loopback = url.host_str().is_some_and(|host| { + let host = host.trim_start_matches('[').trim_end_matches(']'); + host.eq_ignore_ascii_case(LOOPBACK_HOST) + || host.parse::().is_ok_and(|ip| ip.is_loopback()) + }); + if url.scheme() == HTTP_SCHEME && !loopback { + WARNED.call_once(|| { + eprintln!( + "warning: the bearer token for {} travels unencrypted over HTTP; put TLS or an HTTPS reverse proxy in front of the server", + url.origin().ascii_serialization() + ); + }); + } +} diff --git a/crates/sqlbuild-orchestration/src/api/constants.rs b/crates/sqlbuild-orchestration/src/api/constants.rs index 89bb616723..d88b1b5a6b 100644 --- a/crates/sqlbuild-orchestration/src/api/constants.rs +++ b/crates/sqlbuild-orchestration/src/api/constants.rs @@ -17,6 +17,8 @@ pub(crate) const OIDC_RESPONSE_BYTES: usize = 1024 * 1024; #[cfg(unix)] pub(crate) const DOCKER_IMAGE_HEX_LENGTH: usize = 64; pub(crate) const HTTPS_SCHEME: &str = "https"; +pub(crate) const HTTP_SCHEME: &str = "http"; +pub(crate) const LOOPBACK_HOST: &str = "localhost"; pub(crate) const ROOT_PATH: &str = "/"; pub(crate) const UI_INDEX_PATH: &str = "/index.html"; pub(crate) const UI_IMMUTABLE_PREFIX: &str = "/_app/immutable/"; diff --git a/crates/sqlbuild-orchestration/src/api/main/connect_client.rs b/crates/sqlbuild-orchestration/src/api/main/connect_client.rs index 3c717eb4f6..1e8d6b919d 100644 --- a/crates/sqlbuild-orchestration/src/api/main/connect_client.rs +++ b/crates/sqlbuild-orchestration/src/api/main/connect_client.rs @@ -1,8 +1,9 @@ -//! Validate the API origin before attaching a sensitive bearer credential. +//! Validate the API origin, and warn about plain HTTP, before attaching a sensitive bearer credential. use reqwest::header::HeaderValue; use crate::api::_helpers::resources::client::http_client; +use crate::api::_helpers::resources::plain_http::warn_plain_http; use crate::api::errors::ClientError; use crate::api::models::ControlClient; @@ -20,6 +21,7 @@ pub(crate) fn connect_client(server_url: &str, bearer: &str) -> Result, + (job, dependency): (&str, &str), + reason: impl FnOnce() -> String, +) -> BTreeMap<(String, String), String> { + reasons + .entry((job.to_owned(), dependency.to_owned())) + .or_insert_with(reason); + reasons +} + +/// Every job left waits on another, so following first dependencies from any of them closes a cycle. +pub(crate) fn describe_cycle( + remaining: &BTreeMap>, + reasons: &BTreeMap<(String, String), String>, +) -> String { + let mut path: Vec<&str> = Vec::new(); + let mut current = remaining.keys().next().map(String::as_str); + while let Some(name) = current { + if let Some(start) = path.iter().position(|seen| *seen == name) { + path.drain(..start); + break; + } + path.push(name); + current = remaining + .get(name) + .and_then(|deps| deps.iter().next()) + .map(String::as_str); + } + path.iter() + .zip(path.iter().cycle().skip(1)) + .map(|(job, dependency)| { + let reason = reasons + .get(&((*job).to_owned(), (*dependency).to_owned())) + .map_or("", String::as_str); + format!("{job} waits for {dependency} ({reason})") + }) + .collect::>() + .join("; ") +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs index c0e012e8e8..f8b81e37f0 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod development; pub(crate) mod documents; +pub(crate) mod group_waits; pub(crate) mod job_detail; pub(crate) mod operator_change; pub(crate) mod operator_state; diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs index aa906942d8..f0f5be3920 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs @@ -50,8 +50,12 @@ pub(crate) async fn trigger_scope( exclude: Vec::new(), }; let (transaction, frozen) = frozen_job_request(transaction, (context, None), &job).await?; - let (transaction, nodes) = - job_write_set(transaction, frozen.snapshot_id, &frozen.selection).await?; + let (transaction, nodes) = job_write_set( + transaction, + frozen.snapshot_id, + (&frozen.selection, &frozen.settings), + ) + .await?; Ok(( transaction, PermissionScope { diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs index e896fba0a5..d2103f2439 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs @@ -2,12 +2,13 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::declarations::models::JobDefinition; +use crate::declarations::_helpers::group_waits::{describe_cycle, record_wait}; +use crate::declarations::models::{GroupWaits, JobDefinition}; use crate::snapshots::models::SnapshotInput; use crate::store::errors::ControlError; pub fn group_dependencies( - input: &SnapshotInput, + (group, input): (&str, &SnapshotInput), jobs: &BTreeMap, (members, after): (&BTreeSet, &BTreeMap>), ) -> Result>, ControlError> { @@ -21,16 +22,17 @@ pub fn group_dependencies( "group needs known jobs and dependencies within its membership".into(), )); } - let edges = lineage_dependencies(input, jobs, (members, after)); - ensure_acyclic(&edges)?; - Ok(edges) + let waits = lineage_dependencies(input, jobs, (members, after)); + ensure_acyclic(group, &waits.edges, &waits.reasons)?; + Ok(waits.edges) } +/// A job that builds a node itself, such as a shared function, never waits for its other builders. fn lineage_dependencies( input: &SnapshotInput, jobs: &BTreeMap, (members, after): (&BTreeSet, &BTreeMap>), -) -> BTreeMap> { +) -> GroupWaits { let mut owners: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); for name in members { for key in &jobs[name].selection { @@ -38,8 +40,14 @@ fn lineage_dependencies( } } let mut edges: BTreeMap> = BTreeMap::new(); + let mut reasons: BTreeMap<(String, String), String> = BTreeMap::new(); for name in members { let mut dependencies = after.get(name).cloned().unwrap_or_default(); + for dependency in &dependencies { + reasons = record_wait(reasons, (name, dependency), || { + "declared in after".to_owned() + }); + } let mut seen: BTreeSet<&str> = BTreeSet::new(); let mut pending: Vec<&str> = jobs[name].selection.iter().map(String::as_str).collect(); while let Some(key) = pending.pop() { @@ -53,13 +61,15 @@ fn lineage_dependencies( .and_then(serde_json::Value::as_array) { for upstream in upstream.iter().filter_map(serde_json::Value::as_str) { - if let Some(owners) = owners.get(upstream) { - dependencies.extend( - owners - .iter() - .filter(|owner| **owner != name) - .map(|owner| (*owner).to_owned()), - ); + if let Some(owners) = owners.get(upstream) + && !owners.contains(name.as_str()) + { + for owner in owners { + dependencies.insert((*owner).to_owned()); + reasons = record_wait(reasons, (name, owner), || { + format!("{key} reads {upstream}") + }); + } } pending.push(upstream); } @@ -67,10 +77,14 @@ fn lineage_dependencies( } edges.insert(name.clone(), dependencies); } - edges + GroupWaits { edges, reasons } } -fn ensure_acyclic(edges: &BTreeMap>) -> Result<(), ControlError> { +fn ensure_acyclic( + group: &str, + edges: &BTreeMap>, + reasons: &BTreeMap<(String, String), String>, +) -> Result<(), ControlError> { let mut remaining = edges.clone(); while !remaining.is_empty() { let ready: BTreeSet = remaining @@ -79,9 +93,10 @@ fn ensure_acyclic(edges: &BTreeMap>) -> Result<(), Cont .map(|(name, _)| name.clone()) .collect(); if ready.is_empty() { - return Err(ControlError::Invalid( - "group dependencies contain a cycle".into(), - )); + return Err(ControlError::Invalid(format!( + "group dependencies contain a cycle in job group {group}: {}", + describe_cycle(&remaining, reasons) + ))); } remaining.retain(|name, deps| { deps.retain(|dependency| !ready.contains(dependency)); diff --git a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs index b89bfb316c..a415f62d13 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs @@ -30,7 +30,8 @@ pub fn snapshot_definitions(input: &SnapshotInput) -> Result = group.jobs.keys().cloned().collect(); - let dependencies = group_dependencies(input, &definitions.jobs, (&members, &group.jobs))?; + let dependencies = + group_dependencies((name, input), &definitions.jobs, (&members, &group.jobs))?; let selection: BTreeSet = group .jobs .keys() diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index b67dadf618..fecd2146a4 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -249,3 +249,10 @@ pub(crate) struct PreviousTriggerState { pub(crate) last_tick_for: Option>, pub(crate) status_override: Option, } + +/// Each member job's waits, and why it waits: an explicit `after`, or the first lineage edge found. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct GroupWaits { + pub(crate) edges: BTreeMap>, + pub(crate) reasons: BTreeMap<(String, String), String>, +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index 137c35efee..082f491538 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -59,8 +59,12 @@ pub(crate) async fn frozen_adhoc_request( let refusal = format!("workers do not run sqb {}", plan.verb); return Ok((transaction, Err(refusal))); }; - let (transaction, writers) = - job_write_set(transaction, snapshot.snapshot_id, &plan.selection).await?; + let (transaction, writers) = job_write_set( + transaction, + snapshot.snapshot_id, + (&plan.selection, &plan.settings), + ) + .await?; let (transaction, images) = query( transaction, "SELECT image_id FROM sqb_snapshots WHERE snapshot_id=$1", diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs index 939a2c7c4f..e5d1050ee3 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs @@ -31,8 +31,7 @@ pub(crate) async fn frozen_group_children( "group scope differs from its pinned membership".into(), )); } - let (transaction, writers) = - job_write_set(transaction, request.snapshot_id, &request.selection).await?; + let mut transaction = transaction; let mut children: Vec = Vec::new(); for name in group.jobs.keys() { let job = &definitions.jobs[name]; @@ -48,6 +47,9 @@ pub(crate) async fn frozen_group_children( if let Some(full_refresh) = request.settings.get(FULL_REFRESH_SETTING) { settings.insert(FULL_REFRESH_SETTING.into(), full_refresh.clone()); } + let write_set; + (transaction, write_set) = + job_write_set(transaction, request.snapshot_id, (&selection, &settings)).await?; children.push(RunRequest { run_id: Uuid::new_v5(&request.run_id, name.as_bytes()), kind: RunKind::Job, @@ -57,7 +59,7 @@ pub(crate) async fn frozen_group_children( rerun_of_run_id: request .rerun_of_run_id .map(|id| Uuid::new_v5(&id, name.as_bytes())), - write_set: selection.intersection(&writers).cloned().collect(), + write_set, selection, settings, concurrency_groups: job.concurrency_groups.clone(), diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs index 4a9acad803..16bdf00f11 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs @@ -58,7 +58,11 @@ pub(crate) fn execution_scopes( action: Action, ) -> Vec { let nodes: BTreeSet = if request.kind == RunKind::Adhoc { - request.selection.clone() + request + .selection + .union(&request.write_set) + .cloned() + .collect() } else { executions .iter() diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs index 9b5d67be48..4eb9c02062 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs @@ -8,6 +8,7 @@ use uuid::Uuid; use crate::credentials::models::AuditContext; use crate::runs::main::queue_run::queue_run; use crate::runs::models::RunRequest; +use crate::snapshots::constants::SOURCE_KEY_PREFIX; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; @@ -59,10 +60,13 @@ pub(crate) async fn publish_request( ) -> Result<(ControlTransaction, bool), ControlError> { if request.requested_by.trim().is_empty() || request.selection.is_empty() - || !request.write_set.is_subset(&request.selection) + || request + .write_set + .iter() + .any(|key| !request.selection.contains(key) && !key.starts_with(SOURCE_KEY_PREFIX)) { return Err(ControlError::Invalid( - "run needs an actor, selection and a contained write set".into(), + "run needs an actor, selection and a write set within it or its loaded sources".into(), )); } let (transaction, previous) = query( diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs index 30ae961508..6642c2afbe 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs @@ -9,7 +9,9 @@ use crate::authorization::main::authorize_listing::authorize_listing; use crate::authorization::main::authorize_person::authorize_person; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::runs::_helpers::requests::permissions::authorize_execution; -use crate::runs::models::{RerunRequest, RunOrigin, RunRequest, RunStatus}; +use crate::runs::models::{RerunRequest, RunCommand, RunKind, RunOrigin, RunRequest, RunStatus}; +use crate::snapshots::constants::SOURCE_KEY_PREFIX; +use crate::snapshots::main::job_write_set::job_write_set; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; @@ -58,9 +60,6 @@ pub(crate) async fn frozen_rerun( .await?; } frozen.selection.retain(|key| named.contains(key.as_str())); - frozen - .write_set - .retain(|key| frozen.selection.contains(key)); } if request.from_failure { let rows; @@ -75,9 +74,21 @@ pub(crate) async fn frozen_rerun( .filter_map(|row| row["node_key"].as_str()) .collect(); frozen.selection.retain(|key| failed.contains(key.as_str())); - frozen - .write_set - .retain(|key| frozen.selection.contains(key)); + } + let writers; + (transaction, writers) = job_write_set( + transaction, + frozen.snapshot_id, + (&frozen.selection, &frozen.settings), + ) + .await?; + frozen.write_set.retain(|key| writers.contains(key)); + if frozen.command == RunCommand::Build && frozen.kind != RunKind::Group { + frozen.write_set.extend( + writers + .into_iter() + .filter(|key| key.starts_with(SOURCE_KEY_PREFIX)), + ); } frozen.run_id = request.run_id; frozen.origin = RunOrigin::Rerun; diff --git a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs index dd3ca6fc2f..cdcc2afaaa 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs @@ -60,8 +60,12 @@ pub(crate) async fn frozen_job_request( ) .await?; } - let (transaction, write_set) = - job_write_set(transaction, snapshot_id, &definition.selection).await?; + let (transaction, write_set) = job_write_set( + transaction, + snapshot_id, + (&definition.selection, &definition.settings), + ) + .await?; Ok(( transaction, RunRequest { diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs index 29ee7f287e..8d4d589957 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -17,7 +17,7 @@ pub(crate) async fn persist_ticks( let (mut transaction, write_set) = job_write_set( transaction, trigger.snapshot_id, - &trigger.job_json.selection, + (&trigger.job_json.selection, &trigger.job_json.settings), ) .await?; for action in &plan.actions { diff --git a/crates/sqlbuild-orchestration/src/snapshots/constants.rs b/crates/sqlbuild-orchestration/src/snapshots/constants.rs index f931b29c58..cf3e62ba43 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/constants.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/constants.rs @@ -48,3 +48,12 @@ pub(crate) const CONTENT: [((&str, &str), &str, &str); 3] = [ "", ), ]; +/// Node kinds a build writes when they are selected. +pub(crate) const WRITER_NODE_KINDS: [&str; 4] = ["model", "seed", "udf", "table_fn"]; +pub(crate) const MODEL_NODE_KIND: &str = "model"; +pub(crate) const SOURCE_NODE_KIND: &str = "source"; +pub(crate) const SOURCE_KEY_PREFIX: &str = "source:"; +/// A source with a non-null loader here is managed, so automatic source loading writes it. +pub(crate) const LOADER_POINTER: &str = "/entry/loader"; +/// The frozen setting for `--no-load-sources`; `false` stops a build loading sources its models read. +pub(crate) const LOAD_SOURCES_SETTING: &str = "load_sources"; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs b/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs index 3060d1b261..a8f24c6600 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs @@ -1,30 +1,61 @@ //! Resolve a job's physical writers from its pinned snapshot. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; +use serde_json::Value; use uuid::Uuid; +use crate::snapshots::constants::{ + LOAD_SOURCES_SETTING, LOADER_POINTER, MODEL_NODE_KIND, SOURCE_KEY_PREFIX, SOURCE_NODE_KIND, + WRITER_NODE_KINDS, +}; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; use crate::store::errors::ControlError; -use crate::store::main::query::query; -use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::models::ControlTransaction; +/// Selected writers plus the managed sources a build loads: selected ones, and parents of selected models unless loading is off. pub(crate) async fn job_write_set( transaction: ControlTransaction, snapshot_id: Uuid, - selection: &BTreeSet, + (selection, settings): (&BTreeSet, &BTreeMap), ) -> Result<(ControlTransaction, BTreeSet), ControlError> { - let (transaction, rows) = query(transaction, - "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND node_kind IN ('model','seed','udf','table_fn')", - vec![Parameter::Id(snapshot_id)]).await?; - Ok(( - transaction, - rows.into_iter() - .filter_map(|row| { - row.get("node_key") - .and_then(serde_json::Value::as_str) - .map(str::to_owned) + let (transaction, selected) = + read_snapshot_nodes(transaction, snapshot_id, Some(selection)).await?; + let loads_model_sources = settings.get(LOAD_SOURCES_SETTING) != Some(&Value::Bool(false)); + let mut writers: BTreeSet = BTreeSet::new(); + let mut sources: BTreeSet = BTreeSet::new(); + for (key, node) in &selected { + if WRITER_NODE_KINDS.contains(&node.kind.as_str()) { + writers.insert(key.clone()); + } + if node.kind == SOURCE_NODE_KIND { + sources.insert(key.clone()); + } else if node.kind == MODEL_NODE_KIND && loads_model_sources { + sources.extend( + node.definition + .get("deps") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .filter(|key| key.starts_with(SOURCE_KEY_PREFIX)) + .map(str::to_owned), + ); + } + } + let (transaction, sources) = + read_snapshot_nodes(transaction, snapshot_id, Some(&sources)).await?; + writers.extend( + sources + .into_iter() + .filter(|(_, node)| { + node.kind == SOURCE_NODE_KIND + && node + .definition + .pointer(LOADER_POINTER) + .is_some_and(|loader| !loader.is_null()) }) - .filter(|key| selection.contains(key)) - .collect(), - )) + .map(|(key, _)| key), + ); + Ok((transaction, writers)) } diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index b8aab47a25..9fd636bed0 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -135,7 +135,7 @@ pub(crate) fn json_column(value: &Value) -> Value { }) } -/// A publication of two models, a seed and a source, with one person per role under test. +/// Two models, a seed, a read-only source, a loaded customers source and one person per role under test. pub(crate) async fn adhoc_fixture( store: &ControlStore, context: &AuditContext, @@ -153,6 +153,14 @@ pub(crate) async fn adhoc_fixture( .into_iter() .map(published_node) .collect(); + let (key, mut managed) = published_node("source:shop.customers"); + managed.definition = json!({"entry": {"loader": "load_customers"}}); + input.nodes.insert(key, managed); + input + .nodes + .get_mut("model:customers") + .expect("customers model") + .definition = json!({"materialization": "table", "deps": ["source:shop.customers"]}); let published = publish_snapshot(store, &input).await.expect("publish"); let admin = bootstrap_user(store, context, "orders-admin") .await diff --git a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs index 24e87728a1..1662bd3364 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs @@ -32,6 +32,26 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ "permissions": [["run.adhoc", true]], "error": null, "launchable": true}), expected_launch: Ok(true), }, + AdhocCommandTestCase { + description: "a build claims the managed source it loads for a selected model", + person: AdhocPerson::Developer, + command: &["build", "model:customers"], + priority: 0, + reason: "Refresh customers", + expected_check: json!({"selection": ["model:customers"], "write_set": ["model:customers", "source:shop.customers"], + "permissions": [["run.adhoc", true]], "error": null, "launchable": true}), + expected_launch: Ok(true), + }, + AdhocCommandTestCase { + description: "a build that does not load sources claims only its own writers", + person: AdhocPerson::Developer, + command: &["build", "model:customers", "--no-load-sources"], + priority: 0, + reason: "Refresh customers only", + expected_check: json!({"selection": ["model:customers"], "write_set": ["model:customers"], + "permissions": [["run.adhoc", true]], "error": null, "launchable": true}), + expected_launch: Ok(true), + }, AdhocCommandTestCase { description: "every gated flag is its own permission, and Developer holds none of them", person: AdhocPerson::Developer, @@ -272,6 +292,8 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ .collect::>(), vec![ json!(["build", {}, "Refresh orders"]), + json!(["build", {}, "Refresh customers"]), + json!(["build", {"load_sources": false}, "Refresh customers only"]), json!(["build", {"full_refresh": true, "warehouse": "BUILD_XS"}, "Rebuild orders now"]), json!(["test", {}, "Check orders"]), ], diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index 86490c97bd..7fff917906 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -59,6 +59,15 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ ip_address: None, }; let mut input = snapshot(location); + let mut managed = input.nodes["orders"].clone(); + managed.kind = "source".into(); + managed.definition = json!({"entry": {"loader": "load_raw_orders"}}); + input.nodes.insert("source:raw_orders".into(), managed); + input + .nodes + .get_mut("orders") + .expect("orders model") + .definition = json!({"materialization": "table", "deps": ["source:raw_orders"]}); input.documents.insert("jobs".into(), json!({ "staging": {"description": "Stage orders", "selection": ["orders", "customers"], "concurrency_groups": ["warehouse"], "settings": {"vars": {"batch": 23}}}, "marts": {"description": "Build inventory", "selection": ["inventory"], "concurrency_groups": ["inventory_limit"]}, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 9d38196822..017d2d8798 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -162,7 +162,7 @@ pub(crate) struct AdhocCommandTestCase { pub(crate) expected_launch: Result, } -/// The CLI's planner reduced to exact node keys, `--full-refresh` and `--warehouse NAME`. +/// The CLI's planner reduced to exact node keys, `--full-refresh`, `--no-load-sources` and `--warehouse NAME`. pub(crate) struct ExactKeyPlanner; impl crate::runs::types::CommandPlanner for ExactKeyPlanner { @@ -189,6 +189,11 @@ impl crate::runs::types::CommandPlanner for ExactKeyPlanner { .insert("full_refresh".into(), serde_json::json!(true)); continue; } + "--no-load-sources" => { + plan.settings + .insert("load_sources".into(), serde_json::json!(false)); + continue; + } "--warehouse" => { let name = tokens.next().ok_or("--warehouse needs a name")?; plan.settings diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index c926c807f7..48f7574c19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -163,7 +163,7 @@ pub(crate) fn compile_declarations( .collect(), ); } - let dependencies = group_dependencies(&input, &jobs, (&members, &after)) + let dependencies = group_dependencies((&group.name, &input), &jobs, (&members, &after)) .map_err(|error| PyValueError::new_err(error.to_string()))?; let selection: BTreeSet = members .iter() diff --git a/fensu.toml b/fensu.toml index e9afc99dd8..7fa4ceac3f 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1701,6 +1701,12 @@ path = "crates/sqlbuild-cli/src/person_login/models.rs" symbols = ["PersonCredential", "LoginFlow"] reason = "Person credential and handoff models implement Debug manually to redact bearer, CSRF and approval secrets." +[[targets.rust.rule_exceptions]] +rule = "FPRSH011" +path = "crates/sqlbuild-orchestration/src/api/_helpers/resources/plain_http.rs" +symbols = ["warn_plain_http"] +reason = "Locations, workers and remote publication share this client; it warns once per process on stderr, before any request, when a bearer token would travel over plain HTTP to a non-loopback host." + [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-orchestration/src/webhooks/models.rs" diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index e542ca01a9..482ef37d1d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -54,6 +54,13 @@ class SnapshotPublicationTestCase: expected_nodes: frozenset[str] +@dataclass(frozen=True) +class PlainHttpWarningTestCase: + description: str + host: str + expected_warnings: int + + @dataclass(frozen=True) class SnapshotWorkerTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index 49b4a75a47..bc538509ff 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -1,9 +1,11 @@ """Real compile, native transport, authenticated server and durable publication.""" import json +import os import socket import sqlite3 import subprocess +import sys from pathlib import Path from typing import Any from urllib.parse import quote @@ -27,6 +29,7 @@ TYPE_PROOF_FILES, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + PlainHttpWarningTestCase, SnapshotPublicationTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( @@ -303,3 +306,64 @@ def test_given_a_code_location_when_publishing_from_cli_then_compilation_isolate delivery_output: CaptureResult[str] = capsys.readouterr() assert "was published, but result delivery failed" in delivery_output.err assert current_snapshot(api, test_case.target)["record"]["generation"] == 3 + + +@pytest.mark.parametrize( + "test_case", + [ + PlainHttpWarningTestCase( + description="a loopback server gets the token without a warning", + host="127.0.0.1", + expected_warnings=0, + ), + PlainHttpWarningTestCase( + description="a plain-HTTP server on another address is warned about once per process", + host="0.0.0.0", + expected_warnings=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_plain_http_server_when_a_location_publishes_then_only_a_non_loopback_origin_warns_once( + test_case: PlainHttpWarningTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "project" + write_files( + project_dir=project_dir, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '\n[targets.development]\nschema = "main"\n', + }, + ) + with snapshot_api(tmp_path / "control", server_arguments=("--host", "0.0.0.0")) as api: + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--no-color", + "--project-dir", + str(project_dir), + "location", + "publish", + "--server", + f"http://{test_case.host}:{api.url.rpartition(':')[2]}", + "--location-id", + api.location_id, + "--target", + "development", + "--generation", + "1", + ], + capture_output=True, + text=True, + timeout=120, + env=os.environ + | { + "SQLBUILD_LOCATION_TOKEN": api.bearer, + "SQLBUILD_CONTROL_DATABASE_URL": "unsupported://must-not-open", + }, + ) + assert result.returncode == 0, result.stderr + assert api.bearer not in result.stderr + assert result.stderr.count("travels unencrypted over HTTP") == test_case.expected_warnings diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 96029d4518..dd84ed7816 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -456,6 +456,21 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ "success", [(23,)], ), + WorkerExecutionTestCase( + "a scheduled build loads the managed source its model reads", + 'SELECT CAST(order_id AS BIGINT) AS amount FROM __source("raw_orders")', + "SELECT amount FROM orders", + "success", + [(1,)], + files={ + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/managed_orders.yml" + ).read_text(), + "python/loaders/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/orders_loader.py" + ).read_text(), + }, + ), WorkerExecutionTestCase( "a platform sensor catches up retained events through a real worker", "SELECT 47 AS amount", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 91484c00fb..5bfb5e9d9a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -887,6 +887,35 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject } }, ), + SnapshotDiscoveryTestCase( + description="DuckShop layer jobs that share a function wait only by lineage", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py" + ).read_text(), + "functions/sql/order_cents.sql": ( + "FUNCTION (description 'Amount in cents.'," + " arguments (amount INTEGER), returns INTEGER);\nSELECT amount * 100\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', tags [staging],\n" + " columns (order_id (type INTEGER), cents (type INTEGER)),\n" + ");\n\n" + "SELECT CAST(1 AS INTEGER) AS order_id," + ' __udf("order_cents")(CAST(5 AS INTEGER)) AS cents\n' + ), + "models/orders.sql": ( + "MODEL (description 'Typed orders.', tags [marts],\n" + " columns (order_id (type INTEGER), cents (type INTEGER)),\n" + ");\n\n" + 'SELECT order_id, __udf("order_cents")(cents) AS cents\nFROM __ref("stg_orders")\n' + ), + }, + expected_jobs=("marts_layer", "staging_layer"), + expected_groups={ + "layered_orders": {"marts_layer": ["staging_layer"], "staging_layer": []} + }, + ), SnapshotDiscoveryTestCase( description="explicit empty declarations do not execute project files", files={"orchestration/jobs.py": "raise RuntimeError('must not import')\n"}, @@ -1250,7 +1279,10 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects ), }, expected_error=ValueError, - expected_message="group dependencies contain a cycle", + expected_message=re.escape( + "group dependencies contain a cycle in job group daily: marts waits for staging " + "(model:orders reads model:stg_orders); staging waits for marts (declared in after)" + ), ), SnapshotDiscoveryRejectionTestCase( description="group waits cannot refer to nonmember jobs", diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 7ac984268b..b4bd7d7e5b 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -13,6 +13,19 @@ sqb server Use `sqb server --help` for service, authentication and launcher options. +## Serve over HTTPS + +Code locations and workers send their bearer token with every request to the `--server` URL. +`sqb server` serves plain HTTP, so put TLS or an HTTPS reverse proxy in front of it and give +locations and workers its `https://` URL. When a location or worker connects to an `http://` URL +on another host, it prints this warning once and keeps running: + +```text +warning: the bearer token for http://control.example.com travels unencrypted over HTTP; put TLS or an HTTPS reverse proxy in front of the server +``` + +Connections to `localhost`, `127.0.0.1` or `::1` print no warning. + ## Upgrade Upgrade the server first, then the locations and workers that connect to it. Don't downgrade the From 993c90c51a058dbfe367e7f1a32ca989b589cff1 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 07:36:14 +0000 Subject: [PATCH 2/3] feat(server): send a run-failure event to the platform webhook A failed or partial run now publishes a run_failed platform event naming its job, run, location, target, outcome and first failed nodes, with node error text redacted and bounded as for person reads. It reaches only the platform webhook, never person platform history, because error text can quote warehouse data. The server page documents the webhook stream. --- clients/typescript/src/schema.d.ts | 17 ++- .../src/platform/_helpers/event_history.rs | 1 + .../src/platform/constants.rs | 3 + .../src/platform/models.rs | 16 +++ .../src/runs/_helpers/lifecycle/error_text.rs | 20 ++++ .../src/runs/_helpers/lifecycle/mod.rs | 1 + .../runs/_helpers/lifecycle/publish_change.rs | 8 ++ .../runs/_helpers/lifecycle/run_failure.rs | 106 ++++++++++++++++++ .../tests/runs/test_lifecycle.rs | 32 +++++- .../sqlbuild/references/docs/cli/server.md | 73 ++++++++++++ website/src/content/docs/docs/cli/server.mdx | 58 ++++++++++ 11 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/run_failure.rs diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index ccc5e5cc8a..cd3e2fa5f0 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2292,9 +2292,24 @@ export interface components { kind: "launch_failed"; /** Format: uuid */ run_id: string; + } | { + /** Format: int64 */ + failed_node_count: number; + /** @description The first failed nodes by key, with redacted and bounded error text. */ + failed_nodes: components["schemas"]["RunNodeFailure"][]; + job_key?: string | null; + /** @enum {string} */ + kind: "run_failed"; + /** Format: uuid */ + location_id: string; + outcome: components["schemas"]["RunOutcome"]; + /** Format: uuid */ + run_id: string; + run_kind: components["schemas"]["RunKind"]; + target: string; }; /** @enum {string} */ - PlatformEventKind: "heartbeat" | "scheduler_no_leader" | "tick_overdue" | "calendar_horizon_low" | "location_broken" | "launch_failed"; + PlatformEventKind: "heartbeat" | "scheduler_no_leader" | "tick_overdue" | "calendar_horizon_low" | "location_broken" | "launch_failed" | "run_failed"; /** @description Problems derived from live state when read; nothing here is stored. */ PlatformProblems: { broken_locations: components["schemas"]["BrokenLocation"][]; diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/event_history.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/event_history.rs index b62df08dbf..a10bc5c9fb 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/event_history.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/event_history.rs @@ -65,6 +65,7 @@ fn person_event( Some(scopes.get(trigger_id).cloned()?) } PlatformEvent::LaunchFailed { run_id, .. } => Some(scopes.get(run_id).cloned()?), + PlatformEvent::RunFailed { .. } => return None, PlatformEvent::Heartbeat { .. } | PlatformEvent::SchedulerNoLeader => None, }; let (location_id, target) = scope.map_or((None, None), |(location_id, target)| { diff --git a/crates/sqlbuild-orchestration/src/platform/constants.rs b/crates/sqlbuild-orchestration/src/platform/constants.rs index 2753bdb529..f8833001d1 100644 --- a/crates/sqlbuild-orchestration/src/platform/constants.rs +++ b/crates/sqlbuild-orchestration/src/platform/constants.rs @@ -5,6 +5,9 @@ pub const PLATFORM_WEBHOOK_VARIABLE: &str = "SQLBUILD_PLATFORM_WEBHOOK_URL"; pub(crate) const PLATFORM_EVENT_PAGE_SIZE: i64 = 100; +/// A run-failure event names at most this many failed nodes and reports the full count beside them. +pub(crate) const RUN_FAILURE_NODE_LIMIT: i64 = 20; + /// Replica heartbeats older than this are pruned; a day's date-scoped invocation is complete by then. pub(crate) const HEARTBEAT_RETENTION_DAYS: i64 = 2; diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index d1e91f47d8..541216c440 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -6,6 +6,7 @@ use uuid::Uuid; use crate::api::models::ControlClient; use crate::declarations::models::TriggerDefinition; +use crate::runs::models::{RunKind, RunNodeFailure, RunOutcome}; use crate::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; use crate::warehouse_metadata::models::{WarehouseHistoryKind, WarehouseStateLocation}; @@ -186,6 +187,7 @@ pub enum PlatformEventKind { CalendarHorizonLow, LocationBroken, LaunchFailed, + RunFailed, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] @@ -216,6 +218,18 @@ pub enum PlatformEvent { claim_id: Uuid, code: String, }, + /// A job, group or ad-hoc run ended failed or partial; only the webhook receives it, never person reads. + RunFailed { + run_id: Uuid, + run_kind: RunKind, + job_key: Option, + location_id: Uuid, + target: String, + outcome: RunOutcome, + /// The first failed nodes by key, with redacted and bounded error text. + failed_nodes: Vec, + failed_node_count: i64, + }, } impl PlatformEvent { @@ -228,6 +242,7 @@ impl PlatformEvent { Self::CalendarHorizonLow { .. } => PlatformEventKind::CalendarHorizonLow, Self::LocationBroken { .. } => PlatformEventKind::LocationBroken, Self::LaunchFailed { .. } => PlatformEventKind::LaunchFailed, + Self::RunFailed { .. } => PlatformEventKind::RunFailed, } } @@ -240,6 +255,7 @@ impl PlatformEvent { Self::CalendarHorizonLow { .. } => "platform_calendar_horizon_low", Self::LocationBroken { .. } => "platform_location_broken", Self::LaunchFailed { .. } => "platform_launch_failed", + Self::RunFailed { .. } => "platform_run_failed", } } } diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/error_text.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/error_text.rs index c5914d86e2..8d316dfe63 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/error_text.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/error_text.rs @@ -1,11 +1,31 @@ //! Node error text is normalized while a run change is read, so no caller stores or returns it raw. use serde::{Deserialize, Deserializer}; +use serde_json::Value; use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; +use crate::runs::models::{RunChange, RunChangeRequest}; +use crate::store::errors::ControlError; + pub(crate) fn bounded_error_text<'de, D: Deserializer<'de>>( deserializer: D, ) -> Result, D::Error> { Ok(Option::::deserialize(deserializer)?.map(|text| redact_error_text(&text))) } + +/// The error text of a stored node-finished fact; `None` when there is no fact or it carries none. +pub(crate) fn finished_error_message(payload: &Value) -> Result, ControlError> { + if payload.is_null() { + return Ok(None); + } + Ok( + match serde_json::from_value::(payload["change"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .change + { + RunChange::NodeFinished { error_message, .. } => error_message, + _ => None, + }, + ) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs index 4b79246a14..85454af29d 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs @@ -12,4 +12,5 @@ pub(in crate::runs) mod publish_change; pub(in crate::runs) mod queued; pub(in crate::runs) mod reads; pub(in crate::runs) mod recall; +pub(in crate::runs) mod run_failure; pub(in crate::runs) mod waits; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs index 0ec90d7056..01eaef914b 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs @@ -5,6 +5,7 @@ use serde_json::json; use crate::platform::models::PlatformEvent; use crate::runs::_helpers::lifecycle::events::append_run_event; use crate::runs::_helpers::lifecycle::projection::project_change; +use crate::runs::_helpers::lifecycle::run_failure::publish_run_failure; use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; use crate::store::errors::ControlError; use crate::store::main::append_control_event::append_control_event; @@ -53,6 +54,13 @@ pub(crate) async fn publish_change( return Ok(transaction); } let mut transaction = project_change(transaction, request, state).await?; + if let RunChange::Finished { + outcome: outcome @ (RunOutcome::Failed | RunOutcome::Partial), + .. + } = request.change + { + transaction = publish_run_failure(transaction, request, outcome).await?; + } if let Some((node_key, _)) = request.change.finished_node() { (transaction, _) = query( transaction, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/run_failure.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/run_failure.rs new file mode 100644 index 0000000000..7629020aff --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/run_failure.rs @@ -0,0 +1,106 @@ +//! A failed or partial run becomes one platform event naming its job, scope and first failed nodes. + +use std::collections::BTreeSet; + +use serde_json::{Value, json}; + +use crate::platform::constants::RUN_FAILURE_NODE_LIMIT; +use crate::platform::models::PlatformEvent; +use crate::runs::_helpers::lifecycle::error_text::finished_error_message; +use crate::runs::models::{RunChangeRequest, RunNodeFailure, RunOutcome}; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; + +/// Runs after the terminal projection, so nodes left running already read as failed. +pub(crate) async fn publish_run_failure( + transaction: ControlTransaction, + request: &RunChangeRequest, + outcome: RunOutcome, +) -> Result { + let invalid = |error: serde_json::Error| ControlError::Invalid(error.to_string()); + let (transaction, runs) = query( + transaction, + "SELECT kind, job_key, location_id, target, snapshot_id FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let run = runs + .into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("finished run is missing".into()))?; + let (transaction, counted) = query( + transaction, + "SELECT COUNT(*) AS count FROM sqb_run_nodes WHERE run_id=$1 AND kind='node' AND status='failed'", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let (transaction, rows) = query( + transaction, + "SELECT n.node_key, n.error_code, e.payload FROM sqb_run_nodes n \ + LEFT JOIN recall_events e ON e.storage_id=n.finished_storage_id \ + WHERE n.run_id=$1 AND n.kind='node' AND n.status='failed' ORDER BY n.node_key LIMIT $2", + vec![ + Parameter::Id(request.run_id), + Parameter::Integer(RUN_FAILURE_NODE_LIMIT), + ], + ) + .await?; + let keys: BTreeSet = rows + .iter() + .filter_map(|row| row["node_key"].as_str().map(str::to_owned)) + .collect(); + let snapshot_id = serde_json::from_value(run["snapshot_id"].clone()).map_err(invalid)?; + let (mut transaction, nodes) = + read_snapshot_nodes(transaction, snapshot_id, Some(&keys)).await?; + let mut failed_nodes: Vec = Vec::new(); + for row in rows { + let node_key = row["node_key"].as_str().unwrap_or_default().to_owned(); + failed_nodes.push(RunNodeFailure { + error_code: row["error_code"].as_str().map(str::to_owned), + message: finished_error_message(&row["payload"])?, + path: nodes + .get(&node_key) + .and_then(|node| node.definition.get("path")) + .and_then(Value::as_str) + .map(|path| path.replace('\\', "/")), + node_key, + }); + } + let platform = PlatformEvent::RunFailed { + run_id: request.run_id, + run_kind: serde_json::from_value(run["kind"].clone()).map_err(invalid)?, + job_key: run["job_key"].as_str().map(str::to_owned), + location_id: serde_json::from_value(run["location_id"].clone()).map_err(invalid)?, + target: run["target"].as_str().unwrap_or_default().to_owned(), + outcome, + failed_nodes, + failed_node_count: counted + .first() + .and_then(|row| row["count"].as_i64()) + .unwrap_or_default(), + }; + let occurred_at = transaction.now; + let appended; + (transaction, appended) = append_control_event( + transaction, + ControlEvent { + event_id: uuid::Uuid::new_v5(&request.event_id, b"platform-run-failed"), + invocation_id: format!("platform/{}", request.run_id), + event_type: platform.event_type(), + producer: "platform", + occurred_at, + run_id: Some(request.run_id), + payload: json!({"platform": platform}), + }, + ) + .await?; + if appended.conflict.is_some() { + return Err(ControlError::Invalid( + "run failure conflicts with a durable fact".into(), + )); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index 25e79224a6..fcc3bb4fca 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -8,7 +8,8 @@ use crate::runs::main::heartbeat_run::heartbeat_run; use crate::runs::main::list_run_nodes::list_run_nodes; use crate::runs::main::list_runs::list_runs; use crate::runs::models::{ - NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunKind, RunNodeFailure, RunOutcome, + RunStatus, }; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::snapshots::main::publish_snapshot::publish_snapshot; @@ -438,6 +439,35 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ assert_eq!(nodes[0].status, "success"); assert_eq!(nodes[0].rows_affected, Some(17)); assert_eq!(nodes[1].error_code.as_deref(), Some("E100")); + let failures: Vec = read_recall_events(store, 0, 100) + .await + .expect("run failure facts") + .into_iter() + .filter(|row| row["event_type"] == "platform_run_failed") + .map(|row| { + serde_json::from_value(row["payload"]["platform"].clone()) + .expect("typed run failure") + }) + .collect(); + assert_eq!( + failures, + vec![PlatformEvent::RunFailed { + run_id: started.run_id, + run_kind: RunKind::Adhoc, + job_key: None, + location_id: location, + target: "development".into(), + outcome: RunOutcome::Partial, + failed_nodes: vec![RunNodeFailure { + node_key: "orders".into(), + error_code: Some("E100".into()), + message: None, + path: None, + }], + failed_node_count: 1, + }], + "a partial run publishes one failure event, surviving rolled-back attempts and retries" + ); let late_failure = RunChangeRequest { event_id: Uuid::now_v7(), ..launch_failed.clone() diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index d315dd698d..46e759c54d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -8,6 +8,8 @@ Online: https://sqlbuild.com/docs/cli/server/ ## Contents +- Serve over HTTPS +- Receive platform events - Upgrade - Set retention - Grant roles to OIDC groups (beta) @@ -25,6 +27,77 @@ sqb server Use `sqb server --help` for service, authentication and launcher options. +## Serve over HTTPS + +Code locations and workers send their bearer token with every request to the `--server` URL. +`sqb server` serves plain HTTP, so put TLS or an HTTPS reverse proxy in front of it and give +locations and workers its `https://` URL. When a location or worker connects to an `http://` URL +on another host, it prints this warning once and keeps running: + +```text +warning: the bearer token for http://control.example.com travels unencrypted over HTTP; put TLS or an HTTPS reverse proxy in front of the server +``` + +Connections to `localhost`, `127.0.0.1` or `::1` print no warning. + +## Receive platform events + +Set `SQLBUILD_PLATFORM_WEBHOOK_URL` in the server's environment to have it `POST` each platform +event to that URL as JSON. Leave it unset to turn delivery off. Locations and workers never +receive the URL. + +```json +{ + "storage_id": 4182, + "event_id": "01a1d3c2-6f0e-7a51-9b2e-3c4d5e6f7a80", + "occurred_at": "2026-11-13T07:12:04.381Z", + "payload": { + "platform": { + "kind": "run_failed", + "run_id": "01a1d3b9-2c4e-7f10-8a3b-1d2e3f4a5b6c", + "run_kind": "job", + "job_key": "daily_orders", + "location_id": "01a1d38b-e214-7151-8c8e-8606917c3ee6", + "target": "production", + "outcome": "partial", + "failed_nodes": [ + { + "node_key": "model:fct_orders", + "error_code": "E100", + "message": "Conversion Error: Could not convert string 'n/a' to INT32", + "path": "models/marts/fct_orders.sql" + } + ], + "failed_node_count": 1 + } + } +} +``` + +Delivery is at least once and in order. A response other than `2xx`, or no response within 10 +seconds, stops delivery and retries the same event. Deduplicate by `event_id`, which the +`Idempotency-Key` header repeats. + +| `kind` | Sent when | Fields | +| --- | --- | --- | +| `heartbeat` | Each server replica is alive, periodically | `replica_id` | +| `scheduler_no_leader` | No server holds the scheduler lease | None | +| `tick_overdue` | A schedule tick is late | `trigger_id`, `scheduled_for` | +| `calendar_horizon_low` | A running calendar schedule has a day or less of dates left | `trigger_id`, `refresh_at`, `horizon_end` | +| `location_broken` | Every instance of a location and target has stopped reporting for over 90 seconds | `location_id`, `target`, `last_seen_at` | +| `launch_failed` | Starting a worker failed; it may still start | `run_id`, `claim_id`, `code` | +| `run_failed` | A job, group or ad-hoc run ends `failed` or `partial` | `run_id`, `run_kind`, `job_key`, `location_id`, `target`, `outcome`, `failed_nodes`, `failed_node_count` | + +Each condition is sent once when it starts, not on every 30-second scan. Monitor heartbeats from +outside the server too: a stopped server can't report its own outage. + +`run_failed` lists at most 20 failed nodes, ordered by node key; `failed_node_count` is the full +number. A `partial` run is included because some of its nodes failed. `job_key` is `null` for +ad-hoc runs. Each `message` is the node's error text with credentials redacted, cut to 2,000 +characters, and `null` when the worker reported none. It can quote warehouse data, so treat the +webhook like other run evidence. `run_failed` events appear only in the webhook, not in the +server's platform history. + ## Upgrade Upgrade the server first, then the locations and workers that connect to it. Don't downgrade the diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index b4bd7d7e5b..c273f4963b 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -26,6 +26,64 @@ warning: the bearer token for http://control.example.com travels unencrypted ove Connections to `localhost`, `127.0.0.1` or `::1` print no warning. +## Receive platform events + +Set `SQLBUILD_PLATFORM_WEBHOOK_URL` in the server's environment to have it `POST` each platform +event to that URL as JSON. Leave it unset to turn delivery off. Locations and workers never +receive the URL. + +```json +{ + "storage_id": 4182, + "event_id": "01a1d3c2-6f0e-7a51-9b2e-3c4d5e6f7a80", + "occurred_at": "2026-11-13T07:12:04.381Z", + "payload": { + "platform": { + "kind": "run_failed", + "run_id": "01a1d3b9-2c4e-7f10-8a3b-1d2e3f4a5b6c", + "run_kind": "job", + "job_key": "daily_orders", + "location_id": "01a1d38b-e214-7151-8c8e-8606917c3ee6", + "target": "production", + "outcome": "partial", + "failed_nodes": [ + { + "node_key": "model:fct_orders", + "error_code": "E100", + "message": "Conversion Error: Could not convert string 'n/a' to INT32", + "path": "models/marts/fct_orders.sql" + } + ], + "failed_node_count": 1 + } + } +} +``` + +Delivery is at least once and in order. A response other than `2xx`, or no response within 10 +seconds, stops delivery and retries the same event. Deduplicate by `event_id`, which the +`Idempotency-Key` header repeats. + +| `kind` | Sent when | Fields | +| --- | --- | --- | +| `heartbeat` | Each server replica is alive, periodically | `replica_id` | +| `scheduler_no_leader` | No server holds the scheduler lease | None | +| `tick_overdue` | A schedule tick is late | `trigger_id`, `scheduled_for` | +| `calendar_horizon_low` | A running calendar schedule has a day or less of dates left | `trigger_id`, `refresh_at`, `horizon_end` | +| `location_broken` | Every instance of a location and target has stopped reporting for over 90 seconds | `location_id`, `target`, `last_seen_at` | +| `launch_failed` | Starting a worker failed; it may still start | `run_id`, `claim_id`, `code` | +| `run_failed` | A job, group or ad-hoc run ends `failed` or `partial` | `run_id`, `run_kind`, `job_key`, `location_id`, `target`, `outcome`, `failed_nodes`, `failed_node_count` | + +Each condition is sent once when it starts, not on every 30-second scan. Monitor heartbeats from +outside the server too: a stopped server can't report its own outage. + +`run_failed` lists at most 20 failed nodes, ordered by node key; `failed_node_count` is the full +number. A `partial` run is included because some of its nodes failed. `job_key` is `null` for +ad-hoc runs. Each `message` is the node's error text with credentials redacted, cut to 2,000 +characters, and `null` when the worker reported none. It can quote warehouse data, so treat the +webhook like other run evidence. `run_failed` events appear only in the webhook, not in the +server's platform history. + ## Upgrade Upgrade the server first, then the locations and workers that connect to it. Don't downgrade the From 75e40349e8e22706c6f85a337429d2d1f462c315 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 08:09:14 +0000 Subject: [PATCH 3/3] fix(ui): skip run-failure events in the person event history Person reads never return run_failed, but the schema's event union now includes it, so the UI history mapping and kind filter drop it. --- ui/src/lib/api/http-client.ts | 1 + ui/src/lib/api/http-problems.test.ts | 1 + ui/src/lib/api/http-problems.ts | 7 +++++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index 56cd7f6cac..e4ddf6b8ec 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -597,6 +597,7 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt const limit = query.limit ?? 200; const wanted = (wire: Wire['PersonPlatformEvent']) => (query.locationId === undefined || (wire.location_id && wire.target ? locationKey(wire.location_id, wire.target) : null) === query.locationId) && + wire.event.kind !== 'run_failed' && (!query.kinds?.length || query.kinds.includes(wire.event.kind)); /** Each scan skips the events the person may not see, so one can come back short; keep scanning until enough match. */ async function storedEvents(): Promise { diff --git a/ui/src/lib/api/http-problems.test.ts b/ui/src/lib/api/http-problems.test.ts index 523adcf75d..8c37d458a8 100644 --- a/ui/src/lib/api/http-problems.test.ts +++ b/ui/src/lib/api/http-problems.test.ts @@ -91,6 +91,7 @@ describe('toPlatformEventRecords', () => { open, [ { storage_id: 9, event_id: 'e9', occurred_at: '2026-11-08T11:00:00Z', location_id: 'l1', target: 'development', event: { kind: 'launch_failed', run_id: 'r1', claim_id: 'c1', code: 'image_missing' } }, + { storage_id: 7, event_id: 'e7', occurred_at: '2026-11-08T10:30:00Z', event: { kind: 'run_failed', run_id: 'r2', run_kind: 'job', job_key: 'orders_daily', location_id: 'l1', target: 'development', outcome: 'failed', failed_nodes: [{ node_key: 'model:orders' }], failed_node_count: 1 } }, { storage_id: 4, event_id: 'e4', occurred_at: '2026-11-08T10:00:00Z', event: { kind: 'scheduler_no_leader' } } ], new Map([['l1:development', 'orders']]) diff --git a/ui/src/lib/api/http-problems.ts b/ui/src/lib/api/http-problems.ts index 456997bde2..4c24b27d0d 100644 --- a/ui/src/lib/api/http-problems.ts +++ b/ui/src/lib/api/http-problems.ts @@ -91,7 +91,7 @@ export function toPlatformHealth(problems: Wire['PlatformProblems']): PlatformHe /** Stored events newest first; an open problem marks the newest stored event of its kind and place as ongoing, or comes first when none was stored. */ export function toPlatformEventRecords(open: PlatformEvent[], stored: Wire['PersonPlatformEvent'][], names: Map): PlatformEventRecord[] { - const records = stored.map((wire) => storedRecord(wire, names)); + const records = stored.flatMap((wire) => storedRecord(wire, names) ?? []); const unmatched: PlatformEventRecord[] = []; for (const event of open) { const latest = records.find((r) => !r.active && r.kind === event.kind && r.locationId === event.locationId); @@ -101,7 +101,8 @@ export function toPlatformEventRecords(open: PlatformEvent[], stored: Wire['Pers return [...unmatched, ...records]; } -function storedRecord(wire: Wire['PersonPlatformEvent'], names: Map): PlatformEventRecord { +/** Null for `run_failed`, which only the platform webhook receives; person reads never return it. */ +function storedRecord(wire: Wire['PersonPlatformEvent'], names: Map): PlatformEventRecord | null { const key = wire.location_id && wire.target ? keyOf(wire.location_id, wire.target) : null; const place = key && wire.target ? placeOf(names.get(key) ?? 'A location', wire.target) : null; const base = { id: wire.event_id, occurredAt: wire.occurred_at, active: false, locationId: key, subject: place, resolvedAt: null, reactions: [] }; @@ -119,5 +120,7 @@ function storedRecord(wire: Wire['PersonPlatformEvent'], names: Map