diff --git a/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py new file mode 100644 index 0000000000..77a937120d --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py @@ -0,0 +1,14 @@ +"""Layer jobs selected by tag share the function their models call, and still order by lineage. + +A tag selection brings in the functions its models call, so both jobs build the shared function. A +job that builds a node itself does not wait for that node's other builders, so the group runs the +marts layer after the staging layer instead of reporting a cycle. +""" + +from sqlbuild.orchestration import job, job_group + +staging = job("staging_layer", description="Build the staging layer.", select="tag:staging") +marts = job("marts_layer", description="Build the marts layer.", select="tag:marts") +layers = job_group( + "layered_orders", description="Build staging, then marts.", jobs=[marts, staging] +) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index ccc5e5cc8a..cd3e2fa5f0 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2292,9 +2292,24 @@ export interface components { kind: "launch_failed"; /** Format: uuid */ run_id: string; + } | { + /** Format: int64 */ + failed_node_count: number; + /** @description The first failed nodes by key, with redacted and bounded error text. */ + failed_nodes: components["schemas"]["RunNodeFailure"][]; + job_key?: string | null; + /** @enum {string} */ + kind: "run_failed"; + /** Format: uuid */ + location_id: string; + outcome: components["schemas"]["RunOutcome"]; + /** Format: uuid */ + run_id: string; + run_kind: components["schemas"]["RunKind"]; + target: string; }; /** @enum {string} */ - PlatformEventKind: "heartbeat" | "scheduler_no_leader" | "tick_overdue" | "calendar_horizon_low" | "location_broken" | "launch_failed"; + PlatformEventKind: "heartbeat" | "scheduler_no_leader" | "tick_overdue" | "calendar_horizon_low" | "location_broken" | "launch_failed" | "run_failed"; /** @description Problems derived from live state when read; nothing here is stored. */ PlatformProblems: { broken_locations: components["schemas"]["BrokenLocation"][]; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs index 376e0414f6..fc208658a0 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/mod.rs @@ -14,6 +14,7 @@ pub(crate) mod person_node_sql; pub(crate) mod person_operations; pub(crate) mod person_plans; pub(crate) mod person_reads; +pub(crate) mod plain_http; pub(crate) mod request_json; pub(crate) mod router; pub(crate) mod run_reads; diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/plain_http.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/plain_http.rs new file mode 100644 index 0000000000..60f535063e --- /dev/null +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/plain_http.rs @@ -0,0 +1,24 @@ +//! Warn once per process before a bearer token crosses the network over plain HTTP. + +use std::net::IpAddr; +use std::sync::Once; + +use crate::api::constants::{HTTP_SCHEME, LOOPBACK_HOST}; + +/// Loopback origins stay silent; any other `http://` origin warns on the first connection only. +pub(crate) fn warn_plain_http(url: &reqwest::Url) { + static WARNED: Once = Once::new(); + let loopback = url.host_str().is_some_and(|host| { + let host = host.trim_start_matches('[').trim_end_matches(']'); + host.eq_ignore_ascii_case(LOOPBACK_HOST) + || host.parse::().is_ok_and(|ip| ip.is_loopback()) + }); + if url.scheme() == HTTP_SCHEME && !loopback { + WARNED.call_once(|| { + eprintln!( + "warning: the bearer token for {} travels unencrypted over HTTP; put TLS or an HTTPS reverse proxy in front of the server", + url.origin().ascii_serialization() + ); + }); + } +} diff --git a/crates/sqlbuild-orchestration/src/api/constants.rs b/crates/sqlbuild-orchestration/src/api/constants.rs index 89bb616723..d88b1b5a6b 100644 --- a/crates/sqlbuild-orchestration/src/api/constants.rs +++ b/crates/sqlbuild-orchestration/src/api/constants.rs @@ -17,6 +17,8 @@ pub(crate) const OIDC_RESPONSE_BYTES: usize = 1024 * 1024; #[cfg(unix)] pub(crate) const DOCKER_IMAGE_HEX_LENGTH: usize = 64; pub(crate) const HTTPS_SCHEME: &str = "https"; +pub(crate) const HTTP_SCHEME: &str = "http"; +pub(crate) const LOOPBACK_HOST: &str = "localhost"; pub(crate) const ROOT_PATH: &str = "/"; pub(crate) const UI_INDEX_PATH: &str = "/index.html"; pub(crate) const UI_IMMUTABLE_PREFIX: &str = "/_app/immutable/"; diff --git a/crates/sqlbuild-orchestration/src/api/main/connect_client.rs b/crates/sqlbuild-orchestration/src/api/main/connect_client.rs index 3c717eb4f6..1e8d6b919d 100644 --- a/crates/sqlbuild-orchestration/src/api/main/connect_client.rs +++ b/crates/sqlbuild-orchestration/src/api/main/connect_client.rs @@ -1,8 +1,9 @@ -//! Validate the API origin before attaching a sensitive bearer credential. +//! Validate the API origin, and warn about plain HTTP, before attaching a sensitive bearer credential. use reqwest::header::HeaderValue; use crate::api::_helpers::resources::client::http_client; +use crate::api::_helpers::resources::plain_http::warn_plain_http; use crate::api::errors::ClientError; use crate::api::models::ControlClient; @@ -20,6 +21,7 @@ pub(crate) fn connect_client(server_url: &str, bearer: &str) -> Result, + (job, dependency): (&str, &str), + reason: impl FnOnce() -> String, +) -> BTreeMap<(String, String), String> { + reasons + .entry((job.to_owned(), dependency.to_owned())) + .or_insert_with(reason); + reasons +} + +/// Every job left waits on another, so following first dependencies from any of them closes a cycle. +pub(crate) fn describe_cycle( + remaining: &BTreeMap>, + reasons: &BTreeMap<(String, String), String>, +) -> String { + let mut path: Vec<&str> = Vec::new(); + let mut current = remaining.keys().next().map(String::as_str); + while let Some(name) = current { + if let Some(start) = path.iter().position(|seen| *seen == name) { + path.drain(..start); + break; + } + path.push(name); + current = remaining + .get(name) + .and_then(|deps| deps.iter().next()) + .map(String::as_str); + } + path.iter() + .zip(path.iter().cycle().skip(1)) + .map(|(job, dependency)| { + let reason = reasons + .get(&((*job).to_owned(), (*dependency).to_owned())) + .map_or("", String::as_str); + format!("{job} waits for {dependency} ({reason})") + }) + .collect::>() + .join("; ") +} diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs index c0e012e8e8..f8b81e37f0 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod development; pub(crate) mod documents; +pub(crate) mod group_waits; pub(crate) mod job_detail; pub(crate) mod operator_change; pub(crate) mod operator_state; diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs index aa906942d8..f0f5be3920 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/trigger_scope.rs @@ -50,8 +50,12 @@ pub(crate) async fn trigger_scope( exclude: Vec::new(), }; let (transaction, frozen) = frozen_job_request(transaction, (context, None), &job).await?; - let (transaction, nodes) = - job_write_set(transaction, frozen.snapshot_id, &frozen.selection).await?; + let (transaction, nodes) = job_write_set( + transaction, + frozen.snapshot_id, + (&frozen.selection, &frozen.settings), + ) + .await?; Ok(( transaction, PermissionScope { diff --git a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs index e896fba0a5..d2103f2439 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/group_dependencies.rs @@ -2,12 +2,13 @@ use std::collections::{BTreeMap, BTreeSet}; -use crate::declarations::models::JobDefinition; +use crate::declarations::_helpers::group_waits::{describe_cycle, record_wait}; +use crate::declarations::models::{GroupWaits, JobDefinition}; use crate::snapshots::models::SnapshotInput; use crate::store::errors::ControlError; pub fn group_dependencies( - input: &SnapshotInput, + (group, input): (&str, &SnapshotInput), jobs: &BTreeMap, (members, after): (&BTreeSet, &BTreeMap>), ) -> Result>, ControlError> { @@ -21,16 +22,17 @@ pub fn group_dependencies( "group needs known jobs and dependencies within its membership".into(), )); } - let edges = lineage_dependencies(input, jobs, (members, after)); - ensure_acyclic(&edges)?; - Ok(edges) + let waits = lineage_dependencies(input, jobs, (members, after)); + ensure_acyclic(group, &waits.edges, &waits.reasons)?; + Ok(waits.edges) } +/// A job that builds a node itself, such as a shared function, never waits for its other builders. fn lineage_dependencies( input: &SnapshotInput, jobs: &BTreeMap, (members, after): (&BTreeSet, &BTreeMap>), -) -> BTreeMap> { +) -> GroupWaits { let mut owners: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new(); for name in members { for key in &jobs[name].selection { @@ -38,8 +40,14 @@ fn lineage_dependencies( } } let mut edges: BTreeMap> = BTreeMap::new(); + let mut reasons: BTreeMap<(String, String), String> = BTreeMap::new(); for name in members { let mut dependencies = after.get(name).cloned().unwrap_or_default(); + for dependency in &dependencies { + reasons = record_wait(reasons, (name, dependency), || { + "declared in after".to_owned() + }); + } let mut seen: BTreeSet<&str> = BTreeSet::new(); let mut pending: Vec<&str> = jobs[name].selection.iter().map(String::as_str).collect(); while let Some(key) = pending.pop() { @@ -53,13 +61,15 @@ fn lineage_dependencies( .and_then(serde_json::Value::as_array) { for upstream in upstream.iter().filter_map(serde_json::Value::as_str) { - if let Some(owners) = owners.get(upstream) { - dependencies.extend( - owners - .iter() - .filter(|owner| **owner != name) - .map(|owner| (*owner).to_owned()), - ); + if let Some(owners) = owners.get(upstream) + && !owners.contains(name.as_str()) + { + for owner in owners { + dependencies.insert((*owner).to_owned()); + reasons = record_wait(reasons, (name, owner), || { + format!("{key} reads {upstream}") + }); + } } pending.push(upstream); } @@ -67,10 +77,14 @@ fn lineage_dependencies( } edges.insert(name.clone(), dependencies); } - edges + GroupWaits { edges, reasons } } -fn ensure_acyclic(edges: &BTreeMap>) -> Result<(), ControlError> { +fn ensure_acyclic( + group: &str, + edges: &BTreeMap>, + reasons: &BTreeMap<(String, String), String>, +) -> Result<(), ControlError> { let mut remaining = edges.clone(); while !remaining.is_empty() { let ready: BTreeSet = remaining @@ -79,9 +93,10 @@ fn ensure_acyclic(edges: &BTreeMap>) -> Result<(), Cont .map(|(name, _)| name.clone()) .collect(); if ready.is_empty() { - return Err(ControlError::Invalid( - "group dependencies contain a cycle".into(), - )); + return Err(ControlError::Invalid(format!( + "group dependencies contain a cycle in job group {group}: {}", + describe_cycle(&remaining, reasons) + ))); } remaining.retain(|name, deps| { deps.retain(|dependency| !ready.contains(dependency)); diff --git a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs index b89bfb316c..a415f62d13 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/snapshot_definitions.rs @@ -30,7 +30,8 @@ pub fn snapshot_definitions(input: &SnapshotInput) -> Result = group.jobs.keys().cloned().collect(); - let dependencies = group_dependencies(input, &definitions.jobs, (&members, &group.jobs))?; + let dependencies = + group_dependencies((name, input), &definitions.jobs, (&members, &group.jobs))?; let selection: BTreeSet = group .jobs .keys() diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index b67dadf618..fecd2146a4 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -249,3 +249,10 @@ pub(crate) struct PreviousTriggerState { pub(crate) last_tick_for: Option>, pub(crate) status_override: Option, } + +/// Each member job's waits, and why it waits: an explicit `after`, or the first lineage edge found. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct GroupWaits { + pub(crate) edges: BTreeMap>, + pub(crate) reasons: BTreeMap<(String, String), String>, +} diff --git a/crates/sqlbuild-orchestration/src/platform/_helpers/event_history.rs b/crates/sqlbuild-orchestration/src/platform/_helpers/event_history.rs index b62df08dbf..a10bc5c9fb 100644 --- a/crates/sqlbuild-orchestration/src/platform/_helpers/event_history.rs +++ b/crates/sqlbuild-orchestration/src/platform/_helpers/event_history.rs @@ -65,6 +65,7 @@ fn person_event( Some(scopes.get(trigger_id).cloned()?) } PlatformEvent::LaunchFailed { run_id, .. } => Some(scopes.get(run_id).cloned()?), + PlatformEvent::RunFailed { .. } => return None, PlatformEvent::Heartbeat { .. } | PlatformEvent::SchedulerNoLeader => None, }; let (location_id, target) = scope.map_or((None, None), |(location_id, target)| { diff --git a/crates/sqlbuild-orchestration/src/platform/constants.rs b/crates/sqlbuild-orchestration/src/platform/constants.rs index 2753bdb529..f8833001d1 100644 --- a/crates/sqlbuild-orchestration/src/platform/constants.rs +++ b/crates/sqlbuild-orchestration/src/platform/constants.rs @@ -5,6 +5,9 @@ pub const PLATFORM_WEBHOOK_VARIABLE: &str = "SQLBUILD_PLATFORM_WEBHOOK_URL"; pub(crate) const PLATFORM_EVENT_PAGE_SIZE: i64 = 100; +/// A run-failure event names at most this many failed nodes and reports the full count beside them. +pub(crate) const RUN_FAILURE_NODE_LIMIT: i64 = 20; + /// Replica heartbeats older than this are pruned; a day's date-scoped invocation is complete by then. pub(crate) const HEARTBEAT_RETENTION_DAYS: i64 = 2; diff --git a/crates/sqlbuild-orchestration/src/platform/models.rs b/crates/sqlbuild-orchestration/src/platform/models.rs index d1e91f47d8..541216c440 100644 --- a/crates/sqlbuild-orchestration/src/platform/models.rs +++ b/crates/sqlbuild-orchestration/src/platform/models.rs @@ -6,6 +6,7 @@ use uuid::Uuid; use crate::api::models::ControlClient; use crate::declarations::models::TriggerDefinition; +use crate::runs::models::{RunKind, RunNodeFailure, RunOutcome}; use crate::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest}; use crate::warehouse_metadata::models::{WarehouseHistoryKind, WarehouseStateLocation}; @@ -186,6 +187,7 @@ pub enum PlatformEventKind { CalendarHorizonLow, LocationBroken, LaunchFailed, + RunFailed, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] @@ -216,6 +218,18 @@ pub enum PlatformEvent { claim_id: Uuid, code: String, }, + /// A job, group or ad-hoc run ended failed or partial; only the webhook receives it, never person reads. + RunFailed { + run_id: Uuid, + run_kind: RunKind, + job_key: Option, + location_id: Uuid, + target: String, + outcome: RunOutcome, + /// The first failed nodes by key, with redacted and bounded error text. + failed_nodes: Vec, + failed_node_count: i64, + }, } impl PlatformEvent { @@ -228,6 +242,7 @@ impl PlatformEvent { Self::CalendarHorizonLow { .. } => PlatformEventKind::CalendarHorizonLow, Self::LocationBroken { .. } => PlatformEventKind::LocationBroken, Self::LaunchFailed { .. } => PlatformEventKind::LaunchFailed, + Self::RunFailed { .. } => PlatformEventKind::RunFailed, } } @@ -240,6 +255,7 @@ impl PlatformEvent { Self::CalendarHorizonLow { .. } => "platform_calendar_horizon_low", Self::LocationBroken { .. } => "platform_location_broken", Self::LaunchFailed { .. } => "platform_launch_failed", + Self::RunFailed { .. } => "platform_run_failed", } } } diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/error_text.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/error_text.rs index c5914d86e2..8d316dfe63 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/error_text.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/error_text.rs @@ -1,11 +1,31 @@ //! Node error text is normalized while a run change is read, so no caller stores or returns it raw. use serde::{Deserialize, Deserializer}; +use serde_json::Value; use sqlbuild_core::redaction::main::redact_error_text::redact_error_text; +use crate::runs::models::{RunChange, RunChangeRequest}; +use crate::store::errors::ControlError; + pub(crate) fn bounded_error_text<'de, D: Deserializer<'de>>( deserializer: D, ) -> Result, D::Error> { Ok(Option::::deserialize(deserializer)?.map(|text| redact_error_text(&text))) } + +/// The error text of a stored node-finished fact; `None` when there is no fact or it carries none. +pub(crate) fn finished_error_message(payload: &Value) -> Result, ControlError> { + if payload.is_null() { + return Ok(None); + } + Ok( + match serde_json::from_value::(payload["change"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))? + .change + { + RunChange::NodeFinished { error_message, .. } => error_message, + _ => None, + }, + ) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs index 4b79246a14..85454af29d 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs @@ -12,4 +12,5 @@ pub(in crate::runs) mod publish_change; pub(in crate::runs) mod queued; pub(in crate::runs) mod reads; pub(in crate::runs) mod recall; +pub(in crate::runs) mod run_failure; pub(in crate::runs) mod waits; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs index 0ec90d7056..01eaef914b 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs @@ -5,6 +5,7 @@ use serde_json::json; use crate::platform::models::PlatformEvent; use crate::runs::_helpers::lifecycle::events::append_run_event; use crate::runs::_helpers::lifecycle::projection::project_change; +use crate::runs::_helpers::lifecycle::run_failure::publish_run_failure; use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; use crate::store::errors::ControlError; use crate::store::main::append_control_event::append_control_event; @@ -53,6 +54,13 @@ pub(crate) async fn publish_change( return Ok(transaction); } let mut transaction = project_change(transaction, request, state).await?; + if let RunChange::Finished { + outcome: outcome @ (RunOutcome::Failed | RunOutcome::Partial), + .. + } = request.change + { + transaction = publish_run_failure(transaction, request, outcome).await?; + } if let Some((node_key, _)) = request.change.finished_node() { (transaction, _) = query( transaction, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/run_failure.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/run_failure.rs new file mode 100644 index 0000000000..7629020aff --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/run_failure.rs @@ -0,0 +1,106 @@ +//! A failed or partial run becomes one platform event naming its job, scope and first failed nodes. + +use std::collections::BTreeSet; + +use serde_json::{Value, json}; + +use crate::platform::constants::RUN_FAILURE_NODE_LIMIT; +use crate::platform::models::PlatformEvent; +use crate::runs::_helpers::lifecycle::error_text::finished_error_message; +use crate::runs::models::{RunChangeRequest, RunNodeFailure, RunOutcome}; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; +use crate::store::errors::ControlError; +use crate::store::main::append_control_event::append_control_event; +use crate::store::main::query::query; +use crate::store::models::{ControlEvent, ControlTransaction, Parameter}; + +/// Runs after the terminal projection, so nodes left running already read as failed. +pub(crate) async fn publish_run_failure( + transaction: ControlTransaction, + request: &RunChangeRequest, + outcome: RunOutcome, +) -> Result { + let invalid = |error: serde_json::Error| ControlError::Invalid(error.to_string()); + let (transaction, runs) = query( + transaction, + "SELECT kind, job_key, location_id, target, snapshot_id FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let run = runs + .into_iter() + .next() + .ok_or_else(|| ControlError::Invalid("finished run is missing".into()))?; + let (transaction, counted) = query( + transaction, + "SELECT COUNT(*) AS count FROM sqb_run_nodes WHERE run_id=$1 AND kind='node' AND status='failed'", + vec![Parameter::Id(request.run_id)], + ) + .await?; + let (transaction, rows) = query( + transaction, + "SELECT n.node_key, n.error_code, e.payload FROM sqb_run_nodes n \ + LEFT JOIN recall_events e ON e.storage_id=n.finished_storage_id \ + WHERE n.run_id=$1 AND n.kind='node' AND n.status='failed' ORDER BY n.node_key LIMIT $2", + vec![ + Parameter::Id(request.run_id), + Parameter::Integer(RUN_FAILURE_NODE_LIMIT), + ], + ) + .await?; + let keys: BTreeSet = rows + .iter() + .filter_map(|row| row["node_key"].as_str().map(str::to_owned)) + .collect(); + let snapshot_id = serde_json::from_value(run["snapshot_id"].clone()).map_err(invalid)?; + let (mut transaction, nodes) = + read_snapshot_nodes(transaction, snapshot_id, Some(&keys)).await?; + let mut failed_nodes: Vec = Vec::new(); + for row in rows { + let node_key = row["node_key"].as_str().unwrap_or_default().to_owned(); + failed_nodes.push(RunNodeFailure { + error_code: row["error_code"].as_str().map(str::to_owned), + message: finished_error_message(&row["payload"])?, + path: nodes + .get(&node_key) + .and_then(|node| node.definition.get("path")) + .and_then(Value::as_str) + .map(|path| path.replace('\\', "/")), + node_key, + }); + } + let platform = PlatformEvent::RunFailed { + run_id: request.run_id, + run_kind: serde_json::from_value(run["kind"].clone()).map_err(invalid)?, + job_key: run["job_key"].as_str().map(str::to_owned), + location_id: serde_json::from_value(run["location_id"].clone()).map_err(invalid)?, + target: run["target"].as_str().unwrap_or_default().to_owned(), + outcome, + failed_nodes, + failed_node_count: counted + .first() + .and_then(|row| row["count"].as_i64()) + .unwrap_or_default(), + }; + let occurred_at = transaction.now; + let appended; + (transaction, appended) = append_control_event( + transaction, + ControlEvent { + event_id: uuid::Uuid::new_v5(&request.event_id, b"platform-run-failed"), + invocation_id: format!("platform/{}", request.run_id), + event_type: platform.event_type(), + producer: "platform", + occurred_at, + run_id: Some(request.run_id), + payload: json!({"platform": platform}), + }, + ) + .await?; + if appended.conflict.is_some() { + return Err(ControlError::Invalid( + "run failure conflicts with a durable fact".into(), + )); + } + Ok(transaction) +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index 137c35efee..082f491538 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -59,8 +59,12 @@ pub(crate) async fn frozen_adhoc_request( let refusal = format!("workers do not run sqb {}", plan.verb); return Ok((transaction, Err(refusal))); }; - let (transaction, writers) = - job_write_set(transaction, snapshot.snapshot_id, &plan.selection).await?; + let (transaction, writers) = job_write_set( + transaction, + snapshot.snapshot_id, + (&plan.selection, &plan.settings), + ) + .await?; let (transaction, images) = query( transaction, "SELECT image_id FROM sqb_snapshots WHERE snapshot_id=$1", diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs index 939a2c7c4f..e5d1050ee3 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/job.rs @@ -31,8 +31,7 @@ pub(crate) async fn frozen_group_children( "group scope differs from its pinned membership".into(), )); } - let (transaction, writers) = - job_write_set(transaction, request.snapshot_id, &request.selection).await?; + let mut transaction = transaction; let mut children: Vec = Vec::new(); for name in group.jobs.keys() { let job = &definitions.jobs[name]; @@ -48,6 +47,9 @@ pub(crate) async fn frozen_group_children( if let Some(full_refresh) = request.settings.get(FULL_REFRESH_SETTING) { settings.insert(FULL_REFRESH_SETTING.into(), full_refresh.clone()); } + let write_set; + (transaction, write_set) = + job_write_set(transaction, request.snapshot_id, (&selection, &settings)).await?; children.push(RunRequest { run_id: Uuid::new_v5(&request.run_id, name.as_bytes()), kind: RunKind::Job, @@ -57,7 +59,7 @@ pub(crate) async fn frozen_group_children( rerun_of_run_id: request .rerun_of_run_id .map(|id| Uuid::new_v5(&id, name.as_bytes())), - write_set: selection.intersection(&writers).cloned().collect(), + write_set, selection, settings, concurrency_groups: job.concurrency_groups.clone(), diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs index 4a9acad803..16bdf00f11 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/permissions.rs @@ -58,7 +58,11 @@ pub(crate) fn execution_scopes( action: Action, ) -> Vec { let nodes: BTreeSet = if request.kind == RunKind::Adhoc { - request.selection.clone() + request + .selection + .union(&request.write_set) + .cloned() + .collect() } else { executions .iter() diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs index 9b5d67be48..4eb9c02062 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/publication.rs @@ -8,6 +8,7 @@ use uuid::Uuid; use crate::credentials::models::AuditContext; use crate::runs::main::queue_run::queue_run; use crate::runs::models::RunRequest; +use crate::snapshots::constants::SOURCE_KEY_PREFIX; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; @@ -59,10 +60,13 @@ pub(crate) async fn publish_request( ) -> Result<(ControlTransaction, bool), ControlError> { if request.requested_by.trim().is_empty() || request.selection.is_empty() - || !request.write_set.is_subset(&request.selection) + || request + .write_set + .iter() + .any(|key| !request.selection.contains(key) && !key.starts_with(SOURCE_KEY_PREFIX)) { return Err(ControlError::Invalid( - "run needs an actor, selection and a contained write set".into(), + "run needs an actor, selection and a write set within it or its loaded sources".into(), )); } let (transaction, previous) = query( diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs index 30ae961508..6642c2afbe 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs @@ -9,7 +9,9 @@ use crate::authorization::main::authorize_listing::authorize_listing; use crate::authorization::main::authorize_person::authorize_person; use crate::credentials::models::{Action, AuditContext, DefaultRole, PermissionScope}; use crate::runs::_helpers::requests::permissions::authorize_execution; -use crate::runs::models::{RerunRequest, RunOrigin, RunRequest, RunStatus}; +use crate::runs::models::{RerunRequest, RunCommand, RunKind, RunOrigin, RunRequest, RunStatus}; +use crate::snapshots::constants::SOURCE_KEY_PREFIX; +use crate::snapshots::main::job_write_set::job_write_set; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; @@ -58,9 +60,6 @@ pub(crate) async fn frozen_rerun( .await?; } frozen.selection.retain(|key| named.contains(key.as_str())); - frozen - .write_set - .retain(|key| frozen.selection.contains(key)); } if request.from_failure { let rows; @@ -75,9 +74,21 @@ pub(crate) async fn frozen_rerun( .filter_map(|row| row["node_key"].as_str()) .collect(); frozen.selection.retain(|key| failed.contains(key.as_str())); - frozen - .write_set - .retain(|key| frozen.selection.contains(key)); + } + let writers; + (transaction, writers) = job_write_set( + transaction, + frozen.snapshot_id, + (&frozen.selection, &frozen.settings), + ) + .await?; + frozen.write_set.retain(|key| writers.contains(key)); + if frozen.command == RunCommand::Build && frozen.kind != RunKind::Group { + frozen.write_set.extend( + writers + .into_iter() + .filter(|key| key.starts_with(SOURCE_KEY_PREFIX)), + ); } frozen.run_id = request.run_id; frozen.origin = RunOrigin::Rerun; diff --git a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs index dd3ca6fc2f..cdcc2afaaa 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs @@ -60,8 +60,12 @@ pub(crate) async fn frozen_job_request( ) .await?; } - let (transaction, write_set) = - job_write_set(transaction, snapshot_id, &definition.selection).await?; + let (transaction, write_set) = job_write_set( + transaction, + snapshot_id, + (&definition.selection, &definition.settings), + ) + .await?; Ok(( transaction, RunRequest { diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs index 29ee7f287e..8d4d589957 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -17,7 +17,7 @@ pub(crate) async fn persist_ticks( let (mut transaction, write_set) = job_write_set( transaction, trigger.snapshot_id, - &trigger.job_json.selection, + (&trigger.job_json.selection, &trigger.job_json.settings), ) .await?; for action in &plan.actions { diff --git a/crates/sqlbuild-orchestration/src/snapshots/constants.rs b/crates/sqlbuild-orchestration/src/snapshots/constants.rs index f931b29c58..cf3e62ba43 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/constants.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/constants.rs @@ -48,3 +48,12 @@ pub(crate) const CONTENT: [((&str, &str), &str, &str); 3] = [ "", ), ]; +/// Node kinds a build writes when they are selected. +pub(crate) const WRITER_NODE_KINDS: [&str; 4] = ["model", "seed", "udf", "table_fn"]; +pub(crate) const MODEL_NODE_KIND: &str = "model"; +pub(crate) const SOURCE_NODE_KIND: &str = "source"; +pub(crate) const SOURCE_KEY_PREFIX: &str = "source:"; +/// A source with a non-null loader here is managed, so automatic source loading writes it. +pub(crate) const LOADER_POINTER: &str = "/entry/loader"; +/// The frozen setting for `--no-load-sources`; `false` stops a build loading sources its models read. +pub(crate) const LOAD_SOURCES_SETTING: &str = "load_sources"; diff --git a/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs b/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs index 3060d1b261..a8f24c6600 100644 --- a/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs +++ b/crates/sqlbuild-orchestration/src/snapshots/main/job_write_set.rs @@ -1,30 +1,61 @@ //! Resolve a job's physical writers from its pinned snapshot. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; +use serde_json::Value; use uuid::Uuid; +use crate::snapshots::constants::{ + LOAD_SOURCES_SETTING, LOADER_POINTER, MODEL_NODE_KIND, SOURCE_KEY_PREFIX, SOURCE_NODE_KIND, + WRITER_NODE_KINDS, +}; +use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; use crate::store::errors::ControlError; -use crate::store::main::query::query; -use crate::store::models::{ControlTransaction, Parameter}; +use crate::store::models::ControlTransaction; +/// Selected writers plus the managed sources a build loads: selected ones, and parents of selected models unless loading is off. pub(crate) async fn job_write_set( transaction: ControlTransaction, snapshot_id: Uuid, - selection: &BTreeSet, + (selection, settings): (&BTreeSet, &BTreeMap), ) -> Result<(ControlTransaction, BTreeSet), ControlError> { - let (transaction, rows) = query(transaction, - "SELECT node_key FROM sqb_snapshot_nodes WHERE snapshot_id=$1 AND node_kind IN ('model','seed','udf','table_fn')", - vec![Parameter::Id(snapshot_id)]).await?; - Ok(( - transaction, - rows.into_iter() - .filter_map(|row| { - row.get("node_key") - .and_then(serde_json::Value::as_str) - .map(str::to_owned) + let (transaction, selected) = + read_snapshot_nodes(transaction, snapshot_id, Some(selection)).await?; + let loads_model_sources = settings.get(LOAD_SOURCES_SETTING) != Some(&Value::Bool(false)); + let mut writers: BTreeSet = BTreeSet::new(); + let mut sources: BTreeSet = BTreeSet::new(); + for (key, node) in &selected { + if WRITER_NODE_KINDS.contains(&node.kind.as_str()) { + writers.insert(key.clone()); + } + if node.kind == SOURCE_NODE_KIND { + sources.insert(key.clone()); + } else if node.kind == MODEL_NODE_KIND && loads_model_sources { + sources.extend( + node.definition + .get("deps") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_str) + .filter(|key| key.starts_with(SOURCE_KEY_PREFIX)) + .map(str::to_owned), + ); + } + } + let (transaction, sources) = + read_snapshot_nodes(transaction, snapshot_id, Some(&sources)).await?; + writers.extend( + sources + .into_iter() + .filter(|(_, node)| { + node.kind == SOURCE_NODE_KIND + && node + .definition + .pointer(LOADER_POINTER) + .is_some_and(|loader| !loader.is_null()) }) - .filter(|key| selection.contains(key)) - .collect(), - )) + .map(|(key, _)| key), + ); + Ok((transaction, writers)) } diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index b8aab47a25..9fd636bed0 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -135,7 +135,7 @@ pub(crate) fn json_column(value: &Value) -> Value { }) } -/// A publication of two models, a seed and a source, with one person per role under test. +/// Two models, a seed, a read-only source, a loaded customers source and one person per role under test. pub(crate) async fn adhoc_fixture( store: &ControlStore, context: &AuditContext, @@ -153,6 +153,14 @@ pub(crate) async fn adhoc_fixture( .into_iter() .map(published_node) .collect(); + let (key, mut managed) = published_node("source:shop.customers"); + managed.definition = json!({"entry": {"loader": "load_customers"}}); + input.nodes.insert(key, managed); + input + .nodes + .get_mut("model:customers") + .expect("customers model") + .definition = json!({"materialization": "table", "deps": ["source:shop.customers"]}); let published = publish_snapshot(store, &input).await.expect("publish"); let admin = bootstrap_user(store, context, "orders-admin") .await diff --git a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs index 24e87728a1..1662bd3364 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_adhoc.rs @@ -32,6 +32,26 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ "permissions": [["run.adhoc", true]], "error": null, "launchable": true}), expected_launch: Ok(true), }, + AdhocCommandTestCase { + description: "a build claims the managed source it loads for a selected model", + person: AdhocPerson::Developer, + command: &["build", "model:customers"], + priority: 0, + reason: "Refresh customers", + expected_check: json!({"selection": ["model:customers"], "write_set": ["model:customers", "source:shop.customers"], + "permissions": [["run.adhoc", true]], "error": null, "launchable": true}), + expected_launch: Ok(true), + }, + AdhocCommandTestCase { + description: "a build that does not load sources claims only its own writers", + person: AdhocPerson::Developer, + command: &["build", "model:customers", "--no-load-sources"], + priority: 0, + reason: "Refresh customers only", + expected_check: json!({"selection": ["model:customers"], "write_set": ["model:customers"], + "permissions": [["run.adhoc", true]], "error": null, "launchable": true}), + expected_launch: Ok(true), + }, AdhocCommandTestCase { description: "every gated flag is its own permission, and Developer holds none of them", person: AdhocPerson::Developer, @@ -272,6 +292,8 @@ fn given_ad_hoc_commands_when_people_check_and_launch_them_then_each_gated_flag_ .collect::>(), vec![ json!(["build", {}, "Refresh orders"]), + json!(["build", {}, "Refresh customers"]), + json!(["build", {"load_sources": false}, "Refresh customers only"]), json!(["build", {"full_refresh": true, "warehouse": "BUILD_XS"}, "Rebuild orders now"]), json!(["test", {}, "Check orders"]), ], diff --git a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs index 25e79224a6..fcc3bb4fca 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_lifecycle.rs @@ -8,7 +8,8 @@ use crate::runs::main::heartbeat_run::heartbeat_run; use crate::runs::main::list_run_nodes::list_run_nodes; use crate::runs::main::list_runs::list_runs; use crate::runs::models::{ - NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunOutcome, RunStatus, + NodeOutcome, RunAuthority, RunChange, RunChangeRequest, RunKind, RunNodeFailure, RunOutcome, + RunStatus, }; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::snapshots::main::publish_snapshot::publish_snapshot; @@ -438,6 +439,35 @@ fn given_a_claimed_worker_when_reports_are_retried_and_fail_between_writes_then_ assert_eq!(nodes[0].status, "success"); assert_eq!(nodes[0].rows_affected, Some(17)); assert_eq!(nodes[1].error_code.as_deref(), Some("E100")); + let failures: Vec = read_recall_events(store, 0, 100) + .await + .expect("run failure facts") + .into_iter() + .filter(|row| row["event_type"] == "platform_run_failed") + .map(|row| { + serde_json::from_value(row["payload"]["platform"].clone()) + .expect("typed run failure") + }) + .collect(); + assert_eq!( + failures, + vec![PlatformEvent::RunFailed { + run_id: started.run_id, + run_kind: RunKind::Adhoc, + job_key: None, + location_id: location, + target: "development".into(), + outcome: RunOutcome::Partial, + failed_nodes: vec![RunNodeFailure { + node_key: "orders".into(), + error_code: Some("E100".into()), + message: None, + path: None, + }], + failed_node_count: 1, + }], + "a partial run publishes one failure event, surviving rolled-back attempts and retries" + ); let late_failure = RunChangeRequest { event_id: Uuid::now_v7(), ..launch_failed.clone() diff --git a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs index 86490c97bd..7fff917906 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_reruns.rs @@ -59,6 +59,15 @@ fn given_a_partial_group_when_rerun_after_redeploy_then_only_failed_and_skipped_ ip_address: None, }; let mut input = snapshot(location); + let mut managed = input.nodes["orders"].clone(); + managed.kind = "source".into(); + managed.definition = json!({"entry": {"loader": "load_raw_orders"}}); + input.nodes.insert("source:raw_orders".into(), managed); + input + .nodes + .get_mut("orders") + .expect("orders model") + .definition = json!({"materialization": "table", "deps": ["source:raw_orders"]}); input.documents.insert("jobs".into(), json!({ "staging": {"description": "Stage orders", "selection": ["orders", "customers"], "concurrency_groups": ["warehouse"], "settings": {"vars": {"batch": 23}}}, "marts": {"description": "Build inventory", "selection": ["inventory"], "concurrency_groups": ["inventory_limit"]}, diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 9d38196822..017d2d8798 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -162,7 +162,7 @@ pub(crate) struct AdhocCommandTestCase { pub(crate) expected_launch: Result, } -/// The CLI's planner reduced to exact node keys, `--full-refresh` and `--warehouse NAME`. +/// The CLI's planner reduced to exact node keys, `--full-refresh`, `--no-load-sources` and `--warehouse NAME`. pub(crate) struct ExactKeyPlanner; impl crate::runs::types::CommandPlanner for ExactKeyPlanner { @@ -189,6 +189,11 @@ impl crate::runs::types::CommandPlanner for ExactKeyPlanner { .insert("full_refresh".into(), serde_json::json!(true)); continue; } + "--no-load-sources" => { + plan.settings + .insert("load_sources".into(), serde_json::json!(false)); + continue; + } "--warehouse" => { let name = tokens.next().ok_or("--warehouse needs a name")?; plan.settings diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index c926c807f7..48f7574c19 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -163,7 +163,7 @@ pub(crate) fn compile_declarations( .collect(), ); } - let dependencies = group_dependencies(&input, &jobs, (&members, &after)) + let dependencies = group_dependencies((&group.name, &input), &jobs, (&members, &after)) .map_err(|error| PyValueError::new_err(error.to_string()))?; let selection: BTreeSet = members .iter() diff --git a/fensu.toml b/fensu.toml index e9afc99dd8..7fa4ceac3f 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1701,6 +1701,12 @@ path = "crates/sqlbuild-cli/src/person_login/models.rs" symbols = ["PersonCredential", "LoginFlow"] reason = "Person credential and handoff models implement Debug manually to redact bearer, CSRF and approval secrets." +[[targets.rust.rule_exceptions]] +rule = "FPRSH011" +path = "crates/sqlbuild-orchestration/src/api/_helpers/resources/plain_http.rs" +symbols = ["warn_plain_http"] +reason = "Locations, workers and remote publication share this client; it warns once per process on stderr, before any request, when a bearer token would travel over plain HTTP to a non-loopback host." + [[targets.rust.rule_exceptions]] rule = "FPRSS201" path = "crates/sqlbuild-orchestration/src/webhooks/models.rs" diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index d315dd698d..46e759c54d 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -8,6 +8,8 @@ Online: https://sqlbuild.com/docs/cli/server/ ## Contents +- Serve over HTTPS +- Receive platform events - Upgrade - Set retention - Grant roles to OIDC groups (beta) @@ -25,6 +27,77 @@ sqb server Use `sqb server --help` for service, authentication and launcher options. +## Serve over HTTPS + +Code locations and workers send their bearer token with every request to the `--server` URL. +`sqb server` serves plain HTTP, so put TLS or an HTTPS reverse proxy in front of it and give +locations and workers its `https://` URL. When a location or worker connects to an `http://` URL +on another host, it prints this warning once and keeps running: + +```text +warning: the bearer token for http://control.example.com travels unencrypted over HTTP; put TLS or an HTTPS reverse proxy in front of the server +``` + +Connections to `localhost`, `127.0.0.1` or `::1` print no warning. + +## Receive platform events + +Set `SQLBUILD_PLATFORM_WEBHOOK_URL` in the server's environment to have it `POST` each platform +event to that URL as JSON. Leave it unset to turn delivery off. Locations and workers never +receive the URL. + +```json +{ + "storage_id": 4182, + "event_id": "01a1d3c2-6f0e-7a51-9b2e-3c4d5e6f7a80", + "occurred_at": "2026-11-13T07:12:04.381Z", + "payload": { + "platform": { + "kind": "run_failed", + "run_id": "01a1d3b9-2c4e-7f10-8a3b-1d2e3f4a5b6c", + "run_kind": "job", + "job_key": "daily_orders", + "location_id": "01a1d38b-e214-7151-8c8e-8606917c3ee6", + "target": "production", + "outcome": "partial", + "failed_nodes": [ + { + "node_key": "model:fct_orders", + "error_code": "E100", + "message": "Conversion Error: Could not convert string 'n/a' to INT32", + "path": "models/marts/fct_orders.sql" + } + ], + "failed_node_count": 1 + } + } +} +``` + +Delivery is at least once and in order. A response other than `2xx`, or no response within 10 +seconds, stops delivery and retries the same event. Deduplicate by `event_id`, which the +`Idempotency-Key` header repeats. + +| `kind` | Sent when | Fields | +| --- | --- | --- | +| `heartbeat` | Each server replica is alive, periodically | `replica_id` | +| `scheduler_no_leader` | No server holds the scheduler lease | None | +| `tick_overdue` | A schedule tick is late | `trigger_id`, `scheduled_for` | +| `calendar_horizon_low` | A running calendar schedule has a day or less of dates left | `trigger_id`, `refresh_at`, `horizon_end` | +| `location_broken` | Every instance of a location and target has stopped reporting for over 90 seconds | `location_id`, `target`, `last_seen_at` | +| `launch_failed` | Starting a worker failed; it may still start | `run_id`, `claim_id`, `code` | +| `run_failed` | A job, group or ad-hoc run ends `failed` or `partial` | `run_id`, `run_kind`, `job_key`, `location_id`, `target`, `outcome`, `failed_nodes`, `failed_node_count` | + +Each condition is sent once when it starts, not on every 30-second scan. Monitor heartbeats from +outside the server too: a stopped server can't report its own outage. + +`run_failed` lists at most 20 failed nodes, ordered by node key; `failed_node_count` is the full +number. A `partial` run is included because some of its nodes failed. `job_key` is `null` for +ad-hoc runs. Each `message` is the node's error text with credentials redacted, cut to 2,000 +characters, and `null` when the worker reported none. It can quote warehouse data, so treat the +webhook like other run evidence. `run_failed` events appear only in the webhook, not in the +server's platform history. + ## Upgrade Upgrade the server first, then the locations and workers that connect to it. Don't downgrade the diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py index e542ca01a9..482ef37d1d 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/_test_types.py @@ -54,6 +54,13 @@ class SnapshotPublicationTestCase: expected_nodes: frozenset[str] +@dataclass(frozen=True) +class PlainHttpWarningTestCase: + description: str + host: str + expected_warnings: int + + @dataclass(frozen=True) class SnapshotWorkerTestCase: description: str diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py index 49b4a75a47..bc538509ff 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_publication.py @@ -1,9 +1,11 @@ """Real compile, native transport, authenticated server and durable publication.""" import json +import os import socket import sqlite3 import subprocess +import sys from pathlib import Path from typing import Any from urllib.parse import quote @@ -27,6 +29,7 @@ TYPE_PROOF_FILES, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication._test_types import ( + PlainHttpWarningTestCase, SnapshotPublicationTestCase, ) from tests.integration.src.sqlbuild.compiler.compiled_project.publication.helpers import ( @@ -303,3 +306,64 @@ def test_given_a_code_location_when_publishing_from_cli_then_compilation_isolate delivery_output: CaptureResult[str] = capsys.readouterr() assert "was published, but result delivery failed" in delivery_output.err assert current_snapshot(api, test_case.target)["record"]["generation"] == 3 + + +@pytest.mark.parametrize( + "test_case", + [ + PlainHttpWarningTestCase( + description="a loopback server gets the token without a warning", + host="127.0.0.1", + expected_warnings=0, + ), + PlainHttpWarningTestCase( + description="a plain-HTTP server on another address is warned about once per process", + host="0.0.0.0", + expected_warnings=1, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_plain_http_server_when_a_location_publishes_then_only_a_non_loopback_origin_warns_once( + test_case: PlainHttpWarningTestCase, tmp_path: Path +) -> None: + project_dir: Path = tmp_path / "project" + write_files( + project_dir=project_dir, + files=TYPE_PROOF_FILES + | { + "sqlbuild_project.toml": TYPE_PROOF_FILES["sqlbuild_project.toml"] + + '\n[connections.local]\ndatabase = "warehouse.duckdb"\n' + + '\n[targets.development]\nschema = "main"\n', + }, + ) + with snapshot_api(tmp_path / "control", server_arguments=("--host", "0.0.0.0")) as api: + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + str(Path(sys.executable).with_name("sqb")), + "--no-color", + "--project-dir", + str(project_dir), + "location", + "publish", + "--server", + f"http://{test_case.host}:{api.url.rpartition(':')[2]}", + "--location-id", + api.location_id, + "--target", + "development", + "--generation", + "1", + ], + capture_output=True, + text=True, + timeout=120, + env=os.environ + | { + "SQLBUILD_LOCATION_TOKEN": api.bearer, + "SQLBUILD_CONTROL_DATABASE_URL": "unsupported://must-not-open", + }, + ) + assert result.returncode == 0, result.stderr + assert api.bearer not in result.stderr + assert result.stderr.count("travels unencrypted over HTTP") == test_case.expected_warnings diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index 96029d4518..dd84ed7816 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -456,6 +456,21 @@ def test_given_a_retained_baseline_when_starting_a_worker_then_verification_and_ "success", [(23,)], ), + WorkerExecutionTestCase( + "a scheduled build loads the managed source its model reads", + 'SELECT CAST(order_id AS BIGINT) AS amount FROM __source("raw_orders")', + "SELECT amount FROM orders", + "success", + [(1,)], + files={ + "sources/orders.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/managed_orders.yml" + ).read_text(), + "python/loaders/orders.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/orders_loader.py" + ).read_text(), + }, + ), WorkerExecutionTestCase( "a platform sensor catches up retained events through a real worker", "SELECT 47 AS amount", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 91484c00fb..5bfb5e9d9a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -887,6 +887,35 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject } }, ), + SnapshotDiscoveryTestCase( + description="DuckShop layer jobs that share a function wait only by lineage", + files={ + "orchestration/jobs.py": Path( + "benchmarks/duckshop/edge_cases/orchestration/shared_function_layers.py" + ).read_text(), + "functions/sql/order_cents.sql": ( + "FUNCTION (description 'Amount in cents.'," + " arguments (amount INTEGER), returns INTEGER);\nSELECT amount * 100\n" + ), + "models/stg_orders.sql": ( + "MODEL (description 'Staged orders.', tags [staging],\n" + " columns (order_id (type INTEGER), cents (type INTEGER)),\n" + ");\n\n" + "SELECT CAST(1 AS INTEGER) AS order_id," + ' __udf("order_cents")(CAST(5 AS INTEGER)) AS cents\n' + ), + "models/orders.sql": ( + "MODEL (description 'Typed orders.', tags [marts],\n" + " columns (order_id (type INTEGER), cents (type INTEGER)),\n" + ");\n\n" + 'SELECT order_id, __udf("order_cents")(cents) AS cents\nFROM __ref("stg_orders")\n' + ), + }, + expected_jobs=("marts_layer", "staging_layer"), + expected_groups={ + "layered_orders": {"marts_layer": ["staging_layer"], "staging_layer": []} + }, + ), SnapshotDiscoveryTestCase( description="explicit empty declarations do not execute project files", files={"orchestration/jobs.py": "raise RuntimeError('must not import')\n"}, @@ -1250,7 +1279,10 @@ def test_given_declaration_modules_when_compiling_then_native_discovery_collects ), }, expected_error=ValueError, - expected_message="group dependencies contain a cycle", + expected_message=re.escape( + "group dependencies contain a cycle in job group daily: marts waits for staging " + "(model:orders reads model:stg_orders); staging waits for marts (declared in after)" + ), ), SnapshotDiscoveryRejectionTestCase( description="group waits cannot refer to nonmember jobs", diff --git a/ui/src/lib/api/http-client.ts b/ui/src/lib/api/http-client.ts index 56cd7f6cac..e4ddf6b8ec 100644 --- a/ui/src/lib/api/http-client.ts +++ b/ui/src/lib/api/http-client.ts @@ -597,6 +597,7 @@ export function createHttpClient({ baseUrl = '', onUnauthorized }: HttpClientOpt const limit = query.limit ?? 200; const wanted = (wire: Wire['PersonPlatformEvent']) => (query.locationId === undefined || (wire.location_id && wire.target ? locationKey(wire.location_id, wire.target) : null) === query.locationId) && + wire.event.kind !== 'run_failed' && (!query.kinds?.length || query.kinds.includes(wire.event.kind)); /** Each scan skips the events the person may not see, so one can come back short; keep scanning until enough match. */ async function storedEvents(): Promise { diff --git a/ui/src/lib/api/http-problems.test.ts b/ui/src/lib/api/http-problems.test.ts index 523adcf75d..8c37d458a8 100644 --- a/ui/src/lib/api/http-problems.test.ts +++ b/ui/src/lib/api/http-problems.test.ts @@ -91,6 +91,7 @@ describe('toPlatformEventRecords', () => { open, [ { storage_id: 9, event_id: 'e9', occurred_at: '2026-11-08T11:00:00Z', location_id: 'l1', target: 'development', event: { kind: 'launch_failed', run_id: 'r1', claim_id: 'c1', code: 'image_missing' } }, + { storage_id: 7, event_id: 'e7', occurred_at: '2026-11-08T10:30:00Z', event: { kind: 'run_failed', run_id: 'r2', run_kind: 'job', job_key: 'orders_daily', location_id: 'l1', target: 'development', outcome: 'failed', failed_nodes: [{ node_key: 'model:orders' }], failed_node_count: 1 } }, { storage_id: 4, event_id: 'e4', occurred_at: '2026-11-08T10:00:00Z', event: { kind: 'scheduler_no_leader' } } ], new Map([['l1:development', 'orders']]) diff --git a/ui/src/lib/api/http-problems.ts b/ui/src/lib/api/http-problems.ts index 456997bde2..4c24b27d0d 100644 --- a/ui/src/lib/api/http-problems.ts +++ b/ui/src/lib/api/http-problems.ts @@ -91,7 +91,7 @@ export function toPlatformHealth(problems: Wire['PlatformProblems']): PlatformHe /** Stored events newest first; an open problem marks the newest stored event of its kind and place as ongoing, or comes first when none was stored. */ export function toPlatformEventRecords(open: PlatformEvent[], stored: Wire['PersonPlatformEvent'][], names: Map): PlatformEventRecord[] { - const records = stored.map((wire) => storedRecord(wire, names)); + const records = stored.flatMap((wire) => storedRecord(wire, names) ?? []); const unmatched: PlatformEventRecord[] = []; for (const event of open) { const latest = records.find((r) => !r.active && r.kind === event.kind && r.locationId === event.locationId); @@ -101,7 +101,8 @@ export function toPlatformEventRecords(open: PlatformEvent[], stored: Wire['Pers return [...unmatched, ...records]; } -function storedRecord(wire: Wire['PersonPlatformEvent'], names: Map): PlatformEventRecord { +/** Null for `run_failed`, which only the platform webhook receives; person reads never return it. */ +function storedRecord(wire: Wire['PersonPlatformEvent'], names: Map): PlatformEventRecord | null { const key = wire.location_id && wire.target ? keyOf(wire.location_id, wire.target) : null; const place = key && wire.target ? placeOf(names.get(key) ?? 'A location', wire.target) : null; const base = { id: wire.event_id, occurredAt: wire.occurred_at, active: false, locationId: key, subject: place, resolvedAt: null, reactions: [] }; @@ -119,5 +120,7 @@ function storedRecord(wire: Wire['PersonPlatformEvent'], names: Map