Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
"""Layer jobs selected by tag share the function their models call, and still order by lineage.

A tag selection brings in the functions its models call, so both jobs build the shared function. A
job that builds a node itself does not wait for that node's other builders, so the group runs the
marts layer after the staging layer instead of reporting a cycle.
"""

from sqlbuild.orchestration import job, job_group

staging = job("staging_layer", description="Build the staging layer.", select="tag:staging")
marts = job("marts_layer", description="Build the marts layer.", select="tag:marts")
layers = job_group(
"layered_orders", description="Build staging, then marts.", jobs=[marts, staging]
)
17 changes: 16 additions & 1 deletion clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2292,9 +2292,24 @@ export interface components {
kind: "launch_failed";
/** Format: uuid */
run_id: string;
} | {
/** Format: int64 */
failed_node_count: number;
/** @description The first failed nodes by key, with redacted and bounded error text. */
failed_nodes: components["schemas"]["RunNodeFailure"][];
job_key?: string | null;
/** @enum {string} */
kind: "run_failed";
/** Format: uuid */
location_id: string;
outcome: components["schemas"]["RunOutcome"];
/** Format: uuid */
run_id: string;
run_kind: components["schemas"]["RunKind"];
target: string;
};
/** @enum {string} */
PlatformEventKind: "heartbeat" | "scheduler_no_leader" | "tick_overdue" | "calendar_horizon_low" | "location_broken" | "launch_failed";
PlatformEventKind: "heartbeat" | "scheduler_no_leader" | "tick_overdue" | "calendar_horizon_low" | "location_broken" | "launch_failed" | "run_failed";
/** @description Problems derived from live state when read; nothing here is stored. */
PlatformProblems: {
broken_locations: components["schemas"]["BrokenLocation"][];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ pub(crate) mod person_node_sql;
pub(crate) mod person_operations;
pub(crate) mod person_plans;
pub(crate) mod person_reads;
pub(crate) mod plain_http;
pub(crate) mod request_json;
pub(crate) mod router;
pub(crate) mod run_reads;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
//! Warn once per process before a bearer token crosses the network over plain HTTP.

use std::net::IpAddr;
use std::sync::Once;

use crate::api::constants::{HTTP_SCHEME, LOOPBACK_HOST};

/// Loopback origins stay silent; any other `http://` origin warns on the first connection only.
pub(crate) fn warn_plain_http(url: &reqwest::Url) {
static WARNED: Once = Once::new();
let loopback = url.host_str().is_some_and(|host| {
let host = host.trim_start_matches('[').trim_end_matches(']');
host.eq_ignore_ascii_case(LOOPBACK_HOST)
|| host.parse::<IpAddr>().is_ok_and(|ip| ip.is_loopback())
});
if url.scheme() == HTTP_SCHEME && !loopback {
WARNED.call_once(|| {
eprintln!(
"warning: the bearer token for {} travels unencrypted over HTTP; put TLS or an HTTPS reverse proxy in front of the server",
url.origin().ascii_serialization()
);
});
}
}
2 changes: 2 additions & 0 deletions crates/sqlbuild-orchestration/src/api/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ pub(crate) const OIDC_RESPONSE_BYTES: usize = 1024 * 1024;
#[cfg(unix)]
pub(crate) const DOCKER_IMAGE_HEX_LENGTH: usize = 64;
pub(crate) const HTTPS_SCHEME: &str = "https";
pub(crate) const HTTP_SCHEME: &str = "http";
pub(crate) const LOOPBACK_HOST: &str = "localhost";
pub(crate) const ROOT_PATH: &str = "/";
pub(crate) const UI_INDEX_PATH: &str = "/index.html";
pub(crate) const UI_IMMUTABLE_PREFIX: &str = "/_app/immutable/";
Expand Down
4 changes: 3 additions & 1 deletion crates/sqlbuild-orchestration/src/api/main/connect_client.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
//! Validate the API origin before attaching a sensitive bearer credential.
//! Validate the API origin, and warn about plain HTTP, before attaching a sensitive bearer credential.

use reqwest::header::HeaderValue;

use crate::api::_helpers::resources::client::http_client;
use crate::api::_helpers::resources::plain_http::warn_plain_http;
use crate::api::errors::ClientError;
use crate::api::models::ControlClient;

Expand All @@ -20,6 +21,7 @@ pub(crate) fn connect_client(server_url: &str, bearer: &str) -> Result<ControlCl
"server URL must use HTTP or HTTPS without credentials, query or fragment",
));
}
warn_plain_http(&base_url);
let mut authorization = HeaderValue::from_str(&format!("Bearer {bearer}"))
.map_err(|_| ClientError::Configuration("bearer is not a valid HTTP credential"))?;
authorization.set_sensitive(true);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
//! Record why one group member waits for another, and name the waits that close a cycle.

use std::collections::{BTreeMap, BTreeSet};

/// Keep the first reason found for a wait: an explicit `after`, or the first lineage edge.
pub(crate) fn record_wait(
mut reasons: BTreeMap<(String, String), String>,
(job, dependency): (&str, &str),
reason: impl FnOnce() -> String,
) -> BTreeMap<(String, String), String> {
reasons
.entry((job.to_owned(), dependency.to_owned()))
.or_insert_with(reason);
reasons
}

/// Every job left waits on another, so following first dependencies from any of them closes a cycle.
pub(crate) fn describe_cycle(
remaining: &BTreeMap<String, BTreeSet<String>>,
reasons: &BTreeMap<(String, String), String>,
) -> String {
let mut path: Vec<&str> = Vec::new();
let mut current = remaining.keys().next().map(String::as_str);
while let Some(name) = current {
if let Some(start) = path.iter().position(|seen| *seen == name) {
path.drain(..start);
break;
}
path.push(name);
current = remaining
.get(name)
.and_then(|deps| deps.iter().next())
.map(String::as_str);
}
path.iter()
.zip(path.iter().cycle().skip(1))
.map(|(job, dependency)| {
let reason = reasons
.get(&((*job).to_owned(), (*dependency).to_owned()))
.map_or("", String::as_str);
format!("{job} waits for {dependency} ({reason})")
})
.collect::<Vec<_>>()
.join("; ")
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
pub(crate) mod development;
pub(crate) mod documents;
pub(crate) mod group_waits;
pub(crate) mod job_detail;
pub(crate) mod operator_change;
pub(crate) mod operator_state;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,8 +50,12 @@ pub(crate) async fn trigger_scope(
exclude: Vec::new(),
};
let (transaction, frozen) = frozen_job_request(transaction, (context, None), &job).await?;
let (transaction, nodes) =
job_write_set(transaction, frozen.snapshot_id, &frozen.selection).await?;
let (transaction, nodes) = job_write_set(
transaction,
frozen.snapshot_id,
(&frozen.selection, &frozen.settings),
)
.await?;
Ok((
transaction,
PermissionScope {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@

use std::collections::{BTreeMap, BTreeSet};

use crate::declarations::models::JobDefinition;
use crate::declarations::_helpers::group_waits::{describe_cycle, record_wait};
use crate::declarations::models::{GroupWaits, JobDefinition};
use crate::snapshots::models::SnapshotInput;
use crate::store::errors::ControlError;

pub fn group_dependencies(
input: &SnapshotInput,
(group, input): (&str, &SnapshotInput),
jobs: &BTreeMap<String, JobDefinition>,
(members, after): (&BTreeSet<String>, &BTreeMap<String, BTreeSet<String>>),
) -> Result<BTreeMap<String, BTreeSet<String>>, ControlError> {
Expand All @@ -21,25 +22,32 @@ pub fn group_dependencies(
"group needs known jobs and dependencies within its membership".into(),
));
}
let edges = lineage_dependencies(input, jobs, (members, after));
ensure_acyclic(&edges)?;
Ok(edges)
let waits = lineage_dependencies(input, jobs, (members, after));
ensure_acyclic(group, &waits.edges, &waits.reasons)?;
Ok(waits.edges)
}

/// A job that builds a node itself, such as a shared function, never waits for its other builders.
fn lineage_dependencies(
input: &SnapshotInput,
jobs: &BTreeMap<String, JobDefinition>,
(members, after): (&BTreeSet<String>, &BTreeMap<String, BTreeSet<String>>),
) -> BTreeMap<String, BTreeSet<String>> {
) -> GroupWaits {
let mut owners: BTreeMap<&str, BTreeSet<&str>> = BTreeMap::new();
for name in members {
for key in &jobs[name].selection {
owners.entry(key).or_default().insert(name);
}
}
let mut edges: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
let mut reasons: BTreeMap<(String, String), String> = BTreeMap::new();
for name in members {
let mut dependencies = after.get(name).cloned().unwrap_or_default();
for dependency in &dependencies {
reasons = record_wait(reasons, (name, dependency), || {
"declared in after".to_owned()
});
}
let mut seen: BTreeSet<&str> = BTreeSet::new();
let mut pending: Vec<&str> = jobs[name].selection.iter().map(String::as_str).collect();
while let Some(key) = pending.pop() {
Expand All @@ -53,24 +61,30 @@ fn lineage_dependencies(
.and_then(serde_json::Value::as_array)
{
for upstream in upstream.iter().filter_map(serde_json::Value::as_str) {
if let Some(owners) = owners.get(upstream) {
dependencies.extend(
owners
.iter()
.filter(|owner| **owner != name)
.map(|owner| (*owner).to_owned()),
);
if let Some(owners) = owners.get(upstream)
&& !owners.contains(name.as_str())
{
for owner in owners {
dependencies.insert((*owner).to_owned());
reasons = record_wait(reasons, (name, owner), || {
format!("{key} reads {upstream}")
});
}
}
pending.push(upstream);
}
}
}
edges.insert(name.clone(), dependencies);
}
edges
GroupWaits { edges, reasons }
}

fn ensure_acyclic(edges: &BTreeMap<String, BTreeSet<String>>) -> Result<(), ControlError> {
fn ensure_acyclic(
group: &str,
edges: &BTreeMap<String, BTreeSet<String>>,
reasons: &BTreeMap<(String, String), String>,
) -> Result<(), ControlError> {
let mut remaining = edges.clone();
while !remaining.is_empty() {
let ready: BTreeSet<String> = remaining
Expand All @@ -79,9 +93,10 @@ fn ensure_acyclic(edges: &BTreeMap<String, BTreeSet<String>>) -> Result<(), Cont
.map(|(name, _)| name.clone())
.collect();
if ready.is_empty() {
return Err(ControlError::Invalid(
"group dependencies contain a cycle".into(),
));
return Err(ControlError::Invalid(format!(
"group dependencies contain a cycle in job group {group}: {}",
describe_cycle(&remaining, reasons)
)));
}
remaining.retain(|name, deps| {
deps.retain(|dependency| !ready.contains(dependency));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ pub fn snapshot_definitions(input: &SnapshotInput) -> Result<Definitions, Contro
}
for (name, group) in &definitions.groups {
let members: BTreeSet<String> = group.jobs.keys().cloned().collect();
let dependencies = group_dependencies(input, &definitions.jobs, (&members, &group.jobs))?;
let dependencies =
group_dependencies((name, input), &definitions.jobs, (&members, &group.jobs))?;
let selection: BTreeSet<String> = group
.jobs
.keys()
Expand Down
7 changes: 7 additions & 0 deletions crates/sqlbuild-orchestration/src/declarations/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -249,3 +249,10 @@ pub(crate) struct PreviousTriggerState {
pub(crate) last_tick_for: Option<DateTime<Utc>>,
pub(crate) status_override: Option<TriggerStatus>,
}

/// Each member job's waits, and why it waits: an explicit `after`, or the first lineage edge found.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub(crate) struct GroupWaits {
pub(crate) edges: BTreeMap<String, BTreeSet<String>>,
pub(crate) reasons: BTreeMap<(String, String), String>,
}
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ fn person_event(
Some(scopes.get(trigger_id).cloned()?)
}
PlatformEvent::LaunchFailed { run_id, .. } => Some(scopes.get(run_id).cloned()?),
PlatformEvent::RunFailed { .. } => return None,
PlatformEvent::Heartbeat { .. } | PlatformEvent::SchedulerNoLeader => None,
};
let (location_id, target) = scope.map_or((None, None), |(location_id, target)| {
Expand Down
3 changes: 3 additions & 0 deletions crates/sqlbuild-orchestration/src/platform/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ pub const PLATFORM_WEBHOOK_VARIABLE: &str = "SQLBUILD_PLATFORM_WEBHOOK_URL";

pub(crate) const PLATFORM_EVENT_PAGE_SIZE: i64 = 100;

/// A run-failure event names at most this many failed nodes and reports the full count beside them.
pub(crate) const RUN_FAILURE_NODE_LIMIT: i64 = 20;

/// Replica heartbeats older than this are pruned; a day's date-scoped invocation is complete by then.
pub(crate) const HEARTBEAT_RETENTION_DAYS: i64 = 2;

Expand Down
16 changes: 16 additions & 0 deletions crates/sqlbuild-orchestration/src/platform/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ use uuid::Uuid;

use crate::api::models::ControlClient;
use crate::declarations::models::TriggerDefinition;
use crate::runs::models::{RunKind, RunNodeFailure, RunOutcome};
use crate::sensors::models::{CustomSensorOutcome, SensorEvaluationRequest};
use crate::warehouse_metadata::models::{WarehouseHistoryKind, WarehouseStateLocation};

Expand Down Expand Up @@ -186,6 +187,7 @@ pub enum PlatformEventKind {
CalendarHorizonLow,
LocationBroken,
LaunchFailed,
RunFailed,
}

#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)]
Expand Down Expand Up @@ -216,6 +218,18 @@ pub enum PlatformEvent {
claim_id: Uuid,
code: String,
},
/// A job, group or ad-hoc run ended failed or partial; only the webhook receives it, never person reads.
RunFailed {
run_id: Uuid,
run_kind: RunKind,
job_key: Option<String>,
location_id: Uuid,
target: String,
outcome: RunOutcome,
/// The first failed nodes by key, with redacted and bounded error text.
failed_nodes: Vec<RunNodeFailure>,
failed_node_count: i64,
},
}

impl PlatformEvent {
Expand All @@ -228,6 +242,7 @@ impl PlatformEvent {
Self::CalendarHorizonLow { .. } => PlatformEventKind::CalendarHorizonLow,
Self::LocationBroken { .. } => PlatformEventKind::LocationBroken,
Self::LaunchFailed { .. } => PlatformEventKind::LaunchFailed,
Self::RunFailed { .. } => PlatformEventKind::RunFailed,
}
}

Expand All @@ -240,6 +255,7 @@ impl PlatformEvent {
Self::CalendarHorizonLow { .. } => "platform_calendar_horizon_low",
Self::LocationBroken { .. } => "platform_location_broken",
Self::LaunchFailed { .. } => "platform_launch_failed",
Self::RunFailed { .. } => "platform_run_failed",
}
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,31 @@
//! Node error text is normalized while a run change is read, so no caller stores or returns it raw.

use serde::{Deserialize, Deserializer};
use serde_json::Value;

use sqlbuild_core::redaction::main::redact_error_text::redact_error_text;

use crate::runs::models::{RunChange, RunChangeRequest};
use crate::store::errors::ControlError;

pub(crate) fn bounded_error_text<'de, D: Deserializer<'de>>(
deserializer: D,
) -> Result<Option<String>, D::Error> {
Ok(Option::<String>::deserialize(deserializer)?.map(|text| redact_error_text(&text)))
}

/// The error text of a stored node-finished fact; `None` when there is no fact or it carries none.
pub(crate) fn finished_error_message(payload: &Value) -> Result<Option<String>, ControlError> {
if payload.is_null() {
return Ok(None);
}
Ok(
match serde_json::from_value::<RunChangeRequest>(payload["change"].clone())
.map_err(|error| ControlError::Invalid(error.to_string()))?
.change
{
RunChange::NodeFinished { error_message, .. } => error_message,
_ => None,
},
)
}
Loading
Loading