diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 2548c57842..5c6f5932c2 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1717,6 +1717,9 @@ export interface components { member_of: string[]; /** @description A group's member jobs in run order; empty for a job. */ members: components["schemas"]["JobMember"][]; + on_failure?: null | components["schemas"]["OnFailure"]; + overlap: components["schemas"]["Overlap"]; + owner?: string | null; select?: string | null; selection: string[]; /** @description Command settings; `vars` and a `defer_*` value that is not a target name show as `[REDACTED]`, and text values pass through secret redaction. */ @@ -1741,6 +1744,9 @@ export interface components { job_key: string; /** @description `job` or `group`; a group's selection is the union of its member jobs' selections. */ kind: components["schemas"]["RunKind"]; + on_failure?: null | components["schemas"]["OnFailure"]; + overlap: components["schemas"]["Overlap"]; + owner?: string | null; /** @description A job's selectors as written; absent for a group and for snapshots published before they were recorded. */ select?: string | null; selection: string[]; @@ -2064,6 +2070,13 @@ export interface components { */ sequences: number; }; + /** @description A failed or partial run reruns from failure up to `retries` times, each `retry_delay_seconds` after the last ended. */ + OnFailure: { + /** Format: int32 */ + retries: number; + /** Format: int32 */ + retry_delay_seconds: number; + }; /** @description One exact-byte output chunk. */ OutputChunk: { /** @@ -2112,6 +2125,11 @@ export interface components { /** Format: uuid */ trigger_id: string; }; + /** + * @description What a schedule tick or sensor request does while a run of the same job is active; people's requests always queue. + * @enum {string} + */ + Overlap: "queue" | "skip"; PasswordChangeRequest: { current_password: string; new_password: string; @@ -2187,6 +2205,8 @@ export interface components { /** Format: date-time */ requested_at: string; run_count: number; + /** @description For each matched request the job's `skip` overlap dropped, the run that was active. */ + skipped_for: string[]; }; /** @description A run key is redacted and withheld like the custom cursor it usually derives from: empty without view.evidence. */ PersonSensorRunRequest: { @@ -2434,6 +2454,13 @@ export interface components { limit: string; /** Format: uuid */ run_id?: string | null; + } | { + /** Format: int32 */ + attempt: number; + /** Format: date-time */ + due_at: string; + /** @enum {string} */ + kind: "retry_delay"; }; /** @description One invocation's records; the server owns its claim namespace, start time and version. */ RecallBatch: { @@ -2790,6 +2817,7 @@ export interface components { requested_by: string; /** Format: uuid */ rerun_of_run_id?: string | null; + retry?: null | components["schemas"]["RunRetry"]; /** Format: uuid */ root_run_id: string; /** Format: uuid */ @@ -2808,6 +2836,13 @@ export interface components { window?: null | components["schemas"]["RunWindow"]; write_set: string[]; }; + /** @description Which retry of the job's `on_failure` policy a run is, and when it may start. */ + RunRetry: { + /** Format: int32 */ + attempt: number; + /** Format: date-time */ + not_before: string; + }; /** @description Warehouse statement metadata of a run: kind, timing, counts and outcome, never the SQL text. */ RunStatementRecord: { /** Format: int64 */ @@ -3063,8 +3098,10 @@ export interface components { */ Table: "invocations" | "output" | "events" | "diagnostics" | "artifacts" | "artifact_refs" | "heartbeats" | "delivery" | "meta" | "conflicts"; /** @enum {string} */ - TickOutcome: "requested" | "covered"; + TickOutcome: "requested" | "covered" | "skipped"; TickRecord: { + /** Format: uuid */ + active_run_id?: string | null; /** Format: uuid */ covered_by_run_id?: string | null; /** Format: int64 */ diff --git a/crates/sqlbuild-orchestration/src/declarations/_helpers/reads.rs b/crates/sqlbuild-orchestration/src/declarations/_helpers/reads.rs index ffe4978047..93502a5ec4 100644 --- a/crates/sqlbuild-orchestration/src/declarations/_helpers/reads.rs +++ b/crates/sqlbuild-orchestration/src/declarations/_helpers/reads.rs @@ -88,6 +88,9 @@ pub(crate) async fn jobs_in( snapshot_id: job.snapshot_id, select: job.definition_json.select, declared_at: job.definition_json.declared_at, + owner: job.definition_json.owner, + overlap: job.definition_json.overlap, + on_failure: job.definition_json.on_failure, }) .collect(); Ok((transaction, jobs)) @@ -99,7 +102,7 @@ pub(crate) async fn latest_ticks_in( (location_id, target): (Uuid, &str), ) -> Result<(ControlTransaction, Vec), ControlError> { query_records(transaction, - "SELECT k.tick_id, k.trigger_id, k.scheduled_for, k.evaluated_at, k.epoch, k.outcome, k.run_id, k.covered_by_run_id \ + "SELECT k.tick_id, k.trigger_id, k.scheduled_for, k.evaluated_at, k.epoch, k.outcome, k.run_id, k.covered_by_run_id, k.active_run_id \ FROM sqb_triggers t JOIN sqb_ticks k ON k.tick_id=(SELECT x.tick_id FROM sqb_ticks x WHERE x.trigger_id=t.trigger_id \ ORDER BY x.scheduled_for DESC LIMIT 1) \ WHERE t.location_id=$1 AND t.target=$2 AND t.removed_in_snapshot_id IS NULL", diff --git a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs index e9ec240fc8..43ea9e89ed 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/mod.rs @@ -9,5 +9,6 @@ pub mod list_trigger_events; pub mod list_triggers; pub mod pause_development_triggers; pub(crate) mod publish_definitions; +pub(crate) mod published_job; pub mod read_person_job; pub mod snapshot_definitions; diff --git a/crates/sqlbuild-orchestration/src/declarations/main/published_job.rs b/crates/sqlbuild-orchestration/src/declarations/main/published_job.rs new file mode 100644 index 0000000000..8a12ef7ea9 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/declarations/main/published_job.rs @@ -0,0 +1,27 @@ +//! The job a target currently publishes under a name. + +use uuid::Uuid; + +use crate::declarations::models::StoredJob; +use crate::store::errors::ControlError; +use crate::store::main::query_records::query_records; +use crate::store::models::{ControlTransaction, Parameter}; + +/// `None` when the target publishes no job or group by that name. +pub(crate) async fn published_job( + transaction: ControlTransaction, + (location_id, target, job_key): (Uuid, &str, &str), +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, stored): (_, Vec) = query_records( + transaction, + "SELECT job_key, kind, snapshot_id, definition_json FROM sqb_jobs \ + WHERE location_id=$1 AND target=$2 AND job_key=$3", + vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(job_key.into()), + ], + ) + .await?; + Ok((transaction, stored.into_iter().next())) +} diff --git a/crates/sqlbuild-orchestration/src/declarations/main/read_person_job.rs b/crates/sqlbuild-orchestration/src/declarations/main/read_person_job.rs index 6ffb2a9d36..6101a2f61d 100644 --- a/crates/sqlbuild-orchestration/src/declarations/main/read_person_job.rs +++ b/crates/sqlbuild-orchestration/src/declarations/main/read_person_job.rs @@ -8,12 +8,13 @@ use crate::declarations::_helpers::job_detail::{ critical_nodes, cursor_type, groups_running, members_in_order, shown_settings, }; use crate::declarations::main::frozen_definitions::frozen_definitions; -use crate::declarations::models::{JobDetailRecord, StoredJob}; +use crate::declarations::main::published_job::published_job; +use crate::declarations::models::JobDetailRecord; use crate::runs::models::RunKind; use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; use crate::store::errors::ControlError; -use crate::store::main::{begin::begin, commit::commit, query_records::query_records}; -use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode}; +use crate::store::main::{begin::begin, commit::commit}; +use crate::store::models::{ControlStore, TransactionMode}; /// `None` when the target publishes no job by that name; refusal comes first either way. pub async fn read_person_job( @@ -30,8 +31,8 @@ pub async fn read_person_job( ) .await?; let (transaction, published) = - published_snapshot(transaction, (location_id, target, job_key)).await?; - let Some(snapshot_id) = published else { + published_job(transaction, (location_id, target, job_key)).await?; + let Some(snapshot_id) = published.map(|job| job.snapshot_id) else { commit(transaction).await?; return Ok(None); }; @@ -67,24 +68,8 @@ pub async fn read_person_job( exclude: job.exclude, targets: job.targets, declared_at: job.declared_at, + owner: job.owner, + overlap: job.overlap, + on_failure: job.on_failure, })) } - -/// The snapshot whose job list holds `job_key`, if the target publishes it. -async fn published_snapshot( - transaction: ControlTransaction, - (location_id, target, job_key): (Uuid, &str, &str), -) -> Result<(ControlTransaction, Option), ControlError> { - let (transaction, stored): (_, Vec) = query_records( - transaction, - "SELECT job_key, kind, snapshot_id, definition_json FROM sqb_jobs \ - WHERE location_id=$1 AND target=$2 AND job_key=$3", - vec![ - Parameter::Id(location_id), - Parameter::Text(target.into()), - Parameter::Text(job_key.into()), - ], - ) - .await?; - Ok((transaction, stored.first().map(|job| job.snapshot_id))) -} diff --git a/crates/sqlbuild-orchestration/src/declarations/models.rs b/crates/sqlbuild-orchestration/src/declarations/models.rs index fecd2146a4..46672e89d2 100644 --- a/crates/sqlbuild-orchestration/src/declarations/models.rs +++ b/crates/sqlbuild-orchestration/src/declarations/models.rs @@ -45,6 +45,37 @@ pub struct JobDefinition { pub targets: BTreeSet, #[serde(default, skip_serializing_if = "Option::is_none")] pub declared_at: Option, + /// Who answers for the job, as written. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub owner: Option, + #[serde(default, skip_serializing_if = "Overlap::is_queue")] + pub overlap: Overlap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub on_failure: Option, +} + +/// What a schedule tick or sensor request does while a run of the same job is active; people's requests always queue. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Overlap { + #[default] + Queue, + Skip, +} + +impl Overlap { + #[must_use] + pub const fn is_queue(&self) -> bool { + matches!(self, Self::Queue) + } +} + +/// A failed or partial run reruns from failure up to `retries` times, each `retry_delay_seconds` after the last ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct OnFailure { + pub retries: u8, + pub retry_delay_seconds: u32, } /// Where a declaration was written, relative to the project root. @@ -123,6 +154,9 @@ pub struct JobRecord { /// A job's selectors as written; absent for a group and for snapshots published before they were recorded. pub select: Option, pub declared_at: Option, + pub owner: Option, + pub overlap: Overlap, + pub on_failure: Option, } /// A `sqb_jobs` row as stored. @@ -158,6 +192,10 @@ pub struct JobDetailRecord { pub declared_at: Option, /// The cursor type every selected cursor node shares; absent when none has a cursor or types differ. pub cursor_type: Option, + pub owner: Option, + pub overlap: Overlap, + /// A member job's own policy applies only when it runs alone; inside a group run the group's applies. + pub on_failure: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs index 6a711597c2..ea33e1d6d7 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/admission.rs @@ -30,6 +30,7 @@ pub(crate) async fn limit_waiters( transaction: ControlTransaction, eligible: Option<&BTreeSet<(Uuid, String)>>, ) -> Result<(ControlTransaction, Vec), ControlError> { + let now = transaction.now; let (transaction, rows) = query(transaction, "SELECT r.request_json, r.queued_at FROM sqb_runs r WHERE r.status='queued' AND r.parent_run_id IS NULL \ AND r.cancel_requested_at IS NULL AND r.worker_instance IS NULL", @@ -39,6 +40,7 @@ pub(crate) async fn limit_waiters( .map(decode_row) .collect::, _>>()? .into_iter() + .filter(|run| run.request_json.due(now)) .filter(|run| { eligible.is_none_or(|scopes| { scopes.contains(&( diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs index 66dce5299e..161941386d 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/groups/lifecycle.rs @@ -82,7 +82,7 @@ pub(crate) async fn advance_group( .await?; } } - } else if parent.state.status == RunStatus::Queued { + } else if parent.state.status == RunStatus::Queued && parent.request_json.due(transaction.now) { let admission = RunRequest { concurrency_groups: admission_limits( &parent.request_json, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs index 85454af29d..bf9c1c65ae 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/mod.rs @@ -12,5 +12,6 @@ pub(in crate::runs) mod publish_change; pub(in crate::runs) mod queued; pub(in crate::runs) mod reads; pub(in crate::runs) mod recall; +pub(in crate::runs) mod retries; pub(in crate::runs) mod run_failure; pub(in crate::runs) mod waits; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs index 01eaef914b..05d82a9414 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/publish_change.rs @@ -5,6 +5,7 @@ use serde_json::json; use crate::platform::models::PlatformEvent; use crate::runs::_helpers::lifecycle::events::append_run_event; use crate::runs::_helpers::lifecycle::projection::project_change; +use crate::runs::_helpers::lifecycle::retries::queue_retry; use crate::runs::_helpers::lifecycle::run_failure::publish_run_failure; use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome}; use crate::store::errors::ControlError; @@ -60,6 +61,7 @@ pub(crate) async fn publish_change( } = request.change { transaction = publish_run_failure(transaction, request, outcome).await?; + transaction = queue_retry(transaction, request.run_id).await?; } if let Some((node_key, _)) = request.change.finished_node() { (transaction, _) = query( diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/retries.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/retries.rs new file mode 100644 index 0000000000..a6147f8766 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/retries.rs @@ -0,0 +1,82 @@ +//! A failed or partial top-level run of a job with `on_failure` queues its next retry in the same transaction. + +use chrono::TimeDelta; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::declarations::main::frozen_definitions::frozen_definitions; +use crate::runs::_helpers::requests::rerun::frozen_rerun; +use crate::runs::main::queue_run::queue_run; +use crate::runs::models::{RerunRequest, RunRequest, RunRetry}; +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// Rerun the failed and skipped nodes after the policy's delay, unless the run was cancelled, is a group member, used up its retries, or its retry's ID is already taken. +pub(crate) async fn queue_retry( + transaction: ControlTransaction, + run_id: Uuid, +) -> Result { + let (transaction, rows) = query( + transaction, + "SELECT r.request_json, r.cancel_requested_at, \ + (SELECT COUNT(*) FROM sqb_run_nodes n WHERE n.run_id=r.run_id AND n.kind='node' AND n.status IN ('failed','skipped')) AS retryable \ + FROM sqb_runs r WHERE r.run_id=$1", + vec![Parameter::Id(run_id)], + ) + .await?; + let Some(row) = rows.into_iter().next() else { + return Ok(transaction); + }; + let source: RunRequest = serde_json::from_value(row["request_json"].clone()) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + let retryable = row["retryable"].as_i64().unwrap_or_default() > 0; + let Some(job_key) = source.job_key.as_deref().filter(|_| { + source.parent_run_id.is_none() && row["cancel_requested_at"].is_null() && retryable + }) else { + return Ok(transaction); + }; + let (transaction, definitions) = frozen_definitions(transaction, source.snapshot_id).await?; + let policy = definitions + .jobs + .get(job_key) + .or_else(|| definitions.groups.get(job_key).map(|group| &group.job)) + .and_then(|job| job.on_failure); + let attempt = source + .retry + .map_or(1, |retry| retry.attempt.saturating_add(1)); + let Some(policy) = policy.filter(|policy| attempt <= policy.retries) else { + return Ok(transaction); + }; + let retry_id = Uuid::new_v5(&run_id, b"retry"); + let (transaction, taken) = query( + transaction, + "SELECT 1 FROM sqb_runs WHERE run_id=$1", + vec![Parameter::Id(retry_id)], + ) + .await?; + if !taken.is_empty() { + return Ok(transaction); + } + let context = AuditContext { + actor: "system:retry".into(), + granting_role: "system:scheduler".into(), + request_id: retry_id, + ip_address: None, + }; + let request = RerunRequest { + run_id: retry_id, + source_run_id: run_id, + from_failure: true, + reason: format!("retry {attempt} of {}", policy.retries), + priority: source.priority, + nodes: Vec::new(), + }; + let (transaction, (_, mut frozen)) = + frozen_rerun(transaction, (&context, None), &request).await?; + frozen.retry = Some(RunRetry { + attempt, + not_before: transaction.now + TimeDelta::seconds(i64::from(policy.retry_delay_seconds)), + }); + queue_run(transaction, &frozen).await +} diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/waits.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/waits.rs index 1dac6910b6..e5b16a207e 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/waits.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/waits.rs @@ -52,6 +52,12 @@ pub(crate) async fn queue_states( let request = &candidate.request_json; let mut waits; (transaction, waits) = child_waits(transaction, request).await?; + if let Some(retry) = request.retry.filter(|_| !request.due(transaction.now)) { + waits.push(QueueWait::RetryDelay { + attempt: retry.attempt, + due_at: retry.not_before, + }); + } waits.extend( running .iter() diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index 082f491538..bcb1eddb60 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -110,6 +110,7 @@ pub(crate) async fn frozen_adhoc_request( window: None, concurrency_groups: BTreeSet::new(), narrowing: None, + retry: None, }, )), )) diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs index 6642c2afbe..848cf01692 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/rerun.rs @@ -97,6 +97,7 @@ pub(crate) async fn frozen_rerun( frozen.trigger_id = None; frozen.tick_id = None; frozen.rerun_of_run_id = Some(request.source_run_id); + frozen.retry = None; frozen.requested_by = context.actor.clone(); frozen.priority = request.priority; let mut context = context.clone(); diff --git a/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs index 48adb8bd65..1b7701dd4d 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/claim_next_run.rs @@ -58,6 +58,10 @@ pub async fn claim_next_run<'a>( } for row in rows { let candidate: QueueCandidate = decode_row(row)?; + if !candidate.request_json.due(transaction.now) { + after = Some(candidate); + continue; + } let ready; (transaction, ready) = child_ready(transaction, &candidate.request_json).await?; if !ready { diff --git a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs index cdcc2afaaa..a3fd49acd6 100644 --- a/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs +++ b/crates/sqlbuild-orchestration/src/runs/main/frozen_job_request.rs @@ -99,6 +99,7 @@ pub(crate) async fn frozen_job_request( window: request.window.clone(), concurrency_groups: definition.concurrency_groups, narrowing, + retry: None, }, )) } diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index ebeb34b8ca..cb337eed86 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -163,6 +163,25 @@ pub struct RunRequest { /// The selectors a person narrowed a job run with; reruns copy it with the rest of the request. #[serde(default, skip_serializing_if = "Option::is_none")] pub narrowing: Option, + /// Set on an automatic retry of a failed run, which is not claimed before `not_before`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub retry: Option, +} + +/// Which retry of the job's `on_failure` policy a run is, and when it may start. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +#[serde(deny_unknown_fields)] +pub struct RunRetry { + pub attempt: u8, + pub not_before: DateTime, +} + +impl RunRequest { + /// Whether a queued run may start at `now`: only a retry still waiting out its delay may not. + #[must_use] + pub fn due(&self, now: DateTime) -> bool { + self.retry.is_none_or(|retry| retry.not_before <= now) + } } /// A trusted operator names a published job; execution inputs are never supplied by the caller. @@ -307,6 +326,8 @@ pub enum QueueWait { }, /// An earlier queued run takes the same concurrency limit, so it goes first. EarlierRun { run_id: Option, limit: String }, + /// An automatic retry waits out the job's retry delay. + RetryDelay { attempt: u8, due_at: DateTime }, } /// The `sqb build` selectors one run of a job was narrowed with. diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs index 8d4d589957..455bda2395 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/persistence.rs @@ -21,18 +21,19 @@ pub(crate) async fn persist_ticks( ) .await?; for action in &plan.actions { - let (outcome, run, covered) = match action.decision { - TickDecision::Requested { run_id } => ("requested", Some(run_id), None), + let (outcome, run, covered, active) = match action.decision { + TickDecision::Requested { run_id } => ("requested", Some(run_id), None, None), TickDecision::Covered { covered_by_run_id } => { - ("covered", None, Some(covered_by_run_id)) + ("covered", None, Some(covered_by_run_id), None) } + TickDecision::Skipped { active_run_id } => ("skipped", None, None, Some(active_run_id)), }; let inserted; (transaction, inserted) = query(transaction, - "INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, run_id, covered_by_run_id) \ - VALUES ($1,$2,$3,$4,$5,$6,$7,$8) ON CONFLICT (trigger_id, scheduled_for) DO NOTHING RETURNING tick_id", + "INSERT INTO sqb_ticks (tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, run_id, covered_by_run_id, active_run_id) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT (trigger_id, scheduled_for) DO NOTHING RETURNING tick_id", vec![Parameter::Id(action.tick_id), Parameter::Id(trigger.trigger_id), Parameter::Time(action.scheduled_for), Parameter::Time(now), Parameter::Integer(lease.epoch), - Parameter::Text(outcome.into()), Parameter::OptionalId(run), Parameter::OptionalId(covered)]).await?; + Parameter::Text(outcome.into()), Parameter::OptionalId(run), Parameter::OptionalId(covered), Parameter::OptionalId(active)]).await?; if !inserted.is_empty() { transaction .changes @@ -68,6 +69,7 @@ pub(crate) async fn persist_ticks( window: None, concurrency_groups: trigger.job_json.concurrency_groups.clone(), narrowing: None, + retry: None, }; transaction = queue_run(transaction, &request).await?; } diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/reads.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/reads.rs index ac1d3463e5..9e52455ae9 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/reads.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/reads.rs @@ -13,7 +13,7 @@ pub(crate) async fn ticks_in( limit: u32, ) -> Result<(ControlTransaction, Vec), ControlError> { query_records(transaction, - "SELECT tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, run_id, covered_by_run_id \ + "SELECT tick_id, trigger_id, scheduled_for, evaluated_at, epoch, outcome, run_id, covered_by_run_id, active_run_id \ FROM sqb_ticks WHERE trigger_id=$1 ORDER BY scheduled_for DESC LIMIT $2", vec![Parameter::Id(trigger_id), Parameter::Integer(i64::from(limit.min(1000)))]).await } diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs index b9091d986b..047cf97d88 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/tick_retention.rs @@ -37,7 +37,7 @@ pub(crate) async fn prune_ticks_when_due( } } -/// One bounded pass; requested ticks stay, as does any tick a run points at. +/// One bounded pass over covered and skipped ticks; requested ticks stay, as does any tick a run points at. pub(crate) async fn prune_ticks( store: &ControlStore, lease: &SchedulerLease, @@ -51,7 +51,7 @@ pub(crate) async fn prune_ticks( (transaction, rows) = query( transaction, "DELETE FROM sqb_ticks WHERE tick_id IN (SELECT k.tick_id FROM sqb_ticks k \ - WHERE k.outcome='covered' AND k.scheduled_for<$1 \ + WHERE k.outcome IN ('covered','skipped') AND k.scheduled_for<$1 \ AND k.scheduled_for<(SELECT MAX(l.scheduled_for) FROM sqb_ticks l WHERE l.trigger_id=k.trigger_id) \ AND NOT EXISTS (SELECT 1 FROM sqb_runs r WHERE r.tick_id=k.tick_id) \ ORDER BY k.scheduled_for, k.tick_id LIMIT $2) RETURNING trigger_id", diff --git a/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs index 605c3ee155..5250350af3 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/_helpers/ticks.rs @@ -18,15 +18,17 @@ pub(crate) fn requested_run(tick_id: Uuid) -> Uuid { Uuid::new_v5(&tick_id, b"run") } +/// A waiting run covers the tick before an active one skips it. pub(crate) fn action( trigger_id: Uuid, scheduled_for: DateTime, - covering_run: Option, + (covering_run, active_run): (Option, Option), ) -> TickAction { let tick_id = tick_id(trigger_id, scheduled_for); - let decision = match covering_run { - Some(covered_by_run_id) => TickDecision::Covered { covered_by_run_id }, - None => TickDecision::Requested { + let decision = match (covering_run, active_run) { + (Some(covered_by_run_id), _) => TickDecision::Covered { covered_by_run_id }, + (None, Some(active_run_id)) => TickDecision::Skipped { active_run_id }, + (None, None) => TickDecision::Requested { run_id: requested_run(tick_id), }, }; diff --git a/crates/sqlbuild-orchestration/src/scheduler/constants.rs b/crates/sqlbuild-orchestration/src/scheduler/constants.rs index c905cbad1c..20c6343462 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/constants.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/constants.rs @@ -6,7 +6,7 @@ pub const RENEW_INTERVAL_SECONDS: u64 = 5; pub const LEASE_TTL_SECONDS: i64 = 30; /// Bound a calendar declaration, including due times carried over a republish. pub const MAX_TICKS_PER_BATCH: usize = 10_000; -/// Covered ticks older than this are deleted. +/// Covered and skipped ticks older than this are deleted. pub(crate) const TICK_RETENTION_DAYS: i64 = 30; pub(crate) const TICK_RETENTION_BATCH: usize = 100; /// Batches per cleanup pass; a longer backlog resumes next cycle, after the lease renews. diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs index b1a8a114be..41021a5a56 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_evaluate_trigger.rs @@ -5,6 +5,7 @@ use uuid::Uuid; use crate::declarations::models::TriggerStatus; use crate::scheduler::_helpers::persistence::persist_ticks; use crate::scheduler::main::_plan_ticks::plan_ticks; +use crate::scheduler::main::active_job_run::active_job_run; use crate::scheduler::main::check_lease::check_lease; use crate::scheduler::models::{DueTrigger, SchedulerLease, TickPlan, TickState}; use crate::store::errors::ControlError; @@ -42,9 +43,9 @@ pub(crate) async fn evaluate_trigger( let now = transaction.now; let (transaction, rows) = query( transaction, - "SELECT r.run_id FROM sqb_runs r WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 \ + "SELECT r.run_id, r.request_json FROM sqb_runs r WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 \ AND r.status='queued' AND r.cancel_requested_at IS NULL \ - AND r.worker_instance IS NULL ORDER BY r.queued_at, r.run_id LIMIT 1", + AND r.worker_instance IS NULL ORDER BY r.queued_at, r.run_id", vec![ Parameter::Id(trigger.location_id), Parameter::Text(trigger.target.clone()), @@ -52,13 +53,25 @@ pub(crate) async fn evaluate_trigger( ], ) .await?; - let waiting_run = rows - .first() - .and_then(|row| row.get("run_id")) - .and_then(serde_json::Value::as_str) - .map(Uuid::parse_str) - .transpose() - .map_err(|error| ControlError::Invalid(error.to_string()))?; + let covering: Vec<_> = rows + .into_iter() + .filter(|row| { + row["request_json"] + .get("retry") + .is_none_or(serde_json::Value::is_null) + }) + .take(1) + .collect(); + let waiting_run = first_run(&covering)?; + let (transaction, active_run) = if trigger.job_json.overlap.is_queue() { + (transaction, None) + } else { + active_job_run( + transaction, + (trigger.location_id, &trigger.target, &trigger.job_key), + ) + .await? + }; let plan = plan_ticks( &trigger.definition_json, &TickState { @@ -66,6 +79,7 @@ pub(crate) async fn evaluate_trigger( started_at: trigger.started_at, last_tick_for: trigger.last_tick_for, waiting_run, + active_run, }, now, )?; @@ -74,3 +88,12 @@ pub(crate) async fn evaluate_trigger( commit(transaction).await?; Ok(Some(plan)) } + +fn first_run(rows: &[crate::store::types::Row]) -> Result, ControlError> { + rows.first() + .and_then(|row| row.get("run_id")) + .and_then(serde_json::Value::as_str) + .map(Uuid::parse_str) + .transpose() + .map_err(|error| ControlError::Invalid(error.to_string())) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs index 9a10a16ce3..263bd7a006 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/_plan_ticks.rs @@ -9,7 +9,7 @@ use crate::scheduler::main::next_schedule_time::next_schedule_time; use crate::scheduler::models::{TickPlan, TickState}; use crate::store::errors::ControlError; -/// Only the latest missed occurrence ticks; a waiting unclaimed run covers it instead of queueing another. +/// Only the latest missed occurrence ticks; a waiting unclaimed run covers it, and an active run skips it unless earlier occurrences were missed. pub(in crate::scheduler) fn plan_ticks( trigger: &TriggerDefinition, state: &TickState, @@ -23,9 +23,16 @@ pub(in crate::scheduler) fn plan_ticks( .ok_or_else(|| ControlError::Invalid("trigger start is out of range".into()))?, }; let latest = latest_occurrence(trigger, now)?.filter(|time| *time > after); + let catching_up = match latest { + Some(time) => { + next_schedule_time(&trigger.schedule, after)?.is_some_and(|first| first < time) + } + None => false, + }; + let active_run = state.active_run.filter(|_| !catching_up); Ok(TickPlan { actions: latest - .map(|time| action(state.trigger_id, time, state.waiting_run)) + .map(|time| action(state.trigger_id, time, (state.waiting_run, active_run))) .into_iter() .collect(), last_tick_for: latest.or(state.last_tick_for), diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/active_job_run.rs b/crates/sqlbuild-orchestration/src/scheduler/main/active_job_run.rs new file mode 100644 index 0000000000..32b421c304 --- /dev/null +++ b/crates/sqlbuild-orchestration/src/scheduler/main/active_job_run.rs @@ -0,0 +1,34 @@ +//! The run a `skip` overlap yields to. + +use uuid::Uuid; + +use crate::store::errors::ControlError; +use crate::store::main::query::query; +use crate::store::models::{ControlTransaction, Parameter}; + +/// The oldest top-level run of the job that is queued or executing and not being cancelled. +pub(crate) async fn active_job_run( + transaction: ControlTransaction, + (location_id, target, job_key): (Uuid, &str, &str), +) -> Result<(ControlTransaction, Option), ControlError> { + let (transaction, rows) = query( + transaction, + "SELECT r.run_id FROM sqb_runs r WHERE r.location_id=$1 AND r.target=$2 AND r.job_key=$3 \ + AND r.parent_run_id IS NULL AND r.status IN ('queued','launched','running','unresponsive') \ + AND r.cancel_requested_at IS NULL ORDER BY r.queued_at, r.run_id LIMIT 1", + vec![ + Parameter::Id(location_id), + Parameter::Text(target.into()), + Parameter::Text(job_key.into()), + ], + ) + .await?; + let run_id = rows + .first() + .and_then(|row| row.get("run_id")) + .and_then(serde_json::Value::as_str) + .map(Uuid::parse_str) + .transpose() + .map_err(|error| ControlError::Invalid(error.to_string()))?; + Ok((transaction, run_id)) +} diff --git a/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs index 5875aa88b5..fe456129d1 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/main/mod.rs @@ -4,6 +4,7 @@ pub(in crate::scheduler) mod _next_cron_time; pub(in crate::scheduler) mod _plan_ticks; pub(in crate::scheduler) mod _previous_cron_time; pub(in crate::scheduler) mod _scheduler_cycle; +pub(crate) mod active_job_run; pub(crate) mod check_lease; pub mod list_person_ticks; pub mod list_ticks; diff --git a/crates/sqlbuild-orchestration/src/scheduler/models.rs b/crates/sqlbuild-orchestration/src/scheduler/models.rs index adc5f713d5..c448b4138e 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/models.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/models.rs @@ -57,6 +57,8 @@ pub struct TickState { pub last_tick_for: Option>, /// Only unclaimed waiting work can cover a later tick; executing work cannot cover future data. pub waiting_run: Option, + /// An active run of the job when its overlap is `skip`; a tick that missed earlier occurrences still queues. + pub active_run: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -64,6 +66,7 @@ pub struct TickState { pub enum TickDecision { Requested { run_id: Uuid }, Covered { covered_by_run_id: Uuid }, + Skipped { active_run_id: Uuid }, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -101,6 +104,8 @@ pub(crate) struct DueTrigger { pub enum TickOutcome { Requested, Covered, + /// The job's overlap is `skip` and `active_run_id` was still active. + Skipped, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] @@ -113,4 +118,5 @@ pub struct TickRecord { pub outcome: TickOutcome, pub run_id: Option, pub covered_by_run_id: Option, + pub active_run_id: Option, } diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs index af58285254..c671642dc6 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/helpers.rs @@ -25,6 +25,7 @@ pub(super) fn state(test_case: &TickTestCase) -> (TriggerDefinition, TickState) started_at: time(0), last_tick_for: test_case.last_second.map(time), waiting_run: test_case.active.then_some(Uuid::from_u128(31)), + active_run: test_case.running.then_some(Uuid::from_u128(41)), }, ) } @@ -60,6 +61,14 @@ pub(super) fn covered_seconds(plan: &TickPlan) -> Vec { .collect() } +pub(super) fn skipped_seconds(plan: &TickPlan) -> Vec { + plan.actions + .iter() + .filter(|action| matches!(action.decision, TickDecision::Skipped { .. })) + .map(|action| action.scheduled_for.timestamp()) + .collect() +} + /// Under crashes and clock jumps a tick is never early, repeated or invented, and the latest due time always ticks. fn exercise_history(due: Vec, history: Vec<(i64, bool)>) -> TestCaseResult { let expected: BTreeSet = due.iter().copied().collect(); @@ -69,6 +78,7 @@ fn exercise_history(due: Vec, history: Vec<(i64, bool)>) -> TestCaseResult started_at: time(0), last_tick_for: None, waiting_run: None, + active_run: None, }; let mut committed: BTreeMap = BTreeMap::new(); let final_step = std::iter::once((1001, true)); @@ -163,6 +173,7 @@ fn cron_ticks( started_at: start, last_tick_for: None, waiting_run: None, + active_run: None, }; let mut committed: BTreeSet> = BTreeSet::new(); let mut clock = start; diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs index 0dfbe8087b..1758be00e2 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_ticks.rs @@ -1,5 +1,7 @@ use crate::scheduler::main::_plan_ticks::plan_ticks; -use crate::scheduler::tests::helpers::{covered_seconds, request_seconds, state, time}; +use crate::scheduler::tests::helpers::{ + covered_seconds, request_seconds, skipped_seconds, state, time, +}; use crate::scheduler::tests::test_types::TickTestCase; #[test] @@ -11,8 +13,10 @@ fn given_durable_schedule_state_when_time_advances_then_decisions_preserve_due_t last_second: None, now_second: 30, active: false, + running: false, expected_requests: vec![30], expected_covered: vec![], + expected_skipped: vec![], expected_last: Some(30), expected_next: Some(40), }, @@ -22,8 +26,10 @@ fn given_durable_schedule_state_when_time_advances_then_decisions_preserve_due_t last_second: Some(10), now_second: 30, active: true, + running: false, expected_requests: vec![], expected_covered: vec![30], + expected_skipped: vec![], expected_last: Some(30), expected_next: Some(40), }, @@ -33,8 +39,49 @@ fn given_durable_schedule_state_when_time_advances_then_decisions_preserve_due_t last_second: Some(30), now_second: 20, active: false, + running: false, expected_requests: vec![], expected_covered: vec![], + expected_skipped: vec![], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "an on-time tick of a skip job yields to its running run", + due_seconds: vec![10, 20, 30, 40], + last_second: Some(20), + now_second: 30, + active: false, + running: true, + expected_requests: vec![], + expected_covered: vec![], + expected_skipped: vec![30], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "a catch-up tick that collapsed missed times still queues while a skip job runs", + due_seconds: vec![10, 20, 30, 40], + last_second: Some(10), + now_second: 30, + active: false, + running: true, + expected_requests: vec![30], + expected_covered: vec![], + expected_skipped: vec![], + expected_last: Some(30), + expected_next: Some(40), + }, + TickTestCase { + description: "a waiting run covers the tick before a running run can skip it", + due_seconds: vec![10, 20, 30, 40], + last_second: Some(20), + now_second: 30, + active: true, + running: true, + expected_requests: vec![], + expected_covered: vec![30], + expected_skipped: vec![], expected_last: Some(30), expected_next: Some(40), }, @@ -55,6 +102,12 @@ fn given_durable_schedule_state_when_time_advances_then_decisions_preserve_due_t "{}", test_case.description ); + assert_eq!( + skipped_seconds(&plan), + test_case.expected_skipped, + "{}", + test_case.description + ); assert_eq!( plan.last_tick_for.map(|time| time.timestamp()), test_case.expected_last @@ -73,7 +126,9 @@ fn given_durable_schedule_state_when_time_advances_then_decisions_preserve_due_t plan.actions.iter().map(|action| action.tick_id).collect(); assert_eq!( tick_ids.len(), - test_case.expected_requests.len() + test_case.expected_covered.len() + test_case.expected_requests.len() + + test_case.expected_covered.len() + + test_case.expected_skipped.len() ); let advanced = crate::scheduler::models::TickState { last_tick_for: plan.last_tick_for, diff --git a/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs index c58abe4898..0d8ba029a4 100644 --- a/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs +++ b/crates/sqlbuild-orchestration/src/scheduler/tests/test_types.rs @@ -17,8 +17,11 @@ pub(super) struct TickTestCase { pub(super) last_second: Option, pub(super) now_second: i64, pub(super) active: bool, + /// Another run of the job is running and the job's overlap is `skip`. + pub(super) running: bool, pub(super) expected_requests: Vec, pub(super) expected_covered: Vec, + pub(super) expected_skipped: Vec, pub(super) expected_last: Option, pub(super) expected_next: Option, } diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs index 6b5ba9f245..2fa31fabdc 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/conditions.rs @@ -1,20 +1,20 @@ //! Each native condition consumes its bounded page inside the evaluation transaction. -use uuid::Uuid; - use crate::sensors::_helpers::nodes::{ consume_node_events, consume_source_events, consume_stale_sources, }; use crate::sensors::_helpers::platform::consume_platform_events; use crate::sensors::_helpers::runs::consume_run_events; -use crate::sensors::models::{SensorCondition, SensorCursor, SensorEvaluationRequest}; +use crate::sensors::models::{ + SensorCondition, SensorCursor, SensorEvaluationRequest, SensorRequests, +}; use crate::store::errors::ControlError; use crate::store::models::ControlTransaction; pub(crate) async fn consume_condition( transaction: ControlTransaction, evaluation: &SensorEvaluationRequest, -) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { +) -> Result<(ControlTransaction, (SensorCursor, SensorRequests)), ControlError> { match &evaluation.definition.when { SensorCondition::Custom { .. } => Err(ControlError::Invalid( "custom sensors require a location result".into(), diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs index c2e65bb427..6918d453ff 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/custom.rs @@ -4,11 +4,11 @@ use std::collections::BTreeSet; use chrono::TimeDelta; use serde_json::json; -use uuid::Uuid; use crate::platform::main::read_location_work::read_location_work; use crate::platform::models::{LocationWork, LocationWorkOutcome, LocationWorkRequest}; use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::models::SensorRequests; use crate::sensors::models::{ CustomSensorOutcome, SensorCondition, SensorEvaluationOutcome, SensorEvaluationRequest, }; @@ -90,15 +90,17 @@ async fn apply_result( }; let mut cursor = evaluation.cursor.clone(); cursor.custom = custom; - let mut run_ids: Vec = Vec::new(); + let mut requests = SensorRequests::default(); let run_keys: BTreeSet = run_keys.into_iter().collect(); for key in run_keys { - let run_id; - (transaction, run_id) = queue_sensor_run(transaction, evaluation, &key).await?; - run_ids.push(run_id); + (transaction, requests) = queue_sensor_run(transaction, evaluation, &key, requests).await?; } Ok(( transaction, - Some(SensorEvaluationOutcome::Completed { cursor, run_ids }), + Some(SensorEvaluationOutcome::Completed { + cursor, + run_ids: requests.run_ids, + skipped: requests.skipped, + }), )) } diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs index 32e4853032..87e6ea7618 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/nodes.rs @@ -5,11 +5,11 @@ use std::num::NonZeroU32; use chrono::{DateTime, NaiveDateTime, TimeDelta, Utc}; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; -use uuid::Uuid; use crate::sensors::_helpers::requests::queue_sensor_run; use crate::sensors::_helpers::transitions::matches_transition; use crate::sensors::constants::TIMESTAMP_VALUE_KIND; +use crate::sensors::models::SensorRequests; use crate::sensors::models::{ NodeEventKind, NodeSensorEvent, SensorCursor, SensorEvaluationRequest, }; @@ -26,7 +26,7 @@ pub(crate) async fn consume_node_events( evaluation: &SensorEvaluationRequest, selection: &BTreeSet, kinds: &BTreeSet, -) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { +) -> Result<(ControlTransaction, (SensorCursor, SensorRequests)), ControlError> { let (mut transaction, rows) = query( transaction, "SELECT n.run_id,n.node_key,n.status,n.finished_storage_id FROM sqb_run_nodes n \ @@ -42,7 +42,7 @@ pub(crate) async fn consume_node_events( ) .await?; let mut cursor = evaluation.cursor.clone(); - let mut run_ids: Vec = Vec::new(); + let mut requests = SensorRequests::default(); for row in rows { let node: NodeSensorEvent = decode_row(row)?; cursor.event_storage_id = node.finished_storage_id; @@ -52,17 +52,16 @@ pub(crate) async fn consume_node_events( let matches; (transaction, matches) = matches_transition(transaction, evaluation, &node, kinds).await?; if matches { - let run_id; - (transaction, run_id) = queue_sensor_run( + (transaction, requests) = queue_sensor_run( transaction, evaluation, &format!("{}/{}", node.run_id, node.node_key), + requests, ) .await?; - run_ids.push(run_id); } } - Ok((transaction, (cursor, run_ids))) + Ok((transaction, (cursor, requests))) } /// A source's newest version queues one run per streak, keyed on the streak's first row so pruning never reuses a key. @@ -70,9 +69,9 @@ pub(crate) async fn consume_source_events( transaction: ControlTransaction, evaluation: &SensorEvaluationRequest, selection: &BTreeSet, -) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { +) -> Result<(ControlTransaction, (SensorCursor, SensorRequests)), ControlError> { let (mut transaction, (nodes, next)) = source_page(transaction, evaluation, selection).await?; - let mut run_ids: Vec = Vec::new(); + let mut requests = SensorRequests::default(); for (key, definition) in nodes { let (parameters, predicate) = source_scope(evaluation, &key, &definition); let rows; @@ -121,13 +120,12 @@ pub(crate) async fn consume_source_events( if !requested.is_empty() { continue; } - let run_id; - (transaction, run_id) = queue_sensor_run(transaction, evaluation, &run_key).await?; - run_ids.push(run_id); + (transaction, requests) = + queue_sensor_run(transaction, evaluation, &run_key, requests).await?; } let mut cursor = evaluation.cursor.clone(); cursor.custom = next; - Ok((transaction, (cursor, run_ids))) + Ok((transaction, (cursor, requests))) } /// One page of selected source definitions after the cursor's last key, and the next page's start. @@ -235,9 +233,9 @@ pub(crate) async fn consume_stale_sources( evaluation: &SensorEvaluationRequest, selection: &BTreeSet, after_seconds: NonZeroU32, -) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { +) -> Result<(ControlTransaction, (SensorCursor, SensorRequests)), ControlError> { let (mut transaction, (nodes, next)) = source_page(transaction, evaluation, selection).await?; - let mut run_ids: Vec = Vec::new(); + let mut requests = SensorRequests::default(); for (key, definition) in nodes { let (parameters, predicate) = source_scope(evaluation, &key, &definition); let rows; @@ -261,12 +259,10 @@ pub(crate) async fn consume_stale_sources( after_seconds, )) .map_err(|error| ControlError::Invalid(error.to_string()))?; - let run_id; - (transaction, run_id) = - queue_sensor_run(transaction, evaluation, &sha256_hex(&identity)).await?; - run_ids.push(run_id); + (transaction, requests) = + queue_sensor_run(transaction, evaluation, &sha256_hex(&identity), requests).await?; } let mut cursor = evaluation.cursor.clone(); cursor.custom = next; - Ok((transaction, (cursor, run_ids))) + Ok((transaction, (cursor, requests))) } diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs index 6ce610bf03..277a88763b 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/platform.rs @@ -1,10 +1,9 @@ //! One bounded platform page yields matched run requests and a storage-order cursor. -use uuid::Uuid; - use crate::platform::main::read_platform_events::read_platform_events; use crate::platform::models::PlatformEventKind; use crate::sensors::_helpers::requests::queue_sensor_run; +use crate::sensors::models::SensorRequests; use crate::sensors::models::{SensorCursor, SensorEvaluationRequest}; use crate::store::errors::ControlError; use crate::store::models::ControlTransaction; @@ -13,7 +12,7 @@ pub(crate) async fn consume_platform_events( transaction: ControlTransaction, evaluation: &SensorEvaluationRequest, kinds: &std::collections::BTreeSet, -) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { +) -> Result<(ControlTransaction, (SensorCursor, SensorRequests)), ControlError> { let (mut transaction, events) = read_platform_events(transaction, evaluation.cursor.event_storage_id).await?; let cursor = SensorCursor { @@ -22,15 +21,18 @@ pub(crate) async fn consume_platform_events( .map_or(evaluation.cursor.event_storage_id, |event| event.storage_id), ..evaluation.cursor.clone() }; - let mut run_ids: Vec = Vec::new(); + let mut requests = SensorRequests::default(); for event in events .iter() .filter(|event| kinds.contains(&event.payload.platform.kind())) { - let run_id; - (transaction, run_id) = - queue_sensor_run(transaction, evaluation, &event.event_id.to_string()).await?; - run_ids.push(run_id); + (transaction, requests) = queue_sensor_run( + transaction, + evaluation, + &event.event_id.to_string(), + requests, + ) + .await?; } - Ok((transaction, (cursor, run_ids))) + Ok((transaction, (cursor, requests))) } diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs index 5c7c6de009..77ea2cab17 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/requests.rs @@ -3,19 +3,24 @@ use uuid::Uuid; use crate::credentials::models::AuditContext; +use crate::declarations::main::published_job::published_job; +use crate::declarations::models::Overlap; use crate::runs::main::frozen_job_request::frozen_job_request; use crate::runs::main::queue_run::queue_run; use crate::runs::models::{JobRunRequest, RunOrigin}; -use crate::sensors::models::SensorEvaluationRequest; +use crate::scheduler::main::active_job_run::active_job_run; +use crate::sensors::models::{SensorEvaluationRequest, SensorRequests, SkippedRequest}; use crate::store::errors::ControlError; use crate::store::main::query::query; use crate::store::models::{ControlTransaction, Parameter}; +/// Queue the run for `run_key` once, or record it skipped while a run of a `skip` job is active. pub(crate) async fn queue_sensor_run( transaction: ControlTransaction, evaluation: &SensorEvaluationRequest, run_key: &str, -) -> Result<(ControlTransaction, Uuid), ControlError> { + mut requests: SensorRequests, +) -> Result<(ControlTransaction, SensorRequests), ControlError> { let (transaction, rows) = query( transaction, "SELECT run_id FROM sqb_run_requests WHERE trigger_id=$1 AND run_key=$2", @@ -26,11 +31,29 @@ pub(crate) async fn queue_sensor_run( ) .await?; if let Some(row) = rows.into_iter().next() { - return Ok(( - transaction, + requests.run_ids.push( serde_json::from_value(row["run_id"].clone()) .map_err(|error| ControlError::Invalid(error.to_string()))?, - )); + ); + return Ok((transaction, requests)); + } + let job_key = &evaluation.definition.job_key; + let (transaction, overlap) = job_overlap(transaction, evaluation).await?; + let (transaction, active) = if overlap.is_queue() { + (transaction, None) + } else { + active_job_run( + transaction, + (evaluation.location_id, &evaluation.target, job_key), + ) + .await? + }; + if let Some(active_run_id) = active { + requests.skipped.push(SkippedRequest { + run_key: run_key.into(), + active_run_id, + }); + return Ok((transaction, requests)); } let run_id = Uuid::new_v5(&evaluation.trigger_id, run_key.as_bytes()); let context = AuditContext { @@ -62,5 +85,27 @@ pub(crate) async fn queue_sensor_run( let (transaction, _) = query(transaction, "INSERT INTO sqb_run_requests (trigger_id,run_key,evaluation_id,run_id) VALUES ($1,$2,$3,$4)", vec![Parameter::Id(evaluation.trigger_id), Parameter::Text(run_key.into()), Parameter::Id(evaluation.evaluation_id), Parameter::Id(run_id)]).await?; - Ok((transaction, run_id)) + requests.run_ids.push(run_id); + Ok((transaction, requests)) +} + +/// The overlap the sensor's job declares in the evaluated snapshot, which the target currently publishes. +async fn job_overlap( + transaction: ControlTransaction, + evaluation: &SensorEvaluationRequest, +) -> Result<(ControlTransaction, Overlap), ControlError> { + let (transaction, job) = published_job( + transaction, + ( + evaluation.location_id, + &evaluation.target, + &evaluation.definition.job_key, + ), + ) + .await?; + Ok(( + transaction, + job.filter(|job| job.snapshot_id == evaluation.snapshot_id) + .map_or(Overlap::Queue, |job| job.definition_json.overlap), + )) } diff --git a/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs index 63ab8aaf02..791ed0093d 100644 --- a/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs +++ b/crates/sqlbuild-orchestration/src/sensors/_helpers/runs.rs @@ -6,6 +6,7 @@ use uuid::Uuid; use crate::sensors::_helpers::requests::queue_sensor_run; use crate::sensors::constants::EXECUTION_NODE_BATCH_SIZE; +use crate::sensors::models::SensorRequests; use crate::sensors::models::{ RunEventKind, SensorCursor, SensorEvaluationRequest, TerminalSensorRun, }; @@ -19,7 +20,7 @@ pub(crate) async fn consume_run_events( evaluation: &SensorEvaluationRequest, (job_key, kinds): (&str, &BTreeSet), executed_successfully: Option<&BTreeSet>, -) -> Result<(ControlTransaction, (SensorCursor, Vec)), ControlError> { +) -> Result<(ControlTransaction, (SensorCursor, SensorRequests)), ControlError> { let (mut transaction, rows) = query( transaction, "SELECT run_id,status,last_event_storage_id FROM sqb_runs \ @@ -41,17 +42,16 @@ pub(crate) async fn consume_run_events( (transaction, eligible) = matching_executions(transaction, eligible, nodes).await?; } let mut cursor = evaluation.cursor.clone(); - let mut run_ids: Vec = Vec::new(); + let mut requests = SensorRequests::default(); for run in runs { cursor.event_storage_id = run.last_event_storage_id; if eligible.contains(&run.run_id) && kinds.iter().any(|kind| kind.matches(run.status)) { - let run_id; - (transaction, run_id) = - queue_sensor_run(transaction, evaluation, &run.run_id.to_string()).await?; - run_ids.push(run_id); + (transaction, requests) = + queue_sensor_run(transaction, evaluation, &run.run_id.to_string(), requests) + .await?; } } - Ok((transaction, (cursor, run_ids))) + Ok((transaction, (cursor, requests))) } /// Each chunk intersects the previous matches: every selected node must qualify. diff --git a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs index 5fbd35834d..390f81a82c 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/complete_builtin_sensor.rs @@ -15,13 +15,17 @@ pub async fn complete_builtin_sensor( let Some(transaction) = pending_evaluation(store, lease, evaluation).await? else { return Ok(0); }; - let (transaction, (cursor, run_ids)) = consume_condition(transaction, evaluation).await?; - let count = run_ids.len(); + let (transaction, (cursor, requests)) = consume_condition(transaction, evaluation).await?; + let count = requests.run_ids.len(); finish_evaluation( transaction, lease, evaluation, - SensorEvaluationOutcome::Completed { cursor, run_ids }, + SensorEvaluationOutcome::Completed { + cursor, + run_ids: requests.run_ids, + skipped: requests.skipped, + }, ) .await?; Ok(count) diff --git a/crates/sqlbuild-orchestration/src/sensors/main/read_person_sensor.rs b/crates/sqlbuild-orchestration/src/sensors/main/read_person_sensor.rs index 5a29f75caa..14ab096e2d 100644 --- a/crates/sqlbuild-orchestration/src/sensors/main/read_person_sensor.rs +++ b/crates/sqlbuild-orchestration/src/sensors/main/read_person_sensor.rs @@ -118,6 +118,7 @@ async fn evaluations( cursor_after: None, run_count: 0, failure_code: None, + skipped_for: Vec::new(), definition_fingerprint: request.definition_fingerprint.clone(), }); let outcome = event.fact_json.outcome; @@ -127,9 +128,14 @@ async fn evaluations( check.phase = outcome.phase().into(); check.finished_at = Some(event.occurred_at); match outcome { - SensorEvaluationOutcome::Completed { cursor, run_ids } => { + SensorEvaluationOutcome::Completed { + cursor, + run_ids, + skipped, + } => { check.cursor_after = Some(shown_cursor(&cursor, evidence)); check.run_count = run_ids.len(); + check.skipped_for = skipped.iter().map(|skip| skip.active_run_id).collect(); } SensorEvaluationOutcome::Failed { code } => check.failure_code = Some(code), _ => {} diff --git a/crates/sqlbuild-orchestration/src/sensors/models.rs b/crates/sqlbuild-orchestration/src/sensors/models.rs index 33e8c7c098..936bddc237 100644 --- a/crates/sqlbuild-orchestration/src/sensors/models.rs +++ b/crates/sqlbuild-orchestration/src/sensors/models.rs @@ -158,6 +158,8 @@ pub enum SensorEvaluationOutcome { Completed { cursor: SensorCursor, run_ids: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + skipped: Vec, }, Failed { code: String, @@ -166,6 +168,21 @@ pub enum SensorEvaluationOutcome { Abandoned, } +/// A run key the sensor matched but did not queue: the job's overlap is `skip` and `active_run_id` was active. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SkippedRequest { + pub run_key: String, + pub active_run_id: Uuid, +} + +/// The runs one evaluation queued and the requests it skipped, in match order. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct SensorRequests { + pub run_ids: Vec, + pub skipped: Vec, +} + impl SensorEvaluationOutcome { #[must_use] pub const fn phase(&self) -> &'static str { @@ -229,6 +246,8 @@ pub struct PersonSensorEvaluation { pub run_count: usize, /// The failure code of a failed check. pub failure_code: Option, + /// For each matched request the job's `skip` overlap dropped, the run that was active. + pub skipped_for: Vec, pub definition_fingerprint: String, } diff --git a/crates/sqlbuild-orchestration/src/store/schema.sql b/crates/sqlbuild-orchestration/src/store/schema.sql index eb7b972977..fafa971546 100644 --- a/crates/sqlbuild-orchestration/src/store/schema.sql +++ b/crates/sqlbuild-orchestration/src/store/schema.sql @@ -120,9 +120,10 @@ CREATE TABLE sqb_ticks ( scheduled_for {time} NOT NULL, evaluated_at {time} NOT NULL, epoch BIGINT NOT NULL, - outcome TEXT NOT NULL CHECK (outcome IN ('requested', 'covered')), + outcome TEXT NOT NULL CHECK (outcome IN ('requested', 'covered', 'skipped')), run_id {uuid}, covered_by_run_id {uuid}, + active_run_id {uuid}, UNIQUE (trigger_id, scheduled_for) ); CREATE INDEX sqb_ticks_trigger_time ON sqb_ticks(trigger_id, scheduled_for); diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs index c01025625a..fb05efb12f 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs @@ -419,10 +419,10 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ let expected_jobs = [ json!({"job_key": "nightly", "kind": "group", "description": "Nightly orders", "selection": ["orders"], "concurrency_groups": [], "snapshot_id": published.snapshot_id, - "select": null, "declared_at": null}), + "select": null, "declared_at": null, "owner": null, "overlap": "queue", "on_failure": null}), json!({"job_key": "orders", "kind": "job", "description": "Refresh orders", "selection": ["orders"], "concurrency_groups": ["warehouse_heavy"], "snapshot_id": published.snapshot_id, - "select": null, "declared_at": null}), + "select": null, "declared_at": null, "owner": null, "overlap": "queue", "on_failure": null}), ]; assert_eq!( listed_jobs, diff --git a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs index f761083db0..a9b30517ba 100644 --- a/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/claims/helpers.rs @@ -33,6 +33,15 @@ pub(crate) async fn slot_counts(store: &ControlStore) -> BTreeMap { } pub(crate) async fn succeed(store: &ControlStore, claim: &RunClaim) { + finish(store, claim, (NodeOutcome::Success, RunOutcome::Success)).await; +} + +/// Every selected node and the run end with the given outcomes. +pub(crate) async fn finish( + store: &ControlStore, + claim: &RunClaim, + (node_outcome, run_outcome): (NodeOutcome, RunOutcome), +) { let mut changes = vec![RunChange::Started]; changes.extend( claim @@ -41,7 +50,7 @@ pub(crate) async fn succeed(store: &ControlStore, claim: &RunClaim) { .iter() .map(|node| RunChange::NodeFinished { node_key: node.clone(), - outcome: NodeOutcome::Success, + outcome: node_outcome, rows_affected: Some(1), error_code: None, skipped_because_of: None, @@ -49,7 +58,7 @@ pub(crate) async fn succeed(store: &ControlStore, claim: &RunClaim) { }), ); changes.push(RunChange::Finished { - outcome: RunOutcome::Success, + outcome: run_outcome, reason: "synthetic child completed".into(), exit_code: Some(0), }); diff --git a/crates/sqlbuild-orchestration/tests/runs/helpers.rs b/crates/sqlbuild-orchestration/tests/runs/helpers.rs index 5dec2a4ee4..bdf42f332a 100644 --- a/crates/sqlbuild-orchestration/tests/runs/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/runs/helpers.rs @@ -94,6 +94,7 @@ pub(crate) fn request( window: None, concurrency_groups: BTreeSet::new(), narrowing: None, + retry: None, } } diff --git a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs index ba16619fc2..64b25eae06 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_coalesce.rs @@ -3,7 +3,9 @@ use std::collections::BTreeMap; use crate::declarations::main::list_triggers::list_triggers; use crate::runs::main::claim_next_run::claim_next_run; use crate::runs::main::list_runs::list_runs; -use crate::scheduler::models::TickDecision; +use crate::runs::tests::claim_helpers::succeed; +use crate::scheduler::main::list_ticks::list_ticks; +use crate::scheduler::models::{TickDecision, TickOutcome}; use crate::scheduler::tests::contracts::helpers::{acquire_lease, evaluate_trigger}; use crate::scheduler::tests::contracts::ticks::helpers::calendar_text; use crate::snapshots::main::publish_snapshot::publish_snapshot; @@ -112,3 +114,111 @@ fn given_coalescing_triggers_when_a_previous_run_has_been_claimed_then_new_ticks }); } } + +#[test] +fn given_a_skip_overlap_job_when_a_tick_falls_due_during_its_run_then_the_tick_is_skipped_and_the_next_one_runs() + { + let test_cases = [ + CoalescedRunTestCase { + description: "SQLite skips an on-time tick while the job runs", + backend: Backend::Sqlite, + expected_runs: 2, + }, + CoalescedRunTestCase { + description: "PostgreSQL skips an on-time tick while the job runs", + backend: Backend::Postgres, + expected_runs: 2, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"], "overlap": "skip"}}), + ); + input.documents.insert("triggers".into(), json!({"daily_orders": {"description": "Refresh orders", "job_key": "orders", "starts_at": calendar_text(0), + "schedule": {"kind": "calendar", "times": [calendar_text(0)], "refresh_at": "2030-01-01T00:00:00Z"}}})); + publish_snapshot(store, &input).await.expect("publish"); + let trigger = list_triggers(store, location, "development") + .await + .expect("trigger")[0] + .trigger_id; + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + evaluate_trigger(store, &lease, trigger) + .await + .expect("first tick"); + let claim = claim_next_run(store, &lease, &BTreeMap::new(), None) + .await + .expect("claim first run") + .expect("scheduled run"); + input.generation = 2; + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!((0..2).map(calendar_text).collect::>()); + publish_snapshot(store, &input) + .await + .expect("add a second calendar time"); + let plan = evaluate_trigger(store, &lease, trigger) + .await + .expect("tick while the first run executes") + .expect("active trigger"); + assert_eq!( + plan.actions[0].decision, + TickDecision::Skipped { + active_run_id: claim.request.run_id + }, + "{}", + test_case.description + ); + succeed(store, &claim).await; + input.generation = 3; + input.documents.get_mut("triggers").expect("triggers")["daily_orders"]["schedule"]["times"] = + json!((0..3).map(calendar_text).collect::>()); + publish_snapshot(store, &input) + .await + .expect("add a third calendar time"); + let plan = evaluate_trigger(store, &lease, trigger) + .await + .expect("tick after the first run ended") + .expect("active trigger"); + assert!( + matches!(plan.actions[0].decision, TickDecision::Requested { .. }), + "{}: the job ended, so the next tick queues", + test_case.description + ); + let ticks = list_ticks(store, trigger, 10).await.expect("ticks"); + let skipped: Vec<_> = ticks + .iter() + .filter(|tick| tick.outcome == TickOutcome::Skipped) + .map(|tick| (tick.active_run_id, tick.run_id)) + .collect(); + assert_eq!( + skipped, + vec![(Some(claim.request.run_id), None)], + "{}", + test_case.description + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("runs") + .len(), + test_case.expected_runs, + "{}: a skipped tick creates no run", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs index 04088727a6..263d09eee7 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_contracts.rs @@ -36,5 +36,7 @@ mod test_permissions; mod test_requests; #[path = "test_reruns.rs"] mod test_reruns; +#[path = "test_retries.rs"] +mod test_retries; #[path = "test_types.rs"] pub(crate) mod test_types; diff --git a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs index 11b8b13088..e8586b2850 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_job_detail.rs @@ -10,6 +10,7 @@ use crate::credentials::models::{BuiltinRole, DefaultRole, RoleAssignmentRequest use crate::credentials::tests::role_helpers::{ SUPPORT_ROLE, assign, credentials, orders_grant, role_fixture, }; +use crate::declarations::main::list_person_jobs::list_person_jobs; use crate::declarations::main::read_person_job::read_person_job; use crate::runs::_helpers::requests::job::frozen_group_children; use crate::runs::main::{ @@ -65,7 +66,9 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r "warehouse": "postgres://orders:orders-db-pass@warehouse.example.test/orders", "concurrency": 4}, "select": "orders", "targets": ["development"], - "declared_at": {"path": "orchestration/jobs.py", "line": 7}}, + "declared_at": {"path": "orchestration/jobs.py", "line": 7}, + "owner": "Orders team", "overlap": "skip", + "on_failure": {"retries": 2, "retry_delay_seconds": 300}}, "customers": {"description": "Build customers", "selection": ["customers"]}, })); input.documents.insert( @@ -138,6 +141,41 @@ fn given_published_jobs_when_people_read_and_launch_them_then_listing_view_and_r "{}: the selector as written, its targets, where it is declared and its nodes' cursor type", test_case.description ); + let listed = list_person_jobs( + store, + administrator, + (location, "development"), + DefaultRole::None, + ) + .await + .expect("administrator lists jobs"); + let policies = + |owner: &Option, overlap, on_failure| json!([owner, overlap, on_failure]); + assert_eq!( + ( + policies(&job.owner, job.overlap, job.on_failure), + listed + .iter() + .map(|record| ( + record.job_key.as_str(), + policies(&record.owner, record.overlap, record.on_failure) + )) + .collect::>() + ), + ( + json!(["Orders team", "skip", {"retries": 2, "retry_delay_seconds": 300}]), + vec![ + ("customers", json!([null, "queue", null])), + ( + "daily_orders", + json!(["Orders team", "skip", {"retries": 2, "retry_delay_seconds": 300}]) + ), + ("orders_group", json!([null, "queue", null])) + ] + ), + "{}: owner, overlap and retries read as declared, and an undeclared job queues without retries", + test_case.description + ); let group = read(administrator, "orders_group") .await .expect("administrator") diff --git a/crates/sqlbuild-orchestration/tests/runs/test_retries.rs b/crates/sqlbuild-orchestration/tests/runs/test_retries.rs new file mode 100644 index 0000000000..d3cf93578f --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/runs/test_retries.rs @@ -0,0 +1,223 @@ +//! A job's `on_failure` policy reruns failed work from failure until its retries are spent or a person cancels. + +use std::collections::BTreeMap; + +use serde_json::json; +use uuid::Uuid; + +use crate::credentials::models::AuditContext; +use crate::credentials::tests::location_helpers::create_location; +use crate::runs::_helpers::lifecycle::reads::list_runs_in; +use crate::runs::main::{ + change_run::change_run, claim_next_run::claim_next_run, list_runs::list_runs, + request_job_run::request_job_run, request_rerun::request_rerun, +}; +use crate::runs::models::{ + JobRunRequest, NodeOutcome, QueueWait, RerunRequest, RunAuthority, RunChange, RunChangeRequest, + RunOrigin, RunOutcome, RunStatus, +}; +use crate::runs::tests::claim_helpers::finish; +use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::test_types::RetryChainTestCase; +use crate::runs::types::RunsListing; +use crate::scheduler::tests::contracts::helpers::acquire_lease; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::{begin::begin, commit::commit, migrate_store::migrate_store}; +use crate::store::models::TransactionMode; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_job_with_on_failure_when_its_runs_fail_then_retries_rerun_failed_work_until_spent_or_cancelled() + { + let test_cases = [ + RetryChainTestCase { + description: "SQLite retry chain", + backend: Backend::Sqlite, + expected_attempts: vec![1, 2], + expected_delayed_attempt: 1, + }, + RetryChainTestCase { + description: "PostgreSQL retry chain", + backend: Backend::Postgres, + expected_attempts: vec![1, 2], + expected_delayed_attempt: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let context = AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({ + "orders": {"description": "Build orders", "selection": ["orders", "customers"], + "on_failure": {"retries": 2, "retry_delay_seconds": 0}}, + "inventory": {"description": "Build inventory", "selection": ["inventory"], + "on_failure": {"retries": 3, "retry_delay_seconds": 3600}} + }), + ); + publish_snapshot(store, &input).await.expect("publish"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + let limits = BTreeMap::new(); + let scope = (location, "development"); + let requested = |job_key: &str| JobRunRequest { + run_id: Uuid::now_v7(), + location_id: location, + target: "development".into(), + job_key: job_key.into(), + reason: "Build".into(), + priority: 0, + window: None, + full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), + }; + let source = requested("orders"); + request_job_run(store, (&context, None), &source) + .await + .expect("request orders"); + let first = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect("source run"); + finish(store, &first, (NodeOutcome::Failed, RunOutcome::Failed)).await; + let retry_one = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim first retry") + .expect("first retry"); + finish(store, &retry_one, (NodeOutcome::Failed, RunOutcome::Failed)).await; + let retry_two = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim second retry") + .expect("second retry"); + finish(store, &retry_two, (NodeOutcome::Failed, RunOutcome::Failed)).await; + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim after retries are spent") + .is_none(), + "{}: no retry is queued once retries are spent", + test_case.description + ); + assert_eq!( + [&retry_one, &retry_two] + .map(|claim| (claim.request.origin, claim.request.rerun_of_run_id)), + [ + (RunOrigin::Rerun, Some(source.run_id)), + (RunOrigin::Rerun, Some(retry_one.request.run_id)), + ], + "{}: each retry reruns the run before it", + test_case.description + ); + assert_eq!( + [&retry_one, &retry_two] + .map(|claim| claim.request.retry.as_ref().map(|retry| retry.attempt)) + .to_vec(), + test_case + .expected_attempts + .iter() + .copied() + .map(Some) + .collect::>(), + "{}", + test_case.description + ); + + let delayed = requested("inventory"); + request_job_run(store, (&context, None), &delayed) + .await + .expect("request inventory"); + let claim = claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim") + .expect("inventory run"); + finish(store, &claim, (NodeOutcome::Failed, RunOutcome::Failed)).await; + assert!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim before the delay") + .is_none(), + "a retry is not claimed before its delay" + ); + let transaction = begin(store, TransactionMode::Read).await.expect("begin"); + let (transaction, queued) = list_runs_in(transaction, scope, RunsListing::Queue(20)) + .await + .expect("queue"); + commit(transaction).await.expect("commit"); + let [waiting] = queued.as_slice() else { + panic!("one waiting retry, not {queued:?}"); + }; + let waits = &waiting.queue.as_ref().expect("queue state").waits; + assert!( + matches!(waits.as_slice(), [QueueWait::RetryDelay { attempt, .. }] if *attempt == test_case.expected_delayed_attempt), + "{}: {waits:?}", + test_case.description + ); + change_run( + store, + RunAuthority::Operator(&context), + &RunChangeRequest { + event_id: Uuid::now_v7(), + run_id: waiting.run_id, + claim_id: None, + change: RunChange::CancelRequested { + actor: "test-operator".into(), + reason: "stop retrying".into(), + }, + }, + ) + .await + .expect("cancel the pending retry"); + let runs = list_runs(store, scope, 20).await.expect("list"); + let inventory: Vec = runs + .iter() + .filter(|run| run.job_key.as_deref() == Some("inventory")) + .map(|run| run.status) + .collect(); + assert_eq!( + inventory, + vec![RunStatus::Cancelled, RunStatus::Failed], + "{}: cancelling the pending retry ends the chain", + test_case.description + ); + let rerun = RerunRequest { + run_id: Uuid::now_v7(), + source_run_id: waiting.run_id, + from_failure: false, + reason: "Retry by hand".into(), + priority: 0, + nodes: Vec::new(), + }; + request_rerun(store, (&context, None), &rerun) + .await + .expect("rerun the cancelled retry"); + assert_eq!( + claim_next_run(store, &lease, &limits, Some(scope)) + .await + .expect("claim the rerun") + .map(|claim| claim.request.run_id), + Some(rerun.run_id), + "{}: a person's rerun of a retry starts at once, without the retry's delay", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/runs/test_types.rs b/crates/sqlbuild-orchestration/tests/runs/test_types.rs index 017d2d8798..578a9e52df 100644 --- a/crates/sqlbuild-orchestration/tests/runs/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/runs/test_types.rs @@ -265,3 +265,13 @@ pub(crate) struct JobDetailTestCase { pub(crate) backend: Backend, pub(crate) expected_order: [&'static str; 2], } + +#[derive(Debug)] +pub(crate) struct RetryChainTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + /// Attempt numbers of the retries claimed, in order, while every run fails. + pub(crate) expected_attempts: Vec, + /// The wait a one-hour retry delay shows while the retry is queued. + pub(crate) expected_delayed_attempt: u8, +} diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs b/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs index e0fde5327c..d21d8b9706 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_custom.rs @@ -291,8 +291,9 @@ fn given_custom_sensor_work_when_results_retry_then_cursor_runs_and_outcome_comm ); } let facts = evaluations(store).await; - let SensorEvaluationOutcome::Completed { cursor, run_ids } = - &facts.last().expect("completed").outcome + let SensorEvaluationOutcome::Completed { + cursor, run_ids, .. + } = &facts.last().expect("completed").outcome else { panic!("completed phase required"); }; diff --git a/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs b/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs index 1e132090e2..2116afd00d 100644 --- a/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs +++ b/crates/sqlbuild-orchestration/tests/sensors/test_delivery.rs @@ -2,12 +2,16 @@ use std::collections::BTreeSet; use std::time::Duration; use chrono::{TimeDelta, Utc}; +use serde_json::json; use uuid::Uuid; +use crate::credentials::models::AuditContext; use crate::credentials::tests::location_helpers::create_location; use crate::declarations::main::list_triggers::list_triggers; use crate::declarations::models::TriggerChange; use crate::runs::main::list_runs::list_runs; +use crate::runs::main::request_job_run::request_job_run; +use crate::runs::models::JobRunRequest; use crate::scheduler::tests::contracts::helpers::acquire_lease; use crate::sensors::main::begin_sensor_evaluation::begin_sensor_evaluation; use crate::sensors::main::complete_builtin_sensor::complete_builtin_sensor; @@ -180,3 +184,116 @@ fn given_paged_platform_events_when_sensor_publication_fails_then_retry_is_atomi }); } } + +#[test] +fn given_a_skip_overlap_job_with_an_active_run_when_a_sensor_matches_then_it_records_skips_instead_of_queueing() + { + let test_cases = [ + SensorTestCase { + description: "SQLite sensor skip", + backend: Backend::Sqlite, + expected_runs: 1, + }, + SensorTestCase { + description: "PostgreSQL sensor skip", + backend: Backend::Postgres, + expected_runs: 1, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let location = create_location(store, "orders", "test-admin") + .await + .expect("location"); + let mut input = snapshot(location); + input.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Refresh orders", "selection": ["orders"], "overlap": "skip"}}), + ); + publish_snapshot(store, &input) + .await + .expect("publish sensor"); + let trigger = list_triggers(store, location, "development") + .await + .expect("triggers") + .remove(0); + let active = Uuid::now_v7(); + request_job_run( + store, + ( + &AuditContext { + actor: "test-operator".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }, + None, + ), + &JobRunRequest { + run_id: active, + location_id: location, + target: "development".into(), + job_key: "orders".into(), + reason: "Refresh by hand".into(), + priority: 0, + window: None, + full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), + }, + ) + .await + .expect("a person's run queues regardless of overlap"); + let lease = acquire_lease(store, Uuid::now_v7()) + .await + .expect("elect") + .expect("leader"); + // The first fixture event is a tick-overdue event, which the sensor matches. + append_events(store, 0..1, Utc::now()).await; + let requested = begin_sensor_evaluation(store, &lease, trigger.trigger_id) + .await + .expect("request") + .expect("due"); + assert_eq!( + complete_builtin_sensor(store, &lease, &requested) + .await + .expect("complete"), + 0, + "{}", + test_case.description + ); + let facts = evaluations(store).await; + let Some(SensorEvaluationOutcome::Completed { + run_ids, skipped, .. + }) = facts.last().map(|fact| fact.outcome.clone()) + else { + panic!("a completed evaluation, not {facts:?}"); + }; + assert!(run_ids.is_empty(), "{}", test_case.description); + assert_eq!( + skipped + .iter() + .map(|skip| skip.active_run_id) + .collect::>(), + vec![active], + "{}: the matched event names the active run", + test_case.description + ); + assert_eq!( + list_runs(store, (location, "development"), 100) + .await + .expect("runs") + .len(), + test_case.expected_runs, + "{}", + test_case.description + ); + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs index 48f7574c19..a63c08cd12 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/compiled_project/snapshot_declarations.rs @@ -22,6 +22,7 @@ use sqlbuild_orchestration::snapshots::models::SnapshotInput; use crate::bindings::_helpers::native_host::orchestration_declarations::{ AuthoredDeclaration, AuthoredGroup, AuthoredJob, AuthoredSensor, declared_at, module_jobs, + run_controls, }; use crate::bindings::_helpers::native_host::orchestration_triggers::{ compile_calendars, compile_cron, compile_sensors, @@ -119,6 +120,14 @@ pub(crate) fn compile_declarations( .map(|(kind, name)| format!("{kind}:{name}")) .collect(); let declared_at: Option = declared_at(project_dir, job.declared_at)?; + let controls = run_controls( + &job.name, + ( + job.owner, + &job.overlap, + job.on_failure.as_ref().map(|policy| policy.bind(py)), + ), + )?; let targets: BTreeSet = job.targets.into_iter().collect(); if let Some(schedule) = job.schedule { triggers.insert( @@ -141,6 +150,9 @@ pub(crate) fn compile_declarations( exclude: Some(job.exclude).filter(|exclude| !exclude.trim().is_empty()), targets, declared_at, + owner: controls.owner, + overlap: controls.overlap, + on_failure: controls.on_failure, }, ); } @@ -170,6 +182,14 @@ pub(crate) fn compile_declarations( .flat_map(|name| jobs[name].selection.iter().cloned()) .collect(); let declared_at: Option = declared_at(project_dir, group.declared_at)?; + let controls = run_controls( + &group.name, + ( + group.owner, + &group.overlap, + group.on_failure.as_ref().map(|policy| policy.bind(py)), + ), + )?; let targets: BTreeSet = group.targets.into_iter().collect(); if let Some(schedule) = group.schedule { triggers.insert( @@ -193,6 +213,9 @@ pub(crate) fn compile_declarations( exclude: None, targets, declared_at, + owner: controls.owner, + overlap: controls.overlap, + on_failure: controls.on_failure, }, jobs: dependencies, }, diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs index 84742b4d46..d19d662071 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/constants.rs @@ -24,3 +24,7 @@ pub(crate) const CALENDAR_LABEL: &str = "calendar schedule"; /// Pages one bounded state-history SELECT streams before the scan continues from its cursor. pub(crate) const HISTORY_STREAM_PAGES: usize = 50; +/// Inclusive bounds on a job's `on_failure` retries. +pub(crate) const MAX_RETRIES: (u64, u64) = (1, 10); +/// Inclusive bounds on a job's `on_failure` retry delay, in seconds: up to 24 hours. +pub(crate) const MAX_RETRY_DELAY: (u64, u64) = (0, 86_400); diff --git a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs index 1c4dda6f85..e465889596 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/native_host/orchestration_declarations.rs @@ -11,11 +11,11 @@ use serde_json::Value; use sqlbuild_analysis::graph::models::JobSelection; use sqlbuild_discovery::project_validation::models::OrchestrationNameFact; use sqlbuild_host::entries::models::UserCodeHost; -use sqlbuild_orchestration::declarations::models::DeclaredAt; +use sqlbuild_orchestration::declarations::models::{DeclaredAt, OnFailure, Overlap}; use crate::bindings::_helpers::native_host::constants::{ - AUTHORING_MODULE, CALENDAR_LABEL, CONSTRUCTING_MODULES, GROUP_LABEL, JOB_LABEL, - RUN_EVENT_CONDITION, SENSOR_LABEL, + AUTHORING_MODULE, CALENDAR_LABEL, CONSTRUCTING_MODULES, GROUP_LABEL, JOB_LABEL, MAX_RETRIES, + MAX_RETRY_DELAY, RUN_EVENT_CONDITION, SENSOR_LABEL, }; use crate::bindings::_helpers::native_host::host::entry; @@ -34,6 +34,9 @@ pub(crate) struct AuthoredJob { pub(crate) targets: Vec, #[pyo3(attribute("_declared_at"))] pub(crate) declared_at: Option<(String, u32)>, + pub(crate) owner: Option, + pub(crate) overlap: String, + pub(crate) on_failure: Option>, } #[derive(FromPyObject, PartialEq, Eq, PartialOrd, Ord)] @@ -52,6 +55,16 @@ pub(crate) struct AuthoredGroup { pub(crate) targets: Vec, #[pyo3(attribute("_declared_at"))] pub(crate) declared_at: Option<(String, u32)>, + pub(crate) owner: Option, + pub(crate) overlap: String, + pub(crate) on_failure: Option>, +} + +/// A job's or group's owner, overlap and failure policy, validated where they enter native code. +pub(crate) struct RunControls { + pub(crate) owner: Option, + pub(crate) overlap: Overlap, + pub(crate) on_failure: Option, } #[derive(FromPyObject)] @@ -152,6 +165,53 @@ pub(crate) fn authored_setting(value: &Bound<'_, PyAny>) -> PyResult, &str, Option<&Bound<'_, PyAny>>), +) -> PyResult { + if owner + .as_deref() + .is_some_and(|owner| owner.trim().is_empty()) + { + return Err(PyValueError::new_err(format!( + "job {name} owner must not be blank" + ))); + } + let overlap = match overlap { + "queue" => Overlap::Queue, + "skip" => Overlap::Skip, + other => { + return Err(PyValueError::new_err(format!( + "job {name} overlap must be queue or skip, not {other}" + ))); + } + }; + let on_failure = if let Some(policy) = policy { + let bounded = |field: &str, (low, high): (u64, u64)| -> PyResult { + authored_setting(&policy.getattr(field)?)? + .as_u64() + .filter(|value| (low..=high).contains(value)) + .ok_or_else(|| { + PyValueError::new_err(format!( + "job {name} on_failure {field} must be a whole number from {low} to {high}" + )) + }) + }; + Some(OnFailure { + retries: u8::try_from(bounded("retries", MAX_RETRIES)?)?, + retry_delay_seconds: u32::try_from(bounded("retry_delay_seconds", MAX_RETRY_DELAY)?)?, + }) + } else { + None + }; + Ok(RunControls { + owner, + overlap, + on_failure, + }) +} + /// Only module execution crosses into Python; discovery and collection stay native. pub(crate) fn module_jobs<'py>( py: Python<'py>, diff --git a/fensu.toml b/fensu.toml index 66dcc9a651..a0ca84706d 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1386,7 +1386,7 @@ paths = ["crates/sqlbuild-orchestration/src/api/_helpers/resources/administrator reason = "Each function is one utoipa-annotated axum route: OpenAPI needs one annotated handler per operation, and each body only checks the caller's scope and calls one store operation. Scope checks and store work live in their shared owners." [[dupes.allowlist]] -paths = ["crates/sqlbuild-orchestration/src/authorization/main/list_role_definitions.rs", "crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs", "crates/sqlbuild-orchestration/src/credentials/_helpers/active_session.rs", "crates/sqlbuild-orchestration/src/declarations/_helpers/reads.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_jobs.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_trigger_changes.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_triggers.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs", "crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs", "crates/sqlbuild-orchestration/src/platform/main/read_person_location_health.rs", "crates/sqlbuild-orchestration/src/platform/main/location_health.rs", "crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs", "crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs", "crates/sqlbuild-orchestration/src/runs/main/list_person_run_nodes.rs", "crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs", "crates/sqlbuild-orchestration/src/runs/main/list_runs.rs", "crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs", "crates/sqlbuild-orchestration/src/warehouse_history/_helpers/checks.rs", "crates/sqlbuild-orchestration/src/scheduler/main/list_person_ticks.rs", "crates/sqlbuild-orchestration/src/store/main/read_records.rs", "crates/sqlbuild-orchestration/src/platform/main/location_work.rs", "crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs", "crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs", "crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs", "crates/sqlbuild-orchestration/src/snapshots/main/read_current_record.rs", "crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs", "crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_catalog.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_check.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_checks.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_maintenance.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_microbatch_coverage.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_statistics.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_node_runs.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_events.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_statements.rs", "crates/sqlbuild-orchestration/src/warehouse_history/_helpers/node_runs.rs"] +paths = ["crates/sqlbuild-orchestration/src/authorization/main/list_role_definitions.rs", "crates/sqlbuild-orchestration/src/authorization/main/read_security_audit.rs", "crates/sqlbuild-orchestration/src/credentials/_helpers/active_session.rs", "crates/sqlbuild-orchestration/src/declarations/_helpers/reads.rs", "crates/sqlbuild-orchestration/src/declarations/main/published_job.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_jobs.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_trigger_changes.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_person_triggers.rs", "crates/sqlbuild-orchestration/src/declarations/main/list_triggers.rs", "crates/sqlbuild-orchestration/src/platform/main/read_location_work.rs", "crates/sqlbuild-orchestration/src/platform/main/read_person_location_health.rs", "crates/sqlbuild-orchestration/src/platform/main/location_health.rs", "crates/sqlbuild-orchestration/src/snapshots/main/current_snapshot_record.rs", "crates/sqlbuild-orchestration/src/runs/main/list_person_runs.rs", "crates/sqlbuild-orchestration/src/runs/main/list_person_run_nodes.rs", "crates/sqlbuild-orchestration/src/runs/main/list_run_nodes.rs", "crates/sqlbuild-orchestration/src/runs/main/list_runs.rs", "crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs", "crates/sqlbuild-orchestration/src/warehouse_history/_helpers/checks.rs", "crates/sqlbuild-orchestration/src/scheduler/main/list_person_ticks.rs", "crates/sqlbuild-orchestration/src/store/main/read_records.rs", "crates/sqlbuild-orchestration/src/platform/main/location_work.rs", "crates/sqlbuild-orchestration/src/scheduler/main/list_ticks.rs", "crates/sqlbuild-orchestration/src/sensors/_helpers/attempts.rs", "crates/sqlbuild-orchestration/src/sensors/_helpers/facts.rs", "crates/sqlbuild-orchestration/src/snapshots/main/read_current_record.rs", "crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_documents.rs", "crates/sqlbuild-orchestration/src/snapshots/main/read_snapshot_nodes.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_catalog.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_check.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_checks.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_maintenance.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_microbatch_coverage.rs", "crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/read_person_statistics.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_node_runs.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_events.rs", "crates/sqlbuild-orchestration/src/warehouse_history/main/read_person_run_statements.rs", "crates/sqlbuild-orchestration/src/warehouse_history/_helpers/node_runs.rs"] reason = "Each is one control-store read with its own SQL and record type. Transactions, person authorization and decoding go through the shared begin_authorized, row_in_target, query_records, read_records and query_bodies helpers; what remains alike is the call shape." [[dupes.allowlist]] @@ -1685,6 +1685,7 @@ exceptions = [ "ScopeDiagnosticCode=Wire enum that serde builds from Python scope diagnostics.", "TickOutcome::Covered=The scheduler writes it as SQL text and serde reads it back.", "TickOutcome::Requested=The scheduler writes it as SQL text and serde reads it back.", + "TickOutcome::Skipped=The scheduler writes it as SQL text and serde reads it back.", "UsageKind=Wire enum that serde builds from Python usage records.", ] diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 06b138a6f6..7ba0b23f30 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -15,6 +15,7 @@ Online: https://sqlbuild.com/docs/cli/server/ - Grant roles to OIDC groups (beta) - Inspect published SQL - Sync from the warehouse now +- Skip overlapping runs and retry failures - Run ad-hoc commands - Plan builds - Define project roles @@ -147,7 +148,7 @@ snapshot only when no retained run, job, trigger, trigger change or location req deletes its node definitions and SQL text when no other snapshot shares them. Repeating or interrupting this cleanup is safe. -The scheduler deletes covered ticks older than 30 days. It keeps each trigger's latest tick and +The scheduler deletes covered and skipped ticks older than 30 days. It keeps each trigger's latest tick and every tick that started a run. A new schedule, a resumed trigger or a cursor reset catches up only on the latest missed time, so deleted ticks never come back. @@ -287,6 +288,53 @@ Details: haven't changed since the last sync, so a sync with nothing new doesn't resume the warehouse. - `[control]` holds no secrets. The session stays in the login `sqb login` saved for that server. +## Skip overlapping runs and retry failures + +Declare who owns a job, whether scheduled requests wait for a running copy, and how often a failed +run retries: + +```python +from sqlbuild.orchestration import job, on_failure, schedule + +nightly_orders = job( + "nightly_orders", + description="Orders and their daily totals", + select="+order_totals", + owner="Orders team", + overlap="skip", + on_failure=on_failure(retries=2, retry_delay_seconds=600), + schedule=schedule(cron="0 2 * * *", timezone="UTC"), +) +``` + +| Setting | Values | Default | Effect | +| --- | --- | --- | --- | +| `owner` | Any non-blank text | None | Shown on the job's page and in the job list | +| `overlap` | `queue` or `skip` | `queue` | With `skip`, a schedule tick or sensor request that arrives while a run of the job is queued or running starts no run | +| `on_failure` | `retries` from 1 to 10, `retry_delay_seconds` from 0 to 86400 | None | A failed or partial run reruns its failed and skipped nodes after the delay, until it succeeds or the retries run out | + +A skipped tick shows as **skipped** in the trigger's tick log, with the run it yielded to, and +creates no run. A sensor check lists the requests it skipped the same way. + +| Sensor watches | A request skipped while the job runs | +| --- | --- | +| Run, node or platform events | Is dropped | +| New source data or stale sources | Is checked again on the next check and starts a run once the job is idle, if the source is still new or stale | +| Custom code | Starts a run once the job is idle if the code returns the same run key again | + +Details: + +- Runs people start, reruns and backfills always queue, whatever the job's `overlap`. +- A tick that makes up for several missed times, after the scheduler was down or the trigger was + paused, still queues. +- A retry waits in the queue until its delay ends, and the run list shows it as a rerun of the run + that failed. Cancelling a run stops its retries. +- A group retries at group level, with the group's own `on_failure`: each retry is one rerun of the + group's failed and skipped nodes. A member job's `on_failure` applies only when that job runs on + its own. +- An invalid value fails publication with the job's name, such as + `job nightly_orders overlap must be queue or skip, not coalesce`. + ## Run ad-hoc commands People with `run.adhoc` can run a `sqb` command on the current publication without declaring a diff --git a/src/sqlbuild/orchestration.py b/src/sqlbuild/orchestration.py index f6f022e65d..265d16d608 100644 --- a/src/sqlbuild/orchestration.py +++ b/src/sqlbuild/orchestration.py @@ -12,6 +12,7 @@ from sqlbuild.orchestration_declarations.models.authoring import Job as job from sqlbuild.orchestration_declarations.models.authoring import JobGroup as job_group from sqlbuild.orchestration_declarations.models.authoring import NodeEventCondition as node_event +from sqlbuild.orchestration_declarations.models.authoring import OnFailure as on_failure from sqlbuild.orchestration_declarations.models.authoring import ( PlatformEventCondition as platform_event, ) @@ -40,6 +41,7 @@ "job", "job_group", "node_event", + "on_failure", "platform_event", "run_event", "schedule", diff --git a/src/sqlbuild/orchestration_declarations/models/authoring.py b/src/sqlbuild/orchestration_declarations/models/authoring.py index a904d71243..dbc3f4e8c1 100644 --- a/src/sqlbuild/orchestration_declarations/models/authoring.py +++ b/src/sqlbuild/orchestration_declarations/models/authoring.py @@ -21,6 +21,14 @@ class Schedule: default_status: Literal["running", "paused"] = "running" +@dataclass(frozen=True, kw_only=True) +class OnFailure: + """Rerun a failed or partial run from failure, `retries` times, `retry_delay_seconds` apart.""" + + retries: int + retry_delay_seconds: int = 0 + + @dataclass(frozen=True, eq=False) class Job: """A named selection to execute, optionally on a wall-clock schedule.""" @@ -34,6 +42,9 @@ class Job: concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) schedule: Schedule | None = field(default=None, kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) + owner: str | None = field(default=None, kw_only=True) + overlap: Literal["queue", "skip"] = field(default="queue", kw_only=True) + on_failure: OnFailure | None = field(default=None, kw_only=True) _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) def __post_init__(self) -> None: @@ -51,6 +62,9 @@ class JobGroup: concurrency_groups: tuple[str, ...] = field(default=(), kw_only=True) schedule: Schedule | None = field(default=None, kw_only=True) targets: tuple[str, ...] = field(default=(), kw_only=True) + owner: str | None = field(default=None, kw_only=True) + overlap: Literal["queue", "skip"] = field(default="queue", kw_only=True) + on_failure: OnFailure | None = field(default=None, kw_only=True) _declared_at: tuple[str, int] | None = field(default=None, init=False, repr=False) def __post_init__(self) -> None: diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py index a11bcbf230..dfb4ed431a 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/_test_types.py @@ -3,12 +3,14 @@ from __future__ import annotations from dataclasses import dataclass, field +from typing import Any from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter from sqlbuild.orchestration_declarations.models.authoring import ( CalendarSchedule, Job, JobGroup, + OnFailure, Schedule, Sensor, ) @@ -277,6 +279,9 @@ class SnapshotDeclarationRejectionTestCase: copies: int = 1 expected_message: str = "" schedule: Schedule | None = None + owner: str | None = None + overlap: Any = "queue" + on_failure: OnFailure | None = None @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py index 5bfb5e9d9a..e0874641ed 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/test_orchestration_snapshot.py @@ -27,6 +27,7 @@ compile_snapshot, job, job_group, + on_failure, run_event, schedule, sensor, @@ -676,6 +677,9 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select concurrency_groups=("order_reporting",), schedule=test_case.schedule, targets=("development",), + owner="Orders team", + overlap="skip", + on_failure=on_failure(retries=2, retry_delay_seconds=600), ) excluded: Job = replace(nightly, name="other_target", targets=("production",)) first: dict[str, Any] = json.loads( @@ -701,6 +705,9 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select "select": "+orders", "exclude": "stg_orders", "targets": ["development"], + "owner": "Orders team", + "overlap": "skip", + "on_failure": {"retries": 2, "retry_delay_seconds": 600}, } } trigger: dict[str, Any] = first["documents"]["triggers"]["nightly_orders"] @@ -802,6 +809,29 @@ def test_given_authored_jobs_when_compiling_then_native_snapshots_resolve_select settings={"start_cursor": "2", "end_cursor_int": "4"}, expected_message="cannot be combined", ), + SnapshotDeclarationRejectionTestCase( + description="blank owner", owner=" ", expected_message="owner must not be blank" + ), + SnapshotDeclarationRejectionTestCase( + description="unknown overlap", + overlap="coalesce", + expected_message="overlap must be queue or skip, not coalesce", + ), + SnapshotDeclarationRejectionTestCase( + description="no retries", + on_failure=on_failure(retries=0), + expected_message="retries must be a whole number from 1 to 10", + ), + SnapshotDeclarationRejectionTestCase( + description="too many retries", + on_failure=on_failure(retries=11), + expected_message="retries must be a whole number from 1 to 10", + ), + SnapshotDeclarationRejectionTestCase( + description="retry delay over a day", + on_failure=on_failure(retries=1, retry_delay_seconds=86_401), + expected_message="retry_delay_seconds must be a whole number from 0 to 86400", + ), SnapshotDeclarationRejectionTestCase( description="duplicate concurrency declarations", concurrency=3, @@ -830,6 +860,9 @@ def test_given_invalid_declarations_when_compiling_then_native_validation_reject concurrency=test_case.concurrency, schedule=test_case.schedule or schedule(cron=test_case.cron, timezone=test_case.timezone), + owner=test_case.owner, + overlap=test_case.overlap, + on_failure=test_case.on_failure, ) ] * test_case.copies with pytest.raises(ValueError, match=test_case.expected_message): diff --git a/ui/src/lib/api/http-catalog.test.ts b/ui/src/lib/api/http-catalog.test.ts index c1b1b75592..eda8ccc575 100644 --- a/ui/src/lib/api/http-catalog.test.ts +++ b/ui/src/lib/api/http-catalog.test.ts @@ -157,7 +157,7 @@ describe('catalog node detail', () => { describe('catalog node runs and jobs', () => { const snapshot = '01a1b000-0000-7000-8000-000000000001'; - const job = (key: string, kind: 'job' | 'group', selection: string[]): W['JobRecord'] => ({ job_key: key, kind, description: key, selection, concurrency_groups: [], snapshot_id: snapshot }); + const job = (key: string, kind: 'job' | 'group', selection: string[]): W['JobRecord'] => ({ job_key: key, kind, description: key, selection, concurrency_groups: [], snapshot_id: snapshot, overlap: 'queue' }); const trigger = (id: string, jobKey: string) => ({ trigger_id: id, job_key: jobKey }) as W['TriggerRecord']; const run = (id: string, status: string, extra: Partial = {}): W['NodeRunRecord'] => ({ run_id: id, job_key: 'orders', queued_at: '2026-11-08T05:59:00Z', status, ...extra }); diff --git a/ui/src/lib/api/http-jobs.test.ts b/ui/src/lib/api/http-jobs.test.ts index 2056dcf665..dbe843d7e0 100644 --- a/ui/src/lib/api/http-jobs.test.ts +++ b/ui/src/lib/api/http-jobs.test.ts @@ -61,7 +61,9 @@ describe('trigger mapping', () => { it('reports lateness from evaluation time and hides sensor parts the server lacks', () => { const tick = toTick({ tick_id: 'k1', trigger_id: 't-daily', epoch: 1, scheduled_for: '2026-11-07T06:00:00Z', evaluated_at: '2026-11-07T06:02:30Z', outcome: 'requested', run_id: 'r-new' }); - expect(tick.lateBySeconds).toBe(150); + expect([tick.lateBySeconds, tick.activeRunId]).toEqual([150, null]); + const skipped = toTick({ tick_id: 'k2', trigger_id: 't-daily', epoch: 1, scheduled_for: '2026-11-07T07:00:00Z', evaluated_at: '2026-11-07T07:00:01Z', outcome: 'skipped', active_run_id: 'r-busy' }); + expect([skipped.outcome, skipped.runId, skipped.activeRunId]).toEqual(['skipped', null, 'r-busy']); const detail = triggerDetailOf(toTrigger(onInventory, [], tick), onInventory, [tick], []); expect(detail.sensor).toMatchObject({ minimumIntervalSeconds: 60, cursorKnown: false }); expect([detail.declaration, detail.upcoming, detail.catchUp, detail.evaluations, detail.stateHistory]).toEqual([null, null, null, null, []]); @@ -79,6 +81,7 @@ describe('trigger mapping', () => { cursor_after: after, run_count: after ? 2 : 0, failure_code: after ? null : 'location_unreachable', + skipped_for: after ? ['r-busy', 'r-busy'] : [], definition_fingerprint: fingerprint }); const state: W['PersonSensorState'] = { @@ -94,9 +97,9 @@ describe('trigger mapping', () => { const detail = triggerDetailOf(toTrigger(onInventory, [], null), onInventory, [], [reset], state); expect(detail.sensor).toMatchObject({ cursor: 'feed/page-[REDACTED]', cursorKnown: true, cursorUpdatedAt: '2026-11-07T06:00:05Z', currentFingerprint: 'v2', lastEvaluatedFingerprint: 'v1' }); expect(detail.definitionChanged).toBe(true); - expect(detail.evaluations?.map((e) => [e.id, e.outcome, e.cursorBefore, e.cursorAfter, e.error])).toEqual([ - ['e-new', 'failed', 'event 40', null, 'location_unreachable'], - ['e-old', 'completed', null, 'event 40', null] + expect(detail.evaluations?.map((e) => [e.id, e.outcome, e.cursorBefore, e.cursorAfter, e.error, e.skippedFor])).toEqual([ + ['e-new', 'failed', 'event 40', null, 'location_unreachable', []], + ['e-old', 'completed', null, 'event 40', null, ['r-busy', 'r-busy']] ]); expect(detail.runRequests).toEqual([{ runKey: 'stock-7', runId: 'r-7', evaluationId: 'e-old', requestedAt: '2026-11-07T06:00:05Z', runStatus: 'success' }]); const fresh = triggerDetailOf(toTrigger(onInventory, [], null), onInventory, [], [], { ...state, cursor: null, evaluations: [] }); @@ -105,7 +108,7 @@ describe('trigger mapping', () => { }); describe('job mapping', () => { - it('links a job to its triggers and newest run, shows where it is declared, and leaves unreported policy null', () => { + it('links a job to its triggers and newest run, and shows where it is declared, its owner, overlap and retries', () => { const wire: W['JobRecord'] = { job_key: 'orders', kind: 'job', @@ -114,11 +117,15 @@ describe('job mapping', () => { concurrency_groups: ['warehouse_heavy'], snapshot_id: SNAPSHOT, select: '+orders', - declared_at: { path: 'orchestration/jobs.py', line: 12 } + declared_at: { path: 'orchestration/jobs.py', line: 12 }, + owner: ' Orders team ', + overlap: 'skip', + on_failure: { retries: 2, retry_delay_seconds: 300 } }; const triggers = [toTrigger(daily, [], null), toTrigger(onInventory, [], null)]; const job = toJob(wire, triggers, [run('r-inv', 'inventory', null), run('r-ord', 'orders', null)]); - expect(job).toMatchObject({ key: 'orders', kind: 'job', nodeCount: 2, triggerIds: ['t-daily'], concurrencyGroups: ['warehouse_heavy'], select: '+orders', sourceFile: 'orchestration/jobs.py:12', overlap: null, priority: null }); + expect(job).toMatchObject({ key: 'orders', kind: 'job', nodeCount: 2, triggerIds: ['t-daily'], concurrencyGroups: ['warehouse_heavy'], select: '+orders', sourceFile: 'orchestration/jobs.py:12', owner: 'Orders team', overlap: 'skip', onFailure: { retries: 2, retryDelaySeconds: 300 }, priority: null }); + expect(toJob({ ...wire, owner: ' ', overlap: 'queue', on_failure: null }, triggers, [])).toMatchObject({ owner: null, overlap: 'queue', onFailure: null }); expect(job.lastRun?.id).toBe('r-ord'); const detail = jobDetailOf(job, { ...wire, settings: { vars: '[REDACTED]', concurrency: 4, empty_ok: true, models: ['a', 'b'] }, members: [], member_of: ['nightly'], critical_nodes: ['model:orders'], exclude: ' stg_orders audit_orders', targets: ['production'], cursor_type: 'integer' }); expect(detail.resolvedNodes).toEqual(['model:customers', 'model:orders']); @@ -145,6 +152,7 @@ describe('job mapping', () => { targets: [], declared_at: null, cursor_type: null, + overlap: 'queue', members: [ { job_key: 'stock', waits_for: [] }, { job_key: 'orders', waits_for: [] }, diff --git a/ui/src/lib/api/http-jobs.ts b/ui/src/lib/api/http-jobs.ts index 5bc6963257..e424ad66c9 100644 --- a/ui/src/lib/api/http-jobs.ts +++ b/ui/src/lib/api/http-jobs.ts @@ -49,6 +49,7 @@ export function toTick(wire: Wire['TickRecord']): Tick { outcome: wire.outcome, runId: wire.run_id ?? null, coveredByRunId: wire.covered_by_run_id ?? null, + activeRunId: wire.active_run_id ?? null, reason: null, lateBySeconds: Math.max(0, Math.round((Date.parse(wire.evaluated_at) - Date.parse(wire.scheduled_for)) / 1000)), catchUp: false @@ -115,6 +116,7 @@ function toEvaluation(wire: Wire['PersonSensorEvaluation']): SensorEvaluation { cursorAfter: wire.cursor_after ? shownCursor(wire.cursor_after) : null, runRequests: wire.run_count, skipReason: null, + skippedFor: wire.skipped_for, error: wire.failure_code ?? null, definitionFingerprint: wire.definition_fingerprint }; @@ -192,11 +194,11 @@ export function toJob(wire: Wire['JobRecord'], triggers: Trigger[], runs: Run[]) select: wire.select ?? null, members: null, nodeCount: wire.selection.length, - overlap: null, - onFailure: null, + overlap: wire.overlap, + onFailure: wire.on_failure ? { retries: wire.on_failure.retries, retryDelaySeconds: wire.on_failure.retry_delay_seconds } : null, concurrencyGroups: wire.concurrency_groups, priority: null, - owner: null, + owner: wire.owner?.trim() || null, sourceFile: wire.declared_at ? `${wire.declared_at.path}:${wire.declared_at.line}` : null, triggerIds: triggers.filter((t) => t.jobKey === wire.job_key).map((t) => t.id), lastRun: newest(runs, (r) => r.jobKey === wire.job_key) diff --git a/ui/src/lib/api/http-runs.ts b/ui/src/lib/api/http-runs.ts index cc9dbc24d3..d5f6d4a9e8 100644 --- a/ui/src/lib/api/http-runs.ts +++ b/ui/src/lib/api/http-runs.ts @@ -118,7 +118,7 @@ export function toRun(wire: Wire['RunRecord'], locationId: string, childRunCount /** The queue in claim order; a run the read sent without queue state (it started meanwhile) is left out. */ export function queueEntriesOf(wires: Wire['RunRecord'][], locationId: string): QueueEntry[] { const runs = toRuns(wires, locationId); - return wires.flatMap((wire, i) => (wire.queue ? [{ run: runs[i], position: wire.queue.position, waits: shownWaits(wire.queue.waits.map(toQueueWait)), overlap: null }] : [])); + return wires.flatMap((wire, i) => (wire.queue ? [{ run: runs[i], position: wire.queue.position, waits: shownWaits(wire.queue.waits.map(toQueueWait)) }] : [])); } /** A running run of the same job also holds the job's write claims; name it once. */ @@ -140,6 +140,8 @@ function toQueueWait(wait: Wire['QueueWait']): QueueWait { case 'job_running': case 'group_parent': return { kind: wait.kind, runId: wait.run_id }; + case 'retry_delay': + return { kind: wait.kind, attempt: wait.attempt, dueAt: wait.due_at }; } } diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index c03592d390..ec6a9e1ec3 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -19,7 +19,6 @@ import type { JobGroupMember, JobLaunchOptions, LaunchVerb, - OverlapOutcome, Principal, QueueEntry, QueueWait, @@ -337,8 +336,7 @@ export class MockOperate { queued.map((rec, i) => ({ run: this.#view(rec, scope), position: i + 1, - waits: waits.get(rec.run.id) ?? [], - overlap: this.#overlap(rec, queued) + waits: waits.get(rec.run.id) ?? [] })) ); } @@ -884,23 +882,6 @@ export class MockOperate { return out; } - #overlap(rec: RunRecord, queued: RunRecord[]): OverlapOutcome | null { - const job = jobDef(rec.run.jobKey); - if (!job || rec.run.parentRunId) return null; - const covered = this.#fx.ticks.filter((t) => t.coveredByRunId === rec.run.id).length; - if (job.overlap === 'coalesce') - return { - policy: 'coalesce', - coveredTicks: covered, - detail: covered ? `${covered} later request${covered === 1 ? '' : 's'} merged into this run` : 'New requests merge into this run while it waits' - }; - if (job.overlap === 'queue') { - const before = queued.filter((q) => q.run.jobKey === job.key && q.run.queuedAt < rec.run.queuedAt); - return { policy: 'queue', coveredTicks: 0, detail: before.length ? `Goes after ${before.map((b) => b.run.id.replace(/^run-/, '')).join(', ')}` : 'Next in line for this job' }; - } - return { policy: 'allow', coveredTicks: 0, detail: 'Waits only for runs updating the same data' }; - } - #newId(): string { let id: string; do id = `run-${(0xa1000 + ++this.#seq * 0x9e37).toString(16).slice(-5)}`; diff --git a/ui/src/lib/api/mock/operate.ts b/ui/src/lib/api/mock/operate.ts index 2c63a8ac7b..28c0da7c9f 100644 --- a/ui/src/lib/api/mock/operate.ts +++ b/ui/src/lib/api/mock/operate.ts @@ -36,7 +36,7 @@ type Known = Omit & { [P in K]-?: NonNullable type JobDef = Known< Omit, - 'members' | 'overlap' | 'onFailure' | 'priority' | 'sourceFile' | 'declaration' | 'settings' + 'members' | 'overlap' | 'priority' | 'sourceFile' | 'declaration' | 'settings' > & { after: Record; }; @@ -54,7 +54,7 @@ export const JOB_DEFS: JobDef[] = [ exclude: [], members: [], overlap: 'queue', - onFailure: 'skip_dependents', + onFailure: { retries: 1, retryDelaySeconds: 300 }, concurrencyGroups: [], priority: 5, sourceFile: 'orchestration/jobs.py', @@ -70,7 +70,6 @@ export const JOB_DEFS: JobDef[] = [ "load_sources", description="Ingest partner order files into raw__orders", select="tag:ingest", - overlap="queue", priority=5, owner="ingest", )` @@ -83,8 +82,8 @@ export const JOB_DEFS: JobDef[] = [ select: 'group:staging group:marts', exclude: [], members: [], - overlap: 'coalesce', - onFailure: 'skip_dependents', + overlap: 'skip', + onFailure: { retries: 2, retryDelaySeconds: 600 }, concurrencyGroups: ['warehouse_heavy'], priority: 0, sourceFile: 'orchestration/jobs.py', @@ -100,8 +99,8 @@ export const JOB_DEFS: JobDef[] = [ "core_build", description="Staging and marts models", select=["group:staging", "group:marts"], - overlap="coalesce", - on_failure="skip_dependents", + overlap="skip", + on_failure=on_failure(retries=2, retry_delay_seconds=600), concurrency_groups=[warehouse_heavy], settings={"--concurrency": 8}, owner="data-platform", @@ -116,8 +115,8 @@ export const JOB_DEFS: JobDef[] = [ select: 'group:marts', exclude: [], members: [], - overlap: 'coalesce', - onFailure: 'skip_dependents', + overlap: 'queue', + onFailure: null, concurrencyGroups: [], priority: 0, sourceFile: 'orchestration/jobs.py', @@ -145,8 +144,8 @@ export const JOB_DEFS: JobDef[] = [ select: 'group:activity', exclude: [], members: [], - overlap: 'coalesce', - onFailure: 'skip_dependents', + overlap: 'queue', + onFailure: null, concurrencyGroups: [], priority: 0, sourceFile: 'orchestration/activity.py', @@ -176,8 +175,8 @@ export const JOB_DEFS: JobDef[] = [ select: 'daily_revenue revenue_report export_revenue_csv', exclude: [], members: [], - overlap: 'coalesce', - onFailure: 'skip_dependents', + overlap: 'queue', + onFailure: null, concurrencyGroups: [], priority: 0, sourceFile: 'orchestration/finance.py', @@ -207,8 +206,8 @@ export const JOB_DEFS: JobDef[] = [ select: 'customer_status', exclude: [], members: [], - overlap: 'allow', - onFailure: 'skip_dependents', + overlap: 'queue', + onFailure: null, concurrencyGroups: [], priority: 0, sourceFile: 'orchestration/finance.py', @@ -224,7 +223,6 @@ export const JOB_DEFS: JobDef[] = [ "customer_segments", description="Customer status segments; hourly and in the close-of-business group", select="customer_status", - overlap="allow", owner="analytics", )` ) @@ -237,7 +235,7 @@ export const JOB_DEFS: JobDef[] = [ exclude: [], members: [], overlap: 'queue', - onFailure: 'skip_dependents', + onFailure: null, concurrencyGroups: ['warehouse_heavy'], priority: 10, sourceFile: 'orchestration/reporting.py', @@ -253,7 +251,6 @@ export const JOB_DEFS: JobDef[] = [ "reporting_exports", description="Reporting tables and the revenue export", select=["group:reporting", "group:exports"], - overlap="queue", concurrency_groups=[warehouse_heavy], priority=10, targets=["prod"], @@ -267,8 +264,8 @@ export const JOB_DEFS: JobDef[] = [ select: null, exclude: [], members: ['load_sources', 'core_build', 'activity_microbatch', 'reporting_exports'], - overlap: 'coalesce', - onFailure: 'skip_dependents', + overlap: 'queue', + onFailure: null, concurrencyGroups: [], priority: 0, sourceFile: 'orchestration/groups.py', @@ -297,8 +294,8 @@ export const JOB_DEFS: JobDef[] = [ select: null, exclude: [], members: ['revenue_refresh', 'activity_microbatch', 'customer_segments'], - overlap: 'coalesce', - onFailure: 'stop_group', + overlap: 'queue', + onFailure: { retries: 1, retryDelaySeconds: 900 }, concurrencyGroups: ['warehouse_heavy'], priority: 20, sourceFile: 'orchestration/groups.py', @@ -316,7 +313,7 @@ export const JOB_DEFS: JobDef[] = [ jobs=[revenue_refresh, activity_microbatch, customer_segments], # Segments read closed revenue through an external lookup lineage can't see. after={customer_segments: [revenue_refresh]}, - on_failure="stop_group", + on_failure=on_failure(retries=1, retry_delay_seconds=900), critical=["daily_revenue"], concurrency_groups=[warehouse_heavy], priority=20, @@ -903,6 +900,7 @@ export function createOperateFixtures(now: number): OperateFixtures { outcome, runId: null, coveredByRunId: null, + activeRunId: null, reason: null, lateBySeconds: 0, catchUp: false, @@ -945,10 +943,8 @@ export function createOperateFixtures(now: number): OperateFixtures { continue; } if (missed.length && t.name === 'activity_microbatch') { - // Catch-up after downtime: `all` with max 3; older missed times are skipped. - const cap = t.catchUp?.max ?? 1; - const caught = missed.slice(-cap); - for (const m of missed.slice(0, -cap)) tick(t.name, m, 'skipped', { evaluatedAt: new Date(downtime.to + 2000).toISOString(), lateBySeconds: Math.round((downtime.to - m) / 1000), catchUp: true, reason: `Beyond the catch-up limit of ${cap} missed times` }); + // Catch-up after downtime: the first missed time starts one run and the rest merge into it. + const caught = missed; const runId = genId(rand); const startAgo = (now - downtime.to) / MIN - 0.1; record({ id: runId, origin: 'schedule', job: t.jobKey, trigger: t.name, by: SCHEDULER, queuedAgo: startAgo + 0.05, startedAgo: startAgo, durationMin: 6, reason: 'Made up after the scheduler was down' }, { queued: downtime.to + 1000, started: downtime.to + 5000 }); @@ -956,7 +952,7 @@ export function createOperateFixtures(now: number): OperateFixtures { rec.run.tickScheduledFor = new Date(caught[0]).toISOString(); rec.run.tickId = tickId(t.name, caught[0]); tick(t.name, caught[0], 'requested', { runId, evaluatedAt: new Date(downtime.to + 1000).toISOString(), lateBySeconds: Math.round((downtime.to - caught[0]) / 1000), catchUp: true, reason: 'Missed while no server was running the scheduler' }); - for (const c of caught.slice(1)) tick(t.name, c, 'covered', { coveredByRunId: runId, evaluatedAt: new Date(downtime.to + 1000).toISOString(), lateBySeconds: Math.round((downtime.to - c) / 1000), catchUp: true, reason: 'Missed time merged into the make-up run (overlap = coalesce)' }); + for (const c of caught.slice(1)) tick(t.name, c, 'covered', { coveredByRunId: runId, evaluatedAt: new Date(downtime.to + 1000).toISOString(), lateBySeconds: Math.round((downtime.to - c) / 1000), catchUp: true, reason: 'Missed time merged into the make-up run' }); missed.length = 0; } if (skip.has(due)) { @@ -965,7 +961,12 @@ export function createOperateFixtures(now: number): OperateFixtures { continue; } if (t.name === 'activity_microbatch' && i === 0) { - tick(t.name, due, 'covered', { coveredByRunId: 'run-5f31', reason: 'A run is already waiting (overlap = coalesce)' }); + tick(t.name, due, 'covered', { coveredByRunId: 'run-5f31', reason: 'A run of this job was already waiting' }); + continue; + } + const active = t.jobKey === 'core_build' && i === 3 ? records.findLast((r) => r.run.jobKey === 'core_build' && !r.run.parentRunId) : undefined; + if (active) { + tick(t.name, due, 'skipped', { activeRunId: active.run.id, reason: 'A run of core_build was still active (overlap = skip)' }); continue; } if (t.name === 'nightly_all' && i === 4) { @@ -1012,6 +1013,7 @@ export function createOperateFixtures(now: number): OperateFixtures { cursorAfter: null, runRequests: 0, skipReason: null, + skippedFor: [], error: null, definitionFingerprint: fingerprint(name), ...extra diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index 552b96193d..018089737f 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -406,20 +406,14 @@ export type QueueWait = | { kind: 'job_running'; runId: string } | { kind: 'upstream_job'; runId: string; jobKey: string } /** A child run waits for its group run to start. */ - | { kind: 'group_parent'; runId: string }; - -export type OverlapOutcome = { - policy: OverlapPolicy; - /** Ticks merged into this waiting run (`coalesce`). */ - coveredTicks: number; - detail: string; -}; + | { kind: 'group_parent'; runId: string } + /** A retry of a failed run waits out the job's `on_failure` delay. */ + | { kind: 'retry_delay'; attempt: number; dueAt: Iso }; export type QueueEntry = { run: Run; position: number; waits: QueueWait[]; - overlap: OverlapOutcome | null; }; // ---- timeline ---- @@ -571,8 +565,10 @@ export type RerunMode = 'all' | 'from_failure' | 'selected'; // ---- jobs and job groups ---- -export type OverlapPolicy = 'coalesce' | 'queue' | 'allow'; -export type OnFailure = 'skip_dependents' | 'skip_downstream_jobs' | 'stop_group'; +/** What a schedule tick or sensor request does while a run of the job is active; people's runs always queue. */ +export type OverlapPolicy = 'queue' | 'skip'; +/** A failed or partial run reruns from failure up to `retries` times, `retryDelaySeconds` after the last ended. */ +export type OnFailure = { retries: number; retryDelaySeconds: number }; export type JobKind = 'job' | 'group'; export type RunRef = Pick; @@ -656,6 +652,8 @@ export type Tick = { outcome: TickOutcome; runId: string | null; coveredByRunId: string | null; + /** The run a `skipped` tick yielded to, because the job's overlap is `skip`. */ + activeRunId: string | null; reason: string | null; /** Evaluated after the grace period (scheduler down or behind). */ lateBySeconds: number; @@ -713,6 +711,8 @@ export type SensorEvaluation = { cursorAfter: string | null; runRequests: number; skipReason: string | null; + /** Active runs that matching requests yielded to, because the job's overlap is `skip`. */ + skippedFor: string[]; error: string | null; definitionFingerprint: string; }; diff --git a/ui/src/lib/operate/labels.ts b/ui/src/lib/operate/labels.ts index 4403e79edd..ca7510f699 100644 --- a/ui/src/lib/operate/labels.ts +++ b/ui/src/lib/operate/labels.ts @@ -1,5 +1,7 @@ // Labels and tones for the OPERATE area's vocabulary. Pure; no side effects. +import { formatStamp } from '$lib/format/time'; + import type { LivenessState, NodeCounts, @@ -61,43 +63,36 @@ export function formatCost(cost: RunCost | null): string { return `${cost.value < 0.01 ? cost.value.toFixed(4) : cost.value.toFixed(3)} cr`; } -export const overlapLabel: Record = { - coalesce: 'coalesce', - queue: 'queue', - allow: 'allow' -}; - export const overlapHelp: Record = { - coalesce: 'At most one run waits; new requests merge into it instead of starting another run', - queue: 'Every request gets its own run; they run one after another, in order', - allow: 'Can run alongside other runs of this job, as long as they don’t update the same data' + queue: 'A schedule tick or sensor request while a run of this job is active queues another run after it', + skip: 'A schedule tick or sensor request while a run of this job is active is skipped and recorded on the tick or check' }; /** Short plain wording of an overlap policy; the policy name goes in a tooltip. */ export const overlapPlain: Record = { - coalesce: 'Merge new requests', - queue: 'One run per request, in order', - allow: 'Can run side by side' + queue: 'Queue another run', + skip: 'Skip if running' }; -export const onFailureLabel: Record = { - skip_dependents: 'skip_dependents', - skip_downstream_jobs: 'skip_downstream_jobs', - stop_group: 'stop_group' -}; +/** A whole delay in its largest exact unit, e.g. 300 → `5 min`, 5400 → `90 min`, 7200 → `2 h`. */ +function delayText(seconds: number): string { + if (seconds && seconds % 3600 === 0) return `${seconds / 3600} h`; + if (seconds && seconds % 60 === 0) return `${seconds / 60} min`; + return `${seconds} s`; +} -export const onFailureHelp: Record = { - skip_dependents: 'When a node fails, only what depends on it is skipped, even in other jobs', - skip_downstream_jobs: 'When a job fails, the jobs after it are skipped', - stop_group: 'Any failure stops the whole group' -}; +/** An `on_failure` policy in words, e.g. "Retry twice, 5 min apart" or "Retry once after 15 min". */ +export function onFailureText(policy: OnFailure): string { + const times = policy.retries === 1 ? 'once' : policy.retries === 2 ? 'twice' : `${policy.retries} times`; + if (!policy.retryDelaySeconds) return `Retry ${times} at once`; + const delay = delayText(policy.retryDelaySeconds); + return policy.retries === 1 ? `Retry once after ${delay}` : `Retry ${times}, ${delay} apart`; +} -/** Short plain wording of an on-failure policy. */ -export const onFailurePlain: Record = { - skip_dependents: 'Skip what depends on a failure', - skip_downstream_jobs: 'Skip later jobs', - stop_group: 'Stop the group' -}; +/** A job's overlap, retries and priority for a list cell; empty when it uses the defaults. */ +export function jobPolicyText(job: { overlap: OverlapPolicy | null; onFailure: OnFailure | null; priority: number | null }): string { + return [job.overlap === 'skip' ? overlapPlain.skip : null, job.onFailure ? onFailureText(job.onFailure) : null, job.priority ? `priority ${job.priority}` : null].filter(Boolean).join(' · '); +} export type WaitPart = { text: string; href?: string; code?: boolean }; @@ -145,6 +140,8 @@ export function describeWait(wait: QueueWait): WaitPart[] { return [{ text: 'Waiting for ' }, { text: wait.jobKey || shortRunId(wait.runId), href: `/runs/${wait.runId}` }, { text: ' to finish first' }]; case 'group_parent': return [{ text: 'Waiting for its group run ' }, { text: shortRunId(wait.runId), href: `/runs/${wait.runId}` }, { text: ' to start' }]; + case 'retry_delay': + return [{ text: `Waiting: retry ${wait.attempt} is due at ${formatStamp(wait.dueAt)}` }]; } } @@ -170,6 +167,8 @@ export function waitTechnical(wait: QueueWait): string { return `upstream job ${wait.jobKey} (${wait.runId}) still active`; case 'group_parent': return `group run ${wait.runId} not running yet`; + case 'retry_delay': + return `on_failure retry ${wait.attempt}, not before ${wait.dueAt}`; } } @@ -190,7 +189,7 @@ export const tickOutcomeText: Record = { /** What each tick outcome means, with the scheduler's own name for debugging. */ export const tickOutcomeHelp: Record = { requested: 'It was due and started a run (requested)', - skipped: 'It was due but didn’t start a run, e.g. it was paused or caught up (skipped)', + skipped: 'It was due while a run of this job was still active, and the job’s overlap is skip (skipped)', covered: 'A run of this job was already waiting, so this one merged into it (covered)', failed: 'The scheduler couldn’t start a run for it (failed)' }; diff --git a/ui/src/routes/jobs/+page.svelte b/ui/src/routes/jobs/+page.svelte index ca8dd2d9ef..b017ad081d 100644 --- a/ui/src/routes/jobs/+page.svelte +++ b/ui/src/routes/jobs/+page.svelte @@ -15,7 +15,7 @@ import { createResource } from '$lib/state/resource.svelte'; import { formatAgo, formatStamp } from '$lib/format/time'; import { isActiveRun, isProblemRun, runStatusLabel, runStatusTone } from '$lib/status'; - import { overlapHelp, overlapPlain } from '$lib/operate/labels'; + import { jobPolicyText, overlapHelp } from '$lib/operate/labels'; const jobs = createResource((client, scope) => client.listJobs(scope), ['jobs', 'runs', 'snapshot']); const triggers = createResource((client, scope) => client.listTriggers(scope), ['triggers', 'ticks']); @@ -51,7 +51,7 @@ ); const scheduledCount = $derived(all.filter((j) => j.triggerIds.some((id) => triggerById.get(id)?.state === 'running')).length); // Columns the server doesn't report stay hidden rather than showing dashes. - const showPolicy = $derived(all.some((j) => j.overlap !== null)); + const showPolicy = $derived(all.some((j) => jobPolicyText(j) !== '')); const showOwner = $derived(all.some((j) => j.owner !== null)); const showSelection = $derived(all.some((j) => j.select !== null || j.members !== null)); const failingCount = $derived(all.filter((j) => j.lastRun && isProblemRun(j.lastRun.status)).length); @@ -103,7 +103,7 @@ Job {#if showSelection}Selection{/if} Nodes - {#if showPolicy}If already running{/if} + {#if showPolicy}Overlap and retries{/if} Started by {#if showOwner}Owner{/if} Last run @@ -132,7 +132,7 @@ {/if} {j.nodeCount} - {#if showPolicy}{j.overlap ? overlapPlain[j.overlap] : '—'}{#if j.priority} · priority {j.priority}{/if}{/if} + {#if showPolicy}{jobPolicyText(j) || '—'}{/if} {#if jobTriggers.length === 0} Only by hand diff --git a/ui/src/routes/jobs/[name]/job-view.svelte b/ui/src/routes/jobs/[name]/job-view.svelte index c17941c24a..d50029c56a 100644 --- a/ui/src/routes/jobs/[name]/job-view.svelte +++ b/ui/src/routes/jobs/[name]/job-view.svelte @@ -12,7 +12,7 @@ import { formatNodeCounts, nodeOutcomeLabel, nodeOutcomeTone, runOriginLabel, runStatusLabel, runStatusTone } from '$lib/status'; import { kindLine } from '$lib/build-area/labels'; import { planHref } from '$lib/run/command'; - import { formatCost, narrowingText, onFailurePlain, overlapPlain, shortRunId, slotLabel } from '$lib/operate/labels'; + import { formatCost, narrowingText, onFailureText, overlapHelp, overlapPlain, shortRunId, slotLabel } from '$lib/operate/labels'; import ScopeGraph from '$lib/lineage/scope-graph.svelte'; import InspectorSection from '$lib/components/lineage/inspector-section.svelte'; import OutcomeStrip from '$lib/components/build-area/outcome-strip.svelte'; @@ -219,8 +219,8 @@ {#if hasSettings}
- {#if def.overlap}
If already running
{overlapPlain[def.overlap]}
{/if} - {#if def.onFailure}
If something fails
{onFailurePlain[def.onFailure]}
{/if} + {#if def.overlap}
If already running
{overlapPlain[def.overlap]}
{/if} + {#if def.onFailure}
If it fails
{onFailureText(def.onFailure)}
{/if} {#if def.priority !== null}
Priority
{def.priority}
{/if} {#if def.targets}
Targets
{def.targets.join(', ')}
{/if}
diff --git a/ui/src/routes/runs/run-queue.svelte b/ui/src/routes/runs/run-queue.svelte index 296a45d35a..0383e0b306 100644 --- a/ui/src/routes/runs/run-queue.svelte +++ b/ui/src/routes/runs/run-queue.svelte @@ -7,10 +7,9 @@ import { app } from '$lib/state/app.svelte'; import { formatAgo, formatStamp } from '$lib/format/time'; import { runKindOriginLabel } from '$lib/status'; - import { describeWait, overlapHelp, overlapPlain, shortRunId, waitTechnical } from '$lib/operate/labels'; + import { describeWait, shortRunId, waitTechnical } from '$lib/operate/labels'; let { entries, resource }: { entries: QueueEntry[]; resource: Resource } = $props(); - const showOverlap = $derived(entries.some((q) => q.overlap)); let error = $state(null); let cancelling = $state>(new Set()); @@ -41,7 +40,6 @@ Priority Waiting since Why it waits - {#if showOverlap}If more requests arrive{/if} @@ -68,11 +66,6 @@ Nothing in the way; it starts on the next scheduler pass {/each} - {#if showOverlap} - - {#if q.overlap}{overlapPlain[q.overlap.policy]}
{q.overlap.detail}
{:else}—{/if} - - {/if} @@ -82,7 +75,7 @@ {:else} - + {/each} diff --git a/ui/src/routes/triggers/[id]/sensor-panel.svelte b/ui/src/routes/triggers/[id]/sensor-panel.svelte index 0425be31cf..5a6bd960b1 100644 --- a/ui/src/routes/triggers/[id]/sensor-panel.svelte +++ b/ui/src/routes/triggers/[id]/sensor-panel.svelte @@ -88,6 +88,7 @@ {e.runRequests} {e.error ?? e.skipReason ?? ''} + {#if e.skippedFor.length}{e.skippedFor.length} skipped while {#each [...new Set(e.skippedFor)] as id, i (id)}{i ? ', ' : ''}{shortRunId(id)}{/each} ran{/if} {#if e.definitionFingerprint !== sensor.currentFingerprint}older code{/if} diff --git a/ui/src/routes/triggers/[id]/tick-log.svelte b/ui/src/routes/triggers/[id]/tick-log.svelte index 984ae0d533..f71182d5b1 100644 --- a/ui/src/routes/triggers/[id]/tick-log.svelte +++ b/ui/src/routes/triggers/[id]/tick-log.svelte @@ -41,6 +41,7 @@ {#if t.runId}{shortRunId(t.runId)} {:else if t.coveredByRunId}Merged into run {shortRunId(t.coveredByRunId)} (already waiting) + {:else if t.activeRunId}Run {shortRunId(t.activeRunId)} was still active {:else}—{/if} {t.lateBySeconds > LATE_AFTER_SECONDS ? formatSeconds(t.lateBySeconds) : '—'} diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 4f797e35bd..7b1110a00f 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -133,7 +133,7 @@ snapshot only when no retained run, job, trigger, trigger change or location req deletes its node definitions and SQL text when no other snapshot shares them. Repeating or interrupting this cleanup is safe. -The scheduler deletes covered ticks older than 30 days. It keeps each trigger's latest tick and +The scheduler deletes covered and skipped ticks older than 30 days. It keeps each trigger's latest tick and every tick that started a run. A new schedule, a resumed trigger or a cursor reset catches up only on the latest missed time, so deleted ticks never come back. @@ -273,6 +273,53 @@ Details: haven't changed since the last sync, so a sync with nothing new doesn't resume the warehouse. - `[control]` holds no secrets. The session stays in the login `sqb login` saved for that server. +## Skip overlapping runs and retry failures + +Declare who owns a job, whether scheduled requests wait for a running copy, and how often a failed +run retries: + +```python +from sqlbuild.orchestration import job, on_failure, schedule + +nightly_orders = job( + "nightly_orders", + description="Orders and their daily totals", + select="+order_totals", + owner="Orders team", + overlap="skip", + on_failure=on_failure(retries=2, retry_delay_seconds=600), + schedule=schedule(cron="0 2 * * *", timezone="UTC"), +) +``` + +| Setting | Values | Default | Effect | +| --- | --- | --- | --- | +| `owner` | Any non-blank text | None | Shown on the job's page and in the job list | +| `overlap` | `queue` or `skip` | `queue` | With `skip`, a schedule tick or sensor request that arrives while a run of the job is queued or running starts no run | +| `on_failure` | `retries` from 1 to 10, `retry_delay_seconds` from 0 to 86400 | None | A failed or partial run reruns its failed and skipped nodes after the delay, until it succeeds or the retries run out | + +A skipped tick shows as **skipped** in the trigger's tick log, with the run it yielded to, and +creates no run. A sensor check lists the requests it skipped the same way. + +| Sensor watches | A request skipped while the job runs | +| --- | --- | +| Run, node or platform events | Is dropped | +| New source data or stale sources | Is checked again on the next check and starts a run once the job is idle, if the source is still new or stale | +| Custom code | Starts a run once the job is idle if the code returns the same run key again | + +Details: + +- Runs people start, reruns and backfills always queue, whatever the job's `overlap`. +- A tick that makes up for several missed times, after the scheduler was down or the trigger was + paused, still queues. +- A retry waits in the queue until its delay ends, and the run list shows it as a rerun of the run + that failed. Cancelling a run stops its retries. +- A group retries at group level, with the group's own `on_failure`: each retry is one rerun of the + group's failed and skipped nodes. A member job's `on_failure` applies only when that job runs on + its own. +- An invalid value fails publication with the job's name, such as + `job nightly_orders overlap must be queue or skip, not coalesce`. + ## Run ad-hoc commands People with `run.adhoc` can run a `sqb` command on the current publication without declaring a