Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 38 additions & 1 deletion clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1717,6 +1717,9 @@ export interface components {
member_of: string[];
/** @description A group's member jobs in run order; empty for a job. */
members: components["schemas"]["JobMember"][];
on_failure?: null | components["schemas"]["OnFailure"];
overlap: components["schemas"]["Overlap"];
owner?: string | null;
select?: string | null;
selection: string[];
/** @description Command settings; `vars` and a `defer_*` value that is not a target name show as `[REDACTED]`, and text values pass through secret redaction. */
Expand All @@ -1741,6 +1744,9 @@ export interface components {
job_key: string;
/** @description `job` or `group`; a group's selection is the union of its member jobs' selections. */
kind: components["schemas"]["RunKind"];
on_failure?: null | components["schemas"]["OnFailure"];
overlap: components["schemas"]["Overlap"];
owner?: string | null;
/** @description A job's selectors as written; absent for a group and for snapshots published before they were recorded. */
select?: string | null;
selection: string[];
Expand Down Expand Up @@ -2064,6 +2070,13 @@ export interface components {
*/
sequences: number;
};
/** @description A failed or partial run reruns from failure up to `retries` times, each `retry_delay_seconds` after the last ended. */
OnFailure: {
/** Format: int32 */
retries: number;
/** Format: int32 */
retry_delay_seconds: number;
};
/** @description One exact-byte output chunk. */
OutputChunk: {
/**
Expand Down Expand Up @@ -2112,6 +2125,11 @@ export interface components {
/** Format: uuid */
trigger_id: string;
};
/**
* @description What a schedule tick or sensor request does while a run of the same job is active; people's requests always queue.
* @enum {string}
*/
Overlap: "queue" | "skip";
PasswordChangeRequest: {
current_password: string;
new_password: string;
Expand Down Expand Up @@ -2187,6 +2205,8 @@ export interface components {
/** Format: date-time */
requested_at: string;
run_count: number;
/** @description For each matched request the job's `skip` overlap dropped, the run that was active. */
skipped_for: string[];
};
/** @description A run key is redacted and withheld like the custom cursor it usually derives from: empty without view.evidence. */
PersonSensorRunRequest: {
Expand Down Expand Up @@ -2434,6 +2454,13 @@ export interface components {
limit: string;
/** Format: uuid */
run_id?: string | null;
} | {
/** Format: int32 */
attempt: number;
/** Format: date-time */
due_at: string;
/** @enum {string} */
kind: "retry_delay";
};
/** @description One invocation's records; the server owns its claim namespace, start time and version. */
RecallBatch: {
Expand Down Expand Up @@ -2790,6 +2817,7 @@ export interface components {
requested_by: string;
/** Format: uuid */
rerun_of_run_id?: string | null;
retry?: null | components["schemas"]["RunRetry"];
/** Format: uuid */
root_run_id: string;
/** Format: uuid */
Expand All @@ -2808,6 +2836,13 @@ export interface components {
window?: null | components["schemas"]["RunWindow"];
write_set: string[];
};
/** @description Which retry of the job's `on_failure` policy a run is, and when it may start. */
RunRetry: {
/** Format: int32 */
attempt: number;
/** Format: date-time */
not_before: string;
};
/** @description Warehouse statement metadata of a run: kind, timing, counts and outcome, never the SQL text. */
RunStatementRecord: {
/** Format: int64 */
Expand Down Expand Up @@ -3063,8 +3098,10 @@ export interface components {
*/
Table: "invocations" | "output" | "events" | "diagnostics" | "artifacts" | "artifact_refs" | "heartbeats" | "delivery" | "meta" | "conflicts";
/** @enum {string} */
TickOutcome: "requested" | "covered";
TickOutcome: "requested" | "covered" | "skipped";
TickRecord: {
/** Format: uuid */
active_run_id?: string | null;
/** Format: uuid */
covered_by_run_id?: string | null;
/** Format: int64 */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ pub(crate) async fn jobs_in(
snapshot_id: job.snapshot_id,
select: job.definition_json.select,
declared_at: job.definition_json.declared_at,
owner: job.definition_json.owner,
overlap: job.definition_json.overlap,
on_failure: job.definition_json.on_failure,
})
.collect();
Ok((transaction, jobs))
Expand All @@ -99,7 +102,7 @@ pub(crate) async fn latest_ticks_in(
(location_id, target): (Uuid, &str),
) -> Result<(ControlTransaction, Vec<TickRecord>), ControlError> {
query_records(transaction,
"SELECT k.tick_id, k.trigger_id, k.scheduled_for, k.evaluated_at, k.epoch, k.outcome, k.run_id, k.covered_by_run_id \
"SELECT k.tick_id, k.trigger_id, k.scheduled_for, k.evaluated_at, k.epoch, k.outcome, k.run_id, k.covered_by_run_id, k.active_run_id \
FROM sqb_triggers t JOIN sqb_ticks k ON k.tick_id=(SELECT x.tick_id FROM sqb_ticks x WHERE x.trigger_id=t.trigger_id \
ORDER BY x.scheduled_for DESC LIMIT 1) \
WHERE t.location_id=$1 AND t.target=$2 AND t.removed_in_snapshot_id IS NULL",
Expand Down
1 change: 1 addition & 0 deletions crates/sqlbuild-orchestration/src/declarations/main/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,6 @@ pub mod list_trigger_events;
pub mod list_triggers;
pub mod pause_development_triggers;
pub(crate) mod publish_definitions;
pub(crate) mod published_job;
pub mod read_person_job;
pub mod snapshot_definitions;
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
//! The job a target currently publishes under a name.

use uuid::Uuid;

use crate::declarations::models::StoredJob;
use crate::store::errors::ControlError;
use crate::store::main::query_records::query_records;
use crate::store::models::{ControlTransaction, Parameter};

/// `None` when the target publishes no job or group by that name.
pub(crate) async fn published_job(
transaction: ControlTransaction,
(location_id, target, job_key): (Uuid, &str, &str),
) -> Result<(ControlTransaction, Option<StoredJob>), ControlError> {
let (transaction, stored): (_, Vec<StoredJob>) = query_records(
transaction,
"SELECT job_key, kind, snapshot_id, definition_json FROM sqb_jobs \
WHERE location_id=$1 AND target=$2 AND job_key=$3",
vec![
Parameter::Id(location_id),
Parameter::Text(target.into()),
Parameter::Text(job_key.into()),
],
)
.await?;
Ok((transaction, stored.into_iter().next()))
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,13 @@ use crate::declarations::_helpers::job_detail::{
critical_nodes, cursor_type, groups_running, members_in_order, shown_settings,
};
use crate::declarations::main::frozen_definitions::frozen_definitions;
use crate::declarations::models::{JobDetailRecord, StoredJob};
use crate::declarations::main::published_job::published_job;
use crate::declarations::models::JobDetailRecord;
use crate::runs::models::RunKind;
use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes;
use crate::store::errors::ControlError;
use crate::store::main::{begin::begin, commit::commit, query_records::query_records};
use crate::store::models::{ControlStore, ControlTransaction, Parameter, TransactionMode};
use crate::store::main::{begin::begin, commit::commit};
use crate::store::models::{ControlStore, TransactionMode};

/// `None` when the target publishes no job by that name; refusal comes first either way.
pub async fn read_person_job(
Expand All @@ -30,8 +31,8 @@ pub async fn read_person_job(
)
.await?;
let (transaction, published) =
published_snapshot(transaction, (location_id, target, job_key)).await?;
let Some(snapshot_id) = published else {
published_job(transaction, (location_id, target, job_key)).await?;
let Some(snapshot_id) = published.map(|job| job.snapshot_id) else {
commit(transaction).await?;
return Ok(None);
};
Expand Down Expand Up @@ -67,24 +68,8 @@ pub async fn read_person_job(
exclude: job.exclude,
targets: job.targets,
declared_at: job.declared_at,
owner: job.owner,
overlap: job.overlap,
on_failure: job.on_failure,
}))
}

/// The snapshot whose job list holds `job_key`, if the target publishes it.
async fn published_snapshot(
transaction: ControlTransaction,
(location_id, target, job_key): (Uuid, &str, &str),
) -> Result<(ControlTransaction, Option<Uuid>), ControlError> {
let (transaction, stored): (_, Vec<StoredJob>) = query_records(
transaction,
"SELECT job_key, kind, snapshot_id, definition_json FROM sqb_jobs \
WHERE location_id=$1 AND target=$2 AND job_key=$3",
vec![
Parameter::Id(location_id),
Parameter::Text(target.into()),
Parameter::Text(job_key.into()),
],
)
.await?;
Ok((transaction, stored.first().map(|job| job.snapshot_id)))
}
38 changes: 38 additions & 0 deletions crates/sqlbuild-orchestration/src/declarations/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,37 @@ pub struct JobDefinition {
pub targets: BTreeSet<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub declared_at: Option<DeclaredAt>,
/// Who answers for the job, as written.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub owner: Option<String>,
#[serde(default, skip_serializing_if = "Overlap::is_queue")]
pub overlap: Overlap,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub on_failure: Option<OnFailure>,
}

/// What a schedule tick or sensor request does while a run of the same job is active; people's requests always queue.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Overlap {
#[default]
Queue,
Skip,
}

impl Overlap {
#[must_use]
pub const fn is_queue(&self) -> bool {
matches!(self, Self::Queue)
}
}

/// A failed or partial run reruns from failure up to `retries` times, each `retry_delay_seconds` after the last ended.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)]
#[serde(deny_unknown_fields)]
pub struct OnFailure {
pub retries: u8,
pub retry_delay_seconds: u32,
}

/// Where a declaration was written, relative to the project root.
Expand Down Expand Up @@ -123,6 +154,9 @@ pub struct JobRecord {
/// A job's selectors as written; absent for a group and for snapshots published before they were recorded.
pub select: Option<String>,
pub declared_at: Option<DeclaredAt>,
pub owner: Option<String>,
pub overlap: Overlap,
pub on_failure: Option<OnFailure>,
}

/// A `sqb_jobs` row as stored.
Expand Down Expand Up @@ -158,6 +192,10 @@ pub struct JobDetailRecord {
pub declared_at: Option<DeclaredAt>,
/// The cursor type every selected cursor node shares; absent when none has a cursor or types differ.
pub cursor_type: Option<String>,
pub owner: Option<String>,
pub overlap: Overlap,
/// A member job's own policy applies only when it runs alone; inside a group run the group's applies.
pub on_failure: Option<OnFailure>,
}

#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ pub(crate) async fn limit_waiters(
transaction: ControlTransaction,
eligible: Option<&BTreeSet<(Uuid, String)>>,
) -> Result<(ControlTransaction, Vec<LimitWaiter>), ControlError> {
let now = transaction.now;
let (transaction, rows) = query(transaction,
"SELECT r.request_json, r.queued_at FROM sqb_runs r WHERE r.status='queued' AND r.parent_run_id IS NULL \
AND r.cancel_requested_at IS NULL AND r.worker_instance IS NULL",
Expand All @@ -39,6 +40,7 @@ pub(crate) async fn limit_waiters(
.map(decode_row)
.collect::<Result<Vec<QueueCandidate>, _>>()?
.into_iter()
.filter(|run| run.request_json.due(now))
.filter(|run| {
eligible.is_none_or(|scopes| {
scopes.contains(&(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ pub(crate) async fn advance_group(
.await?;
}
}
} else if parent.state.status == RunStatus::Queued {
} else if parent.state.status == RunStatus::Queued && parent.request_json.due(transaction.now) {
let admission = RunRequest {
concurrency_groups: admission_limits(
&parent.request_json,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,6 @@ pub(in crate::runs) mod publish_change;
pub(in crate::runs) mod queued;
pub(in crate::runs) mod reads;
pub(in crate::runs) mod recall;
pub(in crate::runs) mod retries;
pub(in crate::runs) mod run_failure;
pub(in crate::runs) mod waits;
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use serde_json::json;
use crate::platform::models::PlatformEvent;
use crate::runs::_helpers::lifecycle::events::append_run_event;
use crate::runs::_helpers::lifecycle::projection::project_change;
use crate::runs::_helpers::lifecycle::retries::queue_retry;
use crate::runs::_helpers::lifecycle::run_failure::publish_run_failure;
use crate::runs::models::{ChangingRun, RunChange, RunChangeRequest, RunOutcome};
use crate::store::errors::ControlError;
Expand Down Expand Up @@ -60,6 +61,7 @@ pub(crate) async fn publish_change(
} = request.change
{
transaction = publish_run_failure(transaction, request, outcome).await?;
transaction = queue_retry(transaction, request.run_id).await?;
}
if let Some((node_key, _)) = request.change.finished_node() {
(transaction, _) = query(
Expand Down
Loading
Loading