Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions clients/typescript/src/schema.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1439,15 +1439,26 @@ export interface components {
run_id: string;
version_hash: string;
};
/** @description `success`, `warn` or `failed`; `run_id` is the scheduler run, null for a newer build outside the scheduler. */
CatalogNodeOutcome: {
/** Format: date-time */
at: string;
/** Format: uuid */
run_id?: string | null;
status: string;
};
/** @description A current snapshot node's declared metadata and latest build, without SQL, source or config. */
CatalogNodeRecord: {
column_count?: number | null;
/** @description The current definition differs from the one `last_build` recorded; false when either is unknown. */
definition_changed: boolean;
/** @description Keys of the nodes this node reads. */
dependencies: string[];
description?: string | null;
/** @description The snapshot node kind, such as `model`, `seed`, `source` or `audit`. */
kind: string;
last_build?: null | components["schemas"]["CatalogNodeBuild"];
last_outcome?: null | components["schemas"]["CatalogNodeOutcome"];
materialization?: string | null;
name: string;
node_key: string;
Expand Down Expand Up @@ -2689,6 +2700,10 @@ export interface components {
cancel_requested_at?: string | null;
/** Format: date-time */
ended_at?: string | null;
git_dirty?: boolean | null;
git_revision?: string | null;
/** @description The deployment image the run is pinned to, when its location publishes one. */
image_id?: string | null;
job_key?: string | null;
kind: components["schemas"]["RunKind"];
/** Format: int64 */
Expand Down Expand Up @@ -2721,8 +2736,15 @@ export interface components {
/** Format: uuid */
run_id: string;
selection_json: string[];
/**
* Format: int64
* @description The published version the run executes, read from its snapshot.
*/
snapshot_generation?: number | null;
/** Format: uuid */
snapshot_id: string;
/** @description The `sqb` version that published the snapshot. */
sqb_version?: string | null;
/** Format: date-time */
started_at?: string | null;
status: components["schemas"]["RunStatus"];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,13 @@ pub(crate) async fn list_runs_in(
(_, None) => "ORDER BY r.queued_at DESC, r.run_id DESC",
};
let (transaction, mut runs) = query_records(transaction, &format!(
"SELECT r.run_id, r.location_id, r.target, r.snapshot_id, r.job_key, r.kind, r.origin, r.parent_run_id, r.root_run_id, \
"SELECT r.run_id, r.location_id, r.target, r.snapshot_id, s.generation AS snapshot_generation, s.git_revision, \
s.git_dirty, s.sqb_version, r.image_id, r.job_key, r.kind, r.origin, r.parent_run_id, r.root_run_id, \
r.trigger_id, t.name AS trigger_name, r.tick_id, r.rerun_of_run_id, r.requested_by, p.name AS requester_name, r.priority, \
r.rewrite_history, r.status, r.status_reason, r.queued_at, r.launched_at, r.started_at, r.ended_at, r.cancel_requested_at, \
r.selection_json, r.node_counts_json, r.last_event_storage_id, r.request_json AS narrowing, r.request_json AS backfill_window \
FROM sqb_runs r LEFT JOIN sqb_triggers t ON t.trigger_id=r.trigger_id \
LEFT JOIN sqb_snapshots s ON s.snapshot_id=r.snapshot_id \
LEFT JOIN sqb_principals p ON 'principal:' || CAST(p.principal_id AS TEXT)=r.requested_by \
WHERE r.location_id=$1 AND r.target=$2 {filter} LIMIT $3"),
parameters).await?;
Expand Down
13 changes: 13 additions & 0 deletions crates/sqlbuild-orchestration/src/runs/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,19 @@ pub struct RunRecord {
pub location_id: Uuid,
pub target: String,
pub snapshot_id: Uuid,
/// The published version the run executes, read from its snapshot.
#[serde(default)]
pub snapshot_generation: Option<i64>,
#[serde(default)]
pub git_revision: Option<String>,
#[serde(default)]
pub git_dirty: Option<bool>,
/// The `sqb` version that published the snapshot.
#[serde(default)]
pub sqb_version: Option<String>,
/// The deployment image the run is pinned to, when its location publishes one.
#[serde(default)]
pub image_id: Option<String>,
pub job_key: Option<String>,
pub kind: RunKind,
pub origin: RunOrigin,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,19 +1,24 @@
//! Catalog listing rows and column lineage from snapshot node definitions and published edges.
//! Catalog listing rows with each node's newest attempt, and column lineage from published edges.

use std::collections::{BTreeMap, BTreeSet, HashMap};

use chrono::{DateTime, Utc};
use serde::Deserialize;
use serde_json::Value;
use uuid::Uuid;

use crate::sensors::constants::SOURCE_NODE_KIND;
use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes;
use crate::snapshots::models::SnapshotNode;
use crate::store::errors::ControlError;
use crate::store::models::ControlTransaction;
use crate::warehouse_metadata::constants::LINEAGE_DEFINITION_BATCH;
use crate::store::main::query_records::query_records;
use crate::store::models::{ControlTransaction, Parameter};
use crate::warehouse_metadata::constants::{
LINEAGE_DEFINITION_BATCH, OUTCOME_FAILURE_WINDOW, OUTCOME_RECENT_RUNS,
};
use crate::warehouse_metadata::models::{
CatalogNodeBuild, CatalogNodeRecord, CatalogRelation, ColumnLineageEdge, LineageColumn,
LineageNodeColumns,
CatalogNodeBuild, CatalogNodeOutcome, CatalogNodeRecord, CatalogRelation, ColumnLineageEdge,
LineageColumn, LineageNodeColumns,
};

/// A declared value from the definition itself, its schema entry or its config, in that order.
Expand Down Expand Up @@ -60,7 +65,11 @@ fn relation(node: &SnapshotNode) -> Option<CatalogRelation> {
pub(crate) fn catalog_node(
(node_key, node): (&str, &SnapshotNode),
snapshot_id: Uuid,
last_build: Option<CatalogNodeBuild>,
(last_build, last_outcome, definition_changed): (
Option<CatalogNodeBuild>,
Option<CatalogNodeOutcome>,
bool,
),
) -> CatalogNodeRecord {
let definition = &node.definition;
CatalogNodeRecord {
Expand All @@ -84,10 +93,78 @@ pub(crate) fn catalog_node(
_ => None,
},
last_build,
last_outcome,
definition_changed,
snapshot_id,
}
}

#[derive(Deserialize)]
struct Attempt {
node_key: String,
run_id: Uuid,
status: String,
ended_at: Option<DateTime<Utc>>,
finished_storage_id: i64,
}

/// Each node's newest started `success`, `warn` or `failed` attempt in those runs, excluding group runs.
pub(crate) async fn latest_attempts(
transaction: ControlTransaction,
(location_id, target): (Uuid, &str),
) -> Result<(ControlTransaction, HashMap<String, CatalogNodeOutcome>), ControlError> {
let newest = "SELECT run_id, status, queued_at FROM sqb_runs WHERE location_id=$1 AND target=$2 AND kind<>'group' \
ORDER BY queued_at DESC, run_id DESC";
let (transaction, attempts): (_, Vec<Attempt>) = query_records(transaction,
&format!("SELECT n.node_key,n.run_id,n.status,n.ended_at,n.finished_storage_id FROM (\
SELECT run_id FROM ({newest} LIMIT $3) AS latest UNION \
SELECT run_id FROM (SELECT run_id FROM ({newest} LIMIT $4) AS recent WHERE status IN ('failed','partial','warn') \
ORDER BY queued_at DESC, run_id DESC LIMIT $3) AS failures) AS r \
JOIN sqb_run_nodes n ON n.run_id=r.run_id WHERE n.kind='node' AND n.finished_storage_id IS NOT NULL \
AND n.started_at IS NOT NULL AND n.status IN ('success','warn','failed')"),
vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(OUTCOME_RECENT_RUNS), Parameter::Integer(OUTCOME_FAILURE_WINDOW)]).await?;
let mut newest: HashMap<String, Attempt> = HashMap::new();
for attempt in attempts {
if newest
.get(&attempt.node_key)
.is_none_or(|seen| attempt.finished_storage_id > seen.finished_storage_id)
{
newest.insert(attempt.node_key.clone(), attempt);
}
}
let outcomes: HashMap<String, CatalogNodeOutcome> = newest
.into_iter()
.filter_map(|(key, attempt)| {
Some((
key,
CatalogNodeOutcome {
status: attempt.status,
at: attempt.ended_at?,
run_id: Some(attempt.run_id),
},
))
})
.collect();
Ok((transaction, outcomes))
}

/// The scheduler attempt, unless a fingerprint from another clock was recorded after it ended.
pub(crate) fn last_outcome(
attempt: Option<CatalogNodeOutcome>,
build: Option<&CatalogNodeBuild>,
) -> Option<CatalogNodeOutcome> {
let newer_build =
build.filter(|build| attempt.as_ref().is_none_or(|a| build.observed_at > a.at));
match newer_build {
Some(build) => Some(CatalogNodeOutcome {
status: "success".into(),
at: build.observed_at,
run_id: None,
}),
None => attempt,
}
}

/// Published edge rows whose both ends are current nodes, so the graph never points at a hidden node.
pub(crate) fn published_edges(
document: Option<&Value>,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,8 @@ pub(crate) const PREVIEW_REQUIRED_TABLES: [WarehouseHistoryKind; 3] = [
WarehouseHistoryKind::Fingerprints,
WarehouseHistoryKind::Consumption,
];

/// The target's newest runs read for node outcomes, and as many newest failed or warned runs; never other targets'.
pub(crate) const OUTCOME_RECENT_RUNS: i64 = 25;
/// How far back, in runs, the newest failed or warned runs are looked for.
pub(crate) const OUTCOME_FAILURE_WINDOW: i64 = 500;
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! A person lists the current nodes with view on all of them; builds are batched indexed seeks.
//! A person lists the current nodes with view on all of them; builds are batched indexed seeks, attempts one bounded read.

use uuid::Uuid;

Expand All @@ -9,10 +9,14 @@ use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes;
use crate::store::errors::ControlError;
use crate::store::main::{begin::begin, commit::commit};
use crate::store::models::{ControlStore, TransactionMode};
use crate::warehouse_metadata::_helpers::catalog_nodes::catalog_node;
use crate::warehouse_metadata::_helpers::catalog_nodes::{
catalog_node, last_outcome, latest_attempts,
};
use crate::warehouse_metadata::_helpers::fingerprint_reads::latest_fingerprints;
use crate::warehouse_metadata::_helpers::node_status::destination;
use crate::warehouse_metadata::models::{CatalogNodeBuild, CatalogNodeRecord, FingerprintQuery};
use crate::warehouse_metadata::_helpers::node_status::{definition_evidence, destination};
use crate::warehouse_metadata::models::{
CatalogNodeBuild, CatalogNodeRecord, DefinitionEvidence, FingerprintQuery,
};

pub async fn list_person_catalog_nodes(
store: &ControlStore,
Expand All @@ -33,6 +37,7 @@ pub async fn list_person_catalog_nodes(
return Ok(Vec::new());
};
let (transaction, nodes) = read_snapshot_nodes(transaction, snapshot.snapshot_id, None).await?;
let nodes: Vec<_> = nodes.iter().collect();
let mut positions: Vec<usize> = Vec::new();
let mut queries: Vec<FingerprintQuery> = Vec::new();
for (position, (_, node)) in nodes.iter().enumerate() {
Expand All @@ -43,8 +48,14 @@ pub async fn list_person_catalog_nodes(
}
let (transaction, found) =
latest_fingerprints(transaction, (location_id, target, &queries)).await?;
let (transaction, mut attempts) = latest_attempts(transaction, (location_id, target)).await?;
let mut builds: Vec<Option<CatalogNodeBuild>> = vec![None; nodes.len()];
let mut changed: Vec<bool> = vec![false; nodes.len()];
for (position, build) in positions.iter().zip(found) {
changed[*position] = matches!(
definition_evidence(nodes[*position].1, build.as_ref()),
DefinitionEvidence::Changed { .. }
);
builds[*position] = build.map(|(record, _)| CatalogNodeBuild {
observed_at: record.observed_at,
run_id: record.run_id,
Expand All @@ -53,8 +64,11 @@ pub async fn list_person_catalog_nodes(
}
let records: Vec<CatalogNodeRecord> = nodes
.iter()
.zip(builds)
.map(|((key, node), build)| catalog_node((key, node), snapshot.snapshot_id, build))
.zip(builds.into_iter().zip(changed))
.map(|((key, node), (build, changed))| {
let outcome = last_outcome(attempts.remove(key.as_str()), build.as_ref());
catalog_node((key, node), snapshot.snapshot_id, (build, outcome, changed))
})
.collect();
commit(transaction).await?;
Ok(records)
Expand Down
12 changes: 12 additions & 0 deletions crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -878,9 +878,21 @@ pub struct CatalogNodeRecord {
pub column_count: Option<usize>,
/// The newest fingerprint recorded for the node at its current relation, from any build.
pub last_build: Option<CatalogNodeBuild>,
/// The newest decisive build attempt in the target's newest scheduler runs and newest failed runs, or a newer build outside them.
pub last_outcome: Option<CatalogNodeOutcome>,
/// The current definition differs from the one `last_build` recorded; false when either is unknown.
pub definition_changed: bool,
pub snapshot_id: Uuid,
}

/// `success`, `warn` or `failed`; `run_id` is the scheduler run, null for a newer build outside the scheduler.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)]
pub struct CatalogNodeOutcome {
pub status: String,
pub at: DateTime<Utc>,
pub run_id: Option<Uuid>,
}

#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)]
pub struct CatalogRelation {
pub database: Option<String>,
Expand Down
Loading
Loading