diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 38cff562b5..987403741d 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1439,15 +1439,26 @@ export interface components { run_id: string; version_hash: string; }; + /** @description `success`, `warn` or `failed`; `run_id` is the scheduler run, null for a newer build outside the scheduler. */ + CatalogNodeOutcome: { + /** Format: date-time */ + at: string; + /** Format: uuid */ + run_id?: string | null; + status: string; + }; /** @description A current snapshot node's declared metadata and latest build, without SQL, source or config. */ CatalogNodeRecord: { column_count?: number | null; + /** @description The current definition differs from the one `last_build` recorded; false when either is unknown. */ + definition_changed: boolean; /** @description Keys of the nodes this node reads. */ dependencies: string[]; description?: string | null; /** @description The snapshot node kind, such as `model`, `seed`, `source` or `audit`. */ kind: string; last_build?: null | components["schemas"]["CatalogNodeBuild"]; + last_outcome?: null | components["schemas"]["CatalogNodeOutcome"]; materialization?: string | null; name: string; node_key: string; @@ -2689,6 +2700,10 @@ export interface components { cancel_requested_at?: string | null; /** Format: date-time */ ended_at?: string | null; + git_dirty?: boolean | null; + git_revision?: string | null; + /** @description The deployment image the run is pinned to, when its location publishes one. */ + image_id?: string | null; job_key?: string | null; kind: components["schemas"]["RunKind"]; /** Format: int64 */ @@ -2721,8 +2736,15 @@ export interface components { /** Format: uuid */ run_id: string; selection_json: string[]; + /** + * Format: int64 + * @description The published version the run executes, read from its snapshot. + */ + snapshot_generation?: number | null; /** Format: uuid */ snapshot_id: string; + /** @description The `sqb` version that published the snapshot. */ + sqb_version?: string | null; /** Format: date-time */ started_at?: string | null; status: components["schemas"]["RunStatus"]; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs index f1f686b3fc..c5fdfcf4f6 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs @@ -36,11 +36,13 @@ pub(crate) async fn list_runs_in( (_, None) => "ORDER BY r.queued_at DESC, r.run_id DESC", }; let (transaction, mut runs) = query_records(transaction, &format!( - "SELECT r.run_id, r.location_id, r.target, r.snapshot_id, r.job_key, r.kind, r.origin, r.parent_run_id, r.root_run_id, \ + "SELECT r.run_id, r.location_id, r.target, r.snapshot_id, s.generation AS snapshot_generation, s.git_revision, \ + s.git_dirty, s.sqb_version, r.image_id, r.job_key, r.kind, r.origin, r.parent_run_id, r.root_run_id, \ r.trigger_id, t.name AS trigger_name, r.tick_id, r.rerun_of_run_id, r.requested_by, p.name AS requester_name, r.priority, \ r.rewrite_history, r.status, r.status_reason, r.queued_at, r.launched_at, r.started_at, r.ended_at, r.cancel_requested_at, \ r.selection_json, r.node_counts_json, r.last_event_storage_id, r.request_json AS narrowing, r.request_json AS backfill_window \ FROM sqb_runs r LEFT JOIN sqb_triggers t ON t.trigger_id=r.trigger_id \ + LEFT JOIN sqb_snapshots s ON s.snapshot_id=r.snapshot_id \ LEFT JOIN sqb_principals p ON 'principal:' || CAST(p.principal_id AS TEXT)=r.requested_by \ WHERE r.location_id=$1 AND r.target=$2 {filter} LIMIT $3"), parameters).await?; diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 12d84969d5..ebeb34b8ca 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -217,6 +217,19 @@ pub struct RunRecord { pub location_id: Uuid, pub target: String, pub snapshot_id: Uuid, + /// The published version the run executes, read from its snapshot. + #[serde(default)] + pub snapshot_generation: Option, + #[serde(default)] + pub git_revision: Option, + #[serde(default)] + pub git_dirty: Option, + /// The `sqb` version that published the snapshot. + #[serde(default)] + pub sqb_version: Option, + /// The deployment image the run is pinned to, when its location publishes one. + #[serde(default)] + pub image_id: Option, pub job_key: Option, pub kind: RunKind, pub origin: RunOrigin, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs index eee2ecc3e2..1b56663fec 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs @@ -1,7 +1,9 @@ -//! Catalog listing rows and column lineage from snapshot node definitions and published edges. +//! Catalog listing rows with each node's newest attempt, and column lineage from published edges. use std::collections::{BTreeMap, BTreeSet, HashMap}; +use chrono::{DateTime, Utc}; +use serde::Deserialize; use serde_json::Value; use uuid::Uuid; @@ -9,11 +11,14 @@ use crate::sensors::constants::SOURCE_NODE_KIND; use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; use crate::snapshots::models::SnapshotNode; use crate::store::errors::ControlError; -use crate::store::models::ControlTransaction; -use crate::warehouse_metadata::constants::LINEAGE_DEFINITION_BATCH; +use crate::store::main::query_records::query_records; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::warehouse_metadata::constants::{ + LINEAGE_DEFINITION_BATCH, OUTCOME_FAILURE_WINDOW, OUTCOME_RECENT_RUNS, +}; use crate::warehouse_metadata::models::{ - CatalogNodeBuild, CatalogNodeRecord, CatalogRelation, ColumnLineageEdge, LineageColumn, - LineageNodeColumns, + CatalogNodeBuild, CatalogNodeOutcome, CatalogNodeRecord, CatalogRelation, ColumnLineageEdge, + LineageColumn, LineageNodeColumns, }; /// A declared value from the definition itself, its schema entry or its config, in that order. @@ -60,7 +65,11 @@ fn relation(node: &SnapshotNode) -> Option { pub(crate) fn catalog_node( (node_key, node): (&str, &SnapshotNode), snapshot_id: Uuid, - last_build: Option, + (last_build, last_outcome, definition_changed): ( + Option, + Option, + bool, + ), ) -> CatalogNodeRecord { let definition = &node.definition; CatalogNodeRecord { @@ -84,10 +93,78 @@ pub(crate) fn catalog_node( _ => None, }, last_build, + last_outcome, + definition_changed, snapshot_id, } } +#[derive(Deserialize)] +struct Attempt { + node_key: String, + run_id: Uuid, + status: String, + ended_at: Option>, + finished_storage_id: i64, +} + +/// Each node's newest started `success`, `warn` or `failed` attempt in those runs, excluding group runs. +pub(crate) async fn latest_attempts( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), +) -> Result<(ControlTransaction, HashMap), ControlError> { + let newest = "SELECT run_id, status, queued_at FROM sqb_runs WHERE location_id=$1 AND target=$2 AND kind<>'group' \ + ORDER BY queued_at DESC, run_id DESC"; + let (transaction, attempts): (_, Vec) = query_records(transaction, + &format!("SELECT n.node_key,n.run_id,n.status,n.ended_at,n.finished_storage_id FROM (\ + SELECT run_id FROM ({newest} LIMIT $3) AS latest UNION \ + SELECT run_id FROM (SELECT run_id FROM ({newest} LIMIT $4) AS recent WHERE status IN ('failed','partial','warn') \ + ORDER BY queued_at DESC, run_id DESC LIMIT $3) AS failures) AS r \ + JOIN sqb_run_nodes n ON n.run_id=r.run_id WHERE n.kind='node' AND n.finished_storage_id IS NOT NULL \ + AND n.started_at IS NOT NULL AND n.status IN ('success','warn','failed')"), + vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(OUTCOME_RECENT_RUNS), Parameter::Integer(OUTCOME_FAILURE_WINDOW)]).await?; + let mut newest: HashMap = HashMap::new(); + for attempt in attempts { + if newest + .get(&attempt.node_key) + .is_none_or(|seen| attempt.finished_storage_id > seen.finished_storage_id) + { + newest.insert(attempt.node_key.clone(), attempt); + } + } + let outcomes: HashMap = newest + .into_iter() + .filter_map(|(key, attempt)| { + Some(( + key, + CatalogNodeOutcome { + status: attempt.status, + at: attempt.ended_at?, + run_id: Some(attempt.run_id), + }, + )) + }) + .collect(); + Ok((transaction, outcomes)) +} + +/// The scheduler attempt, unless a fingerprint from another clock was recorded after it ended. +pub(crate) fn last_outcome( + attempt: Option, + build: Option<&CatalogNodeBuild>, +) -> Option { + let newer_build = + build.filter(|build| attempt.as_ref().is_none_or(|a| build.observed_at > a.at)); + match newer_build { + Some(build) => Some(CatalogNodeOutcome { + status: "success".into(), + at: build.observed_at, + run_id: None, + }), + None => attempt, + } +} + /// Published edge rows whose both ends are current nodes, so the graph never points at a hidden node. pub(crate) fn published_edges( document: Option<&Value>, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index 7a15ec7660..a879c1ede9 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -13,3 +13,8 @@ pub(crate) const PREVIEW_REQUIRED_TABLES: [WarehouseHistoryKind; 3] = [ WarehouseHistoryKind::Fingerprints, WarehouseHistoryKind::Consumption, ]; + +/// The target's newest runs read for node outcomes, and as many newest failed or warned runs; never other targets'. +pub(crate) const OUTCOME_RECENT_RUNS: i64 = 25; +/// How far back, in runs, the newest failed or warned runs are looked for. +pub(crate) const OUTCOME_FAILURE_WINDOW: i64 = 500; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs index 1803a07f26..4187567da1 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs @@ -1,4 +1,4 @@ -//! A person lists the current nodes with view on all of them; builds are batched indexed seeks. +//! A person lists the current nodes with view on all of them; builds are batched indexed seeks, attempts one bounded read. use uuid::Uuid; @@ -9,10 +9,14 @@ use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; use crate::store::models::{ControlStore, TransactionMode}; -use crate::warehouse_metadata::_helpers::catalog_nodes::catalog_node; +use crate::warehouse_metadata::_helpers::catalog_nodes::{ + catalog_node, last_outcome, latest_attempts, +}; use crate::warehouse_metadata::_helpers::fingerprint_reads::latest_fingerprints; -use crate::warehouse_metadata::_helpers::node_status::destination; -use crate::warehouse_metadata::models::{CatalogNodeBuild, CatalogNodeRecord, FingerprintQuery}; +use crate::warehouse_metadata::_helpers::node_status::{definition_evidence, destination}; +use crate::warehouse_metadata::models::{ + CatalogNodeBuild, CatalogNodeRecord, DefinitionEvidence, FingerprintQuery, +}; pub async fn list_person_catalog_nodes( store: &ControlStore, @@ -33,6 +37,7 @@ pub async fn list_person_catalog_nodes( return Ok(Vec::new()); }; let (transaction, nodes) = read_snapshot_nodes(transaction, snapshot.snapshot_id, None).await?; + let nodes: Vec<_> = nodes.iter().collect(); let mut positions: Vec = Vec::new(); let mut queries: Vec = Vec::new(); for (position, (_, node)) in nodes.iter().enumerate() { @@ -43,8 +48,14 @@ pub async fn list_person_catalog_nodes( } let (transaction, found) = latest_fingerprints(transaction, (location_id, target, &queries)).await?; + let (transaction, mut attempts) = latest_attempts(transaction, (location_id, target)).await?; let mut builds: Vec> = vec![None; nodes.len()]; + let mut changed: Vec = vec![false; nodes.len()]; for (position, build) in positions.iter().zip(found) { + changed[*position] = matches!( + definition_evidence(nodes[*position].1, build.as_ref()), + DefinitionEvidence::Changed { .. } + ); builds[*position] = build.map(|(record, _)| CatalogNodeBuild { observed_at: record.observed_at, run_id: record.run_id, @@ -53,8 +64,11 @@ pub async fn list_person_catalog_nodes( } let records: Vec = nodes .iter() - .zip(builds) - .map(|((key, node), build)| catalog_node((key, node), snapshot.snapshot_id, build)) + .zip(builds.into_iter().zip(changed)) + .map(|((key, node), (build, changed))| { + let outcome = last_outcome(attempts.remove(key.as_str()), build.as_ref()); + catalog_node((key, node), snapshot.snapshot_id, (build, outcome, changed)) + }) .collect(); commit(transaction).await?; Ok(records) diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 42bc27126a..6484f195e3 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -878,9 +878,21 @@ pub struct CatalogNodeRecord { pub column_count: Option, /// The newest fingerprint recorded for the node at its current relation, from any build. pub last_build: Option, + /// The newest decisive build attempt in the target's newest scheduler runs and newest failed runs, or a newer build outside them. + pub last_outcome: Option, + /// The current definition differs from the one `last_build` recorded; false when either is unknown. + pub definition_changed: bool, pub snapshot_id: Uuid, } +/// `success`, `warn` or `failed`; `run_id` is the scheduler run, null for a newer build outside the scheduler. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct CatalogNodeOutcome { + pub status: String, + pub at: DateTime, + pub run_id: Option, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct CatalogRelation { pub database: Option, diff --git a/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs index 8287aa9c47..cba2166f6f 100644 --- a/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs @@ -5,7 +5,7 @@ use std::cell::Cell; use uuid::Uuid; use crate::credentials::models::{AuditContext, DefaultRole}; -use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::helpers::{request, request_run, snapshot}; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::snapshots::models::SnapshotInput; use crate::snapshots::models::SnapshotNode; @@ -21,8 +21,8 @@ use crate::warehouse_metadata::main::{ }; use crate::warehouse_metadata::models::{MetadataBatch, NodeStatusQuery}; -/// Statements a catalog listing issues besides one build read per 100 nodes: 7 fixed, 1 per extra namespace shape. -const WIDE_LISTING_FIXED_STATEMENTS: usize = 8; +/// Statements a catalog listing issues besides one build read per 100 nodes: 8 fixed, 1 per extra namespace shape. +const WIDE_LISTING_FIXED_STATEMENTS: usize = 9; pub(crate) async fn node_status_reads( store: &ControlStore, @@ -53,7 +53,7 @@ pub(crate) async fn node_status_reads( python_source: None, }, ); - publish_snapshot(store, &input) + let published = publish_snapshot(store, &input) .await .expect("identity snapshot"); let options = NodeStatusQuery { @@ -235,6 +235,15 @@ pub(crate) async fn node_status_reads( json!({"database": null, "schema": "main", "name": "orders"}) ); assert_eq!(orders["dependencies"], json!(["source:raw_orders"])); + assert_eq!( + orders["definition_changed"], true, + "the listing compares definitions like the node status" + ); + assert_eq!( + orders["last_outcome"], + json!({"status": "success", "at": changed.records[0]["ts"], "run_id": null}), + "a build outside the scheduler is a success" + ); assert_eq!( listed_node("udf:orders")["last_build"]["run_id"], "orders-function-run", @@ -248,6 +257,13 @@ pub(crate) async fn node_status_reads( ); assert!(!json!(listed).to_string().contains("metadata_json")); listing_ignores_build_history(store, credentials, (location, &listed)).await; + listing_reports_scheduler_attempts( + store, + credentials, + (location, published.snapshot_id), + serde_json::from_value(changed.records[0]["ts"].clone()).expect("build time"), + ) + .await; listing_wide_targets_is_batched_without_super_linear_growth( store, credentials, @@ -368,6 +384,82 @@ async fn listing_ignores_build_history( ); } +/// A failed scheduler attempt replaces the outcome only when it ended after the newest build, and stays after quicker runs. +async fn listing_reports_scheduler_attempts( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location, snapshot_id): (Uuid, Uuid), + built_at: chrono::DateTime, +) { + let outcome = || async { + let listed = list_person_catalog_nodes( + store, + credentials, + (location, "development"), + DefaultRole::None, + ) + .await + .expect("catalog nodes"); + let orders = listed + .into_iter() + .find(|node| node.node_key == "model:orders") + .expect("orders listed"); + json!(orders.last_outcome) + }; + for (id, ended_at, expected) in [ + ( + 0x0f01, + built_at - chrono::Duration::hours(1), + json!({"status": "success", "at": built_at, "run_id": null}), + ), + ( + 0x0f02, + built_at + chrono::Duration::hours(1), + json!({"status": "failed", "at": built_at + chrono::Duration::hours(1), + "run_id": Uuid::from_u128(0x0f02)}), + ), + ] { + let run = request((location, snapshot_id), id, 0, &["model:orders"]); + request_run(store, &run).await.expect("queue attempt"); + let transaction = begin(store, TransactionMode::Write).await.expect("begin"); + let (transaction, _) = query( + transaction, + "UPDATE sqb_run_nodes SET status='failed',started_at=$1,ended_at=$2,finished_storage_id=$3 WHERE run_id=$4", + vec![ + Parameter::Time(ended_at - chrono::Duration::minutes(5)), + Parameter::Time(ended_at), + Parameter::Integer(i64::try_from(id).expect("storage id")), + Parameter::Id(run.run_id), + ], + ) + .await + .expect("finish attempt"); + let (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET status='failed' WHERE run_id=$1", + vec![Parameter::Id(run.run_id)], + ) + .await + .expect("fail run"); + commit(transaction).await.expect("commit attempt"); + assert_eq!(outcome().await, expected, "attempt ending at {ended_at}"); + } + for id in 0x0f10..0x0f10 + 25 { + request_run( + store, + &request((location, snapshot_id), id, 0, &["customers"]), + ) + .await + .expect("newer run of another node"); + } + assert_eq!( + outcome().await, + json!({"status": "failed", "at": built_at + chrono::Duration::hours(1), + "run_id": Uuid::from_u128(0x0f02)}), + "a failure stays reported after more than 25 newer runs" + ); +} + /// Three thousand nodes, half in a named database, keep their own builds, read 100 per statement without super-linear growth. async fn listing_wide_targets_is_batched_without_super_linear_growth( store: &ControlStore, diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs index 7c90f2ffbf..c01025625a 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs @@ -261,6 +261,10 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ ("origin", json!("manual")), ("job_key", json!("orders")), ("requester_name", json!("orders-person")), + ("snapshot_generation", json!(published.generation)), + ("git_revision", json!(input.git_revision)), + ("git_dirty", json!(input.git_dirty)), + ("sqb_version", json!(input.sqb_version)), ("root_run_id", launched["run_id"].clone()), ("parent_run_id", Value::Null), ("trigger_name", Value::Null), diff --git a/ui/src/lib/api/http-catalog.test.ts b/ui/src/lib/api/http-catalog.test.ts index 14d166ed95..c1b1b75592 100644 --- a/ui/src/lib/api/http-catalog.test.ts +++ b/ui/src/lib/api/http-catalog.test.ts @@ -7,7 +7,7 @@ type W = components['schemas']; function node(key: string, extra: Partial = {}): W['CatalogNodeRecord'] { const [kind, name] = key.split(':'); - return { node_key: key, kind, name, tags: [], dependencies: [], snapshot_id: '01a1b000-0000-7000-8000-000000000001', ...extra }; + return { node_key: key, kind, name, tags: [], dependencies: [], definition_changed: false, snapshot_id: '01a1b000-0000-7000-8000-000000000001', ...extra }; } describe('catalog node mapping', () => { @@ -39,7 +39,7 @@ describe('catalog node mapping', () => { expect(orders.upstream).toEqual(['source:raw_orders']); }); - it('reports the fingerprint as a sqb build and leaves unreported build state unknown, not empty', () => { + it('reports the fingerprint as a sqb build and leaves unreported state unknown, not empty', () => { const [orders] = toCatalogNodes([ node('model:orders', { materialization: 'table', @@ -52,7 +52,28 @@ describe('catalog node mapping', () => { ]); expect(orders.lastBuild).toEqual({ fingerprintRunId: 'sqb-run-1', runId: null, jobKey: null, builtAt: '2026-11-07T06:00:00Z', fingerprint: 'v1' }); expect([orders.description, orders.tags, orders.columnCount, orders.relation]).toEqual(['One row per order', ['marts'], 4, { database: null, schema: 'main', name: 'orders' }]); - expect([orders.stale, orders.checks, orders.lastOutcome, orders.group, orders.owner]).toEqual([null, null, null, null, []]); + expect([orders.stale, orders.checks, orders.lastOutcome, orders.group, orders.owner]).toEqual([[], null, null, null, []]); + }); + + it('marks a changed definition, and a built table older than the data upstream of it, even through a view, as stale', () => { + const built = (observed_at: string) => ({ last_build: { observed_at, run_id: 'sqb', version_hash: 'v' } }); + const table = (key: string, at: string | null, extra: Partial = {}) => + node(key, { materialization: 'table', dependencies: ['model:orders', 'source:raw_orders'], ...(at ? built(at) : {}), ...extra }); + const nodes = toCatalogNodes([ + node('source:raw_orders'), + table('model:orders', '2026-11-07T06:00:00Z', { dependencies: ['source:raw_orders'], definition_changed: true, last_outcome: { status: 'failed', at: '2026-11-07T06:30:00Z', run_id: 'r1' } }), + table('model:order_totals', '2026-11-07T05:00:00Z', { last_outcome: { status: 'success', at: '2026-11-07T05:00:00Z' } }), + table('model:order_daily', '2026-11-07T07:00:00Z'), + table('model:order_view', '2026-11-07T04:00:00Z', { materialization: 'view' }), + table('model:order_drafts', null, { definition_changed: true }), + table('model:order_report', '2026-11-07T05:30:00Z', { dependencies: ['model:order_view'] }) + ]); + const by = Object.fromEntries(nodes.map((n) => [n.key, n])); + expect(by['model:orders'].stale).toEqual([{ reason: 'changed', change: 'definition', detail: 'The definition changed since it was last built here.', since: '2026-11-07T06:00:00Z' }]); + expect(by['model:order_totals'].stale).toEqual([{ reason: 'behind', upstream: ['model:orders'], detail: 'orders has newer data than this build.', since: '2026-11-07T06:00:00.000Z' }]); + expect(by['model:order_report'].stale?.map((r) => r.reason === 'behind' && [r.upstream, r.since])).toEqual([[['model:order_view'], '2026-11-07T06:00:00.000Z']]); + expect(['model:order_daily', 'model:order_view', 'model:order_drafts', 'source:raw_orders'].map((k) => by[k].stale)).toEqual([[], [], [], []]); + expect(nodes.map((n) => n.lastOutcome)).toEqual([null, 'failed', 'success', null, null, null, null]); }); it('finds a node by key, or by bare name preferring the model', () => { @@ -82,8 +103,9 @@ describe('catalog node detail', () => { expect(d.downstream).toEqual(['model:order_totals']); }); - it('reports a changed definition as stale and maps the warehouse columns', () => { - const changed = nodeDetailOf(nodes, nodes[0], status({ status: 'changed', current_local_hash: 'b', recorded_local_hash: 'a' }, observed), null, []); + it('keeps the listing staleness and maps the warehouse columns', () => { + const [listed] = toCatalogNodes([node('model:orders', { materialization: 'table', definition_changed: true, last_build: { observed_at: '2026-11-07T06:00:00Z', run_id: 'sqb', version_hash: 'v' } })]); + const changed = nodeDetailOf(nodes, listed, status({ status: 'changed', current_local_hash: 'b', recorded_local_hash: 'a' }, observed), null, []); expect(changed.stale.map((r) => [r.reason, r.since])).toEqual([['changed', '2026-11-07T06:00:00Z']]); expect(changed.columns).toEqual([{ name: 'order_id', type: 'INTEGER', description: '', audits: [] }]); const matched = nodeDetailOf(nodes, nodes[0], status({ status: 'matched', local_hash: 'a' }, { status: 'unavailable', reason: 'no catalog' }), null, []); diff --git a/ui/src/lib/api/http-catalog.ts b/ui/src/lib/api/http-catalog.ts index 28f72202b5..54cccd81cc 100644 --- a/ui/src/lib/api/http-catalog.ts +++ b/ui/src/lib/api/http-catalog.ts @@ -15,13 +15,16 @@ function kindOf(wire: Wire['CatalogNodeRecord']): NodeKind | null { } /** - * One node of the current snapshot. The server reports declared metadata and the newest - * fingerprint, not run outcomes, staleness, check counts, owners or groups, so those stay null - * or empty and the pages hide them. The fingerprint's run is a `sqb` run, not a scheduler run. + * One node of the current snapshot. The server reports declared metadata, the newest fingerprint, + * the newest decisive outcome and whether the definition changed since that fingerprint, not + * check counts, owners or groups, so those stay null or empty and the pages hide them. The + * fingerprint's run is a `sqb` run, not a scheduler run. Staleness from upstream builds needs the + * whole listing, so `toCatalogNodes` adds it. */ export function toCatalogNode(wire: Wire['CatalogNodeRecord']): CatalogNode | null { const kind = kindOf(wire); if (!kind) return null; + const outcome = wire.last_outcome?.status; return { key: wire.node_key, kind, @@ -37,18 +40,62 @@ export function toCatalogNode(wire: Wire['CatalogNodeRecord']): CatalogNode | nu lastBuild: wire.last_build ? { fingerprintRunId: wire.last_build.run_id, runId: null, jobKey: null, builtAt: wire.last_build.observed_at, fingerprint: wire.last_build.version_hash } : null, - lastOutcome: null, - stale: null, + lastOutcome: outcome === 'success' || outcome === 'warn' || outcome === 'failed' ? outcome : null, + stale: + wire.definition_changed && wire.last_build + ? [{ reason: 'changed', change: 'definition', detail: 'The definition changed since it was last built here.', since: wire.last_build.observed_at }] + : [], sourceFreshness: null, checks: null }; } +/** + * When the data a node reads last changed: its own last build, or for a view, which reads its + * upstream when queried, the newest data behind it. Null when nothing behind it is built. + */ +function dataTimes(listed: Map): (key: string) => number | null { + const memo = new Map(); + const at = (key: string): number | null => { + if (memo.has(key)) return memo.get(key) ?? null; + memo.set(key, null); + const node = listed.get(key); + let time = node?.lastBuild ? Date.parse(node.lastBuild.builtAt) : null; + if (node?.kind === 'view') { + const upstream = node.upstream.flatMap((k) => at(k) ?? []); + time = upstream.length ? Math.max(...upstream) : null; + } + memo.set(key, time); + return time; + }; + return at; +} + +/** + * Upstream nodes with newer data than this node's own last build. Views read their upstream when + * queried, so they are never behind, and an unbuilt node is not built yet rather than behind. + */ +function behindOf(node: CatalogNode, dataAt: (key: string) => number | null): StaleReason[] { + const builtAt = node.lastBuild?.builtAt; + if (!builtAt || node.kind === 'view') return []; + const newer = node.upstream.flatMap((key) => { + const at = dataAt(key); + return at !== null && at > Date.parse(builtAt) ? [{ key, at }] : []; + }); + if (!newer.length) return []; + const since = new Date(Math.min(...newer.map((n) => n.at))).toISOString(); + const names = newer.map((n) => shortName(n.key)); + const detail = `${names.length === 1 ? names[0] : `${names.length} upstream nodes`} ${names.length === 1 ? 'has' : 'have'} newer data than this build.`; + return [{ reason: 'behind', upstream: newer.map((n) => n.key), detail, since }]; +} + /** Catalog nodes in listing order, keeping only dependencies that are themselves listed. */ export function toCatalogNodes(wire: Wire['CatalogNodeRecord'][]): CatalogNode[] { const nodes = wire.flatMap((node) => toCatalogNode(node) ?? []); - const listed = new Set(nodes.map((node) => node.key)); - return nodes.map((node) => ({ ...node, upstream: node.upstream.filter((key) => listed.has(key)) })); + const keys = new Set(nodes.map((node) => node.key)); + const linked = nodes.map((node) => ({ ...node, upstream: node.upstream.filter((key) => keys.has(key)) })); + const dataAt = dataTimes(new Map(linked.map((node) => [node.key, node]))); + return linked.map((node) => ({ ...node, stale: [...(node.stale ?? []), ...behindOf(node, dataAt)] })); } /** Passing, warning and failing by each check's newest result; checks without one count only towards the total. */ @@ -62,13 +109,6 @@ export function checkCountsOf(checks: Check[]): CheckCounts { }; } -/** A definition that changed since its last recorded build here; other staleness is not reported by the server. */ -function staleOf(status: Wire['NodeStatus'] | null): StaleReason[] { - if (status?.definition.status !== 'changed' || !status.build) return []; - return [{ reason: 'changed', change: 'definition', detail: 'The definition changed since it was last built here.', since: status.build.observed_at }]; -} - - /** The listed node a key or bare name refers to, preferring a model; migrations name models without their kind. */ export function findNode(nodes: CatalogNode[], ref: string): CatalogNode | null { return nodes.find((n) => n.key === ref) ?? nodes.find((n) => n.key === `model:${ref}`) ?? nodes.find((n) => shortName(n.key) === ref) ?? null; @@ -160,7 +200,7 @@ export function nodeDetailOf( const physical = status?.physical.status === 'observed' ? status.physical.observation.record : null; return { ...node, - stale: staleOf(status), + stale: node.stale ?? [], checks: checkCountsOf(own), downstream: nodes.filter((n) => n.upstream.includes(node.key)).map((n) => n.key), recentOutcomes: (runs ?? []).slice(0, 40).map((r) => r.outcome).reverse(), diff --git a/ui/src/lib/api/http-overview.test.ts b/ui/src/lib/api/http-overview.test.ts index 2c45f9624e..1983125206 100644 --- a/ui/src/lib/api/http-overview.test.ts +++ b/ui/src/lib/api/http-overview.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import { overviewOf } from './http-overview'; -import type { CatalogNode, Check, NodeChange, PlatformEvent, Run, SnapshotChange } from './types'; +import type { CatalogNode, Check, NodeChange, PlatformEvent, Run, SnapshotChange, StaleReason } from './types'; const now = Date.parse('2026-11-09T12:00:00Z'); const hoursAgo = (h: number) => new Date(now - h * 3_600_000).toISOString(); @@ -9,7 +9,7 @@ function run(id: string, status: Run['status'], queuedHoursAgo: number, endedHou return { id, status, parentRunId, jobKey, narrowing, window: null, kind: jobKey ? 'job' : 'adhoc', statusReason: null, queuedAt: hoursAgo(queuedHoursAgo), startedAt: hoursAgo(queuedHoursAgo), endedAt: endedHoursAgo === null ? null : hoursAgo(endedHoursAgo) } as Run; } -const node = (key: string, kind: CatalogNode['kind']) => ({ key, kind }) as CatalogNode; +const node = (key: string, kind: CatalogNode['kind'], stale: StaleReason['reason'][] = []) => ({ key, kind, stale: stale.map((reason) => ({ reason })) }) as CatalogNode; const check = (id: string, outcome: string | null) => ({ id, latest: outcome ? { outcome } : null }) as unknown as Check; const change = (publishedHoursAgo: number, nodes: [NodeChange['op'], string][]) => ({ snapshotId: `s-${publishedHoursAgo}`, publishedAt: hoursAgo(publishedHoursAgo), nodes: nodes.map(([op, key]) => ({ key, op })) }) as SnapshotChange; @@ -39,7 +39,7 @@ describe('overview from live reads', () => { const overview = overviewOf({ runs, checks: [check('a', 'pass'), check('b', 'fail'), check('c', 'error'), check('d', 'warn'), check('e', null)], - nodes: [node('model:orders', 'table'), node('model:totals', 'view'), node('model:events', 'incremental'), node('source:raw', 'source'), node('seed:regions', 'seed')], + nodes: [node('model:orders', 'table', ['changed', 'behind']), node('model:totals', 'view'), node('model:events', 'incremental', ['behind']), node('source:raw', 'source'), node('seed:regions', 'seed')], changes: [change(1, [['added', 'model:orders'], ['added', 'seed:regions'], ['removed', 'model:old']]), change(24 * 8, [['added', 'model:totals']])], events: [event('scheduler', true), event('cleared', false)], acknowledgements: new Map([['orders-new', { by: 'ada', at: hoursAgo(1) }]]), @@ -59,7 +59,8 @@ describe('overview from live reads', () => { 'run_problem:catalogue-partial:-', 'run_problem:adhoc-old:-', 'run_unresponsive:stuck:-', - 'event:scheduler' + 'event:scheduler', + 'stale_models' ]); }); @@ -75,7 +76,12 @@ describe('overview from live reads', () => { expect(overview.latestRun?.id).toBe('fulfilment-ok'); }); + it('counts each stale node once, and leaves late sources unknown', () => { + expect(overview.metrics.stale).toEqual({ nodes: 2, changed: 1, behind: 2, sources: null }); + expect(overview.attention.find((i) => i.kind === 'stale_models')).toEqual({ kind: 'stale_models', changed: ['model:orders'], behind: ['model:orders', 'model:events'] }); + }); + it('leaves parts without a server read empty instead of zero', () => { - expect([overview.metrics.stale, overview.metrics.cost, overview.groups]).toEqual([null, null, null]); + expect([overview.metrics.cost, overview.groups]).toEqual([null, null]); }); }); diff --git a/ui/src/lib/api/http-overview.ts b/ui/src/lib/api/http-overview.ts index de5a7625b2..dc98e58dd9 100644 --- a/ui/src/lib/api/http-overview.ts +++ b/ui/src/lib/api/http-overview.ts @@ -42,8 +42,9 @@ export function problemRunsOf(topLevel: Run[], now: number): Run[] { const attentionRun = ({ id, jobKey, kind, status, statusReason, nodeCounts, startedAt, endedAt }: Run): AttentionRun => ({ id, jobKey, kind, status, statusReason, nodeCounts, startedAt, endedAt }); /** - * The overview from reads the person already has. The server has no project-wide build state, cost - * or group read yet, so stale counts and group status are null and cost is left out; the page hides them. + * The overview from reads the person already has. Staleness comes from the catalog listing, which + * does not check source freshness, so late sources stay unknown. The server has no cost or group + * read yet, so group status is null and cost is left out; the page hides them. */ export function overviewOf({ runs, checks, nodes, changes, events, acknowledgements, now }: OverviewInputs): Overview { const acknowledged = (r: Run) => acknowledgements.get(r.id) ?? null; @@ -57,6 +58,11 @@ export function overviewOf({ runs, checks, nodes, changes, events, acknowledgeme .map((r): AttentionItem => ({ kind: 'run_unresponsive', run: attentionRun(r), lastHeartbeatAt: null, acknowledged: acknowledged(r) })), ...events.filter((e) => e.active).map((event): AttentionItem => ({ kind: 'platform_event', event })) ]; + const staleKeys = (test: (reason: string) => boolean) => nodes.filter((n) => n.stale?.some((r) => test(r.reason))).map((n) => n.key); + const changed = staleKeys((reason) => reason !== 'behind'); + const behind = staleKeys((reason) => reason === 'behind'); + const staleNodes = new Set([...changed, ...behind]).size; + if (staleNodes) attention.push({ kind: 'stale_models', changed, behind }); const thisWeek = changes.filter((c) => Date.parse(c.publishedAt) >= now - 7 * DAY).flatMap((c) => c.nodes); const latest = topLevel.filter((r) => r.endedAt).sort((a, b) => Date.parse(b.endedAt!) - Date.parse(a.endedAt!))[0]; return { @@ -75,7 +81,7 @@ export function overviewOf({ runs, checks, nodes, changes, events, acknowledgeme active: recent.filter((r) => ACTIVE_STATUSES.has(r.status)).length }, checks: checkCountsOf(checks), - stale: null, + stale: { nodes: staleNodes, changed: changed.length, behind: behind.length, sources: null }, cost: null }, attention, diff --git a/ui/src/lib/api/http-runs.test.ts b/ui/src/lib/api/http-runs.test.ts index cc1056afdb..258a043059 100644 --- a/ui/src/lib/api/http-runs.test.ts +++ b/ui/src/lib/api/http-runs.test.ts @@ -216,6 +216,14 @@ describe('run detail', () => { expect(nodes[4].status).toBe('not_run'); expect(nodes[2].skippedBecauseOf).toEqual({ nodeKey: 'model.n1', runId: 'run', jobKey: 'orders_daily' }); }); + + it("shows the version a run executes from its snapshot, and none when the server doesn't report it", () => { + const built = wire('built', { snapshot_generation: 7, git_revision: 'c31f0b9', git_dirty: true, sqb_version: '0.216.0', image_id: null, selection_json: ['model.orders'] }); + const [run, bare] = toRuns([built, wire('bare')], LOCATION); + expect(runDetailOf(run, built, [run], []).provenance).toMatchObject({ snapshotGeneration: 7, gitRevision: 'c31f0b9', gitDirty: true, sqbVersion: '0.216.0', imageId: null, command: null, selection: ['model.orders'] }); + expect(run.gitRevision).toBe('c31f0b9'); + expect(runDetailOf(bare, wire('bare'), [bare], []).provenance).toBeNull(); + }); }); describe('run Recall reads', () => { @@ -262,10 +270,10 @@ describe('run Recall reads', () => { ], errors: [{ invocation_id: inv, occurred_at: '2026-11-08T06:00:02Z', error_type: 'ValueError', message: 'RuntimeError: bad input' }] }); - expect(output.map((l) => [l.stream, l.text])).toEqual([ - ['stderr', 'warning'], - ['stderr', 'RuntimeError: bad input'], - ['stdout', 'done'] + expect(output.map((l) => [l.stream, l.text, l.error])).toEqual([ + ['stderr', 'warning', false], + ['stderr', 'RuntimeError: bad input', true], + ['stdout', 'done', false] ]); }); }); diff --git a/ui/src/lib/api/http-runs.ts b/ui/src/lib/api/http-runs.ts index 78965a49c0..cc9dbc24d3 100644 --- a/ui/src/lib/api/http-runs.ts +++ b/ui/src/lib/api/http-runs.ts @@ -104,7 +104,7 @@ export function toRun(wire: Wire['RunRecord'], locationId: string, childRunCount locationId, target: wire.target, snapshotId: wire.snapshot_id, - gitRevision: null, + gitRevision: wire.git_revision ?? null, queuedAt: wire.queued_at, launchedAt: wire.launched_at ?? null, startedAt: wire.started_at ?? null, @@ -293,7 +293,20 @@ export function runDetailOf(run: Run, wire: Wire['RunRecord'], listed: Run[], no invocations: [], liveness: null, cancel: null, - provenance: null, + provenance: + wire.snapshot_generation != null + ? { + snapshotId: wire.snapshot_id, + snapshotGeneration: wire.snapshot_generation, + gitRevision: wire.git_revision ?? null, + gitDirty: wire.git_dirty ?? false, + imageId: wire.image_id ?? null, + sqbVersion: wire.sqb_version ?? '', + selection: wire.selection_json, + exclude: [], + command: null + } + : null, selection: wire.selection_json, waits: [], rerunFromFailureNodes: nodes.filter((n) => n.status === 'failed' || n.status === 'skipped').map((n) => n.nodeKey), @@ -388,8 +401,8 @@ export function toRunStatements(wire: Wire['RunStatementRecord'][], runId: strin /** Captured lines and error text in time order; error text reads as stderr of its invocation. */ export function toRunOutput(wire: Wire['RunOutput']): RunOutputLine[] { - const lines: RunOutputLine[] = wire.lines.map((l) => ({ invocationId: l.invocation_id, stream: l.stream === 'stderr' ? 'stderr' : 'stdout', at: l.occurred_at, text: l.text })); - const errors: RunOutputLine[] = wire.errors.map((e) => ({ invocationId: e.invocation_id, stream: 'stderr', at: e.occurred_at ?? '', text: e.message })); + const lines: RunOutputLine[] = wire.lines.map((l) => ({ invocationId: l.invocation_id, stream: l.stream === 'stderr' ? 'stderr' : 'stdout', at: l.occurred_at, text: l.text, error: false })); + const errors: RunOutputLine[] = wire.errors.map((e) => ({ invocationId: e.invocation_id, stream: 'stderr', at: e.occurred_at ?? '', text: e.message, error: true })); return [...lines, ...errors].sort((a, b) => a.at.localeCompare(b.at)); } diff --git a/ui/src/lib/api/mock/operate.ts b/ui/src/lib/api/mock/operate.ts index 15ebe70c1c..2c63a8ac7b 100644 --- a/ui/src/lib/api/mock/operate.ts +++ b/ui/src/lib/api/mock/operate.ts @@ -653,6 +653,7 @@ type Seed = { warn?: string[]; errors?: Record; end?: RunEnd; + crashError?: string; select?: string[]; exclude?: string[]; verb?: LaunchVerb; @@ -753,6 +754,7 @@ export function createOperateFixtures(now: number): OperateFixtures { critical: job?.criticalNodes ?? [], plannedMs, end: seed.end ?? 'complete', + crashError: seed.crashError, cancel, lastHeartbeatAt: seed.lastHeartbeatAgoSec != null ? now - seed.lastHeartbeatAgoSec * 1000 : null, imageId: seed.image ?? IMAGE_ID, @@ -861,6 +863,7 @@ export function createOperateFixtures(now: number): OperateFixtures { record({ id: 'run-c40e', origin: 'manual', job: 'marts_only', by: CI_ACCOUNT, queuedAgo: 1080, startedAgo: 1079.8, durationMin: 4 }); record({ id: 'run-e3f8', origin: 'schedule', job: 'activity_microbatch', trigger: 'activity_microbatch', tick: 101, by: SCHEDULER, queuedAgo: at('activity_microbatch', 101, 0), startedAgo: at('activity_microbatch', 101, 6), durationMin: 11, end: 'presumed_failed', lastHeartbeatAgoSec: (at('activity_microbatch', 101, 6) - 0.6) * 60, reason: 'Lost contact: no heartbeat for 10 minutes, so the run was stopped and marked failed' }); record({ id: 'run-a7c1', origin: 'schedule', job: 'revenue_refresh', trigger: 'revenue_refresh', tick: 0, by: SCHEDULER, queuedAgo: at('revenue_refresh', 0, 0), startedAgo: at('revenue_refresh', 0, 8), durationMin: 2, fail: ['daily_revenue'], errors: { daily_revenue: 'Warehouse error: the warehouse was suspended while the statement ran' } }); + record({ id: 'run-7e14', origin: 'manual', job: 'marts_only', by: DEMO_USER, queuedAgo: 300, startedAgo: 299.8, durationMin: 1, end: 'crashed', reason: 'worker executor completed and closed its connections', crashError: "Couldn't connect to the warehouse: role TRANSFORMER has no access to warehouse REPORTING_WH" }); record({ id: 'run-f1a9', origin: 'sensor', job: 'load_sources', trigger: 'partner_feed_poll', runKey: 'orders_part_0398.csv', by: SENSOR_DAEMON, queuedAgo: 50 * 60, end: 'never_started', reason: 'Couldn’t start: the run didn’t report in within 2 minutes of launch (the image couldn’t be downloaded)', image: PREVIOUS_IMAGE_ID, generation: 41 }); const nightlyStages = [['load_sources'], ['core_build'], ['activity_microbatch'], ['reporting_exports']]; diff --git a/ui/src/lib/api/mock/run-detail.ts b/ui/src/lib/api/mock/run-detail.ts index 49e0fc7b64..3556bd268c 100644 --- a/ui/src/lib/api/mock/run-detail.ts +++ b/ui/src/lib/api/mock/run-detail.ts @@ -20,7 +20,8 @@ import type { } from '../types'; import { ancestors, snapshotNode, sortTopologically } from './operate-graph'; -export type RunEnd = 'complete' | 'cancelled' | 'presumed_failed' | 'never_started'; +/** `crashed`: the worker stopped before building anything, as when its plan is refused; `crashError` says why. */ +export type RunEnd = 'complete' | 'cancelled' | 'presumed_failed' | 'never_started' | 'crashed'; export type RunScript = { verb: LaunchVerb; @@ -36,6 +37,7 @@ export type RunScript = { plannedMs: number; /** How the run ends, once terminal. */ end: RunEnd; + crashError?: string; cancel: CancelProgress | null; /** Last heartbeat; node progress stops here for unresponsive and lost runs. */ lastHeartbeatAt: number | null; @@ -127,6 +129,8 @@ export function computeNodes(record: RunRecord, siblings: RunRecord[], now: numb return sortTopologically(script.nodes).map((k) => base(k, status)); } + if (terminal && script.end === 'crashed') return sortTopologically(script.nodes).map((k) => ({ ...base(k, 'skipped'), endedAt: run.endedAt })); + const { timed, total } = schedule(script.nodes); const t0 = started + 1500; const unit = Math.max(200, script.plannedMs - 3000) / total; @@ -226,7 +230,7 @@ export function countNodes(nodes: RunNode[]): NodeCounts { /** Terminal status from node outcomes: partial when something failed and everything else completed. */ export function deriveTerminalStatus(end: RunEnd, nodes: RunNode[]): RunStatus { - if (end === 'never_started') return 'failed'; + if (end === 'never_started' || end === 'crashed') return 'failed'; if (end === 'cancelled') return 'cancelled'; if (end === 'presumed_failed') return 'presumed_failed'; const c = countNodes(nodes); @@ -354,7 +358,7 @@ export function execution(record: RunRecord, nodes: RunNode[], now?: number): Ru drafts.push({ producer: 'scheduler', type, occurredAt: typeof at === 'number' ? iso(at) : at, invocationId: null, nodeKey: null, statementId: null, attempt: null, message }); const ev = (type: Draft['type'], at: number, nodeKey: string | null, message: string, statementId: string | null = null) => drafts.push({ type, occurredAt: iso(at), invocationId: inv, nodeKey, statementId, attempt: nodeKey ? 1 : null, message }); - const out = (stream: RunOutputLine['stream'], at: number, text: string) => inv && output.push({ invocationId: inv, stream, at: iso(at), text }); + const out = (stream: RunOutputLine['stream'], at: number, text: string, error = false) => inv && output.push({ invocationId: inv, stream, at: iso(at), text, error }); const queued = Date.parse(run.queuedAt); fact('requested', queued - 200, `Requested by ${run.requestedBy.displayName}${run.triggerName ? ` via ${run.triggerName}` : ''}`); @@ -433,6 +437,7 @@ export function execution(record: RunRecord, nodes: RunNode[], now?: number): Ru out('stdout', e, `OK ${n.nodeKey} ${k} ${dur}s`); } } + if (run.endedAt && script.crashError) out('stderr', Date.parse(run.endedAt) - 500, script.crashError, true); if (run.endedAt) { const end = Date.parse(run.endedAt) - 400; const c = countNodes(nodes); diff --git a/ui/src/lib/api/types/build.ts b/ui/src/lib/api/types/build.ts index 9536d87308..0a1380de75 100644 --- a/ui/src/lib/api/types/build.ts +++ b/ui/src/lib/api/types/build.ts @@ -357,8 +357,8 @@ export type OverviewMetrics = { models: { total: number; addedThisWeek: number; removedThisWeek: number }; runs24h: { total: number; success: number; warn: number; partial: number; failed: number; active: number }; checks: CheckCounts; - /** Distinct stale nodes, and how many are changed or behind (a node can be both); null when the server reports no project-wide build state. */ - stale: { nodes: number; changed: number; behind: number; sources: number } | null; + /** Distinct stale nodes, and how many are changed or behind (a node can be both); null when the server reports no project-wide build state, and `sources` null when source freshness is not checked. */ + stale: { nodes: number; changed: number; behind: number; sources: number | null } | null; /** Null when the adapter has no cost data. */ cost: { last24h: number; baseline: number; baselineDays: number; unit: CostUnit } | null; }; diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index 1b41832ec0..552b96193d 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -315,6 +315,8 @@ export type RunOutputLine = { stream: 'stdout' | 'stderr'; at: Iso; text: string; + /** The error the invocation recorded when it failed, rather than a line it printed. */ + error: boolean; }; export type RunInvocation = { @@ -356,14 +358,17 @@ export type CancelProgress = { export type RunProvenance = { snapshotId: string; snapshotGeneration: number; - gitRevision: string; + /** Null when the project was published outside a git checkout. */ + gitRevision: string | null; gitDirty: boolean; - imageId: string; + /** Null when the location publishes no deployment image. */ + imageId: string | null; sqbVersion: string; /** The frozen selection the run was authorized for. */ selection: string[]; exclude: string[]; - command: string; + /** Null where the server does not report the command the worker ran. */ + command: string | null; }; export type RunDetail = { diff --git a/ui/src/lib/build-area/attention.ts b/ui/src/lib/build-area/attention.ts index e18d05838d..14d9dff438 100644 --- a/ui/src/lib/build-area/attention.ts +++ b/ui/src/lib/build-area/attention.ts @@ -5,6 +5,7 @@ import { describeWait, outcomeCounts } from '$lib/operate/labels'; import { eventKindLabel, problemHref } from '$lib/platform/labels'; import { formatAgo } from '$lib/format/time'; import { runStatusLabel } from '$lib/status'; +import { shortName } from '$lib/build-area/labels'; export type AttentionTone = 'error' | 'warning' | 'stale' | 'info'; @@ -51,7 +52,7 @@ export function attentionView(item: AttentionItem, now: number): AttentionView { key: 'stale', tone: 'stale', title: `${plural(total, 'node')} ${total === 1 ? 'is' : 'are'} stale`, - detail: `${parts.join(' · ')} · ${[...new Set([...item.changed, ...item.behind])].slice(0, 4).join(', ')}`, + detail: `${parts.join(' · ')} · ${[...new Set([...item.changed, ...item.behind])].slice(0, 4).map(shortName).join(', ')}`, since: 'since the last build', href: '/catalog?stale=1', action: 'View in catalog' diff --git a/ui/src/lib/status.ts b/ui/src/lib/status.ts index dc7db40e72..d38e0747f0 100644 --- a/ui/src/lib/status.ts +++ b/ui/src/lib/status.ts @@ -1,3 +1,4 @@ +import { shortName } from '$lib/build-area/labels'; import type { CheckOutcome, NodeCounts, NodeOutcome, StaleReason, RunKind, RunOrigin, RunStatus, TickOutcome, TriggerState } from '$lib/api/types'; export const runStatusTone: Record = { @@ -115,8 +116,8 @@ export function staleReasonLabel(reason: StaleReason): string { /** One-line explanation of a stale reason. */ export function staleReasonDetail(reason: StaleReason): string { if (reason.reason === 'changed') return reason.detail; - if (reason.reason === 'changed_upstream') return `Upstream definition changed: ${reason.upstream.join(', ')}`; - return `Newer data in ${reason.upstream.join(', ')}`; + if (reason.reason === 'changed_upstream') return `Upstream definition changed: ${reason.upstream.map(shortName).join(', ')}`; + return `Newer data in ${reason.upstream.map(shortName).join(', ')}`; } /** e.g. `8 ok · 1 warn · 1 failed · 3 skipped`, or `4 running · 10 pending`. */ diff --git a/ui/src/routes/+page.svelte b/ui/src/routes/+page.svelte index 31e259d8b3..6b42cad532 100644 --- a/ui/src/routes/+page.svelte +++ b/ui/src/routes/+page.svelte @@ -198,7 +198,7 @@
Stale
{m.stale.nodes}nodes
- {m.stale.changed} changed · {m.stale.behind} behind · {m.stale.sources} {m.stale.sources === 1 ? 'source' : 'sources'} late + {m.stale.changed} changed · {m.stale.behind} behind{#if m.stale.sources !== null} · {m.stale.sources} {m.stale.sources === 1 ? 'source' : 'sources'} late{/if}
{/if} {#if m.cost} diff --git a/ui/src/routes/runs/[id]/log-tab.svelte b/ui/src/routes/runs/[id]/log-tab.svelte index 976a81a58e..1f2cfde2cf 100644 --- a/ui/src/routes/runs/[id]/log-tab.svelte +++ b/ui/src/routes/runs/[id]/log-tab.svelte @@ -162,8 +162,9 @@ top: 0; z-index: 2; display: flex; + flex-wrap: wrap; align-items: center; - gap: 8px; + gap: 6px 8px; padding: 6px 16px; border-bottom: 1px solid var(--border); background: var(--background); diff --git a/ui/src/routes/runs/[id]/related-runs.svelte b/ui/src/routes/runs/[id]/related-runs.svelte index 99738ff7f3..7541c22356 100644 --- a/ui/src/routes/runs/[id]/related-runs.svelte +++ b/ui/src/routes/runs/[id]/related-runs.svelte @@ -43,19 +43,28 @@ {/if} - {#if view.showDetails} + +{#snippet selection(keys: string[])} + {#if keys.length > 12} +
{keys.length} nodes{keys.join(' ')}
+ {:else} + {keys.join(' ') || '—'} + {/if} +{/snippet} + +{#if view.showDetails} {@const p = d.provenance}
{#if p} -
Project version
Version {p.snapshotGeneration} · commit {p.gitRevision}{p.gitDirty ? ' (with uncommitted changes)' : ''} · snapshot {p.snapshotId}
-
Command
{p.command}
+
Project version
Version {p.snapshotGeneration}{#if p.gitRevision} · commit {p.gitRevision}{/if}{p.gitDirty ? ' (with uncommitted changes)' : ''} · snapshot {p.snapshotId}
+ {#if p.command}
Command
{p.command}
{/if} {#if p.selection.length || p.exclude.length} -
Selection
{p.selection.join(' ') || '—'}{#if p.exclude.length} excluding {p.exclude.join(' ')}{/if}
+
Selection
{@render selection(p.selection)}{#if p.exclude.length} excluding {p.exclude.join(' ')}{/if}
{/if} -
Image
{shortImage(p.imageId)} · sqb {p.sqbVersion}
+
{p.imageId ? 'Image' : 'Published by'}
{#if p.imageId}{shortImage(p.imageId)} · {/if}sqb {p.sqbVersion}
{:else if d.selection.length} -
Selection
{d.selection.join(' ')}
+
Selection
{@render selection(d.selection)}
{/if}
Rebuilds history
{d.run.rewriteHistory ? 'Yes: a full refresh or a set time window, so it needs extra permission to rerun' : 'No'}
Priority
{d.run.priority}
@@ -157,6 +166,11 @@ dt { color: var(--muted-foreground); } + summary { + width: fit-content; + color: var(--primary); + cursor: pointer; + } table { margin-top: 10px; border-collapse: collapse; diff --git a/ui/src/routes/runs/[id]/run-header.svelte b/ui/src/routes/runs/[id]/run-header.svelte index 226254b810..be7b12aef7 100644 --- a/ui/src/routes/runs/[id]/run-header.svelte +++ b/ui/src/routes/runs/[id]/run-header.svelte @@ -67,7 +67,7 @@ {#if view.detail?.provenance} {@const p = view.detail.provenance} · - version {p.snapshotGeneration} + version {p.snapshotGeneration} {/if}
diff --git a/ui/src/routes/runs/[id]/state.svelte.ts b/ui/src/routes/runs/[id]/state.svelte.ts index 2c3eb7c88a..a69387f5a2 100644 --- a/ui/src/routes/runs/[id]/state.svelte.ts +++ b/ui/src/routes/runs/[id]/state.svelte.ts @@ -38,7 +38,8 @@ export function createRunDetailState(runId: string) { const nodes = $derived(detail?.nodes ?? []); const active = $derived(!!run && isActiveRun(run.status)); const progress = $derived(progressOf(nodes)); - const banners = $derived(detail ? bannersFor(detail, app.now) : []); + const cause = $derived(outputRes.data?.find((l) => l.error)?.text ?? null); + const banners = $derived(detail ? bannersFor(detail, app.now, cause) : []); const statements = $derived(statementsRes.data ?? []); // Progress reported by a run with no signal is stale, so the live card only shows while it's running. const runningStatements = $derived(run?.status === 'running' ? statements.filter((s) => s.status === 'running') : []); diff --git a/ui/src/routes/runs/[id]/utils/banners.test.ts b/ui/src/routes/runs/[id]/utils/banners.test.ts new file mode 100644 index 0000000000..036897f771 --- /dev/null +++ b/ui/src/routes/runs/[id]/utils/banners.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from 'vitest'; +import type { RunDetail } from '$lib/api'; +import { bannersFor } from './banners'; + +const failed = (statusReason: string | null, failedNodes = 0) => + ({ run: { status: 'failed', statusReason, nodeCounts: { failed: failedNodes } }, liveness: null, cancel: null, waits: [] }) as unknown as RunDetail; +const text = (detail: RunDetail, cause: string | null) => bannersFor(detail, 0, cause).find((b) => b.key === 'failed')?.body.map((p) => p.text).join(''); + +describe('the banner of a run that failed as a whole', () => { + it('leads with the first recorded error and keeps how the run ended beside it', () => { + expect(text(failed('Timed out after 1h'), 'Connection reset; retrying')).toBe('Connection reset; retrying · Ended: Timed out after 1h'); + }); + + it('falls back to the status reason, and leaves a failed node to explain itself', () => { + expect(text(failed('Timed out after 1h'), null)).toBe('Timed out after 1h'); + expect(text(failed('Worker exited with code 1', 1), 'Binder Error')).toBeUndefined(); + }); +}); diff --git a/ui/src/routes/runs/[id]/utils/banners.ts b/ui/src/routes/runs/[id]/utils/banners.ts index 6265efe441..463d8627e3 100644 --- a/ui/src/routes/runs/[id]/utils/banners.ts +++ b/ui/src/routes/runs/[id]/utils/banners.ts @@ -30,7 +30,8 @@ function cancelTechnical(cancel: CancelProgress): string { ].join('\n'); } -export function bannersFor(detail: RunDetail, now: number): Banner[] { +/** `cause` is the first error the run recorded, when the person may read its output. */ +export function bannersFor(detail: RunDetail, now: number, cause: string | null = null): Banner[] { const out: Banner[] = []; const { run, liveness: live, cancel } = detail; if (live?.state === 'unresponsive' && live.lastHeartbeatAt) { @@ -53,8 +54,11 @@ export function bannersFor(detail: RunDetail, now: number): Banner[] { detail: run.statusReason ?? undefined }); } - if (run.status === 'failed' && run.statusReason && run.nodeCounts.failed === 0) { - out.push({ key: 'failed', tone: 'error', title: 'The run failed.', body: [{ text: run.statusReason }] }); + // With no failed node to explain it, the first recorded error leads. It may be an earlier error the run + // survived, so the status reason, the scheduler's own account of the end, stays beside it. + if (run.status === 'failed' && (cause || run.statusReason) && run.nodeCounts.failed === 0) { + const reason = cause && run.statusReason ? [{ text: ` · Ended: ${run.statusReason}` }] : []; + out.push({ key: 'failed', tone: 'error', title: 'The run failed.', body: [{ text: cause ?? run.statusReason ?? '' }, ...reason] }); } if (live?.state === 'never_started') { out.push({ diff --git a/ui/src/routes/runs/[id]/utils/events.test.ts b/ui/src/routes/runs/[id]/utils/events.test.ts index 9b299f849f..4ee3abada3 100644 --- a/ui/src/routes/runs/[id]/utils/events.test.ts +++ b/ui/src/routes/runs/[id]/utils/events.test.ts @@ -22,7 +22,7 @@ describe('run log', () => { it("interleaves a group's events and printed lines by time and keeps each line's run", () => { const entries = logEntries( [event('e2', '2026-11-09T08:17:38Z', { type: 'resource_attempt_failed', level: 'error', message: 'ConversionError: no', runId: 'staging' }), event('e1', '2026-11-09T08:17:35Z', { nodeKey: 'model:orders' })], - [{ runId: 'orders', invocationId: 'i', stream: 'stderr', at: '2026-11-09T08:17:36Z', text: 'Compiling project...' }] + [{ runId: 'orders', invocationId: 'i', stream: 'stderr', at: '2026-11-09T08:17:36Z', text: 'Compiling project...', error: false }] ); expect(entries.map((e) => [e.id, e.runId, e.category, e.level])).toEqual([ ['e1', 'staging', 'nodes', 'info'], @@ -67,12 +67,15 @@ describe('run log', () => { expect(entries[1].queries.map((q) => q.id)).toEqual(['q']); }); - it("pins only a failing run's last stderr block", () => { - const out = (runId: string, stream: 'stdout' | 'stderr', at: string) => ({ runId, invocationId: 'i', stream, at, text: `${runId} ${stream}` }); + it("pins a failing run's last stderr block and every recorded error, and no other output", () => { + const out = (runId: string, stream: 'stdout' | 'stderr', at: string, error = false) => ({ runId, invocationId: 'i', stream, at, text: `${runId} ${stream}`, error }); const entries = logEntries( [event('fail', '2026-11-09T08:00:04Z', { type: 'resource_attempt_failed', level: 'error', message: 'X: no' }), event('ok', '2026-11-09T08:00:04Z', { runId: 'orders', type: 'resource_attempt_completed' })], - [out('staging', 'stderr', '2026-11-09T08:00:01Z'), out('staging', 'stderr', '2026-11-09T08:00:05Z'), out('staging', 'stdout', '2026-11-09T08:00:06Z'), out('orders', 'stderr', '2026-11-09T08:00:07Z')] + [out('staging', 'stderr', '2026-11-09T08:00:01Z'), out('staging', 'stderr', '2026-11-09T08:00:05Z'), out('staging', 'stdout', '2026-11-09T08:00:06Z'), out('orders', 'stderr', '2026-11-09T08:00:07Z'), out('orders', 'stderr', '2026-11-09T08:00:08Z', true)] ); - expect(entries.filter((e) => e.pinned).map((e) => e.id)).toEqual(['out-staging-1']); + expect(entries.filter((e) => e.pinned).map((e) => [e.id, e.level])).toEqual([ + ['out-staging-1', 'info'], + ['out-orders-4', 'error'] + ]); }); }); diff --git a/ui/src/routes/runs/[id]/utils/events.ts b/ui/src/routes/runs/[id]/utils/events.ts index 25576d00bc..78e7b80ac7 100644 --- a/ui/src/routes/runs/[id]/utils/events.ts +++ b/ui/src/routes/runs/[id]/utils/events.ts @@ -139,15 +139,15 @@ export function logEntries(events: readonly RunEvent[], output: readonly (RunOut runId: l.runId, at: l.at, nodeKey: null, - level: 'info', + level: l.error ? 'error' : 'info', category: 'output', text: l.text, title: l.stream, - tone: l.stream === 'stderr' ? 'var(--sb-warning)' : 'var(--sb-text-faint)', + tone: l.error ? 'var(--sb-error)' : l.stream === 'stderr' ? 'var(--sb-warning)' : 'var(--sb-text-faint)', stream: l.stream, statementId: null, queries: [], - pinned: lastStderr.get(l.runId) === i + pinned: l.error || lastStderr.get(l.runId) === i }) ); // A stable sort keeps each source's own order for equal times.