From 15abd57dd2b78a737febdb7cacab389282924fa6 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 06:26:02 +0000 Subject: [PATCH 1/2] feat(ui): show why a run failed as a whole and which version it ran A run that fails before building any node now leads its banner with the first error it recorded, such as a refused plan, with how the run ended beside it. Recorded errors stay pinned and red in the log. The run read reports the version a run executes from its snapshot: generation, commit, uncommitted changes, the publishing sqb version and the pinned image. The run header links the version and Details shows it, with a long selection folded behind its node count. The log toolbar wraps instead of overflowing on narrower screens. --- clients/typescript/src/schema.d.ts | 11 ++++++++ .../src/runs/_helpers/lifecycle/reads.rs | 4 ++- .../sqlbuild-orchestration/src/runs/models.rs | 13 ++++++++++ .../tests/api/test_person_sessions.rs | 4 +++ ui/src/lib/api/http-runs.test.ts | 16 +++++++++--- ui/src/lib/api/http-runs.ts | 21 ++++++++++++--- ui/src/lib/api/mock/operate.ts | 3 +++ ui/src/lib/api/mock/run-detail.ts | 11 +++++--- ui/src/lib/api/types/operate.ts | 11 +++++--- ui/src/routes/runs/[id]/log-tab.svelte | 3 ++- ui/src/routes/runs/[id]/related-runs.svelte | 26 ++++++++++++++----- ui/src/routes/runs/[id]/run-header.svelte | 2 +- ui/src/routes/runs/[id]/state.svelte.ts | 3 ++- ui/src/routes/runs/[id]/utils/banners.test.ts | 18 +++++++++++++ ui/src/routes/runs/[id]/utils/banners.ts | 10 ++++--- ui/src/routes/runs/[id]/utils/events.test.ts | 13 ++++++---- ui/src/routes/runs/[id]/utils/events.ts | 6 ++--- 17 files changed, 140 insertions(+), 35 deletions(-) create mode 100644 ui/src/routes/runs/[id]/utils/banners.test.ts diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 38cff562b5..db8cbcf18f 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -2689,6 +2689,10 @@ export interface components { cancel_requested_at?: string | null; /** Format: date-time */ ended_at?: string | null; + git_dirty?: boolean | null; + git_revision?: string | null; + /** @description The deployment image the run is pinned to, when its location publishes one. */ + image_id?: string | null; job_key?: string | null; kind: components["schemas"]["RunKind"]; /** Format: int64 */ @@ -2721,8 +2725,15 @@ export interface components { /** Format: uuid */ run_id: string; selection_json: string[]; + /** + * Format: int64 + * @description The published version the run executes, read from its snapshot. + */ + snapshot_generation?: number | null; /** Format: uuid */ snapshot_id: string; + /** @description The `sqb` version that published the snapshot. */ + sqb_version?: string | null; /** Format: date-time */ started_at?: string | null; status: components["schemas"]["RunStatus"]; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs index f1f686b3fc..c5fdfcf4f6 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/lifecycle/reads.rs @@ -36,11 +36,13 @@ pub(crate) async fn list_runs_in( (_, None) => "ORDER BY r.queued_at DESC, r.run_id DESC", }; let (transaction, mut runs) = query_records(transaction, &format!( - "SELECT r.run_id, r.location_id, r.target, r.snapshot_id, r.job_key, r.kind, r.origin, r.parent_run_id, r.root_run_id, \ + "SELECT r.run_id, r.location_id, r.target, r.snapshot_id, s.generation AS snapshot_generation, s.git_revision, \ + s.git_dirty, s.sqb_version, r.image_id, r.job_key, r.kind, r.origin, r.parent_run_id, r.root_run_id, \ r.trigger_id, t.name AS trigger_name, r.tick_id, r.rerun_of_run_id, r.requested_by, p.name AS requester_name, r.priority, \ r.rewrite_history, r.status, r.status_reason, r.queued_at, r.launched_at, r.started_at, r.ended_at, r.cancel_requested_at, \ r.selection_json, r.node_counts_json, r.last_event_storage_id, r.request_json AS narrowing, r.request_json AS backfill_window \ FROM sqb_runs r LEFT JOIN sqb_triggers t ON t.trigger_id=r.trigger_id \ + LEFT JOIN sqb_snapshots s ON s.snapshot_id=r.snapshot_id \ LEFT JOIN sqb_principals p ON 'principal:' || CAST(p.principal_id AS TEXT)=r.requested_by \ WHERE r.location_id=$1 AND r.target=$2 {filter} LIMIT $3"), parameters).await?; diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 12d84969d5..ebeb34b8ca 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -217,6 +217,19 @@ pub struct RunRecord { pub location_id: Uuid, pub target: String, pub snapshot_id: Uuid, + /// The published version the run executes, read from its snapshot. + #[serde(default)] + pub snapshot_generation: Option, + #[serde(default)] + pub git_revision: Option, + #[serde(default)] + pub git_dirty: Option, + /// The `sqb` version that published the snapshot. + #[serde(default)] + pub sqb_version: Option, + /// The deployment image the run is pinned to, when its location publishes one. + #[serde(default)] + pub image_id: Option, pub job_key: Option, pub kind: RunKind, pub origin: RunOrigin, diff --git a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs index 7c90f2ffbf..c01025625a 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_person_sessions.rs @@ -261,6 +261,10 @@ fn given_browser_session_roles_when_work_is_requested_then_scope_revocation_and_ ("origin", json!("manual")), ("job_key", json!("orders")), ("requester_name", json!("orders-person")), + ("snapshot_generation", json!(published.generation)), + ("git_revision", json!(input.git_revision)), + ("git_dirty", json!(input.git_dirty)), + ("sqb_version", json!(input.sqb_version)), ("root_run_id", launched["run_id"].clone()), ("parent_run_id", Value::Null), ("trigger_name", Value::Null), diff --git a/ui/src/lib/api/http-runs.test.ts b/ui/src/lib/api/http-runs.test.ts index cc1056afdb..258a043059 100644 --- a/ui/src/lib/api/http-runs.test.ts +++ b/ui/src/lib/api/http-runs.test.ts @@ -216,6 +216,14 @@ describe('run detail', () => { expect(nodes[4].status).toBe('not_run'); expect(nodes[2].skippedBecauseOf).toEqual({ nodeKey: 'model.n1', runId: 'run', jobKey: 'orders_daily' }); }); + + it("shows the version a run executes from its snapshot, and none when the server doesn't report it", () => { + const built = wire('built', { snapshot_generation: 7, git_revision: 'c31f0b9', git_dirty: true, sqb_version: '0.216.0', image_id: null, selection_json: ['model.orders'] }); + const [run, bare] = toRuns([built, wire('bare')], LOCATION); + expect(runDetailOf(run, built, [run], []).provenance).toMatchObject({ snapshotGeneration: 7, gitRevision: 'c31f0b9', gitDirty: true, sqbVersion: '0.216.0', imageId: null, command: null, selection: ['model.orders'] }); + expect(run.gitRevision).toBe('c31f0b9'); + expect(runDetailOf(bare, wire('bare'), [bare], []).provenance).toBeNull(); + }); }); describe('run Recall reads', () => { @@ -262,10 +270,10 @@ describe('run Recall reads', () => { ], errors: [{ invocation_id: inv, occurred_at: '2026-11-08T06:00:02Z', error_type: 'ValueError', message: 'RuntimeError: bad input' }] }); - expect(output.map((l) => [l.stream, l.text])).toEqual([ - ['stderr', 'warning'], - ['stderr', 'RuntimeError: bad input'], - ['stdout', 'done'] + expect(output.map((l) => [l.stream, l.text, l.error])).toEqual([ + ['stderr', 'warning', false], + ['stderr', 'RuntimeError: bad input', true], + ['stdout', 'done', false] ]); }); }); diff --git a/ui/src/lib/api/http-runs.ts b/ui/src/lib/api/http-runs.ts index 78965a49c0..cc9dbc24d3 100644 --- a/ui/src/lib/api/http-runs.ts +++ b/ui/src/lib/api/http-runs.ts @@ -104,7 +104,7 @@ export function toRun(wire: Wire['RunRecord'], locationId: string, childRunCount locationId, target: wire.target, snapshotId: wire.snapshot_id, - gitRevision: null, + gitRevision: wire.git_revision ?? null, queuedAt: wire.queued_at, launchedAt: wire.launched_at ?? null, startedAt: wire.started_at ?? null, @@ -293,7 +293,20 @@ export function runDetailOf(run: Run, wire: Wire['RunRecord'], listed: Run[], no invocations: [], liveness: null, cancel: null, - provenance: null, + provenance: + wire.snapshot_generation != null + ? { + snapshotId: wire.snapshot_id, + snapshotGeneration: wire.snapshot_generation, + gitRevision: wire.git_revision ?? null, + gitDirty: wire.git_dirty ?? false, + imageId: wire.image_id ?? null, + sqbVersion: wire.sqb_version ?? '', + selection: wire.selection_json, + exclude: [], + command: null + } + : null, selection: wire.selection_json, waits: [], rerunFromFailureNodes: nodes.filter((n) => n.status === 'failed' || n.status === 'skipped').map((n) => n.nodeKey), @@ -388,8 +401,8 @@ export function toRunStatements(wire: Wire['RunStatementRecord'][], runId: strin /** Captured lines and error text in time order; error text reads as stderr of its invocation. */ export function toRunOutput(wire: Wire['RunOutput']): RunOutputLine[] { - const lines: RunOutputLine[] = wire.lines.map((l) => ({ invocationId: l.invocation_id, stream: l.stream === 'stderr' ? 'stderr' : 'stdout', at: l.occurred_at, text: l.text })); - const errors: RunOutputLine[] = wire.errors.map((e) => ({ invocationId: e.invocation_id, stream: 'stderr', at: e.occurred_at ?? '', text: e.message })); + const lines: RunOutputLine[] = wire.lines.map((l) => ({ invocationId: l.invocation_id, stream: l.stream === 'stderr' ? 'stderr' : 'stdout', at: l.occurred_at, text: l.text, error: false })); + const errors: RunOutputLine[] = wire.errors.map((e) => ({ invocationId: e.invocation_id, stream: 'stderr', at: e.occurred_at ?? '', text: e.message, error: true })); return [...lines, ...errors].sort((a, b) => a.at.localeCompare(b.at)); } diff --git a/ui/src/lib/api/mock/operate.ts b/ui/src/lib/api/mock/operate.ts index 15ebe70c1c..2c63a8ac7b 100644 --- a/ui/src/lib/api/mock/operate.ts +++ b/ui/src/lib/api/mock/operate.ts @@ -653,6 +653,7 @@ type Seed = { warn?: string[]; errors?: Record; end?: RunEnd; + crashError?: string; select?: string[]; exclude?: string[]; verb?: LaunchVerb; @@ -753,6 +754,7 @@ export function createOperateFixtures(now: number): OperateFixtures { critical: job?.criticalNodes ?? [], plannedMs, end: seed.end ?? 'complete', + crashError: seed.crashError, cancel, lastHeartbeatAt: seed.lastHeartbeatAgoSec != null ? now - seed.lastHeartbeatAgoSec * 1000 : null, imageId: seed.image ?? IMAGE_ID, @@ -861,6 +863,7 @@ export function createOperateFixtures(now: number): OperateFixtures { record({ id: 'run-c40e', origin: 'manual', job: 'marts_only', by: CI_ACCOUNT, queuedAgo: 1080, startedAgo: 1079.8, durationMin: 4 }); record({ id: 'run-e3f8', origin: 'schedule', job: 'activity_microbatch', trigger: 'activity_microbatch', tick: 101, by: SCHEDULER, queuedAgo: at('activity_microbatch', 101, 0), startedAgo: at('activity_microbatch', 101, 6), durationMin: 11, end: 'presumed_failed', lastHeartbeatAgoSec: (at('activity_microbatch', 101, 6) - 0.6) * 60, reason: 'Lost contact: no heartbeat for 10 minutes, so the run was stopped and marked failed' }); record({ id: 'run-a7c1', origin: 'schedule', job: 'revenue_refresh', trigger: 'revenue_refresh', tick: 0, by: SCHEDULER, queuedAgo: at('revenue_refresh', 0, 0), startedAgo: at('revenue_refresh', 0, 8), durationMin: 2, fail: ['daily_revenue'], errors: { daily_revenue: 'Warehouse error: the warehouse was suspended while the statement ran' } }); + record({ id: 'run-7e14', origin: 'manual', job: 'marts_only', by: DEMO_USER, queuedAgo: 300, startedAgo: 299.8, durationMin: 1, end: 'crashed', reason: 'worker executor completed and closed its connections', crashError: "Couldn't connect to the warehouse: role TRANSFORMER has no access to warehouse REPORTING_WH" }); record({ id: 'run-f1a9', origin: 'sensor', job: 'load_sources', trigger: 'partner_feed_poll', runKey: 'orders_part_0398.csv', by: SENSOR_DAEMON, queuedAgo: 50 * 60, end: 'never_started', reason: 'Couldn’t start: the run didn’t report in within 2 minutes of launch (the image couldn’t be downloaded)', image: PREVIOUS_IMAGE_ID, generation: 41 }); const nightlyStages = [['load_sources'], ['core_build'], ['activity_microbatch'], ['reporting_exports']]; diff --git a/ui/src/lib/api/mock/run-detail.ts b/ui/src/lib/api/mock/run-detail.ts index 49e0fc7b64..3556bd268c 100644 --- a/ui/src/lib/api/mock/run-detail.ts +++ b/ui/src/lib/api/mock/run-detail.ts @@ -20,7 +20,8 @@ import type { } from '../types'; import { ancestors, snapshotNode, sortTopologically } from './operate-graph'; -export type RunEnd = 'complete' | 'cancelled' | 'presumed_failed' | 'never_started'; +/** `crashed`: the worker stopped before building anything, as when its plan is refused; `crashError` says why. */ +export type RunEnd = 'complete' | 'cancelled' | 'presumed_failed' | 'never_started' | 'crashed'; export type RunScript = { verb: LaunchVerb; @@ -36,6 +37,7 @@ export type RunScript = { plannedMs: number; /** How the run ends, once terminal. */ end: RunEnd; + crashError?: string; cancel: CancelProgress | null; /** Last heartbeat; node progress stops here for unresponsive and lost runs. */ lastHeartbeatAt: number | null; @@ -127,6 +129,8 @@ export function computeNodes(record: RunRecord, siblings: RunRecord[], now: numb return sortTopologically(script.nodes).map((k) => base(k, status)); } + if (terminal && script.end === 'crashed') return sortTopologically(script.nodes).map((k) => ({ ...base(k, 'skipped'), endedAt: run.endedAt })); + const { timed, total } = schedule(script.nodes); const t0 = started + 1500; const unit = Math.max(200, script.plannedMs - 3000) / total; @@ -226,7 +230,7 @@ export function countNodes(nodes: RunNode[]): NodeCounts { /** Terminal status from node outcomes: partial when something failed and everything else completed. */ export function deriveTerminalStatus(end: RunEnd, nodes: RunNode[]): RunStatus { - if (end === 'never_started') return 'failed'; + if (end === 'never_started' || end === 'crashed') return 'failed'; if (end === 'cancelled') return 'cancelled'; if (end === 'presumed_failed') return 'presumed_failed'; const c = countNodes(nodes); @@ -354,7 +358,7 @@ export function execution(record: RunRecord, nodes: RunNode[], now?: number): Ru drafts.push({ producer: 'scheduler', type, occurredAt: typeof at === 'number' ? iso(at) : at, invocationId: null, nodeKey: null, statementId: null, attempt: null, message }); const ev = (type: Draft['type'], at: number, nodeKey: string | null, message: string, statementId: string | null = null) => drafts.push({ type, occurredAt: iso(at), invocationId: inv, nodeKey, statementId, attempt: nodeKey ? 1 : null, message }); - const out = (stream: RunOutputLine['stream'], at: number, text: string) => inv && output.push({ invocationId: inv, stream, at: iso(at), text }); + const out = (stream: RunOutputLine['stream'], at: number, text: string, error = false) => inv && output.push({ invocationId: inv, stream, at: iso(at), text, error }); const queued = Date.parse(run.queuedAt); fact('requested', queued - 200, `Requested by ${run.requestedBy.displayName}${run.triggerName ? ` via ${run.triggerName}` : ''}`); @@ -433,6 +437,7 @@ export function execution(record: RunRecord, nodes: RunNode[], now?: number): Ru out('stdout', e, `OK ${n.nodeKey} ${k} ${dur}s`); } } + if (run.endedAt && script.crashError) out('stderr', Date.parse(run.endedAt) - 500, script.crashError, true); if (run.endedAt) { const end = Date.parse(run.endedAt) - 400; const c = countNodes(nodes); diff --git a/ui/src/lib/api/types/operate.ts b/ui/src/lib/api/types/operate.ts index 1b41832ec0..552b96193d 100644 --- a/ui/src/lib/api/types/operate.ts +++ b/ui/src/lib/api/types/operate.ts @@ -315,6 +315,8 @@ export type RunOutputLine = { stream: 'stdout' | 'stderr'; at: Iso; text: string; + /** The error the invocation recorded when it failed, rather than a line it printed. */ + error: boolean; }; export type RunInvocation = { @@ -356,14 +358,17 @@ export type CancelProgress = { export type RunProvenance = { snapshotId: string; snapshotGeneration: number; - gitRevision: string; + /** Null when the project was published outside a git checkout. */ + gitRevision: string | null; gitDirty: boolean; - imageId: string; + /** Null when the location publishes no deployment image. */ + imageId: string | null; sqbVersion: string; /** The frozen selection the run was authorized for. */ selection: string[]; exclude: string[]; - command: string; + /** Null where the server does not report the command the worker ran. */ + command: string | null; }; export type RunDetail = { diff --git a/ui/src/routes/runs/[id]/log-tab.svelte b/ui/src/routes/runs/[id]/log-tab.svelte index 976a81a58e..1f2cfde2cf 100644 --- a/ui/src/routes/runs/[id]/log-tab.svelte +++ b/ui/src/routes/runs/[id]/log-tab.svelte @@ -162,8 +162,9 @@ top: 0; z-index: 2; display: flex; + flex-wrap: wrap; align-items: center; - gap: 8px; + gap: 6px 8px; padding: 6px 16px; border-bottom: 1px solid var(--border); background: var(--background); diff --git a/ui/src/routes/runs/[id]/related-runs.svelte b/ui/src/routes/runs/[id]/related-runs.svelte index 99738ff7f3..7541c22356 100644 --- a/ui/src/routes/runs/[id]/related-runs.svelte +++ b/ui/src/routes/runs/[id]/related-runs.svelte @@ -43,19 +43,28 @@ {/if} - {#if view.showDetails} + +{#snippet selection(keys: string[])} + {#if keys.length > 12} +
{keys.length} nodes{keys.join(' ')}
+ {:else} + {keys.join(' ') || '—'} + {/if} +{/snippet} + +{#if view.showDetails} {@const p = d.provenance}
{#if p} -
Project version
Version {p.snapshotGeneration} · commit {p.gitRevision}{p.gitDirty ? ' (with uncommitted changes)' : ''} · snapshot {p.snapshotId}
-
Command
{p.command}
+
Project version
Version {p.snapshotGeneration}{#if p.gitRevision} · commit {p.gitRevision}{/if}{p.gitDirty ? ' (with uncommitted changes)' : ''} · snapshot {p.snapshotId}
+ {#if p.command}
Command
{p.command}
{/if} {#if p.selection.length || p.exclude.length} -
Selection
{p.selection.join(' ') || '—'}{#if p.exclude.length} excluding {p.exclude.join(' ')}{/if}
+
Selection
{@render selection(p.selection)}{#if p.exclude.length} excluding {p.exclude.join(' ')}{/if}
{/if} -
Image
{shortImage(p.imageId)} · sqb {p.sqbVersion}
+
{p.imageId ? 'Image' : 'Published by'}
{#if p.imageId}{shortImage(p.imageId)} · {/if}sqb {p.sqbVersion}
{:else if d.selection.length} -
Selection
{d.selection.join(' ')}
+
Selection
{@render selection(d.selection)}
{/if}
Rebuilds history
{d.run.rewriteHistory ? 'Yes: a full refresh or a set time window, so it needs extra permission to rerun' : 'No'}
Priority
{d.run.priority}
@@ -157,6 +166,11 @@ dt { color: var(--muted-foreground); } + summary { + width: fit-content; + color: var(--primary); + cursor: pointer; + } table { margin-top: 10px; border-collapse: collapse; diff --git a/ui/src/routes/runs/[id]/run-header.svelte b/ui/src/routes/runs/[id]/run-header.svelte index 226254b810..be7b12aef7 100644 --- a/ui/src/routes/runs/[id]/run-header.svelte +++ b/ui/src/routes/runs/[id]/run-header.svelte @@ -67,7 +67,7 @@ {#if view.detail?.provenance} {@const p = view.detail.provenance} · - version {p.snapshotGeneration} + version {p.snapshotGeneration} {/if}
diff --git a/ui/src/routes/runs/[id]/state.svelte.ts b/ui/src/routes/runs/[id]/state.svelte.ts index 2c3eb7c88a..a69387f5a2 100644 --- a/ui/src/routes/runs/[id]/state.svelte.ts +++ b/ui/src/routes/runs/[id]/state.svelte.ts @@ -38,7 +38,8 @@ export function createRunDetailState(runId: string) { const nodes = $derived(detail?.nodes ?? []); const active = $derived(!!run && isActiveRun(run.status)); const progress = $derived(progressOf(nodes)); - const banners = $derived(detail ? bannersFor(detail, app.now) : []); + const cause = $derived(outputRes.data?.find((l) => l.error)?.text ?? null); + const banners = $derived(detail ? bannersFor(detail, app.now, cause) : []); const statements = $derived(statementsRes.data ?? []); // Progress reported by a run with no signal is stale, so the live card only shows while it's running. const runningStatements = $derived(run?.status === 'running' ? statements.filter((s) => s.status === 'running') : []); diff --git a/ui/src/routes/runs/[id]/utils/banners.test.ts b/ui/src/routes/runs/[id]/utils/banners.test.ts new file mode 100644 index 0000000000..036897f771 --- /dev/null +++ b/ui/src/routes/runs/[id]/utils/banners.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from 'vitest'; +import type { RunDetail } from '$lib/api'; +import { bannersFor } from './banners'; + +const failed = (statusReason: string | null, failedNodes = 0) => + ({ run: { status: 'failed', statusReason, nodeCounts: { failed: failedNodes } }, liveness: null, cancel: null, waits: [] }) as unknown as RunDetail; +const text = (detail: RunDetail, cause: string | null) => bannersFor(detail, 0, cause).find((b) => b.key === 'failed')?.body.map((p) => p.text).join(''); + +describe('the banner of a run that failed as a whole', () => { + it('leads with the first recorded error and keeps how the run ended beside it', () => { + expect(text(failed('Timed out after 1h'), 'Connection reset; retrying')).toBe('Connection reset; retrying · Ended: Timed out after 1h'); + }); + + it('falls back to the status reason, and leaves a failed node to explain itself', () => { + expect(text(failed('Timed out after 1h'), null)).toBe('Timed out after 1h'); + expect(text(failed('Worker exited with code 1', 1), 'Binder Error')).toBeUndefined(); + }); +}); diff --git a/ui/src/routes/runs/[id]/utils/banners.ts b/ui/src/routes/runs/[id]/utils/banners.ts index 6265efe441..463d8627e3 100644 --- a/ui/src/routes/runs/[id]/utils/banners.ts +++ b/ui/src/routes/runs/[id]/utils/banners.ts @@ -30,7 +30,8 @@ function cancelTechnical(cancel: CancelProgress): string { ].join('\n'); } -export function bannersFor(detail: RunDetail, now: number): Banner[] { +/** `cause` is the first error the run recorded, when the person may read its output. */ +export function bannersFor(detail: RunDetail, now: number, cause: string | null = null): Banner[] { const out: Banner[] = []; const { run, liveness: live, cancel } = detail; if (live?.state === 'unresponsive' && live.lastHeartbeatAt) { @@ -53,8 +54,11 @@ export function bannersFor(detail: RunDetail, now: number): Banner[] { detail: run.statusReason ?? undefined }); } - if (run.status === 'failed' && run.statusReason && run.nodeCounts.failed === 0) { - out.push({ key: 'failed', tone: 'error', title: 'The run failed.', body: [{ text: run.statusReason }] }); + // With no failed node to explain it, the first recorded error leads. It may be an earlier error the run + // survived, so the status reason, the scheduler's own account of the end, stays beside it. + if (run.status === 'failed' && (cause || run.statusReason) && run.nodeCounts.failed === 0) { + const reason = cause && run.statusReason ? [{ text: ` · Ended: ${run.statusReason}` }] : []; + out.push({ key: 'failed', tone: 'error', title: 'The run failed.', body: [{ text: cause ?? run.statusReason ?? '' }, ...reason] }); } if (live?.state === 'never_started') { out.push({ diff --git a/ui/src/routes/runs/[id]/utils/events.test.ts b/ui/src/routes/runs/[id]/utils/events.test.ts index 9b299f849f..4ee3abada3 100644 --- a/ui/src/routes/runs/[id]/utils/events.test.ts +++ b/ui/src/routes/runs/[id]/utils/events.test.ts @@ -22,7 +22,7 @@ describe('run log', () => { it("interleaves a group's events and printed lines by time and keeps each line's run", () => { const entries = logEntries( [event('e2', '2026-11-09T08:17:38Z', { type: 'resource_attempt_failed', level: 'error', message: 'ConversionError: no', runId: 'staging' }), event('e1', '2026-11-09T08:17:35Z', { nodeKey: 'model:orders' })], - [{ runId: 'orders', invocationId: 'i', stream: 'stderr', at: '2026-11-09T08:17:36Z', text: 'Compiling project...' }] + [{ runId: 'orders', invocationId: 'i', stream: 'stderr', at: '2026-11-09T08:17:36Z', text: 'Compiling project...', error: false }] ); expect(entries.map((e) => [e.id, e.runId, e.category, e.level])).toEqual([ ['e1', 'staging', 'nodes', 'info'], @@ -67,12 +67,15 @@ describe('run log', () => { expect(entries[1].queries.map((q) => q.id)).toEqual(['q']); }); - it("pins only a failing run's last stderr block", () => { - const out = (runId: string, stream: 'stdout' | 'stderr', at: string) => ({ runId, invocationId: 'i', stream, at, text: `${runId} ${stream}` }); + it("pins a failing run's last stderr block and every recorded error, and no other output", () => { + const out = (runId: string, stream: 'stdout' | 'stderr', at: string, error = false) => ({ runId, invocationId: 'i', stream, at, text: `${runId} ${stream}`, error }); const entries = logEntries( [event('fail', '2026-11-09T08:00:04Z', { type: 'resource_attempt_failed', level: 'error', message: 'X: no' }), event('ok', '2026-11-09T08:00:04Z', { runId: 'orders', type: 'resource_attempt_completed' })], - [out('staging', 'stderr', '2026-11-09T08:00:01Z'), out('staging', 'stderr', '2026-11-09T08:00:05Z'), out('staging', 'stdout', '2026-11-09T08:00:06Z'), out('orders', 'stderr', '2026-11-09T08:00:07Z')] + [out('staging', 'stderr', '2026-11-09T08:00:01Z'), out('staging', 'stderr', '2026-11-09T08:00:05Z'), out('staging', 'stdout', '2026-11-09T08:00:06Z'), out('orders', 'stderr', '2026-11-09T08:00:07Z'), out('orders', 'stderr', '2026-11-09T08:00:08Z', true)] ); - expect(entries.filter((e) => e.pinned).map((e) => e.id)).toEqual(['out-staging-1']); + expect(entries.filter((e) => e.pinned).map((e) => [e.id, e.level])).toEqual([ + ['out-staging-1', 'info'], + ['out-orders-4', 'error'] + ]); }); }); diff --git a/ui/src/routes/runs/[id]/utils/events.ts b/ui/src/routes/runs/[id]/utils/events.ts index 25576d00bc..78e7b80ac7 100644 --- a/ui/src/routes/runs/[id]/utils/events.ts +++ b/ui/src/routes/runs/[id]/utils/events.ts @@ -139,15 +139,15 @@ export function logEntries(events: readonly RunEvent[], output: readonly (RunOut runId: l.runId, at: l.at, nodeKey: null, - level: 'info', + level: l.error ? 'error' : 'info', category: 'output', text: l.text, title: l.stream, - tone: l.stream === 'stderr' ? 'var(--sb-warning)' : 'var(--sb-text-faint)', + tone: l.error ? 'var(--sb-error)' : l.stream === 'stderr' ? 'var(--sb-warning)' : 'var(--sb-text-faint)', stream: l.stream, statementId: null, queries: [], - pinned: lastStderr.get(l.runId) === i + pinned: l.error || lastStderr.get(l.runId) === i }) ); // A stable sort keeps each source's own order for equal times. From ca0239048ac2b5889b83760940c2128926e5f588 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Fri, 13 Nov 2026 06:47:53 +0000 Subject: [PATCH 2/2] feat(ui): show each node's last outcome and why it is stale The catalog listing now reports the newest decisive build attempt of each node, from scheduler runs or a newer build outside them, and whether its definition changed since the last recorded build. The UI derives which built tables have older data than the nodes upstream of them, so the catalog, lineage and Overview show changed and behind nodes, and the Overview offers to rebuild them through the review page. --- clients/typescript/src/schema.d.ts | 11 ++ .../_helpers/catalog_nodes.rs | 89 ++++++++++++++-- .../src/warehouse_metadata/constants.rs | 5 + .../person_reads/list_person_catalog_nodes.rs | 26 +++-- .../src/warehouse_metadata/models.rs | 12 +++ .../tests/api/node_status/helpers.rs | 100 +++++++++++++++++- ui/src/lib/api/http-catalog.test.ts | 32 +++++- ui/src/lib/api/http-catalog.ts | 70 +++++++++--- ui/src/lib/api/http-overview.test.ts | 16 ++- ui/src/lib/api/http-overview.ts | 12 ++- ui/src/lib/api/types/build.ts | 4 +- ui/src/lib/build-area/attention.ts | 3 +- ui/src/lib/status.ts | 5 +- ui/src/routes/+page.svelte | 2 +- 14 files changed, 337 insertions(+), 50 deletions(-) diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index db8cbcf18f..987403741d 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1439,15 +1439,26 @@ export interface components { run_id: string; version_hash: string; }; + /** @description `success`, `warn` or `failed`; `run_id` is the scheduler run, null for a newer build outside the scheduler. */ + CatalogNodeOutcome: { + /** Format: date-time */ + at: string; + /** Format: uuid */ + run_id?: string | null; + status: string; + }; /** @description A current snapshot node's declared metadata and latest build, without SQL, source or config. */ CatalogNodeRecord: { column_count?: number | null; + /** @description The current definition differs from the one `last_build` recorded; false when either is unknown. */ + definition_changed: boolean; /** @description Keys of the nodes this node reads. */ dependencies: string[]; description?: string | null; /** @description The snapshot node kind, such as `model`, `seed`, `source` or `audit`. */ kind: string; last_build?: null | components["schemas"]["CatalogNodeBuild"]; + last_outcome?: null | components["schemas"]["CatalogNodeOutcome"]; materialization?: string | null; name: string; node_key: string; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs index eee2ecc3e2..1b56663fec 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/_helpers/catalog_nodes.rs @@ -1,7 +1,9 @@ -//! Catalog listing rows and column lineage from snapshot node definitions and published edges. +//! Catalog listing rows with each node's newest attempt, and column lineage from published edges. use std::collections::{BTreeMap, BTreeSet, HashMap}; +use chrono::{DateTime, Utc}; +use serde::Deserialize; use serde_json::Value; use uuid::Uuid; @@ -9,11 +11,14 @@ use crate::sensors::constants::SOURCE_NODE_KIND; use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; use crate::snapshots::models::SnapshotNode; use crate::store::errors::ControlError; -use crate::store::models::ControlTransaction; -use crate::warehouse_metadata::constants::LINEAGE_DEFINITION_BATCH; +use crate::store::main::query_records::query_records; +use crate::store::models::{ControlTransaction, Parameter}; +use crate::warehouse_metadata::constants::{ + LINEAGE_DEFINITION_BATCH, OUTCOME_FAILURE_WINDOW, OUTCOME_RECENT_RUNS, +}; use crate::warehouse_metadata::models::{ - CatalogNodeBuild, CatalogNodeRecord, CatalogRelation, ColumnLineageEdge, LineageColumn, - LineageNodeColumns, + CatalogNodeBuild, CatalogNodeOutcome, CatalogNodeRecord, CatalogRelation, ColumnLineageEdge, + LineageColumn, LineageNodeColumns, }; /// A declared value from the definition itself, its schema entry or its config, in that order. @@ -60,7 +65,11 @@ fn relation(node: &SnapshotNode) -> Option { pub(crate) fn catalog_node( (node_key, node): (&str, &SnapshotNode), snapshot_id: Uuid, - last_build: Option, + (last_build, last_outcome, definition_changed): ( + Option, + Option, + bool, + ), ) -> CatalogNodeRecord { let definition = &node.definition; CatalogNodeRecord { @@ -84,10 +93,78 @@ pub(crate) fn catalog_node( _ => None, }, last_build, + last_outcome, + definition_changed, snapshot_id, } } +#[derive(Deserialize)] +struct Attempt { + node_key: String, + run_id: Uuid, + status: String, + ended_at: Option>, + finished_storage_id: i64, +} + +/// Each node's newest started `success`, `warn` or `failed` attempt in those runs, excluding group runs. +pub(crate) async fn latest_attempts( + transaction: ControlTransaction, + (location_id, target): (Uuid, &str), +) -> Result<(ControlTransaction, HashMap), ControlError> { + let newest = "SELECT run_id, status, queued_at FROM sqb_runs WHERE location_id=$1 AND target=$2 AND kind<>'group' \ + ORDER BY queued_at DESC, run_id DESC"; + let (transaction, attempts): (_, Vec) = query_records(transaction, + &format!("SELECT n.node_key,n.run_id,n.status,n.ended_at,n.finished_storage_id FROM (\ + SELECT run_id FROM ({newest} LIMIT $3) AS latest UNION \ + SELECT run_id FROM (SELECT run_id FROM ({newest} LIMIT $4) AS recent WHERE status IN ('failed','partial','warn') \ + ORDER BY queued_at DESC, run_id DESC LIMIT $3) AS failures) AS r \ + JOIN sqb_run_nodes n ON n.run_id=r.run_id WHERE n.kind='node' AND n.finished_storage_id IS NOT NULL \ + AND n.started_at IS NOT NULL AND n.status IN ('success','warn','failed')"), + vec![Parameter::Id(location_id), Parameter::Text(target.into()), Parameter::Integer(OUTCOME_RECENT_RUNS), Parameter::Integer(OUTCOME_FAILURE_WINDOW)]).await?; + let mut newest: HashMap = HashMap::new(); + for attempt in attempts { + if newest + .get(&attempt.node_key) + .is_none_or(|seen| attempt.finished_storage_id > seen.finished_storage_id) + { + newest.insert(attempt.node_key.clone(), attempt); + } + } + let outcomes: HashMap = newest + .into_iter() + .filter_map(|(key, attempt)| { + Some(( + key, + CatalogNodeOutcome { + status: attempt.status, + at: attempt.ended_at?, + run_id: Some(attempt.run_id), + }, + )) + }) + .collect(); + Ok((transaction, outcomes)) +} + +/// The scheduler attempt, unless a fingerprint from another clock was recorded after it ended. +pub(crate) fn last_outcome( + attempt: Option, + build: Option<&CatalogNodeBuild>, +) -> Option { + let newer_build = + build.filter(|build| attempt.as_ref().is_none_or(|a| build.observed_at > a.at)); + match newer_build { + Some(build) => Some(CatalogNodeOutcome { + status: "success".into(), + at: build.observed_at, + run_id: None, + }), + None => attempt, + } +} + /// Published edge rows whose both ends are current nodes, so the graph never points at a hidden node. pub(crate) fn published_edges( document: Option<&Value>, diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs index 7a15ec7660..a879c1ede9 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/constants.rs @@ -13,3 +13,8 @@ pub(crate) const PREVIEW_REQUIRED_TABLES: [WarehouseHistoryKind; 3] = [ WarehouseHistoryKind::Fingerprints, WarehouseHistoryKind::Consumption, ]; + +/// The target's newest runs read for node outcomes, and as many newest failed or warned runs; never other targets'. +pub(crate) const OUTCOME_RECENT_RUNS: i64 = 25; +/// How far back, in runs, the newest failed or warned runs are looked for. +pub(crate) const OUTCOME_FAILURE_WINDOW: i64 = 500; diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs index 1803a07f26..4187567da1 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/main/person_reads/list_person_catalog_nodes.rs @@ -1,4 +1,4 @@ -//! A person lists the current nodes with view on all of them; builds are batched indexed seeks. +//! A person lists the current nodes with view on all of them; builds are batched indexed seeks, attempts one bounded read. use uuid::Uuid; @@ -9,10 +9,14 @@ use crate::snapshots::main::read_snapshot_nodes::read_snapshot_nodes; use crate::store::errors::ControlError; use crate::store::main::{begin::begin, commit::commit}; use crate::store::models::{ControlStore, TransactionMode}; -use crate::warehouse_metadata::_helpers::catalog_nodes::catalog_node; +use crate::warehouse_metadata::_helpers::catalog_nodes::{ + catalog_node, last_outcome, latest_attempts, +}; use crate::warehouse_metadata::_helpers::fingerprint_reads::latest_fingerprints; -use crate::warehouse_metadata::_helpers::node_status::destination; -use crate::warehouse_metadata::models::{CatalogNodeBuild, CatalogNodeRecord, FingerprintQuery}; +use crate::warehouse_metadata::_helpers::node_status::{definition_evidence, destination}; +use crate::warehouse_metadata::models::{ + CatalogNodeBuild, CatalogNodeRecord, DefinitionEvidence, FingerprintQuery, +}; pub async fn list_person_catalog_nodes( store: &ControlStore, @@ -33,6 +37,7 @@ pub async fn list_person_catalog_nodes( return Ok(Vec::new()); }; let (transaction, nodes) = read_snapshot_nodes(transaction, snapshot.snapshot_id, None).await?; + let nodes: Vec<_> = nodes.iter().collect(); let mut positions: Vec = Vec::new(); let mut queries: Vec = Vec::new(); for (position, (_, node)) in nodes.iter().enumerate() { @@ -43,8 +48,14 @@ pub async fn list_person_catalog_nodes( } let (transaction, found) = latest_fingerprints(transaction, (location_id, target, &queries)).await?; + let (transaction, mut attempts) = latest_attempts(transaction, (location_id, target)).await?; let mut builds: Vec> = vec![None; nodes.len()]; + let mut changed: Vec = vec![false; nodes.len()]; for (position, build) in positions.iter().zip(found) { + changed[*position] = matches!( + definition_evidence(nodes[*position].1, build.as_ref()), + DefinitionEvidence::Changed { .. } + ); builds[*position] = build.map(|(record, _)| CatalogNodeBuild { observed_at: record.observed_at, run_id: record.run_id, @@ -53,8 +64,11 @@ pub async fn list_person_catalog_nodes( } let records: Vec = nodes .iter() - .zip(builds) - .map(|((key, node), build)| catalog_node((key, node), snapshot.snapshot_id, build)) + .zip(builds.into_iter().zip(changed)) + .map(|((key, node), (build, changed))| { + let outcome = last_outcome(attempts.remove(key.as_str()), build.as_ref()); + catalog_node((key, node), snapshot.snapshot_id, (build, outcome, changed)) + }) .collect(); commit(transaction).await?; Ok(records) diff --git a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs index 42bc27126a..6484f195e3 100644 --- a/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs +++ b/crates/sqlbuild-orchestration/src/warehouse_metadata/models.rs @@ -878,9 +878,21 @@ pub struct CatalogNodeRecord { pub column_count: Option, /// The newest fingerprint recorded for the node at its current relation, from any build. pub last_build: Option, + /// The newest decisive build attempt in the target's newest scheduler runs and newest failed runs, or a newer build outside them. + pub last_outcome: Option, + /// The current definition differs from the one `last_build` recorded; false when either is unknown. + pub definition_changed: bool, pub snapshot_id: Uuid, } +/// `success`, `warn` or `failed`; `run_id` is the scheduler run, null for a newer build outside the scheduler. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] +pub struct CatalogNodeOutcome { + pub status: String, + pub at: DateTime, + pub run_id: Option, +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] pub struct CatalogRelation { pub database: Option, diff --git a/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs index 8287aa9c47..cba2166f6f 100644 --- a/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs +++ b/crates/sqlbuild-orchestration/tests/api/node_status/helpers.rs @@ -5,7 +5,7 @@ use std::cell::Cell; use uuid::Uuid; use crate::credentials::models::{AuditContext, DefaultRole}; -use crate::runs::tests::helpers::snapshot; +use crate::runs::tests::helpers::{request, request_run, snapshot}; use crate::snapshots::main::publish_snapshot::publish_snapshot; use crate::snapshots::models::SnapshotInput; use crate::snapshots::models::SnapshotNode; @@ -21,8 +21,8 @@ use crate::warehouse_metadata::main::{ }; use crate::warehouse_metadata::models::{MetadataBatch, NodeStatusQuery}; -/// Statements a catalog listing issues besides one build read per 100 nodes: 7 fixed, 1 per extra namespace shape. -const WIDE_LISTING_FIXED_STATEMENTS: usize = 8; +/// Statements a catalog listing issues besides one build read per 100 nodes: 8 fixed, 1 per extra namespace shape. +const WIDE_LISTING_FIXED_STATEMENTS: usize = 9; pub(crate) async fn node_status_reads( store: &ControlStore, @@ -53,7 +53,7 @@ pub(crate) async fn node_status_reads( python_source: None, }, ); - publish_snapshot(store, &input) + let published = publish_snapshot(store, &input) .await .expect("identity snapshot"); let options = NodeStatusQuery { @@ -235,6 +235,15 @@ pub(crate) async fn node_status_reads( json!({"database": null, "schema": "main", "name": "orders"}) ); assert_eq!(orders["dependencies"], json!(["source:raw_orders"])); + assert_eq!( + orders["definition_changed"], true, + "the listing compares definitions like the node status" + ); + assert_eq!( + orders["last_outcome"], + json!({"status": "success", "at": changed.records[0]["ts"], "run_id": null}), + "a build outside the scheduler is a success" + ); assert_eq!( listed_node("udf:orders")["last_build"]["run_id"], "orders-function-run", @@ -248,6 +257,13 @@ pub(crate) async fn node_status_reads( ); assert!(!json!(listed).to_string().contains("metadata_json")); listing_ignores_build_history(store, credentials, (location, &listed)).await; + listing_reports_scheduler_attempts( + store, + credentials, + (location, published.snapshot_id), + serde_json::from_value(changed.records[0]["ts"].clone()).expect("build time"), + ) + .await; listing_wide_targets_is_batched_without_super_linear_growth( store, credentials, @@ -368,6 +384,82 @@ async fn listing_ignores_build_history( ); } +/// A failed scheduler attempt replaces the outcome only when it ended after the newest build, and stays after quicker runs. +async fn listing_reports_scheduler_attempts( + store: &ControlStore, + credentials: (&AuditContext, Uuid), + (location, snapshot_id): (Uuid, Uuid), + built_at: chrono::DateTime, +) { + let outcome = || async { + let listed = list_person_catalog_nodes( + store, + credentials, + (location, "development"), + DefaultRole::None, + ) + .await + .expect("catalog nodes"); + let orders = listed + .into_iter() + .find(|node| node.node_key == "model:orders") + .expect("orders listed"); + json!(orders.last_outcome) + }; + for (id, ended_at, expected) in [ + ( + 0x0f01, + built_at - chrono::Duration::hours(1), + json!({"status": "success", "at": built_at, "run_id": null}), + ), + ( + 0x0f02, + built_at + chrono::Duration::hours(1), + json!({"status": "failed", "at": built_at + chrono::Duration::hours(1), + "run_id": Uuid::from_u128(0x0f02)}), + ), + ] { + let run = request((location, snapshot_id), id, 0, &["model:orders"]); + request_run(store, &run).await.expect("queue attempt"); + let transaction = begin(store, TransactionMode::Write).await.expect("begin"); + let (transaction, _) = query( + transaction, + "UPDATE sqb_run_nodes SET status='failed',started_at=$1,ended_at=$2,finished_storage_id=$3 WHERE run_id=$4", + vec![ + Parameter::Time(ended_at - chrono::Duration::minutes(5)), + Parameter::Time(ended_at), + Parameter::Integer(i64::try_from(id).expect("storage id")), + Parameter::Id(run.run_id), + ], + ) + .await + .expect("finish attempt"); + let (transaction, _) = query( + transaction, + "UPDATE sqb_runs SET status='failed' WHERE run_id=$1", + vec![Parameter::Id(run.run_id)], + ) + .await + .expect("fail run"); + commit(transaction).await.expect("commit attempt"); + assert_eq!(outcome().await, expected, "attempt ending at {ended_at}"); + } + for id in 0x0f10..0x0f10 + 25 { + request_run( + store, + &request((location, snapshot_id), id, 0, &["customers"]), + ) + .await + .expect("newer run of another node"); + } + assert_eq!( + outcome().await, + json!({"status": "failed", "at": built_at + chrono::Duration::hours(1), + "run_id": Uuid::from_u128(0x0f02)}), + "a failure stays reported after more than 25 newer runs" + ); +} + /// Three thousand nodes, half in a named database, keep their own builds, read 100 per statement without super-linear growth. async fn listing_wide_targets_is_batched_without_super_linear_growth( store: &ControlStore, diff --git a/ui/src/lib/api/http-catalog.test.ts b/ui/src/lib/api/http-catalog.test.ts index 14d166ed95..c1b1b75592 100644 --- a/ui/src/lib/api/http-catalog.test.ts +++ b/ui/src/lib/api/http-catalog.test.ts @@ -7,7 +7,7 @@ type W = components['schemas']; function node(key: string, extra: Partial = {}): W['CatalogNodeRecord'] { const [kind, name] = key.split(':'); - return { node_key: key, kind, name, tags: [], dependencies: [], snapshot_id: '01a1b000-0000-7000-8000-000000000001', ...extra }; + return { node_key: key, kind, name, tags: [], dependencies: [], definition_changed: false, snapshot_id: '01a1b000-0000-7000-8000-000000000001', ...extra }; } describe('catalog node mapping', () => { @@ -39,7 +39,7 @@ describe('catalog node mapping', () => { expect(orders.upstream).toEqual(['source:raw_orders']); }); - it('reports the fingerprint as a sqb build and leaves unreported build state unknown, not empty', () => { + it('reports the fingerprint as a sqb build and leaves unreported state unknown, not empty', () => { const [orders] = toCatalogNodes([ node('model:orders', { materialization: 'table', @@ -52,7 +52,28 @@ describe('catalog node mapping', () => { ]); expect(orders.lastBuild).toEqual({ fingerprintRunId: 'sqb-run-1', runId: null, jobKey: null, builtAt: '2026-11-07T06:00:00Z', fingerprint: 'v1' }); expect([orders.description, orders.tags, orders.columnCount, orders.relation]).toEqual(['One row per order', ['marts'], 4, { database: null, schema: 'main', name: 'orders' }]); - expect([orders.stale, orders.checks, orders.lastOutcome, orders.group, orders.owner]).toEqual([null, null, null, null, []]); + expect([orders.stale, orders.checks, orders.lastOutcome, orders.group, orders.owner]).toEqual([[], null, null, null, []]); + }); + + it('marks a changed definition, and a built table older than the data upstream of it, even through a view, as stale', () => { + const built = (observed_at: string) => ({ last_build: { observed_at, run_id: 'sqb', version_hash: 'v' } }); + const table = (key: string, at: string | null, extra: Partial = {}) => + node(key, { materialization: 'table', dependencies: ['model:orders', 'source:raw_orders'], ...(at ? built(at) : {}), ...extra }); + const nodes = toCatalogNodes([ + node('source:raw_orders'), + table('model:orders', '2026-11-07T06:00:00Z', { dependencies: ['source:raw_orders'], definition_changed: true, last_outcome: { status: 'failed', at: '2026-11-07T06:30:00Z', run_id: 'r1' } }), + table('model:order_totals', '2026-11-07T05:00:00Z', { last_outcome: { status: 'success', at: '2026-11-07T05:00:00Z' } }), + table('model:order_daily', '2026-11-07T07:00:00Z'), + table('model:order_view', '2026-11-07T04:00:00Z', { materialization: 'view' }), + table('model:order_drafts', null, { definition_changed: true }), + table('model:order_report', '2026-11-07T05:30:00Z', { dependencies: ['model:order_view'] }) + ]); + const by = Object.fromEntries(nodes.map((n) => [n.key, n])); + expect(by['model:orders'].stale).toEqual([{ reason: 'changed', change: 'definition', detail: 'The definition changed since it was last built here.', since: '2026-11-07T06:00:00Z' }]); + expect(by['model:order_totals'].stale).toEqual([{ reason: 'behind', upstream: ['model:orders'], detail: 'orders has newer data than this build.', since: '2026-11-07T06:00:00.000Z' }]); + expect(by['model:order_report'].stale?.map((r) => r.reason === 'behind' && [r.upstream, r.since])).toEqual([[['model:order_view'], '2026-11-07T06:00:00.000Z']]); + expect(['model:order_daily', 'model:order_view', 'model:order_drafts', 'source:raw_orders'].map((k) => by[k].stale)).toEqual([[], [], [], []]); + expect(nodes.map((n) => n.lastOutcome)).toEqual([null, 'failed', 'success', null, null, null, null]); }); it('finds a node by key, or by bare name preferring the model', () => { @@ -82,8 +103,9 @@ describe('catalog node detail', () => { expect(d.downstream).toEqual(['model:order_totals']); }); - it('reports a changed definition as stale and maps the warehouse columns', () => { - const changed = nodeDetailOf(nodes, nodes[0], status({ status: 'changed', current_local_hash: 'b', recorded_local_hash: 'a' }, observed), null, []); + it('keeps the listing staleness and maps the warehouse columns', () => { + const [listed] = toCatalogNodes([node('model:orders', { materialization: 'table', definition_changed: true, last_build: { observed_at: '2026-11-07T06:00:00Z', run_id: 'sqb', version_hash: 'v' } })]); + const changed = nodeDetailOf(nodes, listed, status({ status: 'changed', current_local_hash: 'b', recorded_local_hash: 'a' }, observed), null, []); expect(changed.stale.map((r) => [r.reason, r.since])).toEqual([['changed', '2026-11-07T06:00:00Z']]); expect(changed.columns).toEqual([{ name: 'order_id', type: 'INTEGER', description: '', audits: [] }]); const matched = nodeDetailOf(nodes, nodes[0], status({ status: 'matched', local_hash: 'a' }, { status: 'unavailable', reason: 'no catalog' }), null, []); diff --git a/ui/src/lib/api/http-catalog.ts b/ui/src/lib/api/http-catalog.ts index 28f72202b5..54cccd81cc 100644 --- a/ui/src/lib/api/http-catalog.ts +++ b/ui/src/lib/api/http-catalog.ts @@ -15,13 +15,16 @@ function kindOf(wire: Wire['CatalogNodeRecord']): NodeKind | null { } /** - * One node of the current snapshot. The server reports declared metadata and the newest - * fingerprint, not run outcomes, staleness, check counts, owners or groups, so those stay null - * or empty and the pages hide them. The fingerprint's run is a `sqb` run, not a scheduler run. + * One node of the current snapshot. The server reports declared metadata, the newest fingerprint, + * the newest decisive outcome and whether the definition changed since that fingerprint, not + * check counts, owners or groups, so those stay null or empty and the pages hide them. The + * fingerprint's run is a `sqb` run, not a scheduler run. Staleness from upstream builds needs the + * whole listing, so `toCatalogNodes` adds it. */ export function toCatalogNode(wire: Wire['CatalogNodeRecord']): CatalogNode | null { const kind = kindOf(wire); if (!kind) return null; + const outcome = wire.last_outcome?.status; return { key: wire.node_key, kind, @@ -37,18 +40,62 @@ export function toCatalogNode(wire: Wire['CatalogNodeRecord']): CatalogNode | nu lastBuild: wire.last_build ? { fingerprintRunId: wire.last_build.run_id, runId: null, jobKey: null, builtAt: wire.last_build.observed_at, fingerprint: wire.last_build.version_hash } : null, - lastOutcome: null, - stale: null, + lastOutcome: outcome === 'success' || outcome === 'warn' || outcome === 'failed' ? outcome : null, + stale: + wire.definition_changed && wire.last_build + ? [{ reason: 'changed', change: 'definition', detail: 'The definition changed since it was last built here.', since: wire.last_build.observed_at }] + : [], sourceFreshness: null, checks: null }; } +/** + * When the data a node reads last changed: its own last build, or for a view, which reads its + * upstream when queried, the newest data behind it. Null when nothing behind it is built. + */ +function dataTimes(listed: Map): (key: string) => number | null { + const memo = new Map(); + const at = (key: string): number | null => { + if (memo.has(key)) return memo.get(key) ?? null; + memo.set(key, null); + const node = listed.get(key); + let time = node?.lastBuild ? Date.parse(node.lastBuild.builtAt) : null; + if (node?.kind === 'view') { + const upstream = node.upstream.flatMap((k) => at(k) ?? []); + time = upstream.length ? Math.max(...upstream) : null; + } + memo.set(key, time); + return time; + }; + return at; +} + +/** + * Upstream nodes with newer data than this node's own last build. Views read their upstream when + * queried, so they are never behind, and an unbuilt node is not built yet rather than behind. + */ +function behindOf(node: CatalogNode, dataAt: (key: string) => number | null): StaleReason[] { + const builtAt = node.lastBuild?.builtAt; + if (!builtAt || node.kind === 'view') return []; + const newer = node.upstream.flatMap((key) => { + const at = dataAt(key); + return at !== null && at > Date.parse(builtAt) ? [{ key, at }] : []; + }); + if (!newer.length) return []; + const since = new Date(Math.min(...newer.map((n) => n.at))).toISOString(); + const names = newer.map((n) => shortName(n.key)); + const detail = `${names.length === 1 ? names[0] : `${names.length} upstream nodes`} ${names.length === 1 ? 'has' : 'have'} newer data than this build.`; + return [{ reason: 'behind', upstream: newer.map((n) => n.key), detail, since }]; +} + /** Catalog nodes in listing order, keeping only dependencies that are themselves listed. */ export function toCatalogNodes(wire: Wire['CatalogNodeRecord'][]): CatalogNode[] { const nodes = wire.flatMap((node) => toCatalogNode(node) ?? []); - const listed = new Set(nodes.map((node) => node.key)); - return nodes.map((node) => ({ ...node, upstream: node.upstream.filter((key) => listed.has(key)) })); + const keys = new Set(nodes.map((node) => node.key)); + const linked = nodes.map((node) => ({ ...node, upstream: node.upstream.filter((key) => keys.has(key)) })); + const dataAt = dataTimes(new Map(linked.map((node) => [node.key, node]))); + return linked.map((node) => ({ ...node, stale: [...(node.stale ?? []), ...behindOf(node, dataAt)] })); } /** Passing, warning and failing by each check's newest result; checks without one count only towards the total. */ @@ -62,13 +109,6 @@ export function checkCountsOf(checks: Check[]): CheckCounts { }; } -/** A definition that changed since its last recorded build here; other staleness is not reported by the server. */ -function staleOf(status: Wire['NodeStatus'] | null): StaleReason[] { - if (status?.definition.status !== 'changed' || !status.build) return []; - return [{ reason: 'changed', change: 'definition', detail: 'The definition changed since it was last built here.', since: status.build.observed_at }]; -} - - /** The listed node a key or bare name refers to, preferring a model; migrations name models without their kind. */ export function findNode(nodes: CatalogNode[], ref: string): CatalogNode | null { return nodes.find((n) => n.key === ref) ?? nodes.find((n) => n.key === `model:${ref}`) ?? nodes.find((n) => shortName(n.key) === ref) ?? null; @@ -160,7 +200,7 @@ export function nodeDetailOf( const physical = status?.physical.status === 'observed' ? status.physical.observation.record : null; return { ...node, - stale: staleOf(status), + stale: node.stale ?? [], checks: checkCountsOf(own), downstream: nodes.filter((n) => n.upstream.includes(node.key)).map((n) => n.key), recentOutcomes: (runs ?? []).slice(0, 40).map((r) => r.outcome).reverse(), diff --git a/ui/src/lib/api/http-overview.test.ts b/ui/src/lib/api/http-overview.test.ts index 2c45f9624e..1983125206 100644 --- a/ui/src/lib/api/http-overview.test.ts +++ b/ui/src/lib/api/http-overview.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import { overviewOf } from './http-overview'; -import type { CatalogNode, Check, NodeChange, PlatformEvent, Run, SnapshotChange } from './types'; +import type { CatalogNode, Check, NodeChange, PlatformEvent, Run, SnapshotChange, StaleReason } from './types'; const now = Date.parse('2026-11-09T12:00:00Z'); const hoursAgo = (h: number) => new Date(now - h * 3_600_000).toISOString(); @@ -9,7 +9,7 @@ function run(id: string, status: Run['status'], queuedHoursAgo: number, endedHou return { id, status, parentRunId, jobKey, narrowing, window: null, kind: jobKey ? 'job' : 'adhoc', statusReason: null, queuedAt: hoursAgo(queuedHoursAgo), startedAt: hoursAgo(queuedHoursAgo), endedAt: endedHoursAgo === null ? null : hoursAgo(endedHoursAgo) } as Run; } -const node = (key: string, kind: CatalogNode['kind']) => ({ key, kind }) as CatalogNode; +const node = (key: string, kind: CatalogNode['kind'], stale: StaleReason['reason'][] = []) => ({ key, kind, stale: stale.map((reason) => ({ reason })) }) as CatalogNode; const check = (id: string, outcome: string | null) => ({ id, latest: outcome ? { outcome } : null }) as unknown as Check; const change = (publishedHoursAgo: number, nodes: [NodeChange['op'], string][]) => ({ snapshotId: `s-${publishedHoursAgo}`, publishedAt: hoursAgo(publishedHoursAgo), nodes: nodes.map(([op, key]) => ({ key, op })) }) as SnapshotChange; @@ -39,7 +39,7 @@ describe('overview from live reads', () => { const overview = overviewOf({ runs, checks: [check('a', 'pass'), check('b', 'fail'), check('c', 'error'), check('d', 'warn'), check('e', null)], - nodes: [node('model:orders', 'table'), node('model:totals', 'view'), node('model:events', 'incremental'), node('source:raw', 'source'), node('seed:regions', 'seed')], + nodes: [node('model:orders', 'table', ['changed', 'behind']), node('model:totals', 'view'), node('model:events', 'incremental', ['behind']), node('source:raw', 'source'), node('seed:regions', 'seed')], changes: [change(1, [['added', 'model:orders'], ['added', 'seed:regions'], ['removed', 'model:old']]), change(24 * 8, [['added', 'model:totals']])], events: [event('scheduler', true), event('cleared', false)], acknowledgements: new Map([['orders-new', { by: 'ada', at: hoursAgo(1) }]]), @@ -59,7 +59,8 @@ describe('overview from live reads', () => { 'run_problem:catalogue-partial:-', 'run_problem:adhoc-old:-', 'run_unresponsive:stuck:-', - 'event:scheduler' + 'event:scheduler', + 'stale_models' ]); }); @@ -75,7 +76,12 @@ describe('overview from live reads', () => { expect(overview.latestRun?.id).toBe('fulfilment-ok'); }); + it('counts each stale node once, and leaves late sources unknown', () => { + expect(overview.metrics.stale).toEqual({ nodes: 2, changed: 1, behind: 2, sources: null }); + expect(overview.attention.find((i) => i.kind === 'stale_models')).toEqual({ kind: 'stale_models', changed: ['model:orders'], behind: ['model:orders', 'model:events'] }); + }); + it('leaves parts without a server read empty instead of zero', () => { - expect([overview.metrics.stale, overview.metrics.cost, overview.groups]).toEqual([null, null, null]); + expect([overview.metrics.cost, overview.groups]).toEqual([null, null]); }); }); diff --git a/ui/src/lib/api/http-overview.ts b/ui/src/lib/api/http-overview.ts index de5a7625b2..dc98e58dd9 100644 --- a/ui/src/lib/api/http-overview.ts +++ b/ui/src/lib/api/http-overview.ts @@ -42,8 +42,9 @@ export function problemRunsOf(topLevel: Run[], now: number): Run[] { const attentionRun = ({ id, jobKey, kind, status, statusReason, nodeCounts, startedAt, endedAt }: Run): AttentionRun => ({ id, jobKey, kind, status, statusReason, nodeCounts, startedAt, endedAt }); /** - * The overview from reads the person already has. The server has no project-wide build state, cost - * or group read yet, so stale counts and group status are null and cost is left out; the page hides them. + * The overview from reads the person already has. Staleness comes from the catalog listing, which + * does not check source freshness, so late sources stay unknown. The server has no cost or group + * read yet, so group status is null and cost is left out; the page hides them. */ export function overviewOf({ runs, checks, nodes, changes, events, acknowledgements, now }: OverviewInputs): Overview { const acknowledged = (r: Run) => acknowledgements.get(r.id) ?? null; @@ -57,6 +58,11 @@ export function overviewOf({ runs, checks, nodes, changes, events, acknowledgeme .map((r): AttentionItem => ({ kind: 'run_unresponsive', run: attentionRun(r), lastHeartbeatAt: null, acknowledged: acknowledged(r) })), ...events.filter((e) => e.active).map((event): AttentionItem => ({ kind: 'platform_event', event })) ]; + const staleKeys = (test: (reason: string) => boolean) => nodes.filter((n) => n.stale?.some((r) => test(r.reason))).map((n) => n.key); + const changed = staleKeys((reason) => reason !== 'behind'); + const behind = staleKeys((reason) => reason === 'behind'); + const staleNodes = new Set([...changed, ...behind]).size; + if (staleNodes) attention.push({ kind: 'stale_models', changed, behind }); const thisWeek = changes.filter((c) => Date.parse(c.publishedAt) >= now - 7 * DAY).flatMap((c) => c.nodes); const latest = topLevel.filter((r) => r.endedAt).sort((a, b) => Date.parse(b.endedAt!) - Date.parse(a.endedAt!))[0]; return { @@ -75,7 +81,7 @@ export function overviewOf({ runs, checks, nodes, changes, events, acknowledgeme active: recent.filter((r) => ACTIVE_STATUSES.has(r.status)).length }, checks: checkCountsOf(checks), - stale: null, + stale: { nodes: staleNodes, changed: changed.length, behind: behind.length, sources: null }, cost: null }, attention, diff --git a/ui/src/lib/api/types/build.ts b/ui/src/lib/api/types/build.ts index 9536d87308..0a1380de75 100644 --- a/ui/src/lib/api/types/build.ts +++ b/ui/src/lib/api/types/build.ts @@ -357,8 +357,8 @@ export type OverviewMetrics = { models: { total: number; addedThisWeek: number; removedThisWeek: number }; runs24h: { total: number; success: number; warn: number; partial: number; failed: number; active: number }; checks: CheckCounts; - /** Distinct stale nodes, and how many are changed or behind (a node can be both); null when the server reports no project-wide build state. */ - stale: { nodes: number; changed: number; behind: number; sources: number } | null; + /** Distinct stale nodes, and how many are changed or behind (a node can be both); null when the server reports no project-wide build state, and `sources` null when source freshness is not checked. */ + stale: { nodes: number; changed: number; behind: number; sources: number | null } | null; /** Null when the adapter has no cost data. */ cost: { last24h: number; baseline: number; baselineDays: number; unit: CostUnit } | null; }; diff --git a/ui/src/lib/build-area/attention.ts b/ui/src/lib/build-area/attention.ts index e18d05838d..14d9dff438 100644 --- a/ui/src/lib/build-area/attention.ts +++ b/ui/src/lib/build-area/attention.ts @@ -5,6 +5,7 @@ import { describeWait, outcomeCounts } from '$lib/operate/labels'; import { eventKindLabel, problemHref } from '$lib/platform/labels'; import { formatAgo } from '$lib/format/time'; import { runStatusLabel } from '$lib/status'; +import { shortName } from '$lib/build-area/labels'; export type AttentionTone = 'error' | 'warning' | 'stale' | 'info'; @@ -51,7 +52,7 @@ export function attentionView(item: AttentionItem, now: number): AttentionView { key: 'stale', tone: 'stale', title: `${plural(total, 'node')} ${total === 1 ? 'is' : 'are'} stale`, - detail: `${parts.join(' · ')} · ${[...new Set([...item.changed, ...item.behind])].slice(0, 4).join(', ')}`, + detail: `${parts.join(' · ')} · ${[...new Set([...item.changed, ...item.behind])].slice(0, 4).map(shortName).join(', ')}`, since: 'since the last build', href: '/catalog?stale=1', action: 'View in catalog' diff --git a/ui/src/lib/status.ts b/ui/src/lib/status.ts index dc7db40e72..d38e0747f0 100644 --- a/ui/src/lib/status.ts +++ b/ui/src/lib/status.ts @@ -1,3 +1,4 @@ +import { shortName } from '$lib/build-area/labels'; import type { CheckOutcome, NodeCounts, NodeOutcome, StaleReason, RunKind, RunOrigin, RunStatus, TickOutcome, TriggerState } from '$lib/api/types'; export const runStatusTone: Record = { @@ -115,8 +116,8 @@ export function staleReasonLabel(reason: StaleReason): string { /** One-line explanation of a stale reason. */ export function staleReasonDetail(reason: StaleReason): string { if (reason.reason === 'changed') return reason.detail; - if (reason.reason === 'changed_upstream') return `Upstream definition changed: ${reason.upstream.join(', ')}`; - return `Newer data in ${reason.upstream.join(', ')}`; + if (reason.reason === 'changed_upstream') return `Upstream definition changed: ${reason.upstream.map(shortName).join(', ')}`; + return `Newer data in ${reason.upstream.map(shortName).join(', ')}`; } /** e.g. `8 ok · 1 warn · 1 failed · 3 skipped`, or `4 running · 10 pending`. */ diff --git a/ui/src/routes/+page.svelte b/ui/src/routes/+page.svelte index 31e259d8b3..6b42cad532 100644 --- a/ui/src/routes/+page.svelte +++ b/ui/src/routes/+page.svelte @@ -198,7 +198,7 @@
Stale
{m.stale.nodes}nodes
- {m.stale.changed} changed · {m.stale.behind} behind · {m.stale.sources} {m.stale.sources === 1 ? 'source' : 'sources'} late + {m.stale.changed} changed · {m.stale.behind} behind{#if m.stale.sources !== null} · {m.stale.sources} {m.stale.sources === 1 ? 'source' : 'sources'} late{/if}
{/if} {#if m.cost}