diff --git a/ui/src/lib/api/mock/build-fixtures.ts b/ui/src/lib/api/mock/build-fixtures.ts index b6635e88e5..bcc0817ff8 100644 --- a/ui/src/lib/api/mock/build-fixtures.ts +++ b/ui/src/lib/api/mock/build-fixtures.ts @@ -365,7 +365,10 @@ export const DEPLOYS: DeploySpec[] = [ gitAuthor: ALEX.displayName, minutesAgo: 6, publishedBy: CI_ACCOUNT, - nodes: [node('fact_orders', 'changed', 'SQL changed: refunded payments now net out of order revenue', { columns: [{ name: 'net_revenue', op: 'changed', detail: 'expression changed' }] })], + nodes: [ + node('fact_orders', 'changed', 'SQL changed: refunded payments now net out of order revenue', { columns: [{ name: 'net_revenue', op: 'changed', detail: 'expression changed' }] }), + node('customer_status', 'changed', 'SQL changed: lapsed customers now need 120 days without an order', { columns: [{ name: 'status', op: 'changed', detail: 'expression changed' }] }) + ], orchestration: [], roles: [], environment: null diff --git a/ui/src/lib/build-area/attention.test.ts b/ui/src/lib/build-area/attention.test.ts new file mode 100644 index 0000000000..0cc0402f25 --- /dev/null +++ b/ui/src/lib/build-area/attention.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from 'vitest'; +import type { AttentionItem, AttentionRun, PlatformEvent } from '$lib/api'; +import { attentionViews } from './attention'; +import { CHANGED_PLAN_HREF, planHref } from '$lib/run/command'; + +const now = Date.parse('2026-01-01T12:00:00Z'); +const ago = (minutes: number) => new Date(now - minutes * 60_000).toISOString(); +const event = (id: string, kind: PlatformEvent['kind'], locationId: string | null, over: Partial = {}): AttentionItem => ({ + kind: 'platform_event', + event: { id, kind, severity: 'warning', occurredAt: ago(5), active: true, locationId, subject: null, message: `${kind} message`, ...over } +}); +const run = (id: string, jobKey: string | null): AttentionRun => ({ id, jobKey, kind: 'job', status: 'unresponsive', statusReason: null, nodeCounts: { success: 0, warn: 0, failed: 0, skipped: 0, running: 1, pending: 0 }, startedAt: ago(30), endedAt: null }); +const unresponsive = (id: string, jobKey: string | null, heardMinutesAgo: number): AttentionItem => ({ kind: 'run_unresponsive', run: run(id, jobKey), lastHeartbeatAt: ago(heardMinutesAgo), acknowledged: null }); +const queued = (runId: string, jobKey: string | null, waits: Extract['waits']): AttentionItem => ({ kind: 'queue_blocked', runId, jobKey, queuedAt: ago(20), waits }); + +describe('overview alerts', () => { + it("lists an overdue schedule under its own location's outage, else under the stopped scheduler", () => { + const views = attentionViews( + [ + event('broken', 'location_broken', 'shop'), + event('late-shop', 'tick_overdue', 'shop', { message: 'hourly is late' }), + event('late-other', 'tick_overdue', 'marketing', { message: 'daily is late' }), + event('stopped', 'scheduler_no_leader', null, { severity: 'critical' }) + ], + now + ); + expect(views.map(({ view }) => [view.key, view.consequences.map((c) => c.text)])).toEqual([ + ['event:stopped', ['daily is late']], + ['event:broken', ['hourly is late']] + ]); + }); + + it('keeps an overdue schedule as its own alert when nothing explains it', () => { + const views = attentionViews([event('late', 'tick_overdue', 'shop'), event('broken', 'location_broken', 'marketing')], now); + expect(views.map(({ view }) => view.key)).toEqual(['event:late', 'event:broken']); + }); + + it('lists a queued run under the unresponsive run of its job or the one it waits for, and the cause takes the more urgent tone', () => { + const views = attentionViews( + [ + unresponsive('r1', 'nightly', 1), + queued('q1', 'nightly', []), + unresponsive('r2', null, 1), + queued('q2', 'exports', [{ kind: 'write_claim', runId: 'r2', nodeKey: 'model:orders', otherNodes: 0 }]), + queued('q3', 'other', []) + ], + now + ); + expect(views.map(({ view }) => [view.key, view.tone, view.consequences.length])).toEqual([ + ['unresponsive:r1', 'warning', 1], + ['unresponsive:r2', 'warning', 1], + ['queue:q3', 'warning', 0] + ]); + }); + + it('keeps a queued run as its own alert when the unresponsive run that holds it up is acknowledged', () => { + const handled: AttentionItem = { ...unresponsive('r1', 'nightly', 1), acknowledged: { by: 'sam', at: ago(1) } } as AttentionItem; + const views = attentionViews([handled, queued('q1', 'nightly', [])], now); + expect(views.map(({ view }) => [view.key, view.consequences.length])).toEqual([ + ['queue:q1', 0], + ['unresponsive:r1', 0] + ]); + }); + + it('words a short heartbeat gap as a pause and a long one as an outage', () => { + const [outage, pause] = attentionViews([unresponsive('short', 'a', 1), unresponsive('long', 'b', 6)], now).sort((x, y) => x.view.key.localeCompare(y.view.key)); + expect([outage.view.title, outage.view.tone]).toEqual(['b is unresponsive', 'warning']); + expect([pause.view.title, pause.view.tone]).toEqual(['a missed a heartbeat', 'info']); + }); + + it('offers one build: what changed when anything changed, else a rebuild of the nodes behind', () => { + const build = (changed: string[], behind: string[]) => attentionViews([{ kind: 'stale_models', changed, behind }], now)[0].view.build; + expect(build(['model:a'], ['model:a', 'model:b'])).toEqual({ label: 'Build what changed', href: CHANGED_PLAN_HREF }); + // `--changed` would build nothing when nodes are only behind, so the rebuild names them. + expect(build([], ['model:b', 'model:c'])).toEqual({ label: 'Rebuild 2', href: planHref(['model:b', 'model:c']) }); + }); + + it('states how many nodes are out of date, counting a node both changed and behind once', () => { + const detail = (changed: string[], behind: string[]) => attentionViews([{ kind: 'stale_models', changed, behind }], now)[0].view; + expect(detail(['model:a', 'model:b'], ['model:b']).title).toBe('2 nodes out of date'); + expect(detail(['model:a', 'model:b'], ['model:b']).detail).toBe('2 changed since their last build, 1 of them also behind an upstream rebuild: a, b'); + expect(detail(['model:a', 'model:b'], ['model:b', 'model:c']).detail).toBe('2 changed since their last build, 1 of them also behind an upstream rebuild, and 1 more behind an upstream rebuild: a, b, c'); + expect(detail(['model:a'], ['model:c']).detail).toBe('1 changed since its last build and 1 behind an upstream rebuild: a, c'); + expect(detail(['model:a'], ['model:a']).detail).toBe('1 changed since its last build, also behind an upstream rebuild: a'); + expect(detail([], ['model:c', 'model:d']).detail).toBe('2 behind an upstream rebuild: c, d'); + }); +}); diff --git a/ui/src/lib/build-area/attention.ts b/ui/src/lib/build-area/attention.ts index 14d9dff438..07dc7066ed 100644 --- a/ui/src/lib/build-area/attention.ts +++ b/ui/src/lib/build-area/attention.ts @@ -1,26 +1,38 @@ // Presentation of the overview's attention items. Pure; no side effects. -import type { AttentionItem } from '$lib/api'; +import type { AttentionItem, QueueWait } from '$lib/api'; import { describeWait, outcomeCounts } from '$lib/operate/labels'; import { eventKindLabel, problemHref } from '$lib/platform/labels'; import { formatAgo } from '$lib/format/time'; import { runStatusLabel } from '$lib/status'; import { shortName } from '$lib/build-area/labels'; +import { CHANGED_PLAN_HREF, planHref } from '$lib/run/command'; export type AttentionTone = 'error' | 'warning' | 'stale' | 'info'; +/** Something an alert's cause also brought about, listed under it instead of as an alert of its own. */ +export type Consequence = { text: string; href: string }; + export type AttentionView = { key: string; tone: AttentionTone; title: string; detail: string; + /** When it started, as a time ago; empty when the alert has no start of its own. */ since: string; href: string; action: string; + consequences: Consequence[]; + /** The one build an out-of-date alert offers: `sqb build --changed` when anything changed, else a rebuild of the nodes behind. */ + build?: { label: string; href: string }; }; +/** A heartbeat gap shorter than this reads as a pause, not an outage: the run carries on when its worker reports again. */ +export const UNRESPONSIVE_OUTAGE_MS = 5 * 60_000; + const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`; const minutes = (seconds: number | null) => (seconds === null ? '—' : seconds % 3600 === 0 ? `${seconds / 3600}h` : `${Math.round(seconds / 60)}m`); +const waitText = (waits: QueueWait[]) => waits.map((w) => describeWait(w).map((p) => p.text).join('')).join(' · '); export function attentionView(item: AttentionItem, now: number): AttentionView { switch (item.kind) { @@ -33,43 +45,62 @@ export function attentionView(item: AttentionItem, now: number): AttentionView { detail: outcomeCounts(item.run.nodeCounts) || item.run.statusReason || item.run.id, since: formatAgo(item.run.endedAt, now), href: `/runs/${item.run.id}`, - action: 'View run' + action: 'View run', + consequences: [] }; - case 'run_unresponsive': + case 'run_unresponsive': { + const heard = item.lastHeartbeatAt ?? item.run.startedAt; + const gap = heard ? now - Date.parse(heard) : Infinity; + const name = item.run.jobKey ?? item.run.id; + const outage = gap >= UNRESPONSIVE_OUTAGE_MS; return { key: `unresponsive:${item.run.id}`, - tone: 'warning', - title: `${item.run.jobKey ?? item.run.id} is unresponsive`, - detail: item.lastHeartbeatAt ? `Not heard from since ${formatAgo(item.lastHeartbeatAt, now)}` : (item.run.statusReason ?? 'Not heard from yet'), - since: formatAgo(item.run.startedAt, now), + tone: outage ? 'warning' : 'info', + title: outage ? `${name} is unresponsive` : `${name} missed a heartbeat`, + detail: item.lastHeartbeatAt + ? `Its worker last reported ${formatAgo(item.lastHeartbeatAt, now)}.` + : (item.run.statusReason ?? 'Its worker hasn’t reported since the run started.'), + since: formatAgo(item.lastHeartbeatAt ?? item.run.startedAt, now), href: `/runs/${item.run.id}`, - action: 'View run' + action: 'View run', + consequences: [] }; + } case 'stale_models': { - const total = new Set([...item.changed, ...item.behind]).size; - const parts = [item.changed.length && `${item.changed.length} changed`, item.behind.length && `${item.behind.length} behind`].filter(Boolean); + const changed = new Set(item.changed); + const both = item.behind.filter((key) => changed.has(key)).length; + const all = [...new Set([...item.changed, ...item.behind])]; + const only = item.behind.length - both; + const alsoBehind = both ? `, ${item.changed.length === 1 ? 'also' : `${both} of them also`} behind an upstream rebuild` : ''; + const onlyBehind = only ? `${both ? `, and ${only} more` : ` and ${only}`} behind an upstream rebuild` : ''; + const counts = item.changed.length + ? `${item.changed.length} changed since ${item.changed.length === 1 ? 'its' : 'their'} last build${alsoBehind}${onlyBehind}` + : `${only} behind an upstream rebuild`; return { key: 'stale', tone: 'stale', - title: `${plural(total, 'node')} ${total === 1 ? 'is' : 'are'} stale`, - detail: `${parts.join(' · ')} · ${[...new Set([...item.changed, ...item.behind])].slice(0, 4).map(shortName).join(', ')}`, - since: 'since the last build', + title: `${plural(all.length, 'node')} out of date`, + detail: `${counts}: ${all.slice(0, 4).map(shortName).join(', ')}${all.length > 4 ? ` and ${all.length - 4} more` : ''}`, + since: '', href: '/catalog?stale=1', - action: 'View in catalog' + action: 'View in catalog', + consequences: [], + build: item.changed.length ? { label: 'Build what changed', href: CHANGED_PLAN_HREF } : { label: `Rebuild ${all.length}`, href: planHref(all) } }; } - case 'stale_sources': { - const first = item.sources[0]; + case 'stale_sources': return { key: 'sources', tone: item.sources.some((s) => s.status === 'error') ? 'error' : 'warning', title: `${plural(item.sources.length, 'source')} past the freshness threshold`, - detail: item.sources.map((s) => `${s.key} loaded ${formatAgo(s.loadedAt, now)}`).join(' · '), - since: `${first.status} after ${minutes(first.status === 'error' ? first.errorAfterSeconds : first.warnAfterSeconds)}`, + detail: item.sources + .map((s) => `${s.key} loaded ${formatAgo(s.loadedAt, now)} (${s.status} after ${minutes(s.status === 'error' ? s.errorAfterSeconds : s.warnAfterSeconds)})`) + .join(' · '), + since: '', href: '/catalog?kind=source', - action: 'View sources' + action: 'View sources', + consequences: [] }; - } case 'platform_event': return { key: `event:${item.event.id}`, @@ -78,17 +109,19 @@ export function attentionView(item: AttentionItem, now: number): AttentionView { detail: item.event.message, since: formatAgo(item.event.occurredAt, now), href: problemHref(item.event), - action: item.event.locationId ? 'View location' : 'View server status' + action: item.event.locationId ? 'View location' : 'View server status', + consequences: [] }; case 'queue_blocked': return { key: `queue:${item.runId}`, tone: 'warning', title: `${item.jobKey ?? item.runId} is waiting in the queue`, - detail: item.waits.map((w) => describeWait(w).map((p) => p.text).join('')).join(' · '), - since: `queued ${formatAgo(item.queuedAt, now)}`, + detail: waitText(item.waits), + since: formatAgo(item.queuedAt, now), href: '/runs?view=queue', - action: 'View queue' + action: 'View queue', + consequences: [] }; case 'calendar_low': return { @@ -98,11 +131,58 @@ export function attentionView(item: AttentionItem, now: number): AttentionView { detail: item.detail, since: '', href: `/triggers/${item.triggerId}`, - action: 'View trigger' + action: 'View trigger', + consequences: [] }; } } +/** The run a wait is held up by, when it names one. */ +const waitRun = (wait: QueueWait): string | null => ('runId' in wait ? wait.runId : null); + +/** + * The cause an item follows from, by the key of the alert that names that cause, or null when it stands alone: + * a schedule is overdue because its location stopped checking in, or else because no server runs the scheduler; + * a run waits in the queue because an unresponsive run of its job, or one it waits for, holds it up. + * An acknowledged cause leaves the list, so nothing is grouped under it. + */ +function causeOf(item: AttentionItem, items: AttentionItem[]): string | null { + if (item.kind === 'platform_event' && item.event.kind === 'tick_overdue') { + const events = items.flatMap((i) => (i.kind === 'platform_event' ? [i.event] : [])); + const location = events.find((e) => e.kind === 'location_broken' && e.locationId === item.event.locationId); + const scheduler = events.find((e) => e.kind === 'scheduler_no_leader'); + const cause = location ?? scheduler; + return cause ? `event:${cause.id}` : null; + } + if (item.kind === 'queue_blocked') { + const held = new Set(item.waits.map(waitRun)); + const stuck = items.find((i) => i.kind === 'run_unresponsive' && !i.acknowledged && ((item.jobKey !== null && i.run.jobKey === item.jobKey) || held.has(i.run.id))); + return stuck && stuck.kind === 'run_unresponsive' ? `unresponsive:${stuck.run.id}` : null; + } + return null; +} + +function consequenceOf(item: AttentionItem, view: AttentionView): Consequence { + if (item.kind === 'queue_blocked') return { text: `${view.title}, queued ${view.since}. ${view.detail}`, href: view.href }; + if (item.kind === 'platform_event') return { text: item.event.message, href: view.href }; + return { text: `${view.title}: ${view.detail}`, href: view.href }; +} + +/** The items as alerts, errors first: an item that follows from another alert's cause is listed under it, not on its own. */ +export function attentionViews(items: AttentionItem[], now: number): { item: AttentionItem; view: AttentionView }[] { + const viewed = items.map((item) => ({ item, view: attentionView(item, now) })); + const byKey = new Map(viewed.map((v) => [v.view.key, v])); + const shown = viewed.filter(({ item, view }) => { + const cause = byKey.get(causeOf(item, items) ?? ''); + if (!cause) return true; + cause.view.consequences.push(consequenceOf(item, view)); + // The cause is at least as urgent as anything it brings about. + if (attentionRank(view.tone) < attentionRank(cause.view.tone)) cause.view.tone = view.tone; + return false; + }); + return shown.sort((a, b) => attentionRank(a.view.tone) - attentionRank(b.view.tone)); +} + /** Error first, then warnings, stale and info. */ export function attentionRank(tone: AttentionTone): number { return { error: 0, warning: 1, stale: 2, info: 3 }[tone]; diff --git a/ui/src/lib/components/app-topbar.svelte b/ui/src/lib/components/app-topbar.svelte index 256fce10bd..256c7dee57 100644 --- a/ui/src/lib/components/app-topbar.svelte +++ b/ui/src/lib/components/app-topbar.svelte @@ -2,7 +2,7 @@ import SunIcon from '@lucide/svelte/icons/sun'; import MoonIcon from '@lucide/svelte/icons/moon'; import type { Snippet } from 'svelte'; - import type { ConnectionState } from '$lib/api'; + import type { ConnectionState, PlatformEvent } from '$lib/api'; import { theme, toggleTheme } from '$lib/theme.svelte'; import { formatAgo } from '$lib/format/time'; import { targetWords } from '$lib/platform/labels'; @@ -10,8 +10,9 @@ import PlatformEventBanner from './shell/platform-event-banner.svelte'; import WarehouseSyncStatus from './shell/warehouse-sync-status.svelte'; - type Props = { title: string; children?: Snippet }; - let { title, children }: Props = $props(); + /** `events` picks the platform problems the banner shows on this page; by default, all of them. */ + type Props = { title: string; children?: Snippet; events?: (event: PlatformEvent) => boolean }; + let { title, children, events }: Props = $props(); const connectionTone: Record = { live: 'var(--sb-success)', @@ -73,7 +74,7 @@ - +