Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion ui/src/lib/api/mock/build-fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,10 @@ export const DEPLOYS: DeploySpec[] = [
gitAuthor: ALEX.displayName,
minutesAgo: 6,
publishedBy: CI_ACCOUNT,
nodes: [node('fact_orders', 'changed', 'SQL changed: refunded payments now net out of order revenue', { columns: [{ name: 'net_revenue', op: 'changed', detail: 'expression changed' }] })],
nodes: [
node('fact_orders', 'changed', 'SQL changed: refunded payments now net out of order revenue', { columns: [{ name: 'net_revenue', op: 'changed', detail: 'expression changed' }] }),
node('customer_status', 'changed', 'SQL changed: lapsed customers now need 120 days without an order', { columns: [{ name: 'status', op: 'changed', detail: 'expression changed' }] })
],
orchestration: [],
roles: [],
environment: null
Expand Down
87 changes: 87 additions & 0 deletions ui/src/lib/build-area/attention.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
import { describe, expect, it } from 'vitest';
import type { AttentionItem, AttentionRun, PlatformEvent } from '$lib/api';
import { attentionViews } from './attention';
import { CHANGED_PLAN_HREF, planHref } from '$lib/run/command';

const now = Date.parse('2026-01-01T12:00:00Z');
const ago = (minutes: number) => new Date(now - minutes * 60_000).toISOString();
const event = (id: string, kind: PlatformEvent['kind'], locationId: string | null, over: Partial<PlatformEvent> = {}): AttentionItem => ({
kind: 'platform_event',
event: { id, kind, severity: 'warning', occurredAt: ago(5), active: true, locationId, subject: null, message: `${kind} message`, ...over }
});
const run = (id: string, jobKey: string | null): AttentionRun => ({ id, jobKey, kind: 'job', status: 'unresponsive', statusReason: null, nodeCounts: { success: 0, warn: 0, failed: 0, skipped: 0, running: 1, pending: 0 }, startedAt: ago(30), endedAt: null });
const unresponsive = (id: string, jobKey: string | null, heardMinutesAgo: number): AttentionItem => ({ kind: 'run_unresponsive', run: run(id, jobKey), lastHeartbeatAt: ago(heardMinutesAgo), acknowledged: null });
const queued = (runId: string, jobKey: string | null, waits: Extract<AttentionItem, { kind: 'queue_blocked' }>['waits']): AttentionItem => ({ kind: 'queue_blocked', runId, jobKey, queuedAt: ago(20), waits });

describe('overview alerts', () => {
it("lists an overdue schedule under its own location's outage, else under the stopped scheduler", () => {
const views = attentionViews(
[
event('broken', 'location_broken', 'shop'),
event('late-shop', 'tick_overdue', 'shop', { message: 'hourly is late' }),
event('late-other', 'tick_overdue', 'marketing', { message: 'daily is late' }),
event('stopped', 'scheduler_no_leader', null, { severity: 'critical' })
],
now
);
expect(views.map(({ view }) => [view.key, view.consequences.map((c) => c.text)])).toEqual([
['event:stopped', ['daily is late']],
['event:broken', ['hourly is late']]
]);
});

it('keeps an overdue schedule as its own alert when nothing explains it', () => {
const views = attentionViews([event('late', 'tick_overdue', 'shop'), event('broken', 'location_broken', 'marketing')], now);
expect(views.map(({ view }) => view.key)).toEqual(['event:late', 'event:broken']);
});

it('lists a queued run under the unresponsive run of its job or the one it waits for, and the cause takes the more urgent tone', () => {
const views = attentionViews(
[
unresponsive('r1', 'nightly', 1),
queued('q1', 'nightly', []),
unresponsive('r2', null, 1),
queued('q2', 'exports', [{ kind: 'write_claim', runId: 'r2', nodeKey: 'model:orders', otherNodes: 0 }]),
queued('q3', 'other', [])
],
now
);
expect(views.map(({ view }) => [view.key, view.tone, view.consequences.length])).toEqual([
['unresponsive:r1', 'warning', 1],
['unresponsive:r2', 'warning', 1],
['queue:q3', 'warning', 0]
]);
});

it('keeps a queued run as its own alert when the unresponsive run that holds it up is acknowledged', () => {
const handled: AttentionItem = { ...unresponsive('r1', 'nightly', 1), acknowledged: { by: 'sam', at: ago(1) } } as AttentionItem;
const views = attentionViews([handled, queued('q1', 'nightly', [])], now);
expect(views.map(({ view }) => [view.key, view.consequences.length])).toEqual([
['queue:q1', 0],
['unresponsive:r1', 0]
]);
});

it('words a short heartbeat gap as a pause and a long one as an outage', () => {
const [outage, pause] = attentionViews([unresponsive('short', 'a', 1), unresponsive('long', 'b', 6)], now).sort((x, y) => x.view.key.localeCompare(y.view.key));
expect([outage.view.title, outage.view.tone]).toEqual(['b is unresponsive', 'warning']);
expect([pause.view.title, pause.view.tone]).toEqual(['a missed a heartbeat', 'info']);
});

it('offers one build: what changed when anything changed, else a rebuild of the nodes behind', () => {
const build = (changed: string[], behind: string[]) => attentionViews([{ kind: 'stale_models', changed, behind }], now)[0].view.build;
expect(build(['model:a'], ['model:a', 'model:b'])).toEqual({ label: 'Build what changed', href: CHANGED_PLAN_HREF });
// `--changed` would build nothing when nodes are only behind, so the rebuild names them.
expect(build([], ['model:b', 'model:c'])).toEqual({ label: 'Rebuild 2', href: planHref(['model:b', 'model:c']) });
});

it('states how many nodes are out of date, counting a node both changed and behind once', () => {
const detail = (changed: string[], behind: string[]) => attentionViews([{ kind: 'stale_models', changed, behind }], now)[0].view;
expect(detail(['model:a', 'model:b'], ['model:b']).title).toBe('2 nodes out of date');
expect(detail(['model:a', 'model:b'], ['model:b']).detail).toBe('2 changed since their last build, 1 of them also behind an upstream rebuild: a, b');
expect(detail(['model:a', 'model:b'], ['model:b', 'model:c']).detail).toBe('2 changed since their last build, 1 of them also behind an upstream rebuild, and 1 more behind an upstream rebuild: a, b, c');
expect(detail(['model:a'], ['model:c']).detail).toBe('1 changed since its last build and 1 behind an upstream rebuild: a, c');
expect(detail(['model:a'], ['model:a']).detail).toBe('1 changed since its last build, also behind an upstream rebuild: a');
expect(detail([], ['model:c', 'model:d']).detail).toBe('2 behind an upstream rebuild: c, d');
});
});
130 changes: 105 additions & 25 deletions ui/src/lib/build-area/attention.ts
Original file line number Diff line number Diff line change
@@ -1,26 +1,38 @@
// Presentation of the overview's attention items. Pure; no side effects.

import type { AttentionItem } from '$lib/api';
import type { AttentionItem, QueueWait } from '$lib/api';
import { describeWait, outcomeCounts } from '$lib/operate/labels';
import { eventKindLabel, problemHref } from '$lib/platform/labels';
import { formatAgo } from '$lib/format/time';
import { runStatusLabel } from '$lib/status';
import { shortName } from '$lib/build-area/labels';
import { CHANGED_PLAN_HREF, planHref } from '$lib/run/command';

export type AttentionTone = 'error' | 'warning' | 'stale' | 'info';

/** Something an alert's cause also brought about, listed under it instead of as an alert of its own. */
export type Consequence = { text: string; href: string };

export type AttentionView = {
key: string;
tone: AttentionTone;
title: string;
detail: string;
/** When it started, as a time ago; empty when the alert has no start of its own. */
since: string;
href: string;
action: string;
consequences: Consequence[];
/** The one build an out-of-date alert offers: `sqb build --changed` when anything changed, else a rebuild of the nodes behind. */
build?: { label: string; href: string };
};

/** A heartbeat gap shorter than this reads as a pause, not an outage: the run carries on when its worker reports again. */
export const UNRESPONSIVE_OUTAGE_MS = 5 * 60_000;

const plural = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
const minutes = (seconds: number | null) => (seconds === null ? '—' : seconds % 3600 === 0 ? `${seconds / 3600}h` : `${Math.round(seconds / 60)}m`);
const waitText = (waits: QueueWait[]) => waits.map((w) => describeWait(w).map((p) => p.text).join('')).join(' · ');

export function attentionView(item: AttentionItem, now: number): AttentionView {
switch (item.kind) {
Expand All @@ -33,43 +45,62 @@ export function attentionView(item: AttentionItem, now: number): AttentionView {
detail: outcomeCounts(item.run.nodeCounts) || item.run.statusReason || item.run.id,
since: formatAgo(item.run.endedAt, now),
href: `/runs/${item.run.id}`,
action: 'View run'
action: 'View run',
consequences: []
};
case 'run_unresponsive':
case 'run_unresponsive': {
const heard = item.lastHeartbeatAt ?? item.run.startedAt;
const gap = heard ? now - Date.parse(heard) : Infinity;
const name = item.run.jobKey ?? item.run.id;
const outage = gap >= UNRESPONSIVE_OUTAGE_MS;
return {
key: `unresponsive:${item.run.id}`,
tone: 'warning',
title: `${item.run.jobKey ?? item.run.id} is unresponsive`,
detail: item.lastHeartbeatAt ? `Not heard from since ${formatAgo(item.lastHeartbeatAt, now)}` : (item.run.statusReason ?? 'Not heard from yet'),
since: formatAgo(item.run.startedAt, now),
tone: outage ? 'warning' : 'info',
title: outage ? `${name} is unresponsive` : `${name} missed a heartbeat`,
detail: item.lastHeartbeatAt
? `Its worker last reported ${formatAgo(item.lastHeartbeatAt, now)}.`
: (item.run.statusReason ?? 'Its worker hasn’t reported since the run started.'),
since: formatAgo(item.lastHeartbeatAt ?? item.run.startedAt, now),
href: `/runs/${item.run.id}`,
action: 'View run'
action: 'View run',
consequences: []
};
}
case 'stale_models': {
const total = new Set([...item.changed, ...item.behind]).size;
const parts = [item.changed.length && `${item.changed.length} changed`, item.behind.length && `${item.behind.length} behind`].filter(Boolean);
const changed = new Set(item.changed);
const both = item.behind.filter((key) => changed.has(key)).length;
const all = [...new Set([...item.changed, ...item.behind])];
const only = item.behind.length - both;
const alsoBehind = both ? `, ${item.changed.length === 1 ? 'also' : `${both} of them also`} behind an upstream rebuild` : '';
const onlyBehind = only ? `${both ? `, and ${only} more` : ` and ${only}`} behind an upstream rebuild` : '';
const counts = item.changed.length
? `${item.changed.length} changed since ${item.changed.length === 1 ? 'its' : 'their'} last build${alsoBehind}${onlyBehind}`
: `${only} behind an upstream rebuild`;
return {
key: 'stale',
tone: 'stale',
title: `${plural(total, 'node')} ${total === 1 ? 'is' : 'are'} stale`,
detail: `${parts.join(' · ')} · ${[...new Set([...item.changed, ...item.behind])].slice(0, 4).map(shortName).join(', ')}`,
since: 'since the last build',
title: `${plural(all.length, 'node')} out of date`,
detail: `${counts}: ${all.slice(0, 4).map(shortName).join(', ')}${all.length > 4 ? ` and ${all.length - 4} more` : ''}`,
since: '',
href: '/catalog?stale=1',
action: 'View in catalog'
action: 'View in catalog',
consequences: [],
build: item.changed.length ? { label: 'Build what changed', href: CHANGED_PLAN_HREF } : { label: `Rebuild ${all.length}`, href: planHref(all) }
};
}
case 'stale_sources': {
const first = item.sources[0];
case 'stale_sources':
return {
key: 'sources',
tone: item.sources.some((s) => s.status === 'error') ? 'error' : 'warning',
title: `${plural(item.sources.length, 'source')} past the freshness threshold`,
detail: item.sources.map((s) => `${s.key} loaded ${formatAgo(s.loadedAt, now)}`).join(' · '),
since: `${first.status} after ${minutes(first.status === 'error' ? first.errorAfterSeconds : first.warnAfterSeconds)}`,
detail: item.sources
.map((s) => `${s.key} loaded ${formatAgo(s.loadedAt, now)} (${s.status} after ${minutes(s.status === 'error' ? s.errorAfterSeconds : s.warnAfterSeconds)})`)
.join(' · '),
since: '',
href: '/catalog?kind=source',
action: 'View sources'
action: 'View sources',
consequences: []
};
}
case 'platform_event':
return {
key: `event:${item.event.id}`,
Expand All @@ -78,17 +109,19 @@ export function attentionView(item: AttentionItem, now: number): AttentionView {
detail: item.event.message,
since: formatAgo(item.event.occurredAt, now),
href: problemHref(item.event),
action: item.event.locationId ? 'View location' : 'View server status'
action: item.event.locationId ? 'View location' : 'View server status',
consequences: []
};
case 'queue_blocked':
return {
key: `queue:${item.runId}`,
tone: 'warning',
title: `${item.jobKey ?? item.runId} is waiting in the queue`,
detail: item.waits.map((w) => describeWait(w).map((p) => p.text).join('')).join(' · '),
since: `queued ${formatAgo(item.queuedAt, now)}`,
detail: waitText(item.waits),
since: formatAgo(item.queuedAt, now),
href: '/runs?view=queue',
action: 'View queue'
action: 'View queue',
consequences: []
};
case 'calendar_low':
return {
Expand All @@ -98,11 +131,58 @@ export function attentionView(item: AttentionItem, now: number): AttentionView {
detail: item.detail,
since: '',
href: `/triggers/${item.triggerId}`,
action: 'View trigger'
action: 'View trigger',
consequences: []
};
}
}

/** The run a wait is held up by, when it names one. */
const waitRun = (wait: QueueWait): string | null => ('runId' in wait ? wait.runId : null);

/**
* The cause an item follows from, by the key of the alert that names that cause, or null when it stands alone:
* a schedule is overdue because its location stopped checking in, or else because no server runs the scheduler;
* a run waits in the queue because an unresponsive run of its job, or one it waits for, holds it up.
* An acknowledged cause leaves the list, so nothing is grouped under it.
*/
function causeOf(item: AttentionItem, items: AttentionItem[]): string | null {
if (item.kind === 'platform_event' && item.event.kind === 'tick_overdue') {
const events = items.flatMap((i) => (i.kind === 'platform_event' ? [i.event] : []));
const location = events.find((e) => e.kind === 'location_broken' && e.locationId === item.event.locationId);
const scheduler = events.find((e) => e.kind === 'scheduler_no_leader');
const cause = location ?? scheduler;
return cause ? `event:${cause.id}` : null;
}
if (item.kind === 'queue_blocked') {
const held = new Set(item.waits.map(waitRun));
const stuck = items.find((i) => i.kind === 'run_unresponsive' && !i.acknowledged && ((item.jobKey !== null && i.run.jobKey === item.jobKey) || held.has(i.run.id)));
return stuck && stuck.kind === 'run_unresponsive' ? `unresponsive:${stuck.run.id}` : null;
}
return null;
}

function consequenceOf(item: AttentionItem, view: AttentionView): Consequence {
if (item.kind === 'queue_blocked') return { text: `${view.title}, queued ${view.since}. ${view.detail}`, href: view.href };
if (item.kind === 'platform_event') return { text: item.event.message, href: view.href };
return { text: `${view.title}: ${view.detail}`, href: view.href };
}

/** The items as alerts, errors first: an item that follows from another alert's cause is listed under it, not on its own. */
export function attentionViews(items: AttentionItem[], now: number): { item: AttentionItem; view: AttentionView }[] {
const viewed = items.map((item) => ({ item, view: attentionView(item, now) }));
const byKey = new Map(viewed.map((v) => [v.view.key, v]));
const shown = viewed.filter(({ item, view }) => {
const cause = byKey.get(causeOf(item, items) ?? '');
if (!cause) return true;
cause.view.consequences.push(consequenceOf(item, view));
// The cause is at least as urgent as anything it brings about.
if (attentionRank(view.tone) < attentionRank(cause.view.tone)) cause.view.tone = view.tone;
return false;
});
return shown.sort((a, b) => attentionRank(a.view.tone) - attentionRank(b.view.tone));
}

/** Error first, then warnings, stale and info. */
export function attentionRank(tone: AttentionTone): number {
return { error: 0, warning: 1, stale: 2, info: 3 }[tone];
Expand Down
9 changes: 5 additions & 4 deletions ui/src/lib/components/app-topbar.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,17 @@
import SunIcon from '@lucide/svelte/icons/sun';
import MoonIcon from '@lucide/svelte/icons/moon';
import type { Snippet } from 'svelte';
import type { ConnectionState } from '$lib/api';
import type { ConnectionState, PlatformEvent } from '$lib/api';
import { theme, toggleTheme } from '$lib/theme.svelte';
import { formatAgo } from '$lib/format/time';
import { targetWords } from '$lib/platform/labels';
import { app } from '$lib/state/app.svelte';
import PlatformEventBanner from './shell/platform-event-banner.svelte';
import WarehouseSyncStatus from './shell/warehouse-sync-status.svelte';

type Props = { title: string; children?: Snippet };
let { title, children }: Props = $props();
/** `events` picks the platform problems the banner shows on this page; by default, all of them. */
type Props = { title: string; children?: Snippet; events?: (event: PlatformEvent) => boolean };
let { title, children, events }: Props = $props();

const connectionTone: Record<ConnectionState, string> = {
live: 'var(--sb-success)',
Expand Down Expand Up @@ -73,7 +74,7 @@
</button>
</div>
</div>
<PlatformEventBanner />
<PlatformEventBanner only={events} />

<style>
.connection {
Expand Down
11 changes: 6 additions & 5 deletions ui/src/lib/components/shell/platform-event-banner.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,11 @@
import { eventKindLabel, problemHref } from '$lib/platform/labels';
import { app } from '$lib/state/app.svelte';

let { only = () => true }: { only?: (event: PlatformEvent) => boolean } = $props();

const rank = (e: PlatformEvent) => (e.severity === 'critical' ? 2 : e.severity === 'warning' ? 1 : 0);
const events = $derived(
app.platformEvents.filter((e) => e.active && e.severity !== 'info').sort((a, b) => rank(b) - rank(a))
app.platformEvents.filter((e) => e.active && e.severity !== 'info' && only(e)).sort((a, b) => rank(b) - rank(a))
);
const critical = $derived(events.some((e) => e.severity === 'critical'));
/** Location problems link to the location's page if it exists, else the Locations list; server problems only to server status. */
Expand All @@ -29,7 +31,7 @@
{@const link = linkOf(event)}
<li>
<strong>{eventKindLabel[event.kind]}</strong>
<span title={event.message}>{event.message}</span>
<span>{event.message}</span>
<small>{formatAgo(event.occurredAt, app.now)}</small>
{#if link}<a href={to(link.href)}>{link.text}</a>{/if}
</li>
Expand Down Expand Up @@ -80,10 +82,9 @@
white-space: nowrap;
}
li span {
overflow: hidden;
min-width: 0;
color: var(--muted-foreground);
text-overflow: ellipsis;
white-space: nowrap;
overflow-wrap: anywhere;
}
li small {
color: var(--sb-text-faint);
Expand Down
Loading
Loading