Skip to content

Commit 1ff49a0

Browse files
authored
feat(ui): build what changed from Overview and Lineage, and show held-back nodes in the plan (#1464)
* feat(ui): build what changed from Overview and Lineage, and show held-back nodes in the plan * fix(ui): word --changed runs by their own reason and refresh held-back nodes by the named job only
1 parent f83b16c commit 1ff49a0

13 files changed

Lines changed: 287 additions & 34 deletions

File tree

‎ui/src/lib/api/http-runs.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ export function planResultOf(outcome: Wire['LocationWorkOutcome']): PlanResult {
3535
column_migrations: list(document.column_migrations),
3636
table_type_conversions: list(document.table_type_conversions),
3737
retention: list(document.retention),
38-
synced_copy: document.synced_copy ?? null
38+
synced_copy: document.synced_copy ?? null,
39+
changed: document.changed ?? null
3940
},
4041
error: null
4142
};

‎ui/src/lib/api/mock/build-fixtures.ts‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,18 @@ const node = (key: string, op: NodeChange['op'], detail: string, extra: Partial<
358358
});
359359

360360
export const DEPLOYS: DeploySpec[] = [
361+
{
362+
generation: 43,
363+
gitRevision: 'c41e9b2',
364+
gitDirty: false,
365+
gitAuthor: ALEX.displayName,
366+
minutesAgo: 6,
367+
publishedBy: CI_ACCOUNT,
368+
nodes: [node('fact_orders', 'changed', 'SQL changed: refunded payments now net out of order revenue', { columns: [{ name: 'net_revenue', op: 'changed', detail: 'expression changed' }] })],
369+
orchestration: [],
370+
roles: [],
371+
environment: null
372+
},
361373
{
362374
generation: 42,
363375
gitRevision: '7d2e41a',

‎ui/src/lib/api/mock/fixtures.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ export function createFixtures(now: number) {
4949
health: 'healthy',
5050
lastSeenAt: ago(0.2),
5151
target: { name: 'prod', adapter: 'snowflake', connectionSummary: 'WAFFLE_SHOP on example-account' },
52-
snapshot: snapshot(42, '7d2e41a', 190, 'Alex Rivera'),
52+
snapshot: snapshot(43, 'c41e9b2', 6, 'Alex Rivera'),
5353
warehouseSync: idleSync(4)
5454
},
5555
{

‎ui/src/lib/api/mock/mock-client.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,11 @@ export class MockApiClient implements ApiClient {
293293

294294
async planBuild(scope: Scope, command: string[]): Promise<PlanResult> {
295295
this.#location(scope);
296-
return this.#operate.planBuild(scope, command);
296+
// `--changed` starts from the nodes whose own definition changed, as the catalog marks them.
297+
const changed = command.some((t) => t === '--changed' || t.startsWith('--changed='))
298+
? new Map((await this.#build.listCatalogNodes(scope)).filter((n) => n.stale?.some((r) => r.reason === 'changed')).map((n) => [n.key, 'definition changed']))
299+
: null;
300+
return this.#operate.planBuild(scope, command, changed);
297301
}
298302

299303
async listJobs(scope: Scope): Promise<Job[]> {

‎ui/src/lib/api/mock/operate-store.ts‎

Lines changed: 56 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import type {
1010
Acknowledgement,
1111
AdhocPermission,
1212
AdhocRunCheck,
13+
PlanChanged,
1314
PlanResult,
1415
AdhocRunRequest,
1516
BuiltInRole,
@@ -68,7 +69,7 @@ import {
6869
type TriggerDef
6970
} from './operate';
7071
import { SCHEDULER } from './principals';
71-
import { ancestors, resolveSelection, SNAPSHOT_NODES, snapshotNode, sortTopologically, edgesWithin, writeSetOf } from './operate-graph';
72+
import { ancestors, isWritable, resolveSelection, SNAPSHOT_NODES, snapshotNode, sortTopologically, edgesWithin, writeSetOf } from './operate-graph';
7273
import {
7374
commandOf,
7475
computeNodes,
@@ -654,21 +655,22 @@ export class MockOperate {
654655
return this.#ctx.reply(this.#view(rec, scope));
655656
}
656657

657-
/** Every selected model is planned as unchanged unless the command asks for a full refresh. */
658-
async planBuild(scope: Scope, command: string[]): Promise<PlanResult> {
658+
/** Every selected model is planned as unchanged unless the command asks for a full refresh; `changed` names the models `--changed` starts from. */
659+
async planBuild(scope: Scope, command: string[], changed: Map<string, string> | null = null): Promise<PlanResult> {
659660
const check = this.#check(scope, { command, priority: 0 });
660661
await new Promise((resolve) => setTimeout(resolve, 1200));
661662
if (check.error) return this.#ctx.reply({ plan: null, error: check.error });
662663
if (check.verb !== 'build') return this.#ctx.reply({ plan: null, error: `sqb plan previews builds; sqb ${check.verb} cannot be planned` });
663664
const reason = command.includes('--full-refresh') ? 'full_refresh' : 'no_change';
665+
const decision = changed ? changedDecision(check.selection, changed) : null;
664666
const modelKinds = new Set(['view', 'table', 'incremental']);
665667
const models = check.selection.filter((key) => modelKinds.has(snapshotNode(key)?.kind ?? '')).map((key) => {
666668
const incremental = snapshotNode(key)?.kind === 'incremental';
667669
return {
668670
name: key,
669671
materialization_type: incremental ? 'incremental' : 'table',
670672
action: incremental ? 'incremental_append' : 'create_table',
671-
reason: incremental && reason === 'no_change' ? 'normal_incremental' : reason,
673+
reason: changed?.get(key) === 'definition changed' && reason === 'no_change' ? 'query_changed' : incremental && reason === 'no_change' ? 'normal_incremental' : reason,
672674
backfill: { action: reason === 'full_refresh' ? 'full' : 'forward', duration: null },
673675
qualified_name: `main.${key}`,
674676
...(incremental ? { incremental_strategy: 'append', cursor_column: 'updated_at', cursor_type: 'timestamp' } : {})
@@ -682,7 +684,7 @@ export class MockOperate {
682684
unknown_until_run: ['source freshness', 'retention', 'column rename support', 'sources outside mirrored schemas']
683685
};
684686
return this.#ctx.reply({
685-
plan: { selected_count: check.selection.length, models, seeds: [], source_loads: [], functions: [], python_nodes: [], warnings: [], migrations: [], column_migrations: [], table_type_conversions: [], retention: [], synced_copy },
687+
plan: { selected_count: check.selection.length, models, seeds: [], source_loads: [], functions: [], python_nodes: [], warnings: [], migrations: [], column_migrations: [], table_type_conversions: [], retention: [], synced_copy, changed: decision },
686688
error: null
687689
});
688690
}
@@ -1342,3 +1344,52 @@ export class MockOperate {
13421344
};
13431345
}
13441346
}
1347+
1348+
/** A node as `sqb plan --changed` names it, such as `model:orders`. */
1349+
function changedKey(key: string): string {
1350+
const kind = snapshotNode(key)?.kind ?? 'table';
1351+
return `${['view', 'table', 'incremental'].includes(kind) ? 'model' : kind}:${key}`;
1352+
}
1353+
1354+
/**
1355+
* The location's `--changed` decision for the mock project, with its default reach: a change runs what reads
1356+
* it, except nodes a triggered job without the change refreshes on its own, and what reads those.
1357+
*/
1358+
function changedDecision(selection: string[], changed: Map<string, string>): PlanChanged {
1359+
// The decision covers only models and seeds, as the CLI's does.
1360+
const selected = new Set(selection.filter((k) => isWritable(k) && ['view', 'table', 'incremental', 'seed'].includes(snapshotNode(k)?.kind ?? '')));
1361+
// A node belongs to jobs, not groups; a job counts as triggered when it or a group it is in has a trigger.
1362+
const triggered = new Set(TRIGGER_DEFS.flatMap((t) => (t.jobKey ? [t.jobKey] : [])));
1363+
const jobs = JOB_DEFS.filter((j) => j.kind === 'job' && (triggered.has(j.key) || JOB_DEFS.some((g) => g.kind === 'group' && triggered.has(g.key) && g.members.includes(j.key)))).map((j) => ({
1364+
key: j.key,
1365+
nodes: new Set(jobNodes(j.key))
1366+
}));
1367+
/** Each running node's changed root. */
1368+
const root = new Map<string, string>([...changed.keys()].filter((k) => selected.has(k)).map((k) => [k, k]));
1369+
/** Each held-back node's changed root, which a node reading it is held from too. */
1370+
const held = new Map<string, string>();
1371+
const decision: PlanChanged = { counts: ['definition'], reach: 'job', runs: [...root.keys()].map((k) => ({ node: changedKey(k), reason: changed.get(k)!, downstream_of: null })), held_back: [], unchanged: [] };
1372+
for (const node of SNAPSHOT_NODES) {
1373+
if (!selected.has(node.key) || root.has(node.key)) continue;
1374+
const heldUpstream = node.upstream.find((u) => held.has(u));
1375+
const runningUpstream = node.upstream.find((u) => root.has(u));
1376+
const origin = heldUpstream ? held.get(heldUpstream) : runningUpstream ? root.get(runningUpstream) : undefined;
1377+
if (!origin) {
1378+
decision.unchanged.push(changedKey(node.key));
1379+
continue;
1380+
}
1381+
// As in the CLI, a triggered job without the change refreshes the node itself before it counts as reading a held node.
1382+
const owners = jobs.filter((j) => j.nodes.has(node.key));
1383+
if (owners.length && !owners.some((j) => j.nodes.has(origin))) {
1384+
held.set(node.key, origin);
1385+
decision.held_back.push({ node: changedKey(node.key), refreshed_by: owners.map((j) => j.key) });
1386+
} else if (heldUpstream) {
1387+
held.set(node.key, origin);
1388+
decision.held_back.push({ node: changedKey(node.key), reads: changedKey(heldUpstream) });
1389+
} else {
1390+
root.set(node.key, origin);
1391+
decision.runs.push({ node: changedKey(node.key), reason: `upstream model ${origin} runs`, downstream_of: changedKey(origin) });
1392+
}
1393+
}
1394+
return decision;
1395+
}

‎ui/src/lib/api/types/operate.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -602,6 +602,20 @@ export type PlanDocument = {
602602
retention: { model: string; action?: string }[];
603603
/** What a location's preview planned from instead of the warehouse; `null` for a live plan. */
604604
synced_copy: PlanSyncedCopy | null;
605+
/** What `--changed` runs, holds back and skips; `null` without `--changed`. */
606+
changed: PlanChanged | null;
607+
};
608+
609+
/** `sqb plan --changed`'s decision. Nodes are keys such as `model:orders`. */
610+
export type PlanChanged = {
611+
counts: ('definition' | 'source_data')[];
612+
reach: 'job' | 'all';
613+
/** Each node that runs, with why; `downstream_of` names the changed upstream it runs for. */
614+
runs: { node: string; reason: string; downstream_of: string | null }[];
615+
/** Nodes left for a job's own run (`refreshed_by`), or that read such a node (`reads`). */
616+
held_back: ({ node: string; refreshed_by: string[] } | { node: string; reads: string })[];
617+
/** Selected nodes that run only when something they read changed; nothing did. */
618+
unchanged: string[];
605619
};
606620

607621
/** The server's synced copy of warehouse metadata a location's plan preview planned from. */

‎ui/src/lib/run/command.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,11 +97,14 @@ export function tokenAt(text: string, caret: number): TokenContext {
9797
}
9898

9999
/** The plan page for a command, opened by every Run button before anything runs; with `job`, a run of that job narrowed by the command. */
100-
export function planHref(selection: string[] = [], verb: LaunchVerb = 'build', narrowing: { job?: string; exclude?: string[] } = {}): string {
101-
const command = buildCommand({ verb, select: selection.map(selectorOf), exclude: (narrowing.exclude ?? []).map(selectorOf), flags: {} });
100+
export function planHref(selection: string[] = [], verb: LaunchVerb = 'build', narrowing: { job?: string; exclude?: string[]; flags?: CommandParts['flags'] } = {}): string {
101+
const command = buildCommand({ verb, select: selection.map(selectorOf), exclude: (narrowing.exclude ?? []).map(selectorOf), flags: narrowing.flags ?? {} });
102102
return `/plan?cmd=${encodeURIComponent(command)}${narrowing.job ? `&job=${encodeURIComponent(narrowing.job)}` : ''}`;
103103
}
104104

105+
/** The plan of `sqb build --changed`: what changed, what runs downstream of it, and what waits for its job. */
106+
export const CHANGED_PLAN_HREF = planHref([], 'build', { flags: { '--changed': true } });
107+
105108
/** The review page for rerunning runs, opened before anything reruns; `nodes` only for one run's selected nodes. */
106109
export function rerunHref(runIds: string[], mode: RerunMode, nodes: string[] = []): string {
107110
const query = new URLSearchParams([...runIds.map((id) => ['run', id]), ['mode', mode], ...nodes.map((n) => ['node', n])]);

‎ui/src/lib/run/plan-report.svelte‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,12 @@
55
import type { PlanDocument } from '$lib/api';
66
import { formatAgo } from '$lib/format/time';
77
import { app } from '$lib/state/app.svelte';
8-
import { planSections, type PlanEntry } from './plan-text';
8+
import { changedPolicyLine, changedSections, planSections, type PlanEntry } from './plan-text';
99
10-
let { plan }: { plan: PlanDocument } = $props();
10+
/** `refresh` words what refreshes the nodes a `--changed` plan holds back for a job. */
11+
let { plan, refresh = (job: string) => `job ${job}` }: { plan: PlanDocument; refresh?: (job: string) => string } = $props();
1112
12-
const sections = $derived(planSections(plan));
13+
const sections = $derived([...planSections(plan), ...(plan.changed ? changedSections(plan.changed, refresh) : [])]);
1314
/** Rows a section shows before "Show all", as the CLI truncates long sections. */
1415
const limit = 50;
1516
let expanded = $state<Record<string, boolean>>({});
@@ -34,6 +35,7 @@
3435
{/if}
3536
</section>
3637
{/if}
38+
{#if plan.changed}<p class="policy" data-testid="changed-policy">{changedPolicyLine(plan.changed)}</p>{/if}
3739
{#each sections as s (s.title)}
3840
{@const shown = expanded[s.title] ? s.entries : s.entries.slice(0, limit)}
3941
<section>
@@ -88,6 +90,12 @@
8890
gap: 18px;
8991
font-size: 12.5px;
9092
}
93+
.policy {
94+
margin: 0;
95+
color: var(--muted-foreground);
96+
font-family: var(--font-mono);
97+
font-size: 12px;
98+
}
9199
h4 {
92100
display: flex;
93101
align-items: center;

‎ui/src/lib/run/plan-text.test.ts‎

Lines changed: 80 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { describe, expect, it } from 'vitest';
22
import type { PlanDocument, PlanModel } from '$lib/api';
3-
import { actionText, planSections, planSentence } from './plan-text';
3+
import { actionText, changedPolicyLine, changedSections, jobRefreshText, planSections, planSentence } from './plan-text';
44

55
const model = (name: string, over: Partial<PlanModel> = {}): PlanModel => ({
66
name,
@@ -23,6 +23,7 @@ const plan = (models: PlanModel[], over: Partial<PlanDocument> = {}): PlanDocume
2323
table_type_conversions: [],
2424
retention: [],
2525
synced_copy: null,
26+
changed: null,
2627
...over
2728
});
2829

@@ -98,4 +99,82 @@ describe('plan sentence', () => {
9899
const functions = [{ name: 'margin', language: 'sql', reason: 'definition_changed' }, { name: 'tax', language: 'sql', reason: 'no_change' }];
99100
expect(planSentence(plan([], { functions }))).toBe('No models build. 1 changed function is recreated.');
100101
});
102+
103+
// The reach fixture's `sqb plan --changed --json` after stg_orders is redefined: finance_daily holds two models back.
104+
const reach = plan(
105+
[
106+
model('stg_customers'),
107+
model('stg_orders', { reason: 'query_changed' }),
108+
model('customer_orders'),
109+
model('int_orders'),
110+
model('fct_revenue'),
111+
model('orders_export'),
112+
model('rpt_pnl'),
113+
model('summary')
114+
],
115+
{
116+
changed: {
117+
counts: ['definition'],
118+
reach: 'job',
119+
runs: [
120+
{ node: 'model:customer_orders', reason: 'upstream model stg_orders runs', downstream_of: 'model:stg_orders' },
121+
{ node: 'model:int_orders', reason: 'upstream model stg_orders runs', downstream_of: 'model:stg_orders' },
122+
{ node: 'model:orders_export', reason: 'upstream model stg_orders runs', downstream_of: 'model:stg_orders' },
123+
{ node: 'model:stg_orders', reason: 'planner change (query_changed)', downstream_of: null }
124+
],
125+
held_back: [
126+
{ node: 'model:fct_revenue', refreshed_by: ['finance_daily'] },
127+
{ node: 'model:rpt_pnl', refreshed_by: ['finance_daily'] },
128+
{ node: 'model:summary', reads: 'model:fct_revenue' }
129+
],
130+
unchanged: ['model:stg_customers']
131+
}
132+
}
133+
);
134+
135+
it('under --changed lists only what runs, then what waits for its job and what is skipped, as sqb plan does', () => {
136+
const sections = [...planSections(reach), ...changedSections(reach.changed!, (job) => `job ${job}`)];
137+
expect(sections.map((s) => [s.title, s.entries.map((e) => `${e.name}: ${e.value}${e.detail.length ? ` (${e.detail.join(', ')})` : ''}`)])).toEqual([
138+
['Query changed', ['stg_orders: continue forward']],
139+
['Models', ['customer_orders: table (reads stg_orders)', 'int_orders: table (reads stg_orders)', 'orders_export: table (reads stg_orders)']],
140+
['Held back', ['fct_revenue: job finance_daily', 'rpt_pnl: job finance_daily', 'summary: reads fct_revenue, which is held back']],
141+
['Unchanged', ['stg_customers: skipped by --changed']]
142+
]);
143+
expect(changedPolicyLine(reach.changed!)).toBe('Counting: definition (source data not counted) · reach: job');
144+
expect(changedPolicyLine({ ...reach.changed!, counts: ['definition', 'source_data'], reach: 'all' })).toBe('Counting: definition, source data · reach: all');
145+
expect(planSentence(reach)).toBe(
146+
'4 models build: stg_orders changed since its last build and the other 3 models read a change upstream. 3 models wait for their job\'s own run. 1 unchanged node is skipped by --changed.'
147+
);
148+
});
149+
150+
it('names the models that run under --changed for their own reason, not as reading a change', () => {
151+
const first = plan([model('orders'), model('order_count', { action: 'incremental_append', reason: 'normal_incremental' })], {
152+
changed: {
153+
counts: ['definition'],
154+
reach: 'job',
155+
runs: [
156+
{ node: 'model:order_count', reason: 'no completed record', downstream_of: null },
157+
{ node: 'model:orders', reason: 'no completed record', downstream_of: null }
158+
],
159+
held_back: [],
160+
unchanged: []
161+
}
162+
});
163+
expect(planSentence(first)).toBe('2 models build: orders and order_count run for their own reason.');
164+
expect(planSections(first).find((s) => s.title === 'Models')?.entries.map((e) => e.detail.at(-1))).toEqual(['no completed record', 'no completed record']);
165+
});
166+
167+
it("says what refreshes a held-back node: the named job's next run, its sensor, or that it is paused", () => {
168+
const now = Date.parse('2026-01-01T00:00:00Z');
169+
type Kind = 'schedule' | 'sensor';
170+
const trigger = (jobKey: string, state: 'running' | 'paused', nextDueAt: string | null, type: Kind = 'schedule') => ({ jobKey, type, state, nextDueAt });
171+
// A group's schedule says nothing about a job whose own trigger is paused; the plan names the group itself when it matters.
172+
expect(jobRefreshText('finance_daily', [trigger('finance_daily', 'paused', '2026-01-01T05:00:00Z'), trigger('nightly', 'running', '2026-01-01T02:00:00Z')], now)).toBe('job finance_daily is paused');
173+
expect(jobRefreshText('nightly', [trigger('nightly', 'running', '2026-01-01T02:00:00Z'), trigger('nightly', 'running', '2026-01-01T05:00:00Z')], now)).toBe('job nightly · next run in 2h');
174+
expect(jobRefreshText('finance_daily', [trigger('finance_daily', 'paused', null), trigger('finance_daily', 'running', null, 'sensor')], now)).toBe('job finance_daily · when its sensor fires');
175+
// A running schedule with no next run, such as a finished calendar, is not a sensor.
176+
expect(jobRefreshText('finance_daily', [trigger('finance_daily', 'running', null)], now)).toBe('job finance_daily');
177+
expect(jobRefreshText('crm', [trigger('finance_daily', 'running', '2026-01-01T05:00:00Z')], now)).toBe('job crm');
178+
});
101179
});
180+

0 commit comments

Comments
 (0)