Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ui/src/lib/api/http-runs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ export function planResultOf(outcome: Wire['LocationWorkOutcome']): PlanResult {
column_migrations: list(document.column_migrations),
table_type_conversions: list(document.table_type_conversions),
retention: list(document.retention),
synced_copy: document.synced_copy ?? null
synced_copy: document.synced_copy ?? null,
changed: document.changed ?? null
},
error: null
};
Expand Down
12 changes: 12 additions & 0 deletions ui/src/lib/api/mock/build-fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,18 @@ const node = (key: string, op: NodeChange['op'], detail: string, extra: Partial<
});

export const DEPLOYS: DeploySpec[] = [
{
generation: 43,
gitRevision: 'c41e9b2',
gitDirty: false,
gitAuthor: ALEX.displayName,
minutesAgo: 6,
publishedBy: CI_ACCOUNT,
nodes: [node('fact_orders', 'changed', 'SQL changed: refunded payments now net out of order revenue', { columns: [{ name: 'net_revenue', op: 'changed', detail: 'expression changed' }] })],
orchestration: [],
roles: [],
environment: null
},
{
generation: 42,
gitRevision: '7d2e41a',
Expand Down
2 changes: 1 addition & 1 deletion ui/src/lib/api/mock/fixtures.ts
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ export function createFixtures(now: number) {
health: 'healthy',
lastSeenAt: ago(0.2),
target: { name: 'prod', adapter: 'snowflake', connectionSummary: 'WAFFLE_SHOP on example-account' },
snapshot: snapshot(42, '7d2e41a', 190, 'Alex Rivera'),
snapshot: snapshot(43, 'c41e9b2', 6, 'Alex Rivera'),
warehouseSync: idleSync(4)
},
{
Expand Down
6 changes: 5 additions & 1 deletion ui/src/lib/api/mock/mock-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,11 @@ export class MockApiClient implements ApiClient {

async planBuild(scope: Scope, command: string[]): Promise<PlanResult> {
this.#location(scope);
return this.#operate.planBuild(scope, command);
// `--changed` starts from the nodes whose own definition changed, as the catalog marks them.
const changed = command.some((t) => t === '--changed' || t.startsWith('--changed='))
? new Map((await this.#build.listCatalogNodes(scope)).filter((n) => n.stale?.some((r) => r.reason === 'changed')).map((n) => [n.key, 'definition changed']))
: null;
return this.#operate.planBuild(scope, command, changed);
}

async listJobs(scope: Scope): Promise<Job[]> {
Expand Down
61 changes: 56 additions & 5 deletions ui/src/lib/api/mock/operate-store.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import type {
Acknowledgement,
AdhocPermission,
AdhocRunCheck,
PlanChanged,
PlanResult,
AdhocRunRequest,
BuiltInRole,
Expand Down Expand Up @@ -65,7 +66,7 @@ import {
type TriggerDef
} from './operate';
import { SCHEDULER } from './principals';
import { ancestors, resolveSelection, SNAPSHOT_NODES, snapshotNode, sortTopologically, edgesWithin, writeSetOf } from './operate-graph';
import { ancestors, isWritable, resolveSelection, SNAPSHOT_NODES, snapshotNode, sortTopologically, edgesWithin, writeSetOf } from './operate-graph';
import {
commandOf,
computeNodes,
Expand Down Expand Up @@ -529,21 +530,22 @@ export class MockOperate {
return this.#ctx.reply(this.#view(rec, scope));
}

/** Every selected model is planned as unchanged unless the command asks for a full refresh. */
async planBuild(scope: Scope, command: string[]): Promise<PlanResult> {
/** Every selected model is planned as unchanged unless the command asks for a full refresh; `changed` names the models `--changed` starts from. */
async planBuild(scope: Scope, command: string[], changed: Map<string, string> | null = null): Promise<PlanResult> {
const check = this.#check(scope, { command, priority: 0 });
await new Promise((resolve) => setTimeout(resolve, 1200));
if (check.error) return this.#ctx.reply({ plan: null, error: check.error });
if (check.verb !== 'build') return this.#ctx.reply({ plan: null, error: `sqb plan previews builds; sqb ${check.verb} cannot be planned` });
const reason = command.includes('--full-refresh') ? 'full_refresh' : 'no_change';
const decision = changed ? changedDecision(check.selection, changed) : null;
const modelKinds = new Set(['view', 'table', 'incremental']);
const models = check.selection.filter((key) => modelKinds.has(snapshotNode(key)?.kind ?? '')).map((key) => {
const incremental = snapshotNode(key)?.kind === 'incremental';
return {
name: key,
materialization_type: incremental ? 'incremental' : 'table',
action: incremental ? 'incremental_append' : 'create_table',
reason: incremental && reason === 'no_change' ? 'normal_incremental' : reason,
reason: changed?.get(key) === 'definition changed' && reason === 'no_change' ? 'query_changed' : incremental && reason === 'no_change' ? 'normal_incremental' : reason,
backfill: { action: reason === 'full_refresh' ? 'full' : 'forward', duration: null },
qualified_name: `main.${key}`,
...(incremental ? { incremental_strategy: 'append', cursor_column: 'updated_at', cursor_type: 'timestamp' } : {})
Expand All @@ -557,7 +559,7 @@ export class MockOperate {
unknown_until_run: ['source freshness', 'retention', 'column rename support', 'sources outside mirrored schemas']
};
return this.#ctx.reply({
plan: { selected_count: check.selection.length, models, seeds: [], source_loads: [], functions: [], python_nodes: [], warnings: [], migrations: [], column_migrations: [], table_type_conversions: [], retention: [], synced_copy },
plan: { selected_count: check.selection.length, models, seeds: [], source_loads: [], functions: [], python_nodes: [], warnings: [], migrations: [], column_migrations: [], table_type_conversions: [], retention: [], synced_copy, changed: decision },
error: null
});
}
Expand Down Expand Up @@ -1217,3 +1219,52 @@ export class MockOperate {
};
}
}

/** A node as `sqb plan --changed` names it, such as `model:orders`. */
function changedKey(key: string): string {
const kind = snapshotNode(key)?.kind ?? 'table';
return `${['view', 'table', 'incremental'].includes(kind) ? 'model' : kind}:${key}`;
}

/**
* The location's `--changed` decision for the mock project, with its default reach: a change runs what reads
* it, except nodes a triggered job without the change refreshes on its own, and what reads those.
*/
function changedDecision(selection: string[], changed: Map<string, string>): PlanChanged {
// The decision covers only models and seeds, as the CLI's does.
const selected = new Set(selection.filter((k) => isWritable(k) && ['view', 'table', 'incremental', 'seed'].includes(snapshotNode(k)?.kind ?? '')));
// A node belongs to jobs, not groups; a job counts as triggered when it or a group it is in has a trigger.
const triggered = new Set(TRIGGER_DEFS.flatMap((t) => (t.jobKey ? [t.jobKey] : [])));
const jobs = JOB_DEFS.filter((j) => j.kind === 'job' && (triggered.has(j.key) || JOB_DEFS.some((g) => g.kind === 'group' && triggered.has(g.key) && g.members.includes(j.key)))).map((j) => ({
key: j.key,
nodes: new Set(jobNodes(j.key))
}));
/** Each running node's changed root. */
const root = new Map<string, string>([...changed.keys()].filter((k) => selected.has(k)).map((k) => [k, k]));
/** Each held-back node's changed root, which a node reading it is held from too. */
const held = new Map<string, string>();
const decision: PlanChanged = { counts: ['definition'], reach: 'job', runs: [...root.keys()].map((k) => ({ node: changedKey(k), reason: changed.get(k)!, downstream_of: null })), held_back: [], unchanged: [] };
for (const node of SNAPSHOT_NODES) {
if (!selected.has(node.key) || root.has(node.key)) continue;
const heldUpstream = node.upstream.find((u) => held.has(u));
const runningUpstream = node.upstream.find((u) => root.has(u));
const origin = heldUpstream ? held.get(heldUpstream) : runningUpstream ? root.get(runningUpstream) : undefined;
if (!origin) {
decision.unchanged.push(changedKey(node.key));
continue;
}
// As in the CLI, a triggered job without the change refreshes the node itself before it counts as reading a held node.
const owners = jobs.filter((j) => j.nodes.has(node.key));
if (owners.length && !owners.some((j) => j.nodes.has(origin))) {
held.set(node.key, origin);
decision.held_back.push({ node: changedKey(node.key), refreshed_by: owners.map((j) => j.key) });
} else if (heldUpstream) {
held.set(node.key, origin);
decision.held_back.push({ node: changedKey(node.key), reads: changedKey(heldUpstream) });
} else {
root.set(node.key, origin);
decision.runs.push({ node: changedKey(node.key), reason: `upstream model ${origin} runs`, downstream_of: changedKey(origin) });
}
}
return decision;
}
14 changes: 14 additions & 0 deletions ui/src/lib/api/types/operate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -530,6 +530,20 @@ export type PlanDocument = {
retention: { model: string; action?: string }[];
/** What a location's preview planned from instead of the warehouse; `null` for a live plan. */
synced_copy: PlanSyncedCopy | null;
/** What `--changed` runs, holds back and skips; `null` without `--changed`. */
changed: PlanChanged | null;
};

/** `sqb plan --changed`'s decision. Nodes are keys such as `model:orders`. */
export type PlanChanged = {
counts: ('definition' | 'source_data')[];
reach: 'job' | 'all';
/** Each node that runs, with why; `downstream_of` names the changed upstream it runs for. */
runs: { node: string; reason: string; downstream_of: string | null }[];
/** Nodes left for a job's own run (`refreshed_by`), or that read such a node (`reads`). */
held_back: ({ node: string; refreshed_by: string[] } | { node: string; reads: string })[];
/** Selected nodes that run only when something they read changed; nothing did. */
unchanged: string[];
};

/** The server's synced copy of warehouse metadata a location's plan preview planned from. */
Expand Down
7 changes: 5 additions & 2 deletions ui/src/lib/run/command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -97,11 +97,14 @@ export function tokenAt(text: string, caret: number): TokenContext {
}

/** The plan page for a command, opened by every Run button before anything runs; with `job`, a run of that job narrowed by the command. */
export function planHref(selection: string[] = [], verb: LaunchVerb = 'build', narrowing: { job?: string; exclude?: string[] } = {}): string {
const command = buildCommand({ verb, select: selection.map(selectorOf), exclude: (narrowing.exclude ?? []).map(selectorOf), flags: {} });
export function planHref(selection: string[] = [], verb: LaunchVerb = 'build', narrowing: { job?: string; exclude?: string[]; flags?: CommandParts['flags'] } = {}): string {
const command = buildCommand({ verb, select: selection.map(selectorOf), exclude: (narrowing.exclude ?? []).map(selectorOf), flags: narrowing.flags ?? {} });
return `/plan?cmd=${encodeURIComponent(command)}${narrowing.job ? `&job=${encodeURIComponent(narrowing.job)}` : ''}`;
}

/** The plan of `sqb build --changed`: what changed, what runs downstream of it, and what waits for its job. */
export const CHANGED_PLAN_HREF = planHref([], 'build', { flags: { '--changed': true } });

/** The review page for rerunning runs, opened before anything reruns; `nodes` only for one run's selected nodes. */
export function rerunHref(runIds: string[], mode: RerunMode, nodes: string[] = []): string {
const query = new URLSearchParams([...runIds.map((id) => ['run', id]), ['mode', mode], ...nodes.map((n) => ['node', n])]);
Expand Down
14 changes: 11 additions & 3 deletions ui/src/lib/run/plan-report.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@
import type { PlanDocument } from '$lib/api';
import { formatAgo } from '$lib/format/time';
import { app } from '$lib/state/app.svelte';
import { planSections, type PlanEntry } from './plan-text';
import { changedPolicyLine, changedSections, planSections, type PlanEntry } from './plan-text';

let { plan }: { plan: PlanDocument } = $props();
/** `refresh` words what refreshes the nodes a `--changed` plan holds back for a job. */
let { plan, refresh = (job: string) => `job ${job}` }: { plan: PlanDocument; refresh?: (job: string) => string } = $props();

const sections = $derived(planSections(plan));
const sections = $derived([...planSections(plan), ...(plan.changed ? changedSections(plan.changed, refresh) : [])]);
/** Rows a section shows before "Show all", as the CLI truncates long sections. */
const limit = 50;
let expanded = $state<Record<string, boolean>>({});
Expand All @@ -34,6 +35,7 @@
{/if}
</section>
{/if}
{#if plan.changed}<p class="policy" data-testid="changed-policy">{changedPolicyLine(plan.changed)}</p>{/if}
{#each sections as s (s.title)}
{@const shown = expanded[s.title] ? s.entries : s.entries.slice(0, limit)}
<section>
Expand Down Expand Up @@ -88,6 +90,12 @@
gap: 18px;
font-size: 12.5px;
}
.policy {
margin: 0;
color: var(--muted-foreground);
font-family: var(--font-mono);
font-size: 12px;
}
h4 {
display: flex;
align-items: center;
Expand Down
81 changes: 80 additions & 1 deletion ui/src/lib/run/plan-text.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { describe, expect, it } from 'vitest';
import type { PlanDocument, PlanModel } from '$lib/api';
import { actionText, planSections, planSentence } from './plan-text';
import { actionText, changedPolicyLine, changedSections, jobRefreshText, planSections, planSentence } from './plan-text';

const model = (name: string, over: Partial<PlanModel> = {}): PlanModel => ({
name,
Expand All @@ -23,6 +23,7 @@ const plan = (models: PlanModel[], over: Partial<PlanDocument> = {}): PlanDocume
table_type_conversions: [],
retention: [],
synced_copy: null,
changed: null,
...over
});

Expand Down Expand Up @@ -98,4 +99,82 @@ describe('plan sentence', () => {
const functions = [{ name: 'margin', language: 'sql', reason: 'definition_changed' }, { name: 'tax', language: 'sql', reason: 'no_change' }];
expect(planSentence(plan([], { functions }))).toBe('No models build. 1 changed function is recreated.');
});

// The reach fixture's `sqb plan --changed --json` after stg_orders is redefined: finance_daily holds two models back.
const reach = plan(
[
model('stg_customers'),
model('stg_orders', { reason: 'query_changed' }),
model('customer_orders'),
model('int_orders'),
model('fct_revenue'),
model('orders_export'),
model('rpt_pnl'),
model('summary')
],
{
changed: {
counts: ['definition'],
reach: 'job',
runs: [
{ node: 'model:customer_orders', reason: 'upstream model stg_orders runs', downstream_of: 'model:stg_orders' },
{ node: 'model:int_orders', reason: 'upstream model stg_orders runs', downstream_of: 'model:stg_orders' },
{ node: 'model:orders_export', reason: 'upstream model stg_orders runs', downstream_of: 'model:stg_orders' },
{ node: 'model:stg_orders', reason: 'planner change (query_changed)', downstream_of: null }
],
held_back: [
{ node: 'model:fct_revenue', refreshed_by: ['finance_daily'] },
{ node: 'model:rpt_pnl', refreshed_by: ['finance_daily'] },
{ node: 'model:summary', reads: 'model:fct_revenue' }
],
unchanged: ['model:stg_customers']
}
}
);

it('under --changed lists only what runs, then what waits for its job and what is skipped, as sqb plan does', () => {
const sections = [...planSections(reach), ...changedSections(reach.changed!, (job) => `job ${job}`)];
expect(sections.map((s) => [s.title, s.entries.map((e) => `${e.name}: ${e.value}${e.detail.length ? ` (${e.detail.join(', ')})` : ''}`)])).toEqual([
['Query changed', ['stg_orders: continue forward']],
['Models', ['customer_orders: table (reads stg_orders)', 'int_orders: table (reads stg_orders)', 'orders_export: table (reads stg_orders)']],
['Held back', ['fct_revenue: job finance_daily', 'rpt_pnl: job finance_daily', 'summary: reads fct_revenue, which is held back']],
['Unchanged', ['stg_customers: skipped by --changed']]
]);
expect(changedPolicyLine(reach.changed!)).toBe('Counting: definition (source data not counted) · reach: job');
expect(changedPolicyLine({ ...reach.changed!, counts: ['definition', 'source_data'], reach: 'all' })).toBe('Counting: definition, source data · reach: all');
expect(planSentence(reach)).toBe(
'4 models build: stg_orders changed since its last build and the other 3 models read a change upstream. 3 models wait for their job\'s own run. 1 unchanged node is skipped by --changed.'
);
});

it('names the models that run under --changed for their own reason, not as reading a change', () => {
const first = plan([model('orders'), model('order_count', { action: 'incremental_append', reason: 'normal_incremental' })], {
changed: {
counts: ['definition'],
reach: 'job',
runs: [
{ node: 'model:order_count', reason: 'no completed record', downstream_of: null },
{ node: 'model:orders', reason: 'no completed record', downstream_of: null }
],
held_back: [],
unchanged: []
}
});
expect(planSentence(first)).toBe('2 models build: orders and order_count run for their own reason.');
expect(planSections(first).find((s) => s.title === 'Models')?.entries.map((e) => e.detail.at(-1))).toEqual(['no completed record', 'no completed record']);
});

it("says what refreshes a held-back node: the named job's next run, its sensor, or that it is paused", () => {
const now = Date.parse('2026-01-01T00:00:00Z');
type Kind = 'schedule' | 'sensor';
const trigger = (jobKey: string, state: 'running' | 'paused', nextDueAt: string | null, type: Kind = 'schedule') => ({ jobKey, type, state, nextDueAt });
// A group's schedule says nothing about a job whose own trigger is paused; the plan names the group itself when it matters.
expect(jobRefreshText('finance_daily', [trigger('finance_daily', 'paused', '2026-01-01T05:00:00Z'), trigger('nightly', 'running', '2026-01-01T02:00:00Z')], now)).toBe('job finance_daily is paused');
expect(jobRefreshText('nightly', [trigger('nightly', 'running', '2026-01-01T02:00:00Z'), trigger('nightly', 'running', '2026-01-01T05:00:00Z')], now)).toBe('job nightly · next run in 2h');
expect(jobRefreshText('finance_daily', [trigger('finance_daily', 'paused', null), trigger('finance_daily', 'running', null, 'sensor')], now)).toBe('job finance_daily · when its sensor fires');
// A running schedule with no next run, such as a finished calendar, is not a sensor.
expect(jobRefreshText('finance_daily', [trigger('finance_daily', 'running', null)], now)).toBe('job finance_daily');
expect(jobRefreshText('crm', [trigger('finance_daily', 'running', '2026-01-01T05:00:00Z')], now)).toBe('job crm');
});
});

Loading
Loading