diff --git a/ui/src/lib/components/lineage/folder-node.svelte b/ui/src/lib/components/lineage/folder-node.svelte new file mode 100644 index 0000000000..c5d5520abd --- /dev/null +++ b/ui/src/lib/components/lineage/folder-node.svelte @@ -0,0 +1,99 @@ + + + +
+ +
+ + {d.group} + {d.count} + +
+
+ {#if d.ok}{d.ok} ok{/if} + {#if d.stale}{d.stale} stale{/if} + {#if d.failed}{d.failed} failed{/if} + {#if !d.ok && !d.stale && !d.failed}No builds{/if} +
+ +
+ + diff --git a/ui/src/lib/lineage/catalog-graph.test.ts b/ui/src/lib/lineage/catalog-graph.test.ts new file mode 100644 index 0000000000..5f89dd4c5a --- /dev/null +++ b/ui/src/lib/lineage/catalog-graph.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; +import type { CatalogNode } from '$lib/api'; +import { folderOf, groupedGraph } from './catalog-graph'; + +function node(key: string, path: string, upstream: string[] = [], extra: Partial = {}): CatalogNode { + const kind = key.startsWith('source:') ? 'source' : key.startsWith('seed:') ? 'seed' : 'table'; + return { key, kind, path, upstream, group: null, stale: [], lastOutcome: null, lastBuild: { builtAt: '2026-11-07T06:00:00Z' }, ...extra } as CatalogNode; +} + +describe('lineage folders', () => { + it('groups models by folder, sources by file, and lets a declared group win', () => { + expect([ + folderOf(node('model:dim_orders', 'models/marts/core/dim_orders.sql')), + folderOf(node('source:shop__orders', 'sources/shop.yml')), + folderOf(node('seed:countries', 'seeds/countries.csv')), + folderOf(node('model:orders', 'orders.sql')), + folderOf(node('model:stg_orders', 'models/staging/stg_orders.sql', [], { group: 'shop' })) + ]).toEqual(['marts/core', 'sources/shop', 'seeds', 'models', 'shop']); + }); + + const nodes = [ + node('source:shop__orders', 'sources/shop.yml'), + node('source:shop__customers', 'sources/shop.yml'), + node('model:stg_orders', 'models/staging/shop/stg_orders.sql', ['source:shop__orders']), + node('model:stg_customers', 'models/staging/crm/stg_customers.sql', ['source:shop__customers'], { stale: [{ reason: 'behind', upstream: [], detail: '', since: '' }] }), + node('model:orders', 'models/marts/orders.sql', ['model:stg_orders', 'model:stg_customers'], { lastOutcome: 'failed' }), + node('model:order_totals', 'models/marts/order_totals.sql', ['model:orders']) + ]; + + it('draws each closed top folder once, with one edge per folder pair and none inside a folder', () => { + const graph = groupedGraph(nodes, new Set()); + expect(graph.nodes.map((n) => [n.id, n.data.count])).toEqual([ + ['group:sources', 2], + ['group:staging', 2], + ['group:marts', 2] + ]); + expect(graph.edges.map((e) => e.id)).toEqual(['group:sources-group:staging', 'group:staging-group:marts']); + expect(graph.nodes.map((n) => [n.data.ok, n.data.stale, n.data.failed])).toEqual([[0, 0, 0], [1, 1, 0], [1, 0, 1]]); + }); + + it('opens a folder into the folders inside it, and those into their nodes, joined to the closed folders around them', () => { + expect(groupedGraph(nodes, new Set(['staging'])).nodes.map((n) => n.id)).toEqual(['group:sources', 'group:staging/shop', 'group:staging/crm', 'group:marts']); + const graph = groupedGraph(nodes, new Set(['staging', 'staging/shop'])); + expect(graph.nodes.map((n) => n.id)).toEqual(['group:sources', 'model:stg_orders', 'group:staging/crm', 'group:marts']); + expect(graph.edges.map((e) => e.id)).toEqual([ + 'group:sources-model:stg_orders', + 'group:sources-group:staging/crm', + 'model:stg_orders-group:marts', + 'group:staging/crm-group:marts' + ]); + }); + + it('draws every node once all folders around it are open', () => { + const graph = groupedGraph(nodes, new Set(['sources', 'sources/shop', 'staging', 'staging/shop', 'staging/crm', 'marts'])); + expect(graph.nodes.every((n) => n.type === 'asset')).toBe(true); + expect(graph.edges).toHaveLength(5); + }); +}); diff --git a/ui/src/lib/lineage/catalog-graph.ts b/ui/src/lib/lineage/catalog-graph.ts index 04ff84ddda..a6117bac33 100644 --- a/ui/src/lib/lineage/catalog-graph.ts +++ b/ui/src/lib/lineage/catalog-graph.ts @@ -57,3 +57,83 @@ export function catalogGraph(nodes: CatalogNode[]): { nodes: Node[]; edges: Edge edges: nodes.flatMap((n) => n.upstream.map((up) => ({ id: `${up}-${n.key}`, source: up, target: n.key, type: 'bezier' }))) }; } + +/** Above this many nodes the lineage opens with each folder collapsed into one card. */ +export const GROUPED_ABOVE = 150; + +/** + * The folder a node lives in, such as `marts/core`, or its source file, such as `sources/orders`; + * a declared group wins. + */ +export function folderOf(node: CatalogNode): string { + if (node.group) return node.group; + const path = node.path.replace(/^models\//, ''); + if (node.kind === 'source') return path.replace(/\.[^./]+$/, '') || 'sources'; + const slash = path.lastIndexOf('/'); + return slash > 0 ? path.slice(0, slash) : node.kind === 'seed' ? 'seeds' : 'models'; +} + +export type GroupCard = { group: string; count: number; ok: number; stale: number; failed: number; dimmed?: boolean }; + +const groupId = (group: string) => `group:${group}`; + +/** A folder's card: how many nodes it holds, and how many last built fine, are stale or failed. */ +function groupCard(group: string, members: CatalogNode[]): GroupCard { + const built = members.filter((n) => n.kind !== 'source'); + return { + group, + count: members.length, + ok: built.filter((n) => cardStatus(n) === 'ok' && !n.stale?.length).length, + stale: built.filter((n) => !!n.stale?.length).length, + failed: built.filter((n) => n.lastOutcome === 'failed').length + }; +} + +/** `expanded` with every folder around the node open, so the node itself is drawn. */ +export function withFoldersOpen(node: CatalogNode, expanded: ReadonlySet): Set { + const parts = folderOf(node).split('/'); + return new Set([...expanded, ...parts.map((_, i) => parts.slice(0, i + 1).join('/'))]); +} + +/** The outermost folder around a node that is not open, or null when every folder around it is. */ +function closedFolder(node: CatalogNode, expanded: ReadonlySet): string | null { + const parts = folderOf(node).split('/'); + for (let depth = 1; depth <= parts.length; depth++) { + const folder = parts.slice(0, depth).join('/'); + if (!expanded.has(folder)) return folder; + } + return null; +} + +/** + * The lineage with every folder that is not open drawn as one card. Folders nest, so opening + * `marts` shows `marts/core` and the other folders inside it. An edge joins whatever stands for + * each end, once per pair, and an edge inside a closed folder disappears. + */ +export function groupedGraph(nodes: CatalogNode[], expanded: ReadonlySet): { nodes: Node[]; edges: Edge[] } { + const standIn = new Map(); + const folders = new Map(); + const cards: Node[] = []; + for (const n of nodes) { + const folder = closedFolder(n, expanded); + standIn.set(n.key, folder === null ? n.key : groupId(folder)); + if (folder === null) cards.push({ id: n.key, type: 'asset', position: { x: 0, y: 0 }, data: cardData(n) }); + else if (folders.has(folder)) folders.get(folder)!.push(n); + else { + folders.set(folder, [n]); + cards.push({ id: groupId(folder), type: 'folder', position: { x: 0, y: 0 }, selectable: false, data: {} }); + } + } + for (const card of cards) if (card.type === 'folder') card.data = groupCard(card.id.slice('group:'.length), folders.get(card.id.slice('group:'.length))!); + const pairs = new Map(); + for (const n of nodes) { + for (const up of n.upstream) { + const source = standIn.get(up); + const target = standIn.get(n.key)!; + if (!source || source === target) continue; + const id = `${source}-${target}`; + if (!pairs.has(id)) pairs.set(id, { id, source, target, type: 'bezier' }); + } + } + return { nodes: cards, edges: [...pairs.values()] }; +} diff --git a/ui/src/lib/lineage/job-graph.test.ts b/ui/src/lib/lineage/job-graph.test.ts index e524ccceca..50c2d1dd20 100644 --- a/ui/src/lib/lineage/job-graph.test.ts +++ b/ui/src/lib/lineage/job-graph.test.ts @@ -72,6 +72,63 @@ describe('job graph', () => { expect(laid.find((n) => n.id === boxId('staging'))!.height).toBeLessThan(61); }); + it('wraps a tall column into several, keeping every node right of what it reads', () => { + // 40 marts with no lineage inside the job, and 30 models off one source: 71 cards, at most 15 to a column. + const wide = [ + ...Array.from({ length: 40 }, (_, i) => node(`model:mart_${i}`)), + node('model:src'), + ...Array.from({ length: 30 }, (_, i) => node(`model:stg_${i}`, ['model:src'])) + ]; + const { nodes, edges } = jobGraph(wide.map((n) => n.key), wide); + const laid = layoutJobGraph(measured(nodes), edges); + const columns = Map.groupBy(laid, (n) => Math.round(n.position.x)); + expect(Math.max(...[...columns.values()].map((c) => c.length))).toBeLessThanOrEqual(15); + // The wrapped columns sit side by side rather than staggered: the graph is no taller than one full column. + const ys = laid.map((n) => n.position.y); + expect(Math.max(...ys) - Math.min(...ys)).toBeLessThan(15 * (61 + 24)); + const x = (id: string) => laid.find((n) => n.id === id)!.position.x; + for (let i = 0; i < 30; i++) expect(x(`model:stg_${i}`)).toBeGreaterThan(x('model:src')); + }); + + it('leaves a graph that needs no wrapping as dagre lays it out', () => { + // `extra` only feeds the end of a long chain, so it sits just before it, not back at the start. + const chain = [node('model:a'), node('model:b', ['model:a']), node('model:c', ['model:b']), node('model:d', ['model:c']), node('model:extra'), node('model:end', ['model:d', 'model:extra'])]; + const { nodes, edges } = jobGraph(chain.map((n) => n.key), chain); + const x = new Map(layoutJobGraph(measured(nodes), edges).map((n) => [n.id, n.position.x])); + expect(x.get('model:extra')).toBe(x.get('model:d')); + }); + + it('keeps every box right of the boxes it runs after when their columns wrap', () => { + const many = (job: string) => Array.from({ length: 30 }, (_, i) => node(`model:${job}_${i}`)); + const wide = [...many('a'), ...many('b')]; + const groupBoxes = [ + { jobKey: 'a', nodes: wide.slice(0, 30).map((n) => n.key) }, + { jobKey: 'b', nodes: wide.slice(30).map((n) => n.key) }, + { jobKey: 'empty', nodes: [] } + ]; + const after = (jobKey: string) => ({ jobKey, upstream: [{ jobKey: 'a', source: 'after' as const }], stage: 1 }); + const { nodes, edges } = jobGraph(wide.map((n) => n.key), wide, groupBoxes, [{ jobKey: 'a', upstream: [], stage: 0 }, after('b'), after('empty')]); + const laid = layoutJobGraph(measured(nodes), edges); + const box = (k: string) => laid.find((n) => n.id === boxId(k))!; + expect(box('a').height).toBeLessThan(15 * (61 + 24) + 60); + expect(box('b').position.x).toBeGreaterThan(box('a').position.x + box('a').width!); + expect(box('empty').position.x).toBeGreaterThan(box('a').position.x + box('a').width!); + }); + + it('keeps lineage left to right when it leaves a box and comes back', () => { + // `broad` builds raw → stg → mart, but `narrow` builds only stg, so stg sits in narrow's box. + const chain = [node('model:raw'), node('model:stg', ['model:raw']), node('model:mart', ['model:stg'])]; + const nested = [{ jobKey: 'broad', nodes: chain.map((n) => n.key) }, { jobKey: 'narrow', nodes: ['model:stg'] }]; + const { nodes, edges } = jobGraph(chain.map((n) => n.key), chain, nested); + const laid = layoutJobGraph(measured(nodes), edges); + const x = (id: string) => { + const card = laid.find((n) => n.id === id)!; + return card.position.x + (laid.find((n) => n.id === card.parentId)?.position.x ?? 0); + }; + expect(x('model:raw')).toBeLessThan(x('model:stg')); + expect(x('model:stg')).toBeLessThan(x('model:mart')); + }); + it('draws a plain job without boxes', () => { const { nodes } = jobGraph(['model:raw', 'model:stg'], catalog); expect(nodes.map((n) => [n.id, n.type, n.parentId])).toEqual([ diff --git a/ui/src/lib/lineage/job-graph.ts b/ui/src/lib/lineage/job-graph.ts index 1cc3f9c5b4..4917d59c7d 100644 --- a/ui/src/lib/lineage/job-graph.ts +++ b/ui/src/lib/lineage/job-graph.ts @@ -16,6 +16,12 @@ const EMPTY_HEIGHT = 44; const COLLAPSED_HEIGHT = HEADER + 2; /** Wide enough for a header's name, node count and last run on one line. */ const COLLAPSED_WIDTH = 380; +/** Space between cards stacked in one column. */ +const STACK_GAP = 20; +/** Space dagre leaves between cards, and between boxes and cards outside them. */ +const CARD_GAP = 24; +const BOX_GAP = 24 + HEADER; +const MARGIN = 24; export type JobBox = { jobKey: string; nodes: string[]; lastRun?: RunRef | null }; @@ -102,32 +108,180 @@ export function jobGraph( return { nodes: [...boxNodes, ...cards], edges: [...lineage, ...after] }; } +/** The most cards a column holds before it wraps, growing with the graph so its shape stays near the canvas's. */ +export function maxColumnCards(count: number): number { + return Math.max(8, Math.round(Math.sqrt(count * 3))); +} + +type Item = { id: string; width: number; height: number }; +type Rect = { x: number; y: number; width: number; height: number }; + /** - * Lays the graph out left to right. Cards use their measured size; boxes are sized to wrap their - * cards, and child positions are relative to their box, as Svelte Flow expects. + * Dagre stacks every node of a rank in one column, so a job whose nodes hang off a few parents, or + * have no lineage inside the job, draws as one column too tall to read. This gives each node a rank + * that spreads such a column over several: the edges into a node get the rank gap as their minimum + * length, and a moved node with no edge in hangs off the column's first node by `holds`, edges that + * are laid out but not drawn. */ -export function layoutJobGraph(nodes: Node[], edges: Edge[]): Node[] { - const g = new dagre.graphlib.Graph({ compound: true }); - g.setGraph({ rankdir: 'LR', ranksep: 64, nodesep: 24 + HEADER, marginx: 24, marginy: 24 }); +function wrapColumns(ids: string[], pairs: ReadonlyArray) { + const preds = new Map(); + for (const [s, t] of pairs) preds.set(t, [...(preds.get(t) ?? []), s]); + // Each node's rank before wrapping: the longest path to it. + const depth = new Map(); + const depthOf = (id: string): number => { + if (depth.has(id)) return depth.get(id)!; + depth.set(id, 0); // A cycle ends here rather than looping; dagre breaks it for the drawing. + const d = Math.max(-1, ...(preds.get(id) ?? []).map(depthOf)) + 1; + depth.set(id, d); + return d; + }; + const columns = new Map(); + for (const id of ids) columns.set(depthOf(id), [...(columns.get(depthOf(id)) ?? []), id]); + + const max = maxColumnCards(ids.length); + const rank = new Map(); + const holds: [string, string][] = []; + let wrapped = false; + const first = (id: string) => [...(preds.get(id) ?? [])].sort()[0] ?? ''; + for (const [, column] of [...columns].sort(([a], [b]) => a - b)) { + // Siblings stay together, so a parent's children fill a column before the next parent's. + column.sort((a, b) => first(a).localeCompare(first(b)) || a.localeCompare(b)); + const base = Math.max(0, ...column.flatMap((id) => (preds.get(id) ?? []).map((p) => (rank.get(p) ?? 0) + 1))); + const per = Math.ceil(column.length / Math.ceil(column.length / max)); + wrapped ||= per < column.length; + column.forEach((id, i) => { + const step = Math.floor(i / per); + rank.set(id, base + step); + if (step && !preds.has(id)) holds.push([column[0], id]); + }); + } + return { rank, holds, wrapped }; +} + +/** + * Lays `items` out left to right from the origin, wrapping tall columns. Dagre staggers the columns + * a tall one wrapped into, lining each up with what it reads, so a graph that wrapped then stacks + * every column from the top: a column keeps dagre's order, and so its edge crossings. A graph that + * didn't wrap keeps dagre's layout. + */ +function layoutLayer(items: Item[], pairs: ReadonlyArray, nodesep: number): Map { + const g = new dagre.graphlib.Graph(); + g.setGraph({ rankdir: 'LR', ranksep: 64, nodesep, marginx: 0, marginy: 0 }); g.setDefaultEdgeLabel(() => ({})); + for (const item of items) g.setNode(item.id, { width: item.width, height: item.height }); + const { rank, holds, wrapped } = wrapColumns(items.map((i) => i.id), pairs); + for (const [s, t] of pairs) g.setEdge(s, t, { minlen: wrapped ? Math.max(1, rank.get(t)! - rank.get(s)!) : 1 }); + for (const [s, t] of holds) if (!g.hasEdge(s, t)) g.setEdge(s, t, { minlen: rank.get(t)! - rank.get(s)! }); + dagre.layout(g); + const rects = new Map(items.map((i) => { + const p = g.node(i.id); + return [i.id, { x: p.x - i.width / 2, y: p.y - i.height / 2, width: i.width, height: i.height }]; + })); + if (wrapped) { + const top = Math.min(...[...rects.values()].map((r) => r.y)); + const stacks = Map.groupBy(rects.values(), (r) => Math.round(r.x + r.width / 2)); + for (const stack of stacks.values()) { + let y = top; + for (const r of stack.sort((a, b) => a.y - b.y)) { + r.y = y; + y += r.height + STACK_GAP; + } + } + } + const left = Math.min(...[...rects.values()].map((r) => r.x)); + const top = Math.min(...[...rects.values()].map((r) => r.y)); + for (const r of rects.values()) { + r.x -= left; + r.y -= top; + } + return rects; +} + +/** + * Lays the graph out left to right: first each box's cards, which sizes the box, then the boxes and + * the cards outside any box, ordered by the lineage and `after` edges between them. Cards use their + * measured size, and a card's position is relative to its box, as Svelte Flow expects. + */ +export function layoutJobGraph(nodes: Node[], edges: Edge[]): Node[] { + if (!nodes.length) return []; const size = (n: Node) => ({ width: n.measured?.width ?? NODE_WIDTH, height: n.measured?.height ?? NODE_HEIGHT }); - const children = new Map(); - for (const n of nodes) if (n.parentId) children.set(n.parentId, [...(children.get(n.parentId) ?? []), n]); + const children = Map.groupBy(nodes.filter((n) => n.parentId), (n) => n.parentId!); + const pairsAmong = (ids: Set, lift: (id: string) => string) => { + const seen = new Set(); + return edges.flatMap((e) => { + const [s, t] = [lift(e.source), lift(e.target)]; + if (s === t || !ids.has(s) || !ids.has(t) || seen.has(`${s}>${t}`)) return []; + seen.add(`${s}>${t}`); + return [[s, t] as const]; + }); + }; + const parentOf = new Map(nodes.map((n) => [n.id, n.parentId ?? n.id])); + const topPairs = pairsAmong(new Set(nodes.filter((n) => !n.parentId).map((n) => n.id)), (id) => parentOf.get(id) ?? id); + // Lineage that leaves a box and comes back, as when a narrow job builds a node in the middle of a + // broad job's chain, can't be drawn box by box without an edge running backwards. + if (loops(topPairs)) return layoutCompound(nodes, edges, size, children); + const inside = new Map>(); + const outer: Item[] = []; for (const n of nodes) { - if (n.type === 'job') { - // An empty box is laid out as a plain node; dagre sizes boxes with children itself. - if (children.has(n.id)) g.setNode(n.id, {}); - else g.setNode(n.id, { width: n.width ?? NODE_WIDTH + 2 * PAD, height: n.height ?? EMPTY_HEIGHT }); - } else g.setNode(n.id, size(n)); + if (n.parentId) continue; + const kids = children.get(n.id); + if (!kids) { + outer.push(n.type === 'job' ? { id: n.id, width: n.width ?? NODE_WIDTH + 2 * PAD, height: n.height ?? EMPTY_HEIGHT } : { id: n.id, ...size(n) }); + continue; + } + const rects = layoutLayer(kids.map((k) => ({ id: k.id, ...size(k) })), pairsAmong(new Set(kids.map((k) => k.id)), (id) => id), CARD_GAP); + inside.set(n.id, rects); + const right = Math.max(...[...rects.values()].map((r) => r.x + r.width)); + const bottom = Math.max(...[...rects.values()].map((r) => r.y + r.height)); + outer.push({ id: n.id, width: right + 2 * PAD, height: bottom + 2 * PAD + HEADER }); } - for (const n of nodes) if (n.parentId) g.setParent(n.id, n.parentId); + const placed = layoutLayer(outer, topPairs, BOX_GAP); - // Dagre can't route edges to a box with children, so an `after` edge orders the boxes through their first cards. + return nodes.map((n) => { + if (n.parentId) { + const r = inside.get(n.parentId)!.get(n.id)!; + return { ...n, position: { x: PAD + r.x, y: PAD + HEADER + r.y } }; + } + const r = placed.get(n.id)!; + const position = { x: MARGIN + r.x, y: MARGIN + r.y }; + return n.type === 'job' ? { ...n, position, width: r.width, height: r.height } : { ...n, position }; + }); +} + +/** Whether the edges form a loop. */ +function loops(pairs: ReadonlyArray): boolean { + const next = Map.groupBy(pairs, ([s]) => s); + const state = new Map(); + const visit = (id: string): boolean => { + if (state.get(id) === 'done') return false; + if (state.get(id) === 'open') return true; + state.set(id, 'open'); + const found = (next.get(id) ?? []).some(([, t]) => visit(t)); + state.set(id, 'done'); + return found; + }; + return [...next.keys()].some(visit); +} + +/** + * Lays the whole graph out at once, boxes and cards together, so lineage between boxes always runs + * left to right. Columns don't wrap, so a large job draws taller. + */ +function layoutCompound(nodes: Node[], edges: Edge[], size: (n: Node) => { width: number; height: number }, children: Map): Node[] { + const g = new dagre.graphlib.Graph({ compound: true }); + g.setGraph({ rankdir: 'LR', ranksep: 64, nodesep: BOX_GAP, marginx: MARGIN, marginy: MARGIN }); + g.setDefaultEdgeLabel(() => ({})); + for (const n of nodes) { + if (n.type === 'job' && children.has(n.id)) g.setNode(n.id, {}); + else if (n.type === 'job') g.setNode(n.id, { width: n.width ?? NODE_WIDTH + 2 * PAD, height: n.height ?? EMPTY_HEIGHT }); + else g.setNode(n.id, size(n)); + } + for (const n of nodes) if (n.parentId) g.setParent(n.id, n.parentId); + // Dagre can't route edges to a box with children, so an edge to a box goes to its first card. const anchor = (id: string) => children.get(id)?.[0]?.id ?? id; for (const e of edges) g.setEdge(anchor(e.source), anchor(e.target)); - dagre.layout(g); const topLeft = (n: Node) => { @@ -135,7 +289,7 @@ export function layoutJobGraph(nodes: Node[], edges: Edge[]): Node[] { const { width, height } = size(n); return { x: p.x - width / 2, y: p.y - height / 2, width, height }; }; - const boxes = new Map(); + const boxes = new Map(); for (const n of nodes) { if (n.type !== 'job') continue; const kids = (children.get(n.id) ?? []).map(topLeft); @@ -152,7 +306,6 @@ export function layoutJobGraph(nodes: Node[], edges: Edge[]): Node[] { const bottom = Math.max(...kids.map((k) => k.y + k.height)) + PAD; boxes.set(n.id, { x: left, y: top, width: right - left, height: bottom - top }); } - return nodes.map((n) => { const box = boxes.get(n.id); if (box) return { ...n, position: { x: box.x, y: box.y }, width: box.width, height: box.height }; diff --git a/ui/src/lib/run/plan-text.test.ts b/ui/src/lib/run/plan-text.test.ts index dd73c21e65..5198692770 100644 --- a/ui/src/lib/run/plan-text.test.ts +++ b/ui/src/lib/run/plan-text.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest'; import type { PlanDocument, PlanModel } from '$lib/api'; -import { actionText, planSections, readySummary } from './plan-text'; +import { actionText, planSections, planSentence } from './plan-text'; const model = (name: string, over: Partial = {}): PlanModel => ({ name, @@ -73,6 +73,29 @@ describe('plan text, as sqb plan prints it', () => { it('collapses a plan that is all full refresh into counts by materialization', () => { const full = plan([model('a', { reason: 'full_refresh' }), model('b', { reason: 'full_refresh' }), model('v', { reason: 'full_refresh', materialization_type: 'view' })]); expect(planSections(full)).toEqual([{ title: 'Full refresh', changed: true, entries: [{ name: 'table', value: '2 models', detail: [] }, { name: 'view', value: '1 model', detail: [] }] }]); - expect(readySummary(full)).toBe('full refresh, 3 selected'); + expect(planSentence(full)).toBe('3 models build from scratch, replacing what was built.'); + }); +}); + +describe('plan sentence', () => { + const incremental = { reason: 'normal_incremental', materialization_type: 'incremental' } as const; + + it('names a changed model, counts the ones taking new rows, and calls the rest usual', () => { + const p = plan([model('orders', { reason: 'query_changed' }), model('events', incremental), model('visits', incremental), model('a'), model('b'), model('c', { action: 'skip' })]); + expect(planSentence(p)).toBe('5 models build: orders changed since its last build, 2 incremental models take new rows and the other 2 are recreated as usual. 1 disabled model is skipped.'); + }); + + it('counts many changes, says first builds apart, and says when nothing changed', () => { + const changed = ['a', 'b', 'c'].map((n) => model(n, { reason: 'config_changed' })); + expect(planSentence(plan([...changed, model('new', { reason: 'first_run' })]))).toBe('4 models build: 3 models changed since their last build and new builds for the first time.'); + expect(planSentence(plan([model('a'), model('b')], { source_loads: [{ name: 'raw', kind: 'source', is_reload: false }] }))).toBe('2 models build: none changed since their last build. 1 source loads first.'); + expect(planSentence(plan([model('a', { action: 'skip', reason: 'disabled' })]))).toBe('Nothing to build. 1 disabled model is skipped.'); + }); + + it('says what runs when no model builds', () => { + const seeds = [{ name: 'carriers', reason: 'first_run' }, { name: 'regions', reason: 'no_change' }]; + expect(planSentence(plan([], { seeds }))).toBe('No models build. 1 seed loads.'); + const functions = [{ name: 'margin', language: 'sql', reason: 'definition_changed' }, { name: 'tax', language: 'sql', reason: 'no_change' }]; + expect(planSentence(plan([], { functions }))).toBe('No models build. 1 changed function is recreated.'); }); }); diff --git a/ui/src/lib/run/plan-text.ts b/ui/src/lib/run/plan-text.ts index 5ebd9bd18c..f999908708 100644 --- a/ui/src/lib/run/plan-text.ts +++ b/ui/src/lib/run/plan-text.ts @@ -132,12 +132,43 @@ export function planSections(plan: PlanDocument): PlanSection[] { return sections; } -/** `N selected`, as the CLI's `Plan ready` line counts it. */ -export function readySummary(plan: PlanDocument): string { - const parts = [isFullRefresh(plan) ? 'full refresh' : '', `${plan.selected_count} selected`]; - const loads = plan.source_loads.length; - if (loads) parts.push(`${loads} source${loads === 1 ? '' : 's'} to load`); - return parts.filter(Boolean).join(', '); +const count = (n: number, one: string, many = `${one}s`) => `${n} ${n === 1 ? one : many}`; + +/** Names a few models, or counts them when there are more than two. */ +const few = (models: PlanModel[], noun: string) => (models.length <= 2 ? models.map((m) => m.name).join(' and ') : count(models.length, noun)); + +/** + * What the run does: how many models build, which changed since their last build, which build for the + * first time, which take new rows and that the rest are recreated as usual, then what else happens. + */ +export function planSentence(plan: PlanDocument): string { + const active = plan.models.filter((m) => m.action !== 'skip'); + // The planner skips only disabled models; seeds it reports as current don't load. + const skipped = plan.models.length - active.length; + const seeds = plan.seeds.filter((s) => s.reason !== 'no_change').length; + const functions = plan.functions.filter((f) => f.reason && f.reason !== 'no_change').length; + const runs = [ + plan.source_loads.length && count(plan.source_loads.length, 'source loads first', 'sources load first'), + seeds && count(seeds, 'seed loads', 'seeds load'), + functions && count(functions, 'changed function is recreated', 'changed functions are recreated'), + plan.python_nodes.length && count(plan.python_nodes.length, 'Python node runs', 'Python nodes run') + ].filter((p): p is string => !!p); + const after = [...runs, ...(skipped ? [count(skipped, 'disabled model is skipped', 'disabled models are skipped')] : [])].map((p) => ` ${p}.`).join(''); + if (!active.length) return `${runs.length ? 'No models build.' : 'Nothing to build.'}${after}`; + const builds = count(active.length, 'model builds', 'models build'); + if (isFullRefresh(plan)) return `${builds} from scratch, replacing what was built.${after}`; + const changed = active.filter((m) => !ROUTINE.has(m.reason) && m.reason !== 'first_run'); + const fresh = active.filter((m) => m.reason === 'first_run'); + const rows = active.filter((m) => m.reason === 'normal_incremental'); + const usual = active.filter((m) => m.reason === 'no_change'); + const parts = [ + changed.length && `${few(changed, 'model')} changed since ${changed.length === 1 ? 'its' : 'their'} last build`, + fresh.length && `${few(fresh, 'model')} ${fresh.length === 1 ? 'builds' : 'build'} for the first time`, + rows.length && `${count(rows.length, 'incremental model takes', 'incremental models take')} new rows` + ].filter((p): p is string => !!p); + if (usual.length) parts.push(parts.length ? `the other ${usual.length} ${usual.length === 1 ? 'is' : 'are'} recreated as usual` : `none changed since ${usual.length === 1 ? 'its' : 'their'} last build`); + const detail = parts.length > 1 ? `${parts.slice(0, -1).join(', ')} and ${parts.at(-1)}` : parts[0]; + return `${builds}: ${detail}.${after}`; } /** The models a plan changes, as catalog keys, for drawing just the part of a large selection that matters. */ diff --git a/ui/src/routes/lineage/+page.svelte b/ui/src/routes/lineage/+page.svelte index f873d077ce..648d7082d0 100644 --- a/ui/src/routes/lineage/+page.svelte +++ b/ui/src/routes/lineage/+page.svelte @@ -5,6 +5,8 @@ import '@xyflow/svelte/dist/style.css'; import { SvelteFlow, Background, Controls, type Node, type Edge } from '@xyflow/svelte'; import AssetNode from '$lib/components/lineage/asset-node.svelte'; + import FolderNode from '$lib/components/lineage/folder-node.svelte'; + import FolderIcon from '@lucide/svelte/icons/folder'; import NodeFieldsPopover from '$lib/components/lineage/node-fields-popover.svelte'; import AppTopbar from '$lib/components/app-topbar.svelte'; import Button from '$lib/components/ui/button/button.svelte'; @@ -43,14 +45,14 @@ import { formatAgo, formatStamp } from '$lib/format/time'; import { checkOutcomeTone, nodeOutcomeLabel, nodeOutcomeTone, staleReasonDetail, staleReasonLabel } from '$lib/status'; import { NODE_KINDS, formatCredits, formatMs, formatNumber, kindLine, nodeKindPlural, relationName, runScopePhaseLabel, shortName, shortRunId, sqlVersions } from '$lib/build-area/labels'; - import { catalogGraph, cardData } from '$lib/lineage/catalog-graph'; + import { GROUPED_ABOVE, catalogGraph, cardData, cardStatus, folderOf, groupedGraph, withFoldersOpen } from '$lib/lineage/catalog-graph'; let browserOpen = $state(false); let inspectorOpen = $state(true); let neighbourhoodOnly = $state(false); let modelQuery = $state(''); - const nodeTypes = { asset: AssetNode }; + const nodeTypes = { asset: AssetNode, folder: FolderNode }; // Automatic fits stop at a zoom where cards stay readable; the fit button and the wheel go further, to the whole graph. const FIT_FLOOR = 0.3; @@ -66,6 +68,10 @@ // paint before the cards are created and laid out, which takes a moment on a large project. let building = false; let fitted = $state(false); + // A large project opens with each folder as one card; opening a folder draws its nodes. + let grouped = $state(false); + let expanded = $state>(new Set()); + const graphOf = (data: CatalogNode[]) => (grouped ? groupedGraph(data, expanded) : catalogGraph(data)); $effect(() => { const data = catalog.data; if (!data || scaleMode) return; @@ -78,22 +84,43 @@ building = false; const latest = catalog.data; if (!latest || nodes.length) return; - const graph = catalogGraph(latest); - const focus = latest.some((n) => n.key === initialFocus) ? initialFocus : null; - nodes = graph.nodes.map((n) => (n.id === focus ? { ...n, selected: true } : n)); + const focus = latest.find((n) => n.key === initialFocus) ?? null; + grouped = latest.length > GROUPED_ABOVE; + if (focus) expanded = withFoldersOpen(focus, expanded); + const graph = graphOf(latest); + nodes = graph.nodes.map((n) => (n.id === focus?.key ? { ...n, selected: true } : n)); edges = graph.edges; }) ); return; } - const byKey = new Map(data.map((n) => [n.key, n])); + const fresh = new Map(graphOf(data).nodes.map((n) => [n.id, n.data])); nodes = nodes.map((n) => { - const fresh = byKey.get(n.id); - return fresh ? { ...n, data: { ...cardData(fresh), dimmed: n.data.dimmed } } : n; + const card = fresh.get(n.id); + return card ? { ...n, data: { ...card, dimmed: n.data.dimmed } } : n; }); }); }); + /** Redraws the graph after folders open or close, keeping the selection; the layout effect then places it. */ + function regroup(next: { grouped?: boolean; expanded?: ReadonlySet }) { + const data = catalog.data; + if (!data) return; + grouped = next.grouped ?? grouped; + expanded = next.expanded ?? expanded; + const selected = new Set(nodes.filter((n) => n.selected).map((n) => n.id)); + const graph = graphOf(data); + neighbourhoodOnly = false; + // The card under the pointer may be gone, and it never reports the pointer leaving. + hoveredId = null; + lastLayoutKey = ''; + nodes = graph.nodes.map((n) => (selected.has(n.id) ? { ...n, selected: true } : n)); + edges = graph.edges; + } + function openFolder(group: string) { + regroup({ expanded: new Set([...expanded, group]) }); + } + // Scale testing: /lineage?models=2000 swaps in a synthetic warehouse-shaped graph. // Positions are precomputed (stand-in for `sqb compile` output) instead of // running dagre in the browser, only on-screen cards are rendered, and when @@ -290,8 +317,11 @@ let statusFilter = $state<'' | 'stale' | 'changed' | 'behind' | 'failed'>(''); const filterActive = $derived(!!(kindFilter || groupFilter || ownerFilter || statusFilter)); function matchesFilter(n: Node): boolean { + if (n.type === 'folder') return catalogNodes.some((c) => `${folderOf(c)}/`.startsWith(`${n.data.group}/`) && matchesCatalog(c)); const c = catalogByKey.get(n.id); - if (!c) return !filterActive; + return c ? matchesCatalog(c) : !filterActive; + } + function matchesCatalog(c: CatalogNode): boolean { if (kindFilter && c.kind !== kindFilter) return false; if (groupFilter && c.group !== groupFilter) return false; if (ownerFilter && !c.owner.includes(ownerFilter)) return false; @@ -315,15 +345,20 @@ { value: 'failed' as const, label: 'Last build failed' } ]; - const groupOf = (node: Node) => String(node.data.group ?? (String(node.data.kind).startsWith('source') ? 'sources' : 'models')); + const groupOf = (node: Pick) => { + const listed = catalogByKey.get(node.id); + return listed ? folderOf(listed) : String(node.data.group ?? (String(node.data.kind).startsWith('source') ? 'sources' : 'models')); + }; let collapsedGroups = $state>(new Set()); function toggleGroup(g: string) { const next = new Set(collapsedGroups); next.has(g) ? next.delete(g) : next.add(g); collapsedGroups = next; } + // The list names every node, including those inside closed folders. + const listNodes = $derived[]>(scaleMode ? nodes : catalogNodes.map((n) => ({ id: n.key, data: cardData(n) }))); const browserNodes = $derived( - [...nodes] + [...listNodes] .filter((node) => String(node.data.name ?? '').toLowerCase().includes(modelQuery.trim().toLowerCase())) .sort((left, right) => String(left.data.name).localeCompare(String(right.data.name))) ); @@ -334,10 +369,19 @@ .filter((g) => g.nodes.length) ); - // immediate upstream/downstream of the selected model (for the Lineage tab) + // immediate upstream/downstream of the selected model (for the Lineage tab); with folders drawn, + // from the catalog, as an edge to a closed folder doesn't name the node it reaches const nameOf = (id: string) => nodes.find((n) => n.id === id)?.data.name ?? id; - const upstream = $derived(primary ? edges.filter((e) => e.target === primary.id).map((e) => nameOf(e.source)) : []); - const downstream = $derived(primary ? edges.filter((e) => e.source === primary.id).map((e) => nameOf(e.target)) : []); + const upstream = $derived( + !primary ? [] : grouped ? (catalogByKey.get(primary.id)?.upstream ?? []) : edges.filter((e) => e.target === primary.id).map((e) => nameOf(e.source)) + ); + const downstream = $derived( + !primary + ? [] + : grouped + ? catalogNodes.filter((n) => n.upstream.includes(primary.id)).map((n) => n.key) + : edges.filter((e) => e.source === primary.id).map((e) => nameOf(e.target)) + ); let sqlMode = $state<'raw' | 'compiled'>('raw'); const sqlRaw = $derived(d?.sql?.raw ?? ''); @@ -363,7 +407,8 @@ sqlOpen = true; } function selectByName(name: string) { - const id = nodes.find((node) => node.data.name === name)?.id; + // A node inside a closed folder isn't drawn; selecting it opens the folders around it. + const id = nodes.find((node) => node.data.name === name)?.id ?? (grouped && catalogByKey.has(name) ? name : undefined); if (id) selectModel(id); } @@ -376,6 +421,8 @@ if (g && setFlowViewport) setFlowViewport({ zoom: 1, x: panelSize.w / 2 - (g.x + 118), y: panelSize.h / 2 - (g.y + 30) }); return; } + const listed = catalogByKey.get(id); + if (grouped && listed && !nodes.some((n) => n.id === id)) regroup({ expanded: withFoldersOpen(listed, expanded) }); nodes = nodes.map((node) => ({ ...node, hidden: false, selected: node.id === id })); edges = edges.map((edge) => ({ ...edge, hidden: false })); await tick(); @@ -493,6 +540,7 @@ fitScaleGraph(); return; } + if (grouped && expanded.size) regroup({ expanded: new Set() }); nodes = nodes.map((node) => ({ ...node, hidden: false })); edges = edges.map((edge) => ({ ...edge, hidden: false })); nodes = layoutGraph(nodes, edges); @@ -547,6 +595,16 @@ {/if} + {#if !scaleMode} + + {/if}
{#each [['ok', 'var(--sb-success)', 'last build succeeded'], ['warn', 'var(--sb-warning)', 'last build warned'], ['stale', 'var(--sb-stale)', 'stale'], ['fail', 'var(--sb-error)', 'last build failed']] as [st, c, label] (st)} - {@const n = nodes.filter((x) => (st === 'stale' ? !!x.data.stale : x.data.status === st)).length} + {@const n = scaleMode ? nodes.filter((x) => (st === 'stale' ? !!x.data.stale : x.data.status === st)).length : catalogNodes.filter((c) => (st === 'stale' ? !!c.stale?.length : cardStatus(c) === st)).length} {#if n}{n}{/if} {/each}
@@ -622,6 +680,7 @@ multiSelectionKey={['Shift', 'Meta', 'Control']} selectionKey="Shift" proOptions={{ hideAttribution: true }} + onnodeclick={({ node }) => node.type === 'folder' && openFolder(String(node.data.group))} onnodepointerenter={({ node }) => (hoveredId = node.id)} onnodepointerleave={() => (hoveredId = null)} > diff --git a/ui/src/routes/plan/+page.svelte b/ui/src/routes/plan/+page.svelte index 92bf838f3b..7702efb5bc 100644 --- a/ui/src/routes/plan/+page.svelte +++ b/ui/src/routes/plan/+page.svelte @@ -14,17 +14,19 @@ import { kindLine } from '$lib/build-area/labels'; import { formatAgo } from '$lib/format/time'; import ScopeGraph from '$lib/lineage/scope-graph.svelte'; - import { PERMISSIONS } from '$lib/platform/permission-catalog'; import { VERBS } from '$lib/run/command-table'; import { applySuggestion, commandTokens, jobBuildTokens, jobRunProblem, parseCommand, settledJobCommand, suggest, windowOf, withFullRefresh, withVerb, withWindow, type Suggestion } from '$lib/run/command'; import DataRange from '$lib/run/data-range.svelte'; import PlanReport from '$lib/run/plan-report.svelte'; - import { changedModelKeys, readySummary } from '$lib/run/plan-text'; + import StatusLabel from '$lib/components/list/status-label.svelte'; + import { checkOutcomeLabel, checkOutcomeTone } from '$lib/status'; + import { changedModelKeys, planSentence } from '$lib/run/plan-text'; import { app } from '$lib/state/app.svelte'; import { createResource } from '$lib/state/resource.svelte'; const snapshotNodes = createResource((client, scope) => client.listSnapshotNodes(scope), ['snapshot']); const catalog = createResource((client, scope) => client.listCatalogNodes(scope), ['snapshot', 'runs', 'node_status']); + const checkList = createResource((client, scope) => client.listChecks(scope), ['checks']); let cmd = $state(page.url.searchParams.get('cmd') ?? 'sqb build'); let cmdEl = $state(); @@ -139,7 +141,15 @@ // ---- what a non-build verb picks, from the catalog ---- const byKey = $derived(new Map((catalog.data ?? []).map((n) => [n.key, n]))); const picked = $derived(shown?.selection ?? []); - let view = $state<'plan' | 'graph'>('plan'); + let view = $state<'plan' | 'graph' | 'checks'>('plan'); + /** The recorded checks on the models and seeds the plan builds, which run after them. */ + const planChecks = $derived.by(() => { + if (!plan) return []; + const built = new Set([...plan.models.filter((m) => m.action !== 'skip').map((m) => m.name), ...plan.seeds.map((s) => s.name)]); + return (checkList.data ?? []).filter((c) => (c.attachment === 'model' || c.attachment === 'seed') && !!c.node && built.has(c.node.slice(c.node.indexOf(':') + 1))); + }); + /** The tab shown: a replan with no checks leaves the Checks tab for the plan. */ + const tab = $derived(view === 'checks' && !(shown?.verb === 'build' && planChecks.length) ? 'plan' : view); /** A large selection draws only what the plan changes; the whole of it would be unreadable. */ const GRAPH_LIMIT = 150; const graphKeys = $derived(picked.length > GRAPH_LIMIT && plan ? changedModelKeys(plan) : picked); @@ -306,26 +316,42 @@ {#if shown?.verb}
{#if shown.verb === 'build'} - {#if plan}{readySummary(plan)}{/if} + {#if plan}{planSentence(plan)}{/if} {:else} {picked.length} node{picked.length === 1 ? '' : 's'}{#if shown.writeSet.length}· updates {shown.writeSet.length}{/if} {/if} + {#each Object.entries(shown.settings) as [name, value] (name)}{value === true ? name : `${name}=${value}`}{/each}
- - + + + {#if shown.verb === 'build' && planChecks.length}{/if}
{#if shown.verb === 'build' && plan} {/if}
{/if} - {#if view === 'graph' && shown?.verb && catalog.data && (shown.verb !== 'build' || plan)} + {#if tab === 'graph' && shown?.verb && catalog.data && (shown.verb !== 'build' || plan)} {#if graphKeys.length < picked.length}

{picked.length} nodes are too many to draw, so the graph shows only the {graphKeys.length === 1 ? 'model' : `${graphKeys.length} models`} this plan changes.

{/if} {#if graphKeys.length}
{}} />
{/if} + {:else if tab === 'checks' && shown?.verb === 'build' && plan} +

These checks run after the models they check, and a failing error check stops what depends on it.

+ + + {#each planChecks as c (c.id)} + + + + + + + {/each} + +
{c.name}{c.node ? c.node.slice(c.node.indexOf(':') + 1) : ''}{c.column ? `.${c.column}` : ''}{c.severity}{#if c.latest}{/if}
{:else if shown?.verb === 'build' && plan} {:else if shown?.verb && shown.verb !== 'build'} @@ -357,25 +383,6 @@ {/if} - -
@@ -523,8 +530,7 @@ padding: 14px 24px; transition: opacity 0.15s; } - .main.stale, - .rail.stale { + .main.stale { opacity: 0.55; } .problem-bar { @@ -580,6 +586,16 @@ color: var(--foreground); font-weight: 600; } + .sentence { + flex-basis: 100%; + margin-bottom: 6px; + color: var(--foreground); + font-size: 13px; + text-wrap: pretty; + } + .sentence + .tabs { + margin-left: 0; + } .summary span { color: var(--sb-text-faint); } @@ -630,38 +646,6 @@ .right { text-align: right; } - .rail { - width: 280px; - flex-shrink: 0; - overflow: auto; - padding: 14px 18px; - border-left: 1px solid var(--border); - } - .rail h3 { - margin: 16px 0 6px; - color: var(--sb-text-faint); - font-size: 12px; - font-weight: 500; - } - .rail h3:first-child { - margin-top: 0; - } - .decision { - display: flex; - align-items: baseline; - gap: 8px; - margin: 3px 0; - font-size: 12.5px; - } - .reason { - margin: 0 0 6px 18px; - font-size: 11.5px; - } - .chips { - display: flex; - flex-wrap: wrap; - gap: 5px; - } .foot { display: flex; align-items: center; @@ -684,12 +668,6 @@ color: var(--sb-text-faint); font-size: 12px; } - .ok-text { - color: var(--sb-success); - } - .bad-text { - color: var(--sb-error); - } .problem { color: var(--sb-error); }