Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 99 additions & 0 deletions ui/src/lib/components/lineage/folder-node.svelte
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
<script lang="ts">
import FolderIcon from '@lucide/svelte/icons/folder';
import ChevronRightIcon from '@lucide/svelte/icons/chevron-right';
import { Handle, Position, type NodeProps } from '@xyflow/svelte';
import type { GroupCard } from '$lib/lineage/catalog-graph';

let { data }: NodeProps = $props();
const d = $derived(data as GroupCard);
</script>

<!-- A collapsed folder; clicking it opens the folder into its nodes. -->
<div class="group" class:dimmed={d.dimmed} title="Show the {d.count} nodes in {d.group}">
<Handle type="target" position={Position.Left} class="node-handle" />
<div class="head">
<FolderIcon size={14} strokeWidth={1.75} class="glyph" />
<span class="name">{d.group}</span>
<span class="count">{d.count}</span>
<ChevronRightIcon size={13} class="glyph" />
</div>
<div class="status">
{#if d.ok}<span><i style:background="var(--sb-success)"></i>{d.ok} ok</span>{/if}
{#if d.stale}<span><i style:background="var(--sb-stale)"></i>{d.stale} stale</span>{/if}
{#if d.failed}<span><i style:background="var(--sb-error)"></i>{d.failed} failed</span>{/if}
{#if !d.ok && !d.stale && !d.failed}<span class="faint">No builds</span>{/if}
</div>
<Handle type="source" position={Position.Right} class="node-handle" />
</div>

<style>
.group {
width: 236px;
border: 1px solid var(--node-border);
border-radius: 2px;
background: color-mix(in srgb, var(--card) 70%, var(--sb-inset));
box-shadow: 3px 3px 0 -1px var(--card), 3px 3px 0 0 var(--node-border);
color: var(--foreground);
cursor: pointer;
font-family: var(--font-sans);
transition: opacity 0.15s, border-color 0.15s;
}
.group:hover {
border-color: color-mix(in srgb, var(--foreground) 35%, transparent);
}
.group.dimmed {
opacity: 0.35;
}
.head {
display: flex;
height: 34px;
align-items: center;
gap: 8px;
padding: 0 10px;
}
.head :global(.glyph) {
flex: 0 0 auto;
color: var(--muted-foreground);
}
.name {
min-width: 0;
flex: 1;
overflow: hidden;
font-size: 12.5px;
font-weight: 600;
text-overflow: ellipsis;
white-space: nowrap;
}
.count {
color: var(--sb-text-faint);
font-size: 11px;
}
.status {
display: flex;
gap: 10px;
padding: 0 10px 8px 32px;
color: var(--muted-foreground);
font-size: 11px;
}
.status span {
display: inline-flex;
align-items: center;
gap: 5px;
}
.status i {
width: 6px;
height: 6px;
border-radius: 50%;
}
.faint {
color: var(--sb-text-faint);
}
.group :global(.node-handle) {
width: 6px;
height: 6px;
min-width: 0;
min-height: 0;
border: 0;
background: transparent;
}
</style>
58 changes: 58 additions & 0 deletions ui/src/lib/lineage/catalog-graph.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import { describe, expect, it } from 'vitest';
import type { CatalogNode } from '$lib/api';
import { folderOf, groupedGraph } from './catalog-graph';

function node(key: string, path: string, upstream: string[] = [], extra: Partial<CatalogNode> = {}): CatalogNode {
const kind = key.startsWith('source:') ? 'source' : key.startsWith('seed:') ? 'seed' : 'table';
return { key, kind, path, upstream, group: null, stale: [], lastOutcome: null, lastBuild: { builtAt: '2026-11-07T06:00:00Z' }, ...extra } as CatalogNode;
}

describe('lineage folders', () => {
it('groups models by folder, sources by file, and lets a declared group win', () => {
expect([
folderOf(node('model:dim_orders', 'models/marts/core/dim_orders.sql')),
folderOf(node('source:shop__orders', 'sources/shop.yml')),
folderOf(node('seed:countries', 'seeds/countries.csv')),
folderOf(node('model:orders', 'orders.sql')),
folderOf(node('model:stg_orders', 'models/staging/stg_orders.sql', [], { group: 'shop' }))
]).toEqual(['marts/core', 'sources/shop', 'seeds', 'models', 'shop']);
});

const nodes = [
node('source:shop__orders', 'sources/shop.yml'),
node('source:shop__customers', 'sources/shop.yml'),
node('model:stg_orders', 'models/staging/shop/stg_orders.sql', ['source:shop__orders']),
node('model:stg_customers', 'models/staging/crm/stg_customers.sql', ['source:shop__customers'], { stale: [{ reason: 'behind', upstream: [], detail: '', since: '' }] }),
node('model:orders', 'models/marts/orders.sql', ['model:stg_orders', 'model:stg_customers'], { lastOutcome: 'failed' }),
node('model:order_totals', 'models/marts/order_totals.sql', ['model:orders'])
];

it('draws each closed top folder once, with one edge per folder pair and none inside a folder', () => {
const graph = groupedGraph(nodes, new Set());
expect(graph.nodes.map((n) => [n.id, n.data.count])).toEqual([
['group:sources', 2],
['group:staging', 2],
['group:marts', 2]
]);
expect(graph.edges.map((e) => e.id)).toEqual(['group:sources-group:staging', 'group:staging-group:marts']);
expect(graph.nodes.map((n) => [n.data.ok, n.data.stale, n.data.failed])).toEqual([[0, 0, 0], [1, 1, 0], [1, 0, 1]]);
});

it('opens a folder into the folders inside it, and those into their nodes, joined to the closed folders around them', () => {
expect(groupedGraph(nodes, new Set(['staging'])).nodes.map((n) => n.id)).toEqual(['group:sources', 'group:staging/shop', 'group:staging/crm', 'group:marts']);
const graph = groupedGraph(nodes, new Set(['staging', 'staging/shop']));
expect(graph.nodes.map((n) => n.id)).toEqual(['group:sources', 'model:stg_orders', 'group:staging/crm', 'group:marts']);
expect(graph.edges.map((e) => e.id)).toEqual([
'group:sources-model:stg_orders',
'group:sources-group:staging/crm',
'model:stg_orders-group:marts',
'group:staging/crm-group:marts'
]);
});

it('draws every node once all folders around it are open', () => {
const graph = groupedGraph(nodes, new Set(['sources', 'sources/shop', 'staging', 'staging/shop', 'staging/crm', 'marts']));
expect(graph.nodes.every((n) => n.type === 'asset')).toBe(true);
expect(graph.edges).toHaveLength(5);
});
});
80 changes: 80 additions & 0 deletions ui/src/lib/lineage/catalog-graph.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,3 +57,83 @@ export function catalogGraph(nodes: CatalogNode[]): { nodes: Node[]; edges: Edge
edges: nodes.flatMap((n) => n.upstream.map((up) => ({ id: `${up}-${n.key}`, source: up, target: n.key, type: 'bezier' })))
};
}

/** Above this many nodes the lineage opens with each folder collapsed into one card. */
export const GROUPED_ABOVE = 150;

/**
* The folder a node lives in, such as `marts/core`, or its source file, such as `sources/orders`;
* a declared group wins.
*/
export function folderOf(node: CatalogNode): string {
if (node.group) return node.group;
const path = node.path.replace(/^models\//, '');
if (node.kind === 'source') return path.replace(/\.[^./]+$/, '') || 'sources';
const slash = path.lastIndexOf('/');
return slash > 0 ? path.slice(0, slash) : node.kind === 'seed' ? 'seeds' : 'models';
}

export type GroupCard = { group: string; count: number; ok: number; stale: number; failed: number; dimmed?: boolean };

const groupId = (group: string) => `group:${group}`;

/** A folder's card: how many nodes it holds, and how many last built fine, are stale or failed. */
function groupCard(group: string, members: CatalogNode[]): GroupCard {
const built = members.filter((n) => n.kind !== 'source');
return {
group,
count: members.length,
ok: built.filter((n) => cardStatus(n) === 'ok' && !n.stale?.length).length,
stale: built.filter((n) => !!n.stale?.length).length,
failed: built.filter((n) => n.lastOutcome === 'failed').length
};
}

/** `expanded` with every folder around the node open, so the node itself is drawn. */
export function withFoldersOpen(node: CatalogNode, expanded: ReadonlySet<string>): Set<string> {
const parts = folderOf(node).split('/');
return new Set([...expanded, ...parts.map((_, i) => parts.slice(0, i + 1).join('/'))]);
}

/** The outermost folder around a node that is not open, or null when every folder around it is. */
function closedFolder(node: CatalogNode, expanded: ReadonlySet<string>): string | null {
const parts = folderOf(node).split('/');
for (let depth = 1; depth <= parts.length; depth++) {
const folder = parts.slice(0, depth).join('/');
if (!expanded.has(folder)) return folder;
}
return null;
}

/**
* The lineage with every folder that is not open drawn as one card. Folders nest, so opening
* `marts` shows `marts/core` and the other folders inside it. An edge joins whatever stands for
* each end, once per pair, and an edge inside a closed folder disappears.
*/
export function groupedGraph(nodes: CatalogNode[], expanded: ReadonlySet<string>): { nodes: Node[]; edges: Edge[] } {
const standIn = new Map<string, string>();
const folders = new Map<string, CatalogNode[]>();
const cards: Node[] = [];
for (const n of nodes) {
const folder = closedFolder(n, expanded);
standIn.set(n.key, folder === null ? n.key : groupId(folder));
if (folder === null) cards.push({ id: n.key, type: 'asset', position: { x: 0, y: 0 }, data: cardData(n) });
else if (folders.has(folder)) folders.get(folder)!.push(n);
else {
folders.set(folder, [n]);
cards.push({ id: groupId(folder), type: 'folder', position: { x: 0, y: 0 }, selectable: false, data: {} });
}
}
for (const card of cards) if (card.type === 'folder') card.data = groupCard(card.id.slice('group:'.length), folders.get(card.id.slice('group:'.length))!);
const pairs = new Map<string, Edge>();
for (const n of nodes) {
for (const up of n.upstream) {
const source = standIn.get(up);
const target = standIn.get(n.key)!;
if (!source || source === target) continue;
const id = `${source}-${target}`;
if (!pairs.has(id)) pairs.set(id, { id, source, target, type: 'bezier' });
}
}
return { nodes: cards, edges: [...pairs.values()] };
}
57 changes: 57 additions & 0 deletions ui/src/lib/lineage/job-graph.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,63 @@ describe('job graph', () => {
expect(laid.find((n) => n.id === boxId('staging'))!.height).toBeLessThan(61);
});

it('wraps a tall column into several, keeping every node right of what it reads', () => {
// 40 marts with no lineage inside the job, and 30 models off one source: 71 cards, at most 15 to a column.
const wide = [
...Array.from({ length: 40 }, (_, i) => node(`model:mart_${i}`)),
node('model:src'),
...Array.from({ length: 30 }, (_, i) => node(`model:stg_${i}`, ['model:src']))
];
const { nodes, edges } = jobGraph(wide.map((n) => n.key), wide);
const laid = layoutJobGraph(measured(nodes), edges);
const columns = Map.groupBy(laid, (n) => Math.round(n.position.x));
expect(Math.max(...[...columns.values()].map((c) => c.length))).toBeLessThanOrEqual(15);
// The wrapped columns sit side by side rather than staggered: the graph is no taller than one full column.
const ys = laid.map((n) => n.position.y);
expect(Math.max(...ys) - Math.min(...ys)).toBeLessThan(15 * (61 + 24));
const x = (id: string) => laid.find((n) => n.id === id)!.position.x;
for (let i = 0; i < 30; i++) expect(x(`model:stg_${i}`)).toBeGreaterThan(x('model:src'));
});

it('leaves a graph that needs no wrapping as dagre lays it out', () => {
// `extra` only feeds the end of a long chain, so it sits just before it, not back at the start.
const chain = [node('model:a'), node('model:b', ['model:a']), node('model:c', ['model:b']), node('model:d', ['model:c']), node('model:extra'), node('model:end', ['model:d', 'model:extra'])];
const { nodes, edges } = jobGraph(chain.map((n) => n.key), chain);
const x = new Map(layoutJobGraph(measured(nodes), edges).map((n) => [n.id, n.position.x]));
expect(x.get('model:extra')).toBe(x.get('model:d'));
});

it('keeps every box right of the boxes it runs after when their columns wrap', () => {
const many = (job: string) => Array.from({ length: 30 }, (_, i) => node(`model:${job}_${i}`));
const wide = [...many('a'), ...many('b')];
const groupBoxes = [
{ jobKey: 'a', nodes: wide.slice(0, 30).map((n) => n.key) },
{ jobKey: 'b', nodes: wide.slice(30).map((n) => n.key) },
{ jobKey: 'empty', nodes: [] }
];
const after = (jobKey: string) => ({ jobKey, upstream: [{ jobKey: 'a', source: 'after' as const }], stage: 1 });
const { nodes, edges } = jobGraph(wide.map((n) => n.key), wide, groupBoxes, [{ jobKey: 'a', upstream: [], stage: 0 }, after('b'), after('empty')]);
const laid = layoutJobGraph(measured(nodes), edges);
const box = (k: string) => laid.find((n) => n.id === boxId(k))!;
expect(box('a').height).toBeLessThan(15 * (61 + 24) + 60);
expect(box('b').position.x).toBeGreaterThan(box('a').position.x + box('a').width!);
expect(box('empty').position.x).toBeGreaterThan(box('a').position.x + box('a').width!);
});

it('keeps lineage left to right when it leaves a box and comes back', () => {
// `broad` builds raw → stg → mart, but `narrow` builds only stg, so stg sits in narrow's box.
const chain = [node('model:raw'), node('model:stg', ['model:raw']), node('model:mart', ['model:stg'])];
const nested = [{ jobKey: 'broad', nodes: chain.map((n) => n.key) }, { jobKey: 'narrow', nodes: ['model:stg'] }];
const { nodes, edges } = jobGraph(chain.map((n) => n.key), chain, nested);
const laid = layoutJobGraph(measured(nodes), edges);
const x = (id: string) => {
const card = laid.find((n) => n.id === id)!;
return card.position.x + (laid.find((n) => n.id === card.parentId)?.position.x ?? 0);
};
expect(x('model:raw')).toBeLessThan(x('model:stg'));
expect(x('model:stg')).toBeLessThan(x('model:mart'));
});

it('draws a plain job without boxes', () => {
const { nodes } = jobGraph(['model:raw', 'model:stg'], catalog);
expect(nodes.map((n) => [n.id, n.type, n.parentId])).toEqual([
Expand Down
Loading
Loading