Skip to content

Commit 82ac670

Browse files
committed
feat(ui): wrap a job graph's tall columns so large jobs stay readable
1 parent b6014fb commit 82ac670

2 files changed

Lines changed: 227 additions & 17 deletions

File tree

‎ui/src/lib/lineage/job-graph.test.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,63 @@ describe('job graph', () => {
7272
expect(laid.find((n) => n.id === boxId('staging'))!.height).toBeLessThan(61);
7373
});
7474

75+
it('wraps a tall column into several, keeping every node right of what it reads', () => {
76+
// 40 marts with no lineage inside the job, and 30 models off one source: 71 cards, at most 15 to a column.
77+
const wide = [
78+
...Array.from({ length: 40 }, (_, i) => node(`model:mart_${i}`)),
79+
node('model:src'),
80+
...Array.from({ length: 30 }, (_, i) => node(`model:stg_${i}`, ['model:src']))
81+
];
82+
const { nodes, edges } = jobGraph(wide.map((n) => n.key), wide);
83+
const laid = layoutJobGraph(measured(nodes), edges);
84+
const columns = Map.groupBy(laid, (n) => Math.round(n.position.x));
85+
expect(Math.max(...[...columns.values()].map((c) => c.length))).toBeLessThanOrEqual(15);
86+
// The wrapped columns sit side by side rather than staggered: the graph is no taller than one full column.
87+
const ys = laid.map((n) => n.position.y);
88+
expect(Math.max(...ys) - Math.min(...ys)).toBeLessThan(15 * (61 + 24));
89+
const x = (id: string) => laid.find((n) => n.id === id)!.position.x;
90+
for (let i = 0; i < 30; i++) expect(x(`model:stg_${i}`)).toBeGreaterThan(x('model:src'));
91+
});
92+
93+
it('leaves a graph that needs no wrapping as dagre lays it out', () => {
94+
// `extra` only feeds the end of a long chain, so it sits just before it, not back at the start.
95+
const chain = [node('model:a'), node('model:b', ['model:a']), node('model:c', ['model:b']), node('model:d', ['model:c']), node('model:extra'), node('model:end', ['model:d', 'model:extra'])];
96+
const { nodes, edges } = jobGraph(chain.map((n) => n.key), chain);
97+
const x = new Map(layoutJobGraph(measured(nodes), edges).map((n) => [n.id, n.position.x]));
98+
expect(x.get('model:extra')).toBe(x.get('model:d'));
99+
});
100+
101+
it('keeps every box right of the boxes it runs after when their columns wrap', () => {
102+
const many = (job: string) => Array.from({ length: 30 }, (_, i) => node(`model:${job}_${i}`));
103+
const wide = [...many('a'), ...many('b')];
104+
const groupBoxes = [
105+
{ jobKey: 'a', nodes: wide.slice(0, 30).map((n) => n.key) },
106+
{ jobKey: 'b', nodes: wide.slice(30).map((n) => n.key) },
107+
{ jobKey: 'empty', nodes: [] }
108+
];
109+
const after = (jobKey: string) => ({ jobKey, upstream: [{ jobKey: 'a', source: 'after' as const }], stage: 1 });
110+
const { nodes, edges } = jobGraph(wide.map((n) => n.key), wide, groupBoxes, [{ jobKey: 'a', upstream: [], stage: 0 }, after('b'), after('empty')]);
111+
const laid = layoutJobGraph(measured(nodes), edges);
112+
const box = (k: string) => laid.find((n) => n.id === boxId(k))!;
113+
expect(box('a').height).toBeLessThan(15 * (61 + 24) + 60);
114+
expect(box('b').position.x).toBeGreaterThan(box('a').position.x + box('a').width!);
115+
expect(box('empty').position.x).toBeGreaterThan(box('a').position.x + box('a').width!);
116+
});
117+
118+
it('keeps lineage left to right when it leaves a box and comes back', () => {
119+
// `broad` builds raw → stg → mart, but `narrow` builds only stg, so stg sits in narrow's box.
120+
const chain = [node('model:raw'), node('model:stg', ['model:raw']), node('model:mart', ['model:stg'])];
121+
const nested = [{ jobKey: 'broad', nodes: chain.map((n) => n.key) }, { jobKey: 'narrow', nodes: ['model:stg'] }];
122+
const { nodes, edges } = jobGraph(chain.map((n) => n.key), chain, nested);
123+
const laid = layoutJobGraph(measured(nodes), edges);
124+
const x = (id: string) => {
125+
const card = laid.find((n) => n.id === id)!;
126+
return card.position.x + (laid.find((n) => n.id === card.parentId)?.position.x ?? 0);
127+
};
128+
expect(x('model:raw')).toBeLessThan(x('model:stg'));
129+
expect(x('model:stg')).toBeLessThan(x('model:mart'));
130+
});
131+
75132
it('draws a plain job without boxes', () => {
76133
const { nodes } = jobGraph(['model:raw', 'model:stg'], catalog);
77134
expect(nodes.map((n) => [n.id, n.type, n.parentId])).toEqual([

‎ui/src/lib/lineage/job-graph.ts‎

Lines changed: 170 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,12 @@ const EMPTY_HEIGHT = 44;
1616
const COLLAPSED_HEIGHT = HEADER + 2;
1717
/** Wide enough for a header's name, node count and last run on one line. */
1818
const COLLAPSED_WIDTH = 380;
19+
/** Space between cards stacked in one column. */
20+
const STACK_GAP = 20;
21+
/** Space dagre leaves between cards, and between boxes and cards outside them. */
22+
const CARD_GAP = 24;
23+
const BOX_GAP = 24 + HEADER;
24+
const MARGIN = 24;
1925

2026
export type JobBox = { jobKey: string; nodes: string[]; lastRun?: RunRef | null };
2127

@@ -102,40 +108,188 @@ export function jobGraph(
102108
return { nodes: [...boxNodes, ...cards], edges: [...lineage, ...after] };
103109
}
104110

111+
/** The most cards a column holds before it wraps, growing with the graph so its shape stays near the canvas's. */
112+
export function maxColumnCards(count: number): number {
113+
return Math.max(8, Math.round(Math.sqrt(count * 3)));
114+
}
115+
116+
type Item = { id: string; width: number; height: number };
117+
type Rect = { x: number; y: number; width: number; height: number };
118+
105119
/**
106-
* Lays the graph out left to right. Cards use their measured size; boxes are sized to wrap their
107-
* cards, and child positions are relative to their box, as Svelte Flow expects.
120+
* Dagre stacks every node of a rank in one column, so a job whose nodes hang off a few parents, or
121+
* have no lineage inside the job, draws as one column too tall to read. This gives each node a rank
122+
* that spreads such a column over several: the edges into a node get the rank gap as their minimum
123+
* length, and a moved node with no edge in hangs off the column's first node by `holds`, edges that
124+
* are laid out but not drawn.
108125
*/
109-
export function layoutJobGraph(nodes: Node[], edges: Edge[]): Node[] {
110-
const g = new dagre.graphlib.Graph({ compound: true });
111-
g.setGraph({ rankdir: 'LR', ranksep: 64, nodesep: 24 + HEADER, marginx: 24, marginy: 24 });
126+
function wrapColumns(ids: string[], pairs: ReadonlyArray<readonly [string, string]>) {
127+
const preds = new Map<string, string[]>();
128+
for (const [s, t] of pairs) preds.set(t, [...(preds.get(t) ?? []), s]);
129+
// Each node's rank before wrapping: the longest path to it.
130+
const depth = new Map<string, number>();
131+
const depthOf = (id: string): number => {
132+
if (depth.has(id)) return depth.get(id)!;
133+
depth.set(id, 0); // A cycle ends here rather than looping; dagre breaks it for the drawing.
134+
const d = Math.max(-1, ...(preds.get(id) ?? []).map(depthOf)) + 1;
135+
depth.set(id, d);
136+
return d;
137+
};
138+
const columns = new Map<number, string[]>();
139+
for (const id of ids) columns.set(depthOf(id), [...(columns.get(depthOf(id)) ?? []), id]);
140+
141+
const max = maxColumnCards(ids.length);
142+
const rank = new Map<string, number>();
143+
const holds: [string, string][] = [];
144+
let wrapped = false;
145+
const first = (id: string) => [...(preds.get(id) ?? [])].sort()[0] ?? '';
146+
for (const [, column] of [...columns].sort(([a], [b]) => a - b)) {
147+
// Siblings stay together, so a parent's children fill a column before the next parent's.
148+
column.sort((a, b) => first(a).localeCompare(first(b)) || a.localeCompare(b));
149+
const base = Math.max(0, ...column.flatMap((id) => (preds.get(id) ?? []).map((p) => (rank.get(p) ?? 0) + 1)));
150+
const per = Math.ceil(column.length / Math.ceil(column.length / max));
151+
wrapped ||= per < column.length;
152+
column.forEach((id, i) => {
153+
const step = Math.floor(i / per);
154+
rank.set(id, base + step);
155+
if (step && !preds.has(id)) holds.push([column[0], id]);
156+
});
157+
}
158+
return { rank, holds, wrapped };
159+
}
160+
161+
/**
162+
* Lays `items` out left to right from the origin, wrapping tall columns. Dagre staggers the columns
163+
* a tall one wrapped into, lining each up with what it reads, so a graph that wrapped then stacks
164+
* every column from the top: a column keeps dagre's order, and so its edge crossings. A graph that
165+
* didn't wrap keeps dagre's layout.
166+
*/
167+
function layoutLayer(items: Item[], pairs: ReadonlyArray<readonly [string, string]>, nodesep: number): Map<string, Rect> {
168+
const g = new dagre.graphlib.Graph();
169+
g.setGraph({ rankdir: 'LR', ranksep: 64, nodesep, marginx: 0, marginy: 0 });
112170
g.setDefaultEdgeLabel(() => ({}));
171+
for (const item of items) g.setNode(item.id, { width: item.width, height: item.height });
172+
const { rank, holds, wrapped } = wrapColumns(items.map((i) => i.id), pairs);
173+
for (const [s, t] of pairs) g.setEdge(s, t, { minlen: wrapped ? Math.max(1, rank.get(t)! - rank.get(s)!) : 1 });
174+
for (const [s, t] of holds) if (!g.hasEdge(s, t)) g.setEdge(s, t, { minlen: rank.get(t)! - rank.get(s)! });
175+
dagre.layout(g);
113176

177+
const rects = new Map(items.map((i) => {
178+
const p = g.node(i.id);
179+
return [i.id, { x: p.x - i.width / 2, y: p.y - i.height / 2, width: i.width, height: i.height }];
180+
}));
181+
if (wrapped) {
182+
const top = Math.min(...[...rects.values()].map((r) => r.y));
183+
const stacks = Map.groupBy(rects.values(), (r) => Math.round(r.x + r.width / 2));
184+
for (const stack of stacks.values()) {
185+
let y = top;
186+
for (const r of stack.sort((a, b) => a.y - b.y)) {
187+
r.y = y;
188+
y += r.height + STACK_GAP;
189+
}
190+
}
191+
}
192+
const left = Math.min(...[...rects.values()].map((r) => r.x));
193+
const top = Math.min(...[...rects.values()].map((r) => r.y));
194+
for (const r of rects.values()) {
195+
r.x -= left;
196+
r.y -= top;
197+
}
198+
return rects;
199+
}
200+
201+
/**
202+
* Lays the graph out left to right: first each box's cards, which sizes the box, then the boxes and
203+
* the cards outside any box, ordered by the lineage and `after` edges between them. Cards use their
204+
* measured size, and a card's position is relative to its box, as Svelte Flow expects.
205+
*/
206+
export function layoutJobGraph(nodes: Node[], edges: Edge[]): Node[] {
207+
if (!nodes.length) return [];
114208
const size = (n: Node) => ({ width: n.measured?.width ?? NODE_WIDTH, height: n.measured?.height ?? NODE_HEIGHT });
115-
const children = new Map<string, Node[]>();
116-
for (const n of nodes) if (n.parentId) children.set(n.parentId, [...(children.get(n.parentId) ?? []), n]);
209+
const children = Map.groupBy(nodes.filter((n) => n.parentId), (n) => n.parentId!);
210+
const pairsAmong = (ids: Set<string>, lift: (id: string) => string) => {
211+
const seen = new Set<string>();
212+
return edges.flatMap((e) => {
213+
const [s, t] = [lift(e.source), lift(e.target)];
214+
if (s === t || !ids.has(s) || !ids.has(t) || seen.has(`${s}>${t}`)) return [];
215+
seen.add(`${s}>${t}`);
216+
return [[s, t] as const];
217+
});
218+
};
219+
const parentOf = new Map(nodes.map((n) => [n.id, n.parentId ?? n.id]));
220+
const topPairs = pairsAmong(new Set(nodes.filter((n) => !n.parentId).map((n) => n.id)), (id) => parentOf.get(id) ?? id);
221+
// Lineage that leaves a box and comes back, as when a narrow job builds a node in the middle of a
222+
// broad job's chain, can't be drawn box by box without an edge running backwards.
223+
if (loops(topPairs)) return layoutCompound(nodes, edges, size, children);
117224

225+
const inside = new Map<string, Map<string, Rect>>();
226+
const outer: Item[] = [];
118227
for (const n of nodes) {
119-
if (n.type === 'job') {
120-
// An empty box is laid out as a plain node; dagre sizes boxes with children itself.
121-
if (children.has(n.id)) g.setNode(n.id, {});
122-
else g.setNode(n.id, { width: n.width ?? NODE_WIDTH + 2 * PAD, height: n.height ?? EMPTY_HEIGHT });
123-
} else g.setNode(n.id, size(n));
228+
if (n.parentId) continue;
229+
const kids = children.get(n.id);
230+
if (!kids) {
231+
outer.push(n.type === 'job' ? { id: n.id, width: n.width ?? NODE_WIDTH + 2 * PAD, height: n.height ?? EMPTY_HEIGHT } : { id: n.id, ...size(n) });
232+
continue;
233+
}
234+
const rects = layoutLayer(kids.map((k) => ({ id: k.id, ...size(k) })), pairsAmong(new Set(kids.map((k) => k.id)), (id) => id), CARD_GAP);
235+
inside.set(n.id, rects);
236+
const right = Math.max(...[...rects.values()].map((r) => r.x + r.width));
237+
const bottom = Math.max(...[...rects.values()].map((r) => r.y + r.height));
238+
outer.push({ id: n.id, width: right + 2 * PAD, height: bottom + 2 * PAD + HEADER });
124239
}
125-
for (const n of nodes) if (n.parentId) g.setParent(n.id, n.parentId);
240+
const placed = layoutLayer(outer, topPairs, BOX_GAP);
126241

127-
// Dagre can't route edges to a box with children, so an `after` edge orders the boxes through their first cards.
242+
return nodes.map((n) => {
243+
if (n.parentId) {
244+
const r = inside.get(n.parentId)!.get(n.id)!;
245+
return { ...n, position: { x: PAD + r.x, y: PAD + HEADER + r.y } };
246+
}
247+
const r = placed.get(n.id)!;
248+
const position = { x: MARGIN + r.x, y: MARGIN + r.y };
249+
return n.type === 'job' ? { ...n, position, width: r.width, height: r.height } : { ...n, position };
250+
});
251+
}
252+
253+
/** Whether the edges form a loop. */
254+
function loops(pairs: ReadonlyArray<readonly [string, string]>): boolean {
255+
const next = Map.groupBy(pairs, ([s]) => s);
256+
const state = new Map<string, 'open' | 'done'>();
257+
const visit = (id: string): boolean => {
258+
if (state.get(id) === 'done') return false;
259+
if (state.get(id) === 'open') return true;
260+
state.set(id, 'open');
261+
const found = (next.get(id) ?? []).some(([, t]) => visit(t));
262+
state.set(id, 'done');
263+
return found;
264+
};
265+
return [...next.keys()].some(visit);
266+
}
267+
268+
/**
269+
* Lays the whole graph out at once, boxes and cards together, so lineage between boxes always runs
270+
* left to right. Columns don't wrap, so a large job draws taller.
271+
*/
272+
function layoutCompound(nodes: Node[], edges: Edge[], size: (n: Node) => { width: number; height: number }, children: Map<string, Node[]>): Node[] {
273+
const g = new dagre.graphlib.Graph({ compound: true });
274+
g.setGraph({ rankdir: 'LR', ranksep: 64, nodesep: BOX_GAP, marginx: MARGIN, marginy: MARGIN });
275+
g.setDefaultEdgeLabel(() => ({}));
276+
for (const n of nodes) {
277+
if (n.type === 'job' && children.has(n.id)) g.setNode(n.id, {});
278+
else if (n.type === 'job') g.setNode(n.id, { width: n.width ?? NODE_WIDTH + 2 * PAD, height: n.height ?? EMPTY_HEIGHT });
279+
else g.setNode(n.id, size(n));
280+
}
281+
for (const n of nodes) if (n.parentId) g.setParent(n.id, n.parentId);
282+
// Dagre can't route edges to a box with children, so an edge to a box goes to its first card.
128283
const anchor = (id: string) => children.get(id)?.[0]?.id ?? id;
129284
for (const e of edges) g.setEdge(anchor(e.source), anchor(e.target));
130-
131285
dagre.layout(g);
132286

133287
const topLeft = (n: Node) => {
134288
const p = g.node(n.id);
135289
const { width, height } = size(n);
136290
return { x: p.x - width / 2, y: p.y - height / 2, width, height };
137291
};
138-
const boxes = new Map<string, { x: number; y: number; width: number; height: number }>();
292+
const boxes = new Map<string, Rect>();
139293
for (const n of nodes) {
140294
if (n.type !== 'job') continue;
141295
const kids = (children.get(n.id) ?? []).map(topLeft);
@@ -152,7 +306,6 @@ export function layoutJobGraph(nodes: Node[], edges: Edge[]): Node[] {
152306
const bottom = Math.max(...kids.map((k) => k.y + k.height)) + PAD;
153307
boxes.set(n.id, { x: left, y: top, width: right - left, height: bottom - top });
154308
}
155-
156309
return nodes.map((n) => {
157310
const box = boxes.get(n.id);
158311
if (box) return { ...n, position: { x: box.x, y: box.y }, width: box.width, height: box.height };

0 commit comments

Comments
 (0)