From 1f4f3a990a0de97cb8a2dbb62a3611329a99553f Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 18 Nov 2026 05:19:48 +0000 Subject: [PATCH] test(ui): hold interactions to counted work budgets Count what each interaction costs instead of timing it: layouts, style recalculations, DOM mutations, DOM nodes, rows or cards drawn, and mock client reads, for graph pan and zoom, Checks, and plan sections on the large mock project. The graph's settle timer is held while a gesture is counted, so DOM changes and reads are the same on every run. Each budget notes where its count comes from, and the budgets run as a required step in the UI job. --- .github/workflows/verify.yml | 2 + ui/e2e/budgets.spec.ts | 289 +++++++++++++++++++++++++++ ui/playwright.config.ts | 7 +- ui/src/lib/api/index.ts | 26 ++- ui/src/lib/api/mock/mock-client.ts | 8 +- ui/src/lib/api/mock/operate-store.ts | 4 +- 6 files changed, 327 insertions(+), 9 deletions(-) create mode 100644 ui/e2e/budgets.spec.ts diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 2f67639bcd..7814fce985 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1434,6 +1434,8 @@ jobs: run: npx playwright install --with-deps chromium - name: Layout test (every page with awkward mock data at 1440 and 1920 px) run: npx playwright test --project=layout + - name: Work budgets (layouts, restyles, DOM changes and reads per interaction, counted) + run: npx playwright test --project=budgets ui_timings: # The timing guards in ui/e2e/perf.spec.ts, with the CPU slowed 6x. They go red over budget and diff --git a/ui/e2e/budgets.spec.ts b/ui/e2e/budgets.spec.ts new file mode 100644 index 0000000000..b65bfb4a9c --- /dev/null +++ b/ui/e2e/budgets.spec.ts @@ -0,0 +1,289 @@ +import { expect, test, type CDPSession, type Page } from '@playwright/test'; + +// Work budgets: what each interaction costs, counted rather than timed, so a slow or busy machine gives +// the same numbers as a fast one. Each gesture is a fixed number of input events, each followed by two +// awaited animation frames. Per gesture the suite counts Chrome's layouts and style recalculations +// (Performance.getMetrics), DOM mutation records (every Svelte re-render that touches the page), the +// DOM nodes and cards or rows on the page afterwards, and the mock client's reads. Budgets sit just +// above the measured values; the breakdown next to each says where the count comes from. + +type Cost = { layouts: number; styles: number; mutations: number; nodes: number; drawn: number; reads: number }; +type Budget = Cost; + +const RATE = Number(process.env.BUDGET_CPU_RATE ?? '1'); +const TRACE = !!process.env.BUDGET_TRACE; +// Not reads: `supports` asks which methods exist. The app shell polls these on a 30 s wall-clock timer +// whatever the page does, so whether one lands inside a gesture depends on when the gesture started. +const NOT_COUNTED = new Set(['supports', 'getWarehouseSync', 'listPlatformEvents', 'navCounts']); + +async function frames(page: Page, count = 2) { + await page.evaluate( + (n) => + new Promise((resolve) => { + let left = n; + const tick = () => (--left <= 0 ? resolve() : requestAnimationFrame(tick)); + requestAnimationFrame(tick); + }), + count + ); +} + +async function counters(page: Page, cdp: CDPSession) { + // Nodes counts detached nodes until they are collected, so collect first. + await cdp.send('HeapProfiler.collectGarbage'); + const { metrics } = await cdp.send('Performance.getMetrics'); + const value = (name: string) => metrics.find((m) => m.name === name)?.value ?? 0; + const page_ = await page.evaluate(() => { + const w = window as unknown as { __sqbMutations?: number; __sqbMockReads?: Record }; + return { mutations: w.__sqbMutations ?? 0, readsByName: { ...(w.__sqbMockReads ?? {}) } }; + }); + const reads = Object.entries(page_.readsByName) + .filter(([name]) => !NOT_COUNTED.has(name)) + .reduce((a, [, n]) => a + n, 0); + return { layouts: value('LayoutCount'), styles: value('RecalcStyleCount'), nodes: value('Nodes'), mutations: page_.mutations, reads, readsByName: page_.readsByName }; +} + +/** + * Runs in the page before the app. While a gesture is counted (`__sqbHold(true)`), the graph canvas's + * settle timer (`settleMs`) is held and run once when counting stops, and interval ticks (the + * one-second clock, the shell's polls) are skipped. Otherwise how often either fires inside a gesture + * depends on how fast the machine runs the steps, and the counts with it. + */ +function holdTimers(settleMs: number) { + const realSetTimeout = window.setTimeout.bind(window); + const realClearTimeout = window.clearTimeout.bind(window); + const realSetInterval = window.setInterval.bind(window); + let holding = false; + let next = 1e9; + const held = new Map void>(); + window.setTimeout = ((handler: TimerHandler, ms?: number, ...args: unknown[]) => { + if (!holding || ms !== settleMs || typeof handler !== 'function') return realSetTimeout(handler, ms, ...args); + held.set(++next, () => handler(...args)); + return next; + }) as typeof window.setTimeout; + window.clearTimeout = ((id?: number) => { + if (id === undefined || !held.delete(id)) realClearTimeout(id); + }) as typeof window.clearTimeout; + window.setInterval = ((handler: TimerHandler, ms?: number, ...args: unknown[]) => + realSetInterval(typeof handler === 'function' ? (...a: unknown[]) => holding || handler(...a) : handler, ms, ...args)) as typeof window.setInterval; + (window as unknown as { __sqbHold: (on: boolean) => void }).__sqbHold = (on) => { + holding = on; + if (on) return; + const due = [...held.values()]; + held.clear(); + for (const run of due) run(); + }; +} + +const hold = (page: Page, on: boolean) => page.evaluate((o) => (window as unknown as { __sqbHold: (on: boolean) => void }).__sqbHold(o), on); + +/** Open `path`, wait for `ready`, start counting mutations and set the CPU rate. */ +async function open(page: Page, path: string, ready: string): Promise { + await page.setViewportSize({ width: 1600, height: 900 }); + // The graph canvas's SETTLE_MS. + await page.addInitScript(holdTimers, 200); + await page.goto(path); + await page.locator(ready).first().waitFor({ timeout: 30_000 }); + await page.waitForLoadState('networkidle'); + await frames(page, 10); + await page.evaluate(() => { + const w = window as unknown as { __sqbMutations: number; __sqbMutationKinds: Record }; + w.__sqbMutations = 0; + w.__sqbMutationKinds = {}; + const kind = (r: MutationRecord) => { + const el = (r.target.nodeType === 1 ? r.target : r.target.parentElement) as Element | null; + const cls = + el + ?.getAttribute('class') + ?.split(/\s+/) + .find((c) => !c.startsWith('svelte-')) ?? + el?.tagName.toLowerCase() ?? + '?'; + return `${r.type}${r.attributeName ? `:${r.attributeName}` : ''} ${cls}`; + }; + new MutationObserver((records) => { + w.__sqbMutations += records.length; + for (const r of records) w.__sqbMutationKinds[kind(r)] = (w.__sqbMutationKinds[kind(r)] ?? 0) + 1; + }).observe(document.body, { subtree: true, childList: true, attributes: true, characterData: true }); + }); + const cdp = await page.context().newCDPSession(page); + await cdp.send('Performance.enable'); + await cdp.send('Emulation.setCPUThrottlingRate', { rate: RATE }); + return cdp; +} + +/** The cost of `act`, which ends with the page settled; `drawn` counts `drawnSelector` afterwards. */ +async function cost(page: Page, cdp: CDPSession, drawnSelector: string, act: () => Promise): Promise { + await frames(page); + if (TRACE) await page.evaluate(() => ((window as unknown as { __sqbMutationKinds: Record }).__sqbMutationKinds = {})); + const before = await counters(page, cdp); + await hold(page, true); + await act(); + await hold(page, false); + await page.waitForLoadState('networkidle'); + await frames(page, 4); + const after = await counters(page, cdp); + const readNames = Object.entries(after.readsByName) + .map(([name, n]) => [name, n - (before.readsByName[name] ?? 0)] as const) + .filter(([, n]) => n > 0) + .map(([name, n]) => `${name}×${n}`); + if (readNames.length) console.info(`READS ${readNames.join(' ')}`); + if (TRACE) { + const kinds = await page.evaluate(() => (window as unknown as { __sqbMutationKinds: Record }).__sqbMutationKinds); + const top = Object.entries(kinds) + .sort((a, b) => b[1] - a[1]) + .slice(0, 8); + console.info(`MUTATIONS ${top.map(([k, n]) => `${k}×${n}`).join(' | ')}`); + } + return { + layouts: after.layouts - before.layouts, + styles: after.styles - before.styles, + mutations: after.mutations - before.mutations, + nodes: after.nodes, + drawn: await page.locator(drawnSelector).count(), + reads: after.reads - before.reads + }; +} + +function judge(name: string, measured: Cost, budget: Budget) { + console.info(`BUDGET ${name} rate=${RATE} ${JSON.stringify(measured)}`); + const over = (Object.keys(budget) as (keyof Cost)[]).filter((k) => measured[k] > budget[k]).map((k) => `${k} ${measured[k]} > ${budget[k]}`); + // Soft, so one test reports every gesture that is over, not only the first. + expect.soft(over, `${name}: over its work budget`).toEqual([]); +} + +async function wheel(page: Page, at: { x: number; y: number }, steps: number, delta: (i: number) => number) { + await page.mouse.move(at.x, at.y); + for (let i = 0; i < steps; i++) { + await page.mouse.wheel(0, delta(i)); + await frames(page); + } +} + +async function drag(page: Page, from: { x: number; y: number }, steps: number) { + await page.mouse.move(from.x, from.y); + await page.mouse.down(); + for (let i = 1; i <= steps; i++) { + await page.mouse.move(from.x + i * 6, from.y + i * 2); + await frames(page); + } + await page.mouse.up(); +} + +/** A point on the canvas with no card under it. */ +async function emptyPoint(page: Page): Promise<{ x: number; y: number }> { + return page.evaluate(() => { + const r = document.querySelector('.svelte-flow')!.getBoundingClientRect(); + for (let y = r.top + 30; y < r.bottom - 30; y += 20) + for (let x = r.left + 80; x < r.right - 30; x += 20) + if (!document.elementFromPoint(x, y)?.closest('.svelte-flow__node, .svelte-flow__controls, .svelte-flow__edge, .svelte-flow__panel')) return { x, y }; + return { x: r.left + 80, y: r.top + 30 }; + }); +} + +const CARD = '.svelte-flow__node'; +const GRAPHS = [ + { name: 'large group', path: '/jobs/warehouse_nightly?mock&still' }, + { name: 'Lineage overview', path: '/lineage?mock&large&still' } +]; + +for (const { name, path } of GRAPHS) { + test(`${name}: wheel zoom and drag-pan`, async ({ page }) => { + test.setTimeout(240_000); + const cdp = await open(page, path, `.graph-canvas:not(.unfitted) ${CARD}`); + const box = (await page.locator('.svelte-flow').first().boundingBox())!; + const centre = { x: box.x + box.width / 2, y: box.y + box.height / 2 }; + judge(`${name} wheel zoom`, await cost(page, cdp, CARD, () => wheel(page, centre, 60, (i) => (i < 30 ? 100 : -100))), BUDGETS[`${name} wheel zoom`]); + const empty = await emptyPoint(page); + judge(`${name} drag-pan`, await cost(page, cdp, CARD, () => drag(page, empty, 40)), BUDGETS[`${name} drag-pan`]); + }); +} + +const ROW = 'table.sb-table tbody tr:not(.spacer)'; + +test('Checks: second visit with thousands of checks, then scrolling the list', async ({ page }) => { + test.setTimeout(180_000); + const nav = (label: string) => page.locator('nav a', { hasText: label }).first(); + const cdp = await open(page, '/?mock&awkward&still', 'nav a'); + await nav('Checks').click(); + await page.locator(ROW).first().waitFor(); + await page.waitForLoadState('networkidle'); + await nav('Overview').click(); + await page.waitForLoadState('networkidle'); + await frames(page, 4); + judge( + 'Checks second visit', + await cost(page, cdp, ROW, async () => { + await nav('Checks').click(); + await page.locator(ROW).first().waitFor(); + }), + BUDGETS['Checks second visit'] + ); + const list = (await page.locator('.sb-table-scroll').first().boundingBox())!; + judge('Checks list scroll', await cost(page, cdp, ROW, () => wheel(page, { x: list.x + list.width / 2, y: list.y + 200 }, 40, () => 400)), BUDGETS['Checks list scroll']); +}); + +test('Plan: switching sections and showing more on a large changed build', async ({ page }) => { + test.setTimeout(180_000); + const PLAN_ROW = '.pane .plan-row'; + const cdp = await open(page, `/plan?mock&large&still&cmd=${encodeURIComponent('sqb build --changed')}`, PLAN_ROW); + for (const key of ['first_run', 'unchanged', 'changed']) { + judge( + `Plan section ${key}`, + await cost(page, cdp, PLAN_ROW, async () => { + await page.locator(`.index .entry[data-section="${key}"]`).click(); + await page.locator(`.pane[data-section="${key}"] .plan-row`).first().waitFor(); + }), + BUDGETS[`Plan section ${key}`] + ); + } + await page.locator('.index .entry[data-section="unchanged"]').click(); + await page.locator(`.pane[data-section="unchanged"] .plan-row`).nth(49).waitFor(); + judge( + 'Plan show more', + await cost(page, cdp, PLAN_ROW, async () => { + await page.getByRole('button', { name: /^Show more/ }).click(); + await page.locator(PLAN_ROW).nth(99).waitFor(); + }), + BUDGETS['Plan show more'] + ); +}); + +// Measured on 10 runs each at 1× and 6× CPU (Chrome throttling), with the breakdown traced from the +// mutation records (`BUDGET_TRACE=1`). The graph's settle timer is held while a gesture is counted and +// run once at its end, so DOM changes and reads are the same on every run and get 2 of headroom. +// Layouts and style recalculations still move by a few, because Chrome batches them per frame however +// fast the machine draws frames, and the Checks page renders a little more at 6×; those budgets are the +// highest count seen plus the spread seen plus 2. A regression that costs one more render per frame adds +// 35 to 45 DOM changes to every gesture; windowing switched off draws 5,000 rows instead of 20. +const BUDGETS: Record = { + // 60 wheel steps: the viewport transform is written once per step that moves (35; the rest stop at + // the zoom limit), the zoom buttons toggle 3 times, and the settle restyles 7 edge paths, redraws the + // grid and sets the canvas classes. Measured layouts 2, styles 43–46, mutations 54, 2,501 nodes, 9 cards. + 'large group wheel zoom': { layouts: 4, styles: 50, mutations: 56, nodes: 2600, drawn: 12, reads: 0 }, + // 40 pointer moves: one transform write and about one restyle each, plus the settle. + // Measured layouts 1, styles 53–56, mutations 51. + 'large group drag-pan': { layouts: 3, styles: 61, mutations: 53, nodes: 2600, drawn: 12, reads: 0 }, + // 45 moving steps, then the settle: canvas classes, the grid redraw and the compact level. + // Measured layouts 1, styles 50–51, mutations 57, 3,313 nodes, 13 cards. + 'Lineage overview wheel zoom': { layouts: 3, styles: 54, mutations: 59, nodes: 3400, drawn: 16, reads: 0 }, + // 40 transform writes, 2 card classes and the settle. Measured layouts 0, styles 43–46, mutations 44. + 'Lineage overview drag-pan': { layouts: 2, styles: 51, mutations: 46, nodes: 3400, drawn: 16, reads: 0 }, + // One listChecks read (filter counts come from it), 20 rows drawn: 72 row insertions and 200 cell + // class writes, the rest the navigation. Measured layouts 2–4, styles 15–22, mutations 350 at 1× and + // 377 at 6×, 3,327–3,545 nodes (as garbage collection finds them). + 'Checks second visit': { layouts: 7, styles: 31, mutations: 410, nodes: 3800, drawn: 24, reads: 1 }, + // 40 wheel steps of 400 px: the window keeps 29 rows (the view and 8 either side); each step swaps + // rows in and out (1,822 insertions and removals), rewrites the recycled cells' classes (11,500) and + // the two spacers' heights (78), with one layout per step. Measured layouts 40–41, styles 81, + // mutations 13,400–13,414, 4,083–4,301 nodes. + 'Checks list scroll': { layouts: 44, styles: 83, mutations: 13430, nodes: 4600, drawn: 32, reads: 0 }, + // Each section switch replaces the 50 rows it shows (100 row insertions and removals) and reads its + // first page once; Changed came with the page, so it reads nothing. Measured the same in every run: + // layouts 2/2/1, styles 5/6/5, mutations 115/115/111. + 'Plan section first_run': { layouts: 4, styles: 7, mutations: 117, nodes: 2500, drawn: 50, reads: 1 }, + 'Plan section unchanged': { layouts: 4, styles: 8, mutations: 117, nodes: 2500, drawn: 50, reads: 1 }, + 'Plan section changed': { layouts: 3, styles: 7, mutations: 113, nodes: 2850, drawn: 50, reads: 0 }, + // Show more reads the next page once and adds its 50 rows. Measured layouts 2, styles 8–9, mutations 54. + 'Plan show more': { layouts: 4, styles: 12, mutations: 56, nodes: 3400, drawn: 100, reads: 1 } +}; diff --git a/ui/playwright.config.ts b/ui/playwright.config.ts index feb0eec911..dbd148a205 100644 --- a/ui/playwright.config.ts +++ b/ui/playwright.config.ts @@ -1,15 +1,16 @@ import { defineConfig } from '@playwright/test'; // The UI's browser suite against the mock client with awkward data; it is not part of `npm test`. -// CI builds the UI (`npm run build`) and runs the layout project as a required step and the perf -// project in its own job, which reports its timings but does not block merges. +// CI builds the UI (`npm run build`) and runs the layout and work-budget projects as required steps, +// and the perf project in its own job, which reports its timings but does not block merges. export default defineConfig({ testDir: './e2e', timeout: 120_000, fullyParallel: true, // Timing guards run once the rest has finished, so they have the machine to themselves. projects: [ - { name: 'layout', testIgnore: /perf\.spec\.ts/ }, + { name: 'layout', testIgnore: /(perf|budgets)\.spec\.ts/ }, + { name: 'budgets', testMatch: /budgets\.spec\.ts/ }, { name: 'perf', testMatch: /perf\.spec\.ts/, dependencies: ['layout'] } ], use: { diff --git a/ui/src/lib/api/index.ts b/ui/src/lib/api/index.ts index e86830c114..ac21520d39 100644 --- a/ui/src/lib/api/index.ts +++ b/ui/src/lib/api/index.ts @@ -36,7 +36,8 @@ async function loadRecording(): Promise { * The client the app uses: the server, or the mock with `?mock` or `VITE_SQB_MOCK=1`. * With the mock, `?as=`, `?events=none|all` and `?live=reconnecting|offline` * adjust it for demos, `?awkward` stretches every list to thousands of rows with very long - * names, for the layout test, and `?large` swaps in a catalog the size of a real project. + * names, for the layout test, `?large` swaps in a catalog the size of a real project, and `?still` + * keeps runs from advancing on their own. */ export function createClient(search: URLSearchParams, onUnauthorized: () => void = () => {}): ApiClient { if (DEMO) return createHttpClient({ onUnauthorized, fetch: replayFetch(loadRecording), changes: new LocalChangeBus() }); @@ -47,8 +48,27 @@ export function createClient(search: URLSearchParams, onUnauthorized: () => void const mock = new MockApiClient({ role: ROLES.find((r) => r === role), events: EVENTS.find((e) => e === events), - connection: LIVE.find((l) => l === live) + connection: LIVE.find((l) => l === live), + still: search.has('still') }); const sized = search.has('large') ? largeProject(mock) : mock; - return search.has('awkward') ? awkward(sized) : sized; + return countedReads(search.has('awkward') ? awkward(sized) : sized); +} + +/** Calls to each mock client method so far, by name, for the browser suite's work budgets. */ +const mockReads: Record = {}; + +/** The mock client, counting each method call in `mockReads` (also `window.__sqbMockReads`). */ +function countedReads(client: ApiClient): ApiClient { + (globalThis as { __sqbMockReads?: Record }).__sqbMockReads = mockReads; + return new Proxy(client, { + get(target, name, receiver) { + const value = Reflect.get(target, name, receiver); + if (typeof value !== 'function' || typeof name !== 'string') return value; + return (...args: unknown[]) => { + mockReads[name] = (mockReads[name] ?? 0) + 1; + return value.apply(target, args); + }; + } + }); } diff --git a/ui/src/lib/api/mock/mock-client.ts b/ui/src/lib/api/mock/mock-client.ts index 8799610cec..db861e33ab 100644 --- a/ui/src/lib/api/mock/mock-client.ts +++ b/ui/src/lib/api/mock/mock-client.ts @@ -56,6 +56,8 @@ export type MockOptions = { role?: BuiltInRole; /** `default` shows three warnings; `none` is all clear; `all` adds critical events. */ events?: 'default' | 'none' | 'all'; + /** Runs never advance on their own (`?still`), for the browser suite's work budgets. */ + still?: boolean; /** Simulated request latency in milliseconds. */ latencyMs?: number; /** Simulated state of the live-signal connection. */ @@ -85,7 +87,8 @@ export class MockApiClient implements ApiClient { role: options.role ?? 'admin', events: options.events ?? 'default', latencyMs: options.latencyMs ?? 120, - connection: options.connection ?? 'live' + connection: options.connection ?? 'live', + still: options.still ?? false }; this.changes.setState(this.#options.connection); this.#fixtures = createFixtures(this.#createdAt); @@ -96,7 +99,8 @@ export class MockApiClient implements ApiClient { user: () => DEMO_USER, adapter: (scope) => this.#location(scope).target.adapter ?? 'duckdb', target: (scope) => this.#location(scope).target.name, - reply: (value) => this.#reply(value) + reply: (value) => this.#reply(value), + still: this.#options.still }); this.#platform = new MockPlatformApi({ now: this.#createdAt, diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index b861e1875a..72914d6ec8 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -132,6 +132,8 @@ export type OperateContext = { /** Name of the location's one target. */ target: (scope: Scope) => string; reply: (value: T) => Promise; + /** Runs stay where they are instead of advancing every 1.5 s, so counted work is the same on every run. */ + still?: boolean; }; /** The end-to-end tests' stand-in for a plan recorded without timings: `window.__sqbPlanWithoutTimings`. */ @@ -166,7 +168,7 @@ export class MockOperate { this.#fx = createOperateFixtures(now); this.#records = this.#fx.records; for (const rec of this.#records) this.#settle(rec, now); - if (typeof window !== 'undefined') setInterval(() => this.#advance(), 1500); + if (typeof window !== 'undefined' && !ctx.still) setInterval(() => this.#advance(), 1500); } // ---- reads: runs ----