diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 2f67639bcd..7814fce985 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -1434,6 +1434,8 @@ jobs: run: npx playwright install --with-deps chromium - name: Layout test (every page with awkward mock data at 1440 and 1920 px) run: npx playwright test --project=layout + - name: Work budgets (layouts, restyles, DOM changes and reads per interaction, counted) + run: npx playwright test --project=budgets ui_timings: # The timing guards in ui/e2e/perf.spec.ts, with the CPU slowed 6x. They go red over budget and diff --git a/ui/e2e/budgets.spec.ts b/ui/e2e/budgets.spec.ts new file mode 100644 index 0000000000..b65bfb4a9c --- /dev/null +++ b/ui/e2e/budgets.spec.ts @@ -0,0 +1,289 @@ +import { expect, test, type CDPSession, type Page } from '@playwright/test'; + +// Work budgets: what each interaction costs, counted rather than timed, so a slow or busy machine gives +// the same numbers as a fast one. Each gesture is a fixed number of input events, each followed by two +// awaited animation frames. Per gesture the suite counts Chrome's layouts and style recalculations +// (Performance.getMetrics), DOM mutation records (every Svelte re-render that touches the page), the +// DOM nodes and cards or rows on the page afterwards, and the mock client's reads. Budgets sit just +// above the measured values; the breakdown next to each says where the count comes from. + +type Cost = { layouts: number; styles: number; mutations: number; nodes: number; drawn: number; reads: number }; +type Budget = Cost; + +const RATE = Number(process.env.BUDGET_CPU_RATE ?? '1'); +const TRACE = !!process.env.BUDGET_TRACE; +// Not reads: `supports` asks which methods exist. The app shell polls these on a 30 s wall-clock timer +// whatever the page does, so whether one lands inside a gesture depends on when the gesture started. +const NOT_COUNTED = new Set(['supports', 'getWarehouseSync', 'listPlatformEvents', 'navCounts']); + +async function frames(page: Page, count = 2) { + await page.evaluate( + (n) => + new Promise((resolve) => { + let left = n; + const tick = () => (--left <= 0 ? resolve() : requestAnimationFrame(tick)); + requestAnimationFrame(tick); + }), + count + ); +} + +async function counters(page: Page, cdp: CDPSession) { + // Nodes counts detached nodes until they are collected, so collect first. + await cdp.send('HeapProfiler.collectGarbage'); + const { metrics } = await cdp.send('Performance.getMetrics'); + const value = (name: string) => metrics.find((m) => m.name === name)?.value ?? 0; + const page_ = await page.evaluate(() => { + const w = window as unknown as { __sqbMutations?: number; __sqbMockReads?: Record }; + return { mutations: w.__sqbMutations ?? 0, readsByName: { ...(w.__sqbMockReads ?? {}) } }; + }); + const reads = Object.entries(page_.readsByName) + .filter(([name]) => !NOT_COUNTED.has(name)) + .reduce((a, [, n]) => a + n, 0); + return { layouts: value('LayoutCount'), styles: value('RecalcStyleCount'), nodes: value('Nodes'), mutations: page_.mutations, reads, readsByName: page_.readsByName }; +} + +/** + * Runs in the page before the app. While a gesture is counted (`__sqbHold(true)`), the graph canvas's + * settle timer (`settleMs`) is held and run once when counting stops, and interval ticks (the + * one-second clock, the shell's polls) are skipped. Otherwise how often either fires inside a gesture + * depends on how fast the machine runs the steps, and the counts with it. + */ +function holdTimers(settleMs: number) { + const realSetTimeout = window.setTimeout.bind(window); + const realClearTimeout = window.clearTimeout.bind(window); + const realSetInterval = window.setInterval.bind(window); + let holding = false; + let next = 1e9; + const held = new Map void>(); + window.setTimeout = ((handler: TimerHandler, ms?: number, ...args: unknown[]) => { + if (!holding || ms !== settleMs || typeof handler !== 'function') return realSetTimeout(handler, ms, ...args); + held.set(++next, () => handler(...args)); + return next; + }) as typeof window.setTimeout; + window.clearTimeout = ((id?: number) => { + if (id === undefined || !held.delete(id)) realClearTimeout(id); + }) as typeof window.clearTimeout; + window.setInterval = ((handler: TimerHandler, ms?: number, ...args: unknown[]) => + realSetInterval(typeof handler === 'function' ? (...a: unknown[]) => holding || handler(...a) : handler, ms, ...args)) as typeof window.setInterval; + (window as unknown as { __sqbHold: (on: boolean) => void }).__sqbHold = (on) => { + holding = on; + if (on) return; + const due = [...held.values()]; + held.clear(); + for (const run of due) run(); + }; +} + +const hold = (page: Page, on: boolean) => page.evaluate((o) => (window as unknown as { __sqbHold: (on: boolean) => void }).__sqbHold(o), on); + +/** Open `path`, wait for `ready`, start counting mutations and set the CPU rate. */ +async function open(page: Page, path: string, ready: string): Promise { + await page.setViewportSize({ width: 1600, height: 900 }); + // The graph canvas's SETTLE_MS. + await page.addInitScript(holdTimers, 200); + await page.goto(path); + await page.locator(ready).first().waitFor({ timeout: 30_000 }); + await page.waitForLoadState('networkidle'); + await frames(page, 10); + await page.evaluate(() => { + const w = window as unknown as { __sqbMutations: number; __sqbMutationKinds: Record }; + w.__sqbMutations = 0; + w.__sqbMutationKinds = {}; + const kind = (r: MutationRecord) => { + const el = (r.target.nodeType === 1 ? r.target : r.target.parentElement) as Element | null; + const cls = + el + ?.getAttribute('class') + ?.split(/\s+/) + .find((c) => !c.startsWith('svelte-')) ?? + el?.tagName.toLowerCase() ?? + '?'; + return `${r.type}${r.attributeName ? `:${r.attributeName}` : ''} ${cls}`; + }; + new MutationObserver((records) => { + w.__sqbMutations += records.length; + for (const r of records) w.__sqbMutationKinds[kind(r)] = (w.__sqbMutationKinds[kind(r)] ?? 0) + 1; + }).observe(document.body, { subtree: true, childList: true, attributes: true, characterData: true }); + }); + const cdp = await page.context().newCDPSession(page); + await cdp.send('Performance.enable'); + await cdp.send('Emulation.setCPUThrottlingRate', { rate: RATE }); + return cdp; +} + +/** The cost of `act`, which ends with the page settled; `drawn` counts `drawnSelector` afterwards. */ +async function cost(page: Page, cdp: CDPSession, drawnSelector: string, act: () => Promise): Promise { + await frames(page); + if (TRACE) await page.evaluate(() => ((window as unknown as { __sqbMutationKinds: Record }).__sqbMutationKinds = {})); + const before = await counters(page, cdp); + await hold(page, true); + await act(); + await hold(page, false); + await page.waitForLoadState('networkidle'); + await frames(page, 4); + const after = await counters(page, cdp); + const readNames = Object.entries(after.readsByName) + .map(([name, n]) => [name, n - (before.readsByName[name] ?? 0)] as const) + .filter(([, n]) => n > 0) + .map(([name, n]) => `${name}×${n}`); + if (readNames.length) console.info(`READS ${readNames.join(' ')}`); + if (TRACE) { + const kinds = await page.evaluate(() => (window as unknown as { __sqbMutationKinds: Record }).__sqbMutationKinds); + const top = Object.entries(kinds) + .sort((a, b) => b[1] - a[1]) + .slice(0, 8); + console.info(`MUTATIONS ${top.map(([k, n]) => `${k}×${n}`).join(' | ')}`); + } + return { + layouts: after.layouts - before.layouts, + styles: after.styles - before.styles, + mutations: after.mutations - before.mutations, + nodes: after.nodes, + drawn: await page.locator(drawnSelector).count(), + reads: after.reads - before.reads + }; +} + +function judge(name: string, measured: Cost, budget: Budget) { + console.info(`BUDGET ${name} rate=${RATE} ${JSON.stringify(measured)}`); + const over = (Object.keys(budget) as (keyof Cost)[]).filter((k) => measured[k] > budget[k]).map((k) => `${k} ${measured[k]} > ${budget[k]}`); + // Soft, so one test reports every gesture that is over, not only the first. + expect.soft(over, `${name}: over its work budget`).toEqual([]); +} + +async function wheel(page: Page, at: { x: number; y: number }, steps: number, delta: (i: number) => number) { + await page.mouse.move(at.x, at.y); + for (let i = 0; i < steps; i++) { + await page.mouse.wheel(0, delta(i)); + await frames(page); + } +} + +async function drag(page: Page, from: { x: number; y: number }, steps: number) { + await page.mouse.move(from.x, from.y); + await page.mouse.down(); + for (let i = 1; i <= steps; i++) { + await page.mouse.move(from.x + i * 6, from.y + i * 2); + await frames(page); + } + await page.mouse.up(); +} + +/** A point on the canvas with no card under it. */ +async function emptyPoint(page: Page): Promise<{ x: number; y: number }> { + return page.evaluate(() => { + const r = document.querySelector('.svelte-flow')!.getBoundingClientRect(); + for (let y = r.top + 30; y < r.bottom - 30; y += 20) + for (let x = r.left + 80; x < r.right - 30; x += 20) + if (!document.elementFromPoint(x, y)?.closest('.svelte-flow__node, .svelte-flow__controls, .svelte-flow__edge, .svelte-flow__panel')) return { x, y }; + return { x: r.left + 80, y: r.top + 30 }; + }); +} + +const CARD = '.svelte-flow__node'; +const GRAPHS = [ + { name: 'large group', path: '/jobs/warehouse_nightly?mock&still' }, + { name: 'Lineage overview', path: '/lineage?mock&large&still' } +]; + +for (const { name, path } of GRAPHS) { + test(`${name}: wheel zoom and drag-pan`, async ({ page }) => { + test.setTimeout(240_000); + const cdp = await open(page, path, `.graph-canvas:not(.unfitted) ${CARD}`); + const box = (await page.locator('.svelte-flow').first().boundingBox())!; + const centre = { x: box.x + box.width / 2, y: box.y + box.height / 2 }; + judge(`${name} wheel zoom`, await cost(page, cdp, CARD, () => wheel(page, centre, 60, (i) => (i < 30 ? 100 : -100))), BUDGETS[`${name} wheel zoom`]); + const empty = await emptyPoint(page); + judge(`${name} drag-pan`, await cost(page, cdp, CARD, () => drag(page, empty, 40)), BUDGETS[`${name} drag-pan`]); + }); +} + +const ROW = 'table.sb-table tbody tr:not(.spacer)'; + +test('Checks: second visit with thousands of checks, then scrolling the list', async ({ page }) => { + test.setTimeout(180_000); + const nav = (label: string) => page.locator('nav a', { hasText: label }).first(); + const cdp = await open(page, '/?mock&awkward&still', 'nav a'); + await nav('Checks').click(); + await page.locator(ROW).first().waitFor(); + await page.waitForLoadState('networkidle'); + await nav('Overview').click(); + await page.waitForLoadState('networkidle'); + await frames(page, 4); + judge( + 'Checks second visit', + await cost(page, cdp, ROW, async () => { + await nav('Checks').click(); + await page.locator(ROW).first().waitFor(); + }), + BUDGETS['Checks second visit'] + ); + const list = (await page.locator('.sb-table-scroll').first().boundingBox())!; + judge('Checks list scroll', await cost(page, cdp, ROW, () => wheel(page, { x: list.x + list.width / 2, y: list.y + 200 }, 40, () => 400)), BUDGETS['Checks list scroll']); +}); + +test('Plan: switching sections and showing more on a large changed build', async ({ page }) => { + test.setTimeout(180_000); + const PLAN_ROW = '.pane .plan-row'; + const cdp = await open(page, `/plan?mock&large&still&cmd=${encodeURIComponent('sqb build --changed')}`, PLAN_ROW); + for (const key of ['first_run', 'unchanged', 'changed']) { + judge( + `Plan section ${key}`, + await cost(page, cdp, PLAN_ROW, async () => { + await page.locator(`.index .entry[data-section="${key}"]`).click(); + await page.locator(`.pane[data-section="${key}"] .plan-row`).first().waitFor(); + }), + BUDGETS[`Plan section ${key}`] + ); + } + await page.locator('.index .entry[data-section="unchanged"]').click(); + await page.locator(`.pane[data-section="unchanged"] .plan-row`).nth(49).waitFor(); + judge( + 'Plan show more', + await cost(page, cdp, PLAN_ROW, async () => { + await page.getByRole('button', { name: /^Show more/ }).click(); + await page.locator(PLAN_ROW).nth(99).waitFor(); + }), + BUDGETS['Plan show more'] + ); +}); + +// Measured on 10 runs each at 1× and 6× CPU (Chrome throttling), with the breakdown traced from the +// mutation records (`BUDGET_TRACE=1`). The graph's settle timer is held while a gesture is counted and +// run once at its end, so DOM changes and reads are the same on every run and get 2 of headroom. +// Layouts and style recalculations still move by a few, because Chrome batches them per frame however +// fast the machine draws frames, and the Checks page renders a little more at 6×; those budgets are the +// highest count seen plus the spread seen plus 2. A regression that costs one more render per frame adds +// 35 to 45 DOM changes to every gesture; windowing switched off draws 5,000 rows instead of 20. +const BUDGETS: Record = { + // 60 wheel steps: the viewport transform is written once per step that moves (35; the rest stop at + // the zoom limit), the zoom buttons toggle 3 times, and the settle restyles 7 edge paths, redraws the + // grid and sets the canvas classes. Measured layouts 2, styles 43–46, mutations 54, 2,501 nodes, 9 cards. + 'large group wheel zoom': { layouts: 4, styles: 50, mutations: 56, nodes: 2600, drawn: 12, reads: 0 }, + // 40 pointer moves: one transform write and about one restyle each, plus the settle. + // Measured layouts 1, styles 53–56, mutations 51. + 'large group drag-pan': { layouts: 3, styles: 61, mutations: 53, nodes: 2600, drawn: 12, reads: 0 }, + // 45 moving steps, then the settle: canvas classes, the grid redraw and the compact level. + // Measured layouts 1, styles 50–51, mutations 57, 3,313 nodes, 13 cards. + 'Lineage overview wheel zoom': { layouts: 3, styles: 54, mutations: 59, nodes: 3400, drawn: 16, reads: 0 }, + // 40 transform writes, 2 card classes and the settle. Measured layouts 0, styles 43–46, mutations 44. + 'Lineage overview drag-pan': { layouts: 2, styles: 51, mutations: 46, nodes: 3400, drawn: 16, reads: 0 }, + // One listChecks read (filter counts come from it), 20 rows drawn: 72 row insertions and 200 cell + // class writes, the rest the navigation. Measured layouts 2–4, styles 15–22, mutations 350 at 1× and + // 377 at 6×, 3,327–3,545 nodes (as garbage collection finds them). + 'Checks second visit': { layouts: 7, styles: 31, mutations: 410, nodes: 3800, drawn: 24, reads: 1 }, + // 40 wheel steps of 400 px: the window keeps 29 rows (the view and 8 either side); each step swaps + // rows in and out (1,822 insertions and removals), rewrites the recycled cells' classes (11,500) and + // the two spacers' heights (78), with one layout per step. Measured layouts 40–41, styles 81, + // mutations 13,400–13,414, 4,083–4,301 nodes. + 'Checks list scroll': { layouts: 44, styles: 83, mutations: 13430, nodes: 4600, drawn: 32, reads: 0 }, + // Each section switch replaces the 50 rows it shows (100 row insertions and removals) and reads its + // first page once; Changed came with the page, so it reads nothing. Measured the same in every run: + // layouts 2/2/1, styles 5/6/5, mutations 115/115/111. + 'Plan section first_run': { layouts: 4, styles: 7, mutations: 117, nodes: 2500, drawn: 50, reads: 1 }, + 'Plan section unchanged': { layouts: 4, styles: 8, mutations: 117, nodes: 2500, drawn: 50, reads: 1 }, + 'Plan section changed': { layouts: 3, styles: 7, mutations: 113, nodes: 2850, drawn: 50, reads: 0 }, + // Show more reads the next page once and adds its 50 rows. Measured layouts 2, styles 8–9, mutations 54. + 'Plan show more': { layouts: 4, styles: 12, mutations: 56, nodes: 3400, drawn: 100, reads: 1 } +}; diff --git a/ui/playwright.config.ts b/ui/playwright.config.ts index feb0eec911..dbd148a205 100644 --- a/ui/playwright.config.ts +++ b/ui/playwright.config.ts @@ -1,15 +1,16 @@ import { defineConfig } from '@playwright/test'; // The UI's browser suite against the mock client with awkward data; it is not part of `npm test`. -// CI builds the UI (`npm run build`) and runs the layout project as a required step and the perf -// project in its own job, which reports its timings but does not block merges. +// CI builds the UI (`npm run build`) and runs the layout and work-budget projects as required steps, +// and the perf project in its own job, which reports its timings but does not block merges. export default defineConfig({ testDir: './e2e', timeout: 120_000, fullyParallel: true, // Timing guards run once the rest has finished, so they have the machine to themselves. projects: [ - { name: 'layout', testIgnore: /perf\.spec\.ts/ }, + { name: 'layout', testIgnore: /(perf|budgets)\.spec\.ts/ }, + { name: 'budgets', testMatch: /budgets\.spec\.ts/ }, { name: 'perf', testMatch: /perf\.spec\.ts/, dependencies: ['layout'] } ], use: { diff --git a/ui/src/lib/api/index.ts b/ui/src/lib/api/index.ts index e86830c114..ac21520d39 100644 --- a/ui/src/lib/api/index.ts +++ b/ui/src/lib/api/index.ts @@ -36,7 +36,8 @@ async function loadRecording(): Promise { * The client the app uses: the server, or the mock with `?mock` or `VITE_SQB_MOCK=1`. * With the mock, `?as=`, `?events=none|all` and `?live=reconnecting|offline` * adjust it for demos, `?awkward` stretches every list to thousands of rows with very long - * names, for the layout test, and `?large` swaps in a catalog the size of a real project. + * names, for the layout test, `?large` swaps in a catalog the size of a real project, and `?still` + * keeps runs from advancing on their own. */ export function createClient(search: URLSearchParams, onUnauthorized: () => void = () => {}): ApiClient { if (DEMO) return createHttpClient({ onUnauthorized, fetch: replayFetch(loadRecording), changes: new LocalChangeBus() }); @@ -47,8 +48,27 @@ export function createClient(search: URLSearchParams, onUnauthorized: () => void const mock = new MockApiClient({ role: ROLES.find((r) => r === role), events: EVENTS.find((e) => e === events), - connection: LIVE.find((l) => l === live) + connection: LIVE.find((l) => l === live), + still: search.has('still') }); const sized = search.has('large') ? largeProject(mock) : mock; - return search.has('awkward') ? awkward(sized) : sized; + return countedReads(search.has('awkward') ? awkward(sized) : sized); +} + +/** Calls to each mock client method so far, by name, for the browser suite's work budgets. */ +const mockReads: Record = {}; + +/** The mock client, counting each method call in `mockReads` (also `window.__sqbMockReads`). */ +function countedReads(client: ApiClient): ApiClient { + (globalThis as { __sqbMockReads?: Record }).__sqbMockReads = mockReads; + return new Proxy(client, { + get(target, name, receiver) { + const value = Reflect.get(target, name, receiver); + if (typeof value !== 'function' || typeof name !== 'string') return value; + return (...args: unknown[]) => { + mockReads[name] = (mockReads[name] ?? 0) + 1; + return value.apply(target, args); + }; + } + }); } diff --git a/ui/src/lib/api/mock/mock-client.ts b/ui/src/lib/api/mock/mock-client.ts index 8799610cec..db861e33ab 100644 --- a/ui/src/lib/api/mock/mock-client.ts +++ b/ui/src/lib/api/mock/mock-client.ts @@ -56,6 +56,8 @@ export type MockOptions = { role?: BuiltInRole; /** `default` shows three warnings; `none` is all clear; `all` adds critical events. */ events?: 'default' | 'none' | 'all'; + /** Runs never advance on their own (`?still`), for the browser suite's work budgets. */ + still?: boolean; /** Simulated request latency in milliseconds. */ latencyMs?: number; /** Simulated state of the live-signal connection. */ @@ -85,7 +87,8 @@ export class MockApiClient implements ApiClient { role: options.role ?? 'admin', events: options.events ?? 'default', latencyMs: options.latencyMs ?? 120, - connection: options.connection ?? 'live' + connection: options.connection ?? 'live', + still: options.still ?? false }; this.changes.setState(this.#options.connection); this.#fixtures = createFixtures(this.#createdAt); @@ -96,7 +99,8 @@ export class MockApiClient implements ApiClient { user: () => DEMO_USER, adapter: (scope) => this.#location(scope).target.adapter ?? 'duckdb', target: (scope) => this.#location(scope).target.name, - reply: (value) => this.#reply(value) + reply: (value) => this.#reply(value), + still: this.#options.still }); this.#platform = new MockPlatformApi({ now: this.#createdAt, diff --git a/ui/src/lib/api/mock/operate-store.ts b/ui/src/lib/api/mock/operate-store.ts index b861e1875a..72914d6ec8 100644 --- a/ui/src/lib/api/mock/operate-store.ts +++ b/ui/src/lib/api/mock/operate-store.ts @@ -132,6 +132,8 @@ export type OperateContext = { /** Name of the location's one target. */ target: (scope: Scope) => string; reply: (value: T) => Promise; + /** Runs stay where they are instead of advancing every 1.5 s, so counted work is the same on every run. */ + still?: boolean; }; /** The end-to-end tests' stand-in for a plan recorded without timings: `window.__sqbPlanWithoutTimings`. */ @@ -166,7 +168,7 @@ export class MockOperate { this.#fx = createOperateFixtures(now); this.#records = this.#fx.records; for (const rec of this.#records) this.#settle(rec, now); - if (typeof window !== 'undefined') setInterval(() => this.#advance(), 1500); + if (typeof window !== 'undefined' && !ctx.still) setInterval(() => this.#advance(), 1500); } // ---- reads: runs ----