From f7d9c8c59012ec611753cdc11a7a85cf5c42b488 Mon Sep 17 00:00:00 2001 From: Brian Love Date: Thu, 3 Sep 2026 07:26:09 -0700 Subject: [PATCH] fix(website): keep the post-promotion e2e run off the local fixture runtime The deploy job re-runs the ordinary Website suite against production with only BASE_URL set, and #982 let that step run for the first time since the custom-target specs landed. Two failure clusters surfaced, both invisible to PR CI by construction because only the push-only deploy job runs the suite against a remote origin: - Twelve custom-runtime-target cases dial a fixture runtime on 127.0.0.1:4399 and the local example apps. Those exist only because the Playwright config starts them, and it starts nothing when BASE_URL is set, so every case failed with ECONNREFUSED after the site had already promoted. The config now ignores the fixture-driven specs whenever it starts no local server, including production-smoke mode. - The reduced-motion check held the runtime in its configuring state by refusing http://localhost:4300. Against the deployed site the frame loads from the production runtime origin, the handshake completes within half a second, and the loader is gone before the assertion. The route now matches the runtime frame by the session params Run mode stamps on every runtime URL, which holds the loader on screen locally and in production alike. Verified by running the full suite against https://threadplane.ai exactly as the deploy step does: 105 passed, 0 failed (was 104 passed, 13 failed). Co-Authored-By: Claude Fable 5.1 --- apps/website/e2e/workspace-shell.spec.ts | 10 ++++- apps/website/playwright.config.ts | 50 ++++++++++++++-------- apps/website/src/playwright-config.spec.ts | 45 ++++++++++++++++++- 3 files changed, 84 insertions(+), 21 deletions(-) diff --git a/apps/website/e2e/workspace-shell.spec.ts b/apps/website/e2e/workspace-shell.spec.ts index 839bef19f..cf55ee6aa 100644 --- a/apps/website/e2e/workspace-shell.spec.ts +++ b/apps/website/e2e/workspace-shell.spec.ts @@ -730,7 +730,15 @@ test.describe('workspace shell', () => { page, }) => { await page.emulateMedia({ reducedMotion: 'reduce' }); - await page.route('http://localhost:4300/**', (request) => request.abort()); + // The loader is on screen only while the runtime is still being + // configured, so refuse the runtime frame itself. Match it by the session + // params Run mode stamps on every runtime URL rather than by host: against + // the deployed site the frame loads from the production runtime origin, + // and a `localhost:4300` route lets it reach ready before the assertion. + await page.route( + (url) => url.searchParams.has('cockpit_cap'), + (route) => route.abort() + ); await page.setViewportSize({ width: 390, height: 844 }); await page.goto(`${streamingDocsPath}?mode=run`); await page.getByRole('button', { name: 'Open navigation' }).click(); diff --git a/apps/website/playwright.config.ts b/apps/website/playwright.config.ts index 2aed767ff..da4036680 100644 --- a/apps/website/playwright.config.ts +++ b/apps/website/playwright.config.ts @@ -22,29 +22,43 @@ export const createWebsitePlaywrightConfig = ( const reuseExistingServer = environment['PLAYWRIGHT_REUSE_EXISTING_SERVER'] === 'true'; + // The public-copy gate crawls every sitemap route. Against a prebuilt + // production server that is seconds; against `next dev` each route compiles + // on demand, which is far too slow to belong in the ordinary suite. It runs + // in production mode only, where its answers are the ones that matter. + const modeIgnores: readonly string[] = productionSmoke + ? ['**/public-copy.spec.ts'] + : productionMode + ? [ + '**/platform-production-smoke.spec.ts', + '**/custom-runtime-bfcache.spec.ts', + ] + : bfcacheRuntimeTest + ? ['**/platform-production-smoke.spec.ts', '**/public-copy.spec.ts'] + : [ + '**/platform-production-smoke.spec.ts', + '**/custom-runtime-bfcache.spec.ts', + '**/public-copy.spec.ts', + ]; + // The custom-target specs drive the fixture runtime on 127.0.0.1:4399 and + // the local example apps, which exist only because this config starts them. + // A run against a deployed BASE_URL — the post-promotion verification, the + // production smoke — starts nothing, so every case would dial a fixture that + // is not there and fail with ECONNREFUSED after the site already promoted. + const fixtureDrivenSpecs: readonly string[] = [ + '**/custom-runtime-targets.spec.ts', + '**/custom-runtime-bfcache.spec.ts', + ]; + const testIgnore = shouldStartLocalServer + ? [...modeIgnores] + : [...new Set([...modeIgnores, ...fixtureDrivenSpecs])]; + return defineConfig({ testDir: './e2e', testMatch: bfcacheRuntimeTest ? '**/custom-runtime-bfcache.spec.ts' : undefined, - // The public-copy gate crawls every sitemap route. Against a prebuilt - // production server that is seconds; against `next dev` each route compiles - // on demand, which is far too slow to belong in the ordinary suite. It runs - // in production mode only, where its answers are the ones that matter. - testIgnore: productionSmoke - ? '**/public-copy.spec.ts' - : productionMode - ? [ - '**/platform-production-smoke.spec.ts', - '**/custom-runtime-bfcache.spec.ts', - ] - : bfcacheRuntimeTest - ? ['**/platform-production-smoke.spec.ts', '**/public-copy.spec.ts'] - : [ - '**/platform-production-smoke.spec.ts', - '**/custom-runtime-bfcache.spec.ts', - '**/public-copy.spec.ts', - ], + testIgnore, fullyParallel: true, // Match the cockpit configs: 2 retries on CI to absorb transient Next.js // dev-server startup flake; 0 locally for fast feedback. diff --git a/apps/website/src/playwright-config.spec.ts b/apps/website/src/playwright-config.spec.ts index c8ac4aee0..8c4317335 100644 --- a/apps/website/src/playwright-config.spec.ts +++ b/apps/website/src/playwright-config.spec.ts @@ -75,8 +75,49 @@ describe('Website Playwright configuration', () => { expect(config.webServer).toBeUndefined(); // The smoke job hits the deployed site, so it runs every spec except the // public-copy gate, which exists to check a locally built production - // server before the code is deployed at all. - expect(config.testIgnore).toBe('**/public-copy.spec.ts'); + // server before the code is deployed at all, and the custom-target specs, + // which drive a fixture runtime this config did not start. + expect(config.testIgnore).toEqual([ + '**/public-copy.spec.ts', + '**/custom-runtime-targets.spec.ts', + '**/custom-runtime-bfcache.spec.ts', + ]); + }); + + it('skips the fixture-driven specs when BASE_URL points at a deployed site', () => { + // The deploy job re-runs the ordinary suite against production with only + // BASE_URL set. No local server starts in that mode, so the custom-target + // specs would dial a fixture on 127.0.0.1:4399 that does not exist and fail + // every case with ECONNREFUSED — after the site was already promoted. + const config = createWebsitePlaywrightConfig({ + BASE_URL: 'https://threadplane.ai', + }); + + expect(config.webServer).toBeUndefined(); + expect(config.use).toEqual( + expect.objectContaining({ baseURL: 'https://threadplane.ai' }) + ); + expect(config.testIgnore).toEqual([ + '**/platform-production-smoke.spec.ts', + '**/custom-runtime-bfcache.spec.ts', + '**/public-copy.spec.ts', + '**/custom-runtime-targets.spec.ts', + ]); + }); + + it('holds the runtime frame by its session params rather than the local host', () => { + // The reduced-motion check needs the runtime to stay in its connecting + // state so the loader is on screen. Refusing `http://localhost:4300` only + // does that against the local example app; against the deployed site the + // frame loads from the production runtime origin, the handshake completes, + // and the loader is gone before the assertion runs. + const shell = readFileSync( + resolve(__dirname, '../e2e/workspace-shell.spec.ts'), + 'utf8' + ); + + expect(shell).not.toContain("page.route('http://localhost:4300/**'"); + expect(shell).toContain("url.searchParams.has('cockpit_cap')"); }); it('starts Website, all migrated runtime apps under custom-runtime E2E, and the fixture', () => {