DO NOT MERGE: email screenshot flake check (wait)

packages/matrix/helpers/index.ts

@@ -425,6 +425,61 @@ export async function getMonacoContent(page: Page): Promise<string> {
   );
 }
 
+// The mail client shows the email in an iframe. The email's text is in the DOM
+// as soon as the iframe is filled, but the email loads web fonts through
+// stylesheets in its <head>, and the browser paints nothing in the frame until
+// those load. Wait for the stylesheets, images and fonts so a screenshot shows
+// the email and not a blank pane.
+async function waitForEmailToRender(emailPage: Page) {
+  let startedAt = Date.now();
+  let firstPending: string[] | undefined;
+  await expect
+    .poll(
+      async () => {
+        let frame = await (
+          await emailPage.locator('.messageview iframe').elementHandle()
+        )?.contentFrame();
+        if (!frame) {
+          return ['the email iframe'];
+        }
+        let pending = await frame.evaluate(() => {
+          let pending: string[] = [];
+          for (let link of document.querySelectorAll<HTMLLinkElement>(
+            'link[rel="stylesheet"]',
+          )) {
+            if (!link.sheet) {
+              pending.push(link.href);
+            }
+          }
+          for (let image of document.images) {
+            if (!image.complete) {
+              pending.push(image.src);
+            }
+          }
+          if (document.fonts.status !== 'loaded') {
+            pending.push('fonts');
+          }
+          return pending;
+        });
+        firstPending ??= pending;
+        return pending;
+      },
+      {
+        message: 'the email has loaded its stylesheets, images and fonts',
+        timeout: 30_000,
+      },
+    )
+    .toEqual([]);
+  let waitedMs = Date.now() - startedAt;
+  if (waitedMs > 1000) {
+    console.log(
+      `waited ${waitedMs}ms for the email to render; pending at first: ${JSON.stringify(
+        firstPending,
+      )}`,
+    );
+  }
+}
+
 export async function validateEmail(
   appPage: Page,
   email: string,
@@ -467,6 +522,7 @@ export async function validateEmail(
   await expect(
     emailPage.locator('.messageview .messageviewheader'),
   ).toContainText(`To:${email}`);
+  await waitForEmailToRender(emailPage);
 
   if (opts?.onEmailPage) {
     await opts.onEmailPage(emailPage);
@@ -525,6 +581,7 @@ export async function validateEmailForResetPassword(
   await expect(
     emailPage.locator('.messageview .messageviewheader'),
   ).toContainText(`To:${email}`);
+  await waitForEmailToRender(emailPage);
 
   if (opts?.onEmailPage) {
     await opts.onEmailPage(emailPage);

packages/matrix/scripts/test.sh

@@ -5,7 +5,13 @@ echo "running tests: ${1}"
 # this and narrows `testMatch` to the spec files bin-packed onto this shard;
 # Playwright's own `--shard` is not used because it balances test count rather
 # than test cost. See packages/matrix/support/shard-spec-files.ts.
-MATRIX_TEST_SHARD="${1:-}"
+# THROWAWAY VERIFICATION: every shard runs only the email screenshot tests,
+# repeated, one worker, no retries. The template job has nothing to run.
+if [ -n "${MATRIX_SYNAPSE_TEMPLATE:-}" ]; then
+  echo "verification run: skipping template job"
+  exit 0
+fi
+MATRIX_TEST_SHARD=""
 export MATRIX_TEST_SHARD
 
 BASE_REALM="https-get://localhost:4201/base/"
@@ -20,4 +26,4 @@ BASE_REALM_READY="$BASE_REALM$READY_PATH"
 WAIT_ON_TIMEOUT=600000 START_SERVER_AND_TEST_INSECURE=1 start-server-and-test \
   'pnpm run wait' \
   "$BASE_REALM_READY" \
-  'pnpm playwright test'
+  'pnpm playwright test tests/profile.spec.ts:90 tests/forgot-password.spec.ts:27 tests/registration-with-token.spec.ts:51 --repeat-each=5 --retries=0 --workers=1'

packages/matrix/tests/fixtures.ts

@@ -1,7 +1,25 @@
 import { test as base, expect } from '@playwright/test';
+import type { BrowserContext } from '@playwright/test';
 import { setRealmRedirects } from '../helpers/index.ts';
 
+// THROWAWAY VERIFICATION: hold the Poppins 700 stylesheet the email templates
+// load, as in the CI failure where it had no response at screenshot time.
+async function stallEmailFontStylesheet(context: BrowserContext) {
+  await context.route(
+    'https://fonts.googleapis.com/css2?family=Poppins:wght@700',
+    async (route) => {
+      console.log('stalling Poppins 700 stylesheet for 8000ms');
+      await new Promise((r) => setTimeout(r, 8000));
+      await route.continue();
+    },
+  );
+}
+
 export const test = base.extend({
+  context: async ({ context }, use) => {
+    await stallEmailFontStylesheet(context);
+    await use(context);
+  },
   page: async ({ page }, use) => {
     await setRealmRedirects(page);
     await use(page);