From a30d79730fd3eae16d5c178983461fdb8c6117ff Mon Sep 17 00:00:00 2001 From: Tim Bradgate Date: Thu, 24 Sep 2026 01:03:03 +0100 Subject: [PATCH] Make flaky Playwright tests visible in CI Retries hide flakiness: a test that fails then passes on retry only shows up as a buried "N flaky" line, and junit/exit-code treat it as a pass. Add CI-only reporters and a job-summary step so flaky tests are actually visible without lowering retries (the suite is globally serial with DB-restore retry hooks). - playwright.config.ts: add the `github` reporter (per-test error annotations + a run-summary notice for flaky/failed tests) and a `json` reporter, both gated on process.env.CI. `json` is listed after `html` because html's onEnd wipes and rebuilds the shared playwright-report/ output folder. - playwright-test.yml: new "Summarize flaky tests" step parses results.json with jq and writes a flaky-test table to $GITHUB_STEP_SUMMARY; guarded with if: always() and never fails the job, including on a missing or unparseable report. Verified: ci-lint and typecheck pass; the jq filter was checked against a real results.json from a full local `CI=true` run of all 221 tests (chromium) and against a mutated copy simulating a flaky result; the json reporter was confirmed to survive alongside the html reporter's output-folder rebuild. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/playwright-test.yml | 43 +++++++++++++++++++++++++++ client-v3/playwright.config.ts | 28 +++++++++++++---- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/.github/workflows/playwright-test.yml b/.github/workflows/playwright-test.yml index 282c8df9e..b1c9c0a73 100644 --- a/.github/workflows/playwright-test.yml +++ b/.github/workflows/playwright-test.yml @@ -49,6 +49,49 @@ jobs: run: npx playwright test --project=${{ matrix.browser }} working-directory: ./client-v3 + # Retries mean a test that fails then passes shows up in the run as + # "flaky", not "failed" — junit/exit-code treat it as a pass, so it's + # easy to miss. This reads the json reporter output (client-v3/playwright.config.ts, + # CI-only) and lists any flaky tests in the job summary. Best-effort: + # guarded with `if: always()` and never fails the job. + - name: Summarize flaky tests + if: always() + working-directory: ./client-v3 + run: | + set +e + REPORT="playwright-report/results.json" + { + echo "### Flaky tests — ${{ matrix.browser }}" + echo "" + } >> "$GITHUB_STEP_SUMMARY" + if [ ! -f "$REPORT" ]; then + echo "_No JSON report found (run may not have completed); skipping._" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + FLAKY=$(jq -r ' + [.. | objects | select(has("tests") and has("ok")) + | . as $spec + | $spec.tests[] + | select(.status == "flaky") + | "| \($spec.file):\($spec.line) | \($spec.title) | \(.projectName) | \(.results | length) |" + ] | join("\n") + ' "$REPORT") + JQ_STATUS=$? + if [ "$JQ_STATUS" -ne 0 ]; then + echo "_Could not parse $REPORT (jq exited $JQ_STATUS) — unable to determine flaky status for this run._" >> "$GITHUB_STEP_SUMMARY" + elif [ -z "$FLAKY" ]; then + echo "None." >> "$GITHUB_STEP_SUMMARY" + else + { + echo "A test that fails at least once but eventually passes on retry still counts as a pass overall — see [issue #1415](https://github.com/dreamteamprod/DigiScript/issues/1415)." + echo "" + echo "| Spec | Test | Project | Attempts |" + echo "|---|---|---|---|" + echo "$FLAKY" + } >> "$GITHUB_STEP_SUMMARY" + fi + exit 0 + - name: Upload Playwright report uses: actions/upload-artifact@v4 if: always() diff --git a/client-v3/playwright.config.ts b/client-v3/playwright.config.ts index ae01964f8..ae35c672a 100644 --- a/client-v3/playwright.config.ts +++ b/client-v3/playwright.config.ts @@ -1,17 +1,35 @@ -import { defineConfig, devices } from '@playwright/test'; +import { defineConfig, devices, type ReporterDescription } from '@playwright/test'; const SERVER_PORT = 8888; +// Retries exist because the suite is globally serial with DB-restore retry +// hooks (see e2e/db-snapshot.ts) — a test that fails then passes on retry is +// "flaky" rather than a hard failure. By default that only shows up as a +// buried "N flaky" line in the run summary. In CI, add reporters that make +// flaky tests visible instead of silent: +// - `github`: emits GitHub Actions annotations — an `::error` annotation +// with the failed attempt's stack trace for each flaky (or failed) test, +// plus a `::notice` run-summary annotation with the flaky/failed counts. +// - `json`: writes a machine-readable report that the CI workflow parses +// with jq to list flaky tests in the job summary. +const reporters: ReporterDescription[] = [ + ['html', { outputFolder: 'playwright-report', open: 'never' }], + ['junit', { outputFile: 'junit/playwright-results.xml' }], +]; +if (process.env.CI) { + // `json` must stay listed after `html`: the html reporter's onEnd wipes + // and rebuilds `outputFolder` ('playwright-report'), so a json report + // written into that same folder before html runs would be deleted. + reporters.push(['github'], ['json', { outputFile: 'playwright-report/results.json' }]); +} + export default defineConfig({ testDir: './e2e/tests', fullyParallel: false, workers: 1, retries: 3, maxFailures: 1, - reporter: [ - ['html', { outputFolder: 'playwright-report', open: 'never' }], - ['junit', { outputFile: 'junit/playwright-results.xml' }], - ], + reporter: reporters, use: { baseURL: `http://localhost:${SERVER_PORT}`, trace: 'retain-on-failure',