diff --git a/.github/workflows/playwright-test.yml b/.github/workflows/playwright-test.yml index 282c8df9..b1c9c0a7 100644 --- a/.github/workflows/playwright-test.yml +++ b/.github/workflows/playwright-test.yml @@ -49,6 +49,49 @@ jobs: run: npx playwright test --project=${{ matrix.browser }} working-directory: ./client-v3 + # Retries mean a test that fails then passes shows up in the run as + # "flaky", not "failed" — junit/exit-code treat it as a pass, so it's + # easy to miss. This reads the json reporter output (client-v3/playwright.config.ts, + # CI-only) and lists any flaky tests in the job summary. Best-effort: + # guarded with `if: always()` and never fails the job. + - name: Summarize flaky tests + if: always() + working-directory: ./client-v3 + run: | + set +e + REPORT="playwright-report/results.json" + { + echo "### Flaky tests — ${{ matrix.browser }}" + echo "" + } >> "$GITHUB_STEP_SUMMARY" + if [ ! -f "$REPORT" ]; then + echo "_No JSON report found (run may not have completed); skipping._" >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + FLAKY=$(jq -r ' + [.. | objects | select(has("tests") and has("ok")) + | . as $spec + | $spec.tests[] + | select(.status == "flaky") + | "| \($spec.file):\($spec.line) | \($spec.title) | \(.projectName) | \(.results | length) |" + ] | join("\n") + ' "$REPORT") + JQ_STATUS=$? + if [ "$JQ_STATUS" -ne 0 ]; then + echo "_Could not parse $REPORT (jq exited $JQ_STATUS) — unable to determine flaky status for this run._" >> "$GITHUB_STEP_SUMMARY" + elif [ -z "$FLAKY" ]; then + echo "None." >> "$GITHUB_STEP_SUMMARY" + else + { + echo "A test that fails at least once but eventually passes on retry still counts as a pass overall — see [issue #1415](https://github.com/dreamteamprod/DigiScript/issues/1415)." + echo "" + echo "| Spec | Test | Project | Attempts |" + echo "|---|---|---|---|" + echo "$FLAKY" + } >> "$GITHUB_STEP_SUMMARY" + fi + exit 0 + - name: Upload Playwright report uses: actions/upload-artifact@v4 if: always() diff --git a/client-v3/playwright.config.ts b/client-v3/playwright.config.ts index ae01964f..ae35c672 100644 --- a/client-v3/playwright.config.ts +++ b/client-v3/playwright.config.ts @@ -1,17 +1,35 @@ -import { defineConfig, devices } from '@playwright/test'; +import { defineConfig, devices, type ReporterDescription } from '@playwright/test'; const SERVER_PORT = 8888; +// Retries exist because the suite is globally serial with DB-restore retry +// hooks (see e2e/db-snapshot.ts) — a test that fails then passes on retry is +// "flaky" rather than a hard failure. By default that only shows up as a +// buried "N flaky" line in the run summary. In CI, add reporters that make +// flaky tests visible instead of silent: +// - `github`: emits GitHub Actions annotations — an `::error` annotation +// with the failed attempt's stack trace for each flaky (or failed) test, +// plus a `::notice` run-summary annotation with the flaky/failed counts. +// - `json`: writes a machine-readable report that the CI workflow parses +// with jq to list flaky tests in the job summary. +const reporters: ReporterDescription[] = [ + ['html', { outputFolder: 'playwright-report', open: 'never' }], + ['junit', { outputFile: 'junit/playwright-results.xml' }], +]; +if (process.env.CI) { + // `json` must stay listed after `html`: the html reporter's onEnd wipes + // and rebuilds `outputFolder` ('playwright-report'), so a json report + // written into that same folder before html runs would be deleted. + reporters.push(['github'], ['json', { outputFile: 'playwright-report/results.json' }]); +} + export default defineConfig({ testDir: './e2e/tests', fullyParallel: false, workers: 1, retries: 3, maxFailures: 1, - reporter: [ - ['html', { outputFolder: 'playwright-report', open: 'never' }], - ['junit', { outputFile: 'junit/playwright-results.xml' }], - ], + reporter: reporters, use: { baseURL: `http://localhost:${SERVER_PORT}`, trace: 'retain-on-failure',