diff --git a/.github/workflows/end-to-end-tests.yml b/.github/workflows/end-to-end-tests.yml index e249f6e25475c..9e03641b978fd 100644 --- a/.github/workflows/end-to-end-tests.yml +++ b/.github/workflows/end-to-end-tests.yml @@ -82,10 +82,13 @@ jobs: fail-fast: false matrix: LOCAL_SCRIPT_DEBUG: [ true, false ] + # See the "Projects" section in tests/e2e/README.md. + test-group: [ 'parallel', 'serial' ] # A matrix value is needed in the 'name' directive for proper grouping in the GitHub UI. label: [ 'E2E Tests' ] with: LOCAL_SCRIPT_DEBUG: ${{ matrix.LOCAL_SCRIPT_DEBUG }} + test-group: ${{ matrix.test-group }} slack-notifications: name: Slack Notifications diff --git a/.github/workflows/reusable-end-to-end-tests.yml b/.github/workflows/reusable-end-to-end-tests.yml index 54bd6b4230fab..40db644c5a2d6 100644 --- a/.github/workflows/reusable-end-to-end-tests.yml +++ b/.github/workflows/reusable-end-to-end-tests.yml @@ -32,6 +32,11 @@ on: required: false type: 'boolean' default: true + test-group: + description: 'The Playwright project to run. Runs the default projects when empty.' + required: false + type: 'string' + default: '' env: LOCAL_DIR: build @@ -62,7 +67,7 @@ jobs: # - Uploads screenshots and HTML snapshots as an artifact. # - Ensures version-controlled files are not modified or deleted. e2e-tests: - name: SCRIPT_DEBUG ${{ inputs.LOCAL_SCRIPT_DEBUG && 'enabled' || 'disabled' }} + name: ${{ inputs.test-group && format( '{0}, ', inputs.test-group ) || '' }}SCRIPT_DEBUG ${{ inputs.LOCAL_SCRIPT_DEBUG && 'enabled' || 'disabled' }} runs-on: ubuntu-24.04 permissions: contents: read @@ -147,8 +152,14 @@ jobs: npm run env:cli -- language theme install de_DE --all --path="/var/www/${LOCAL_DIR}" - name: Run E2E tests - run: npm run test:e2e + run: | + if [ -n "${TEST_GROUP}" ]; then + npm run test:e2e -- --project="${TEST_GROUP}" + else + npm run test:e2e + fi env: + TEST_GROUP: ${{ inputs.test-group }} # Chromium is installed above and is the only browser the suite runs. Without this, # `wp-scripts test-playwright` also downloads Firefox and WebKit on every run. PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: ${{ inputs.install-playwright && '1' || '' }} @@ -157,7 +168,7 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: always() with: - name: failures-artifacts${{ inputs.LOCAL_SCRIPT_DEBUG && '-SCRIPT_DEBUG' || '' }}-${{ github.run_id }} + name: failures-artifacts${{ inputs.test-group && format( '-{0}', inputs.test-group ) || '' }}${{ inputs.LOCAL_SCRIPT_DEBUG && '-SCRIPT_DEBUG' || '' }}-${{ github.run_id }} path: artifacts if-no-files-found: ignore include-hidden-files: true diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 23eb8016710af..597ac99896bd8 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -1 +1 @@ -# E2E Tests End-To-End (E2E) tests for WordPress. ## Running the tests The e2e tests require a production-like environment to run. By default, they will assume an environment is available at `http://localhost:8889`, with username `admin` and password `password`. If you don't already have an environment ready, you can set one up by following [these instructions](https://github.com/WordPress/wordpress-develop/blob/master/README.md). Then you can launch the tests by running: ``` npm run test:e2e ``` which will run the test suite using a headless browser. If your environment has a different url, username or password to the default, you can provide the base URL, username and password like this: ``` WP_BASE_URL=http://mycustomurl WP_USERNAME=username WP_PASSWORD=password npm run test:e2e ``` **DO NOT run these tests in an actual production environment, as they will delete all your content.** For debugging purposes, you might want to follow the test visually. You can do so by running the tests in an interactive mode: ``` npm run test:e2e -- --ui ``` [UI Mode](https://playwright.dev/docs/test-ui-mode) let's you explore, run and debug tests with a time travel experience complete with watch mode. All test files are loaded into the testing sidebar where you can expand each file and describe block to individually run, view, watch and debug each test. You can also run a single test file separately: ``` npm run test:e2e tests/e2e/specs/hello.test.js ``` ## Documentation * Block Editor Handbook end to end testing overview: https://developer.wordpress.org/block-editor/contributors/code/testing-overview/#end-to-end-testing * Gutenberg e2e-test-utils-playwright package API docs: https://github.com/WordPress/gutenberg/tree/trunk/packages/e2e-test-utils-playwright * Playwright API docs: https://playwright.dev/docs (the version we are using is indicated in the `@wordpress/scripts` package: https://github.com/WordPress/gutenberg/blob/trunk/packages/scripts/package.json) \ No newline at end of file +# E2E Tests End-To-End (E2E) tests for WordPress. ## Running the tests The e2e tests require a production-like environment to run. By default, they will assume an environment is available at `http://localhost:8889`, with username `admin` and password `password`. If you don't already have an environment ready, you can set one up by following [these instructions](https://github.com/WordPress/wordpress-develop/blob/master/README.md). Then you can launch the tests by running: ``` npm run test:e2e ``` which will run the test suite using a headless browser. If your environment has a different url, username or password to the default, you can provide the base URL, username and password like this: ``` WP_BASE_URL=http://mycustomurl WP_USERNAME=username WP_PASSWORD=password npm run test:e2e ``` **DO NOT run these tests in an actual production environment, as they will delete all your content.** For debugging purposes, you might want to follow the test visually. You can do so by running the tests in an interactive mode: ``` npm run test:e2e -- --ui ``` [UI Mode](https://playwright.dev/docs/test-ui-mode) let's you explore, run and debug tests with a time travel experience complete with watch mode. All test files are loaded into the testing sidebar where you can expand each file and describe block to individually run, view, watch and debug each test. You can also run a single test file separately: ``` npm run test:e2e tests/e2e/specs/hello.test.js ``` ## Projects The suite has three Playwright projects, defined in `playwright.config.js`: * `all`: every spec, one at a time on one worker. `npm run test:e2e` runs this project. * `parallel`: specs that can share the site with other tests. Spec files run at the same time on several workers, one for every two CPU cores. Tests inside one file still run in order. Run it with `npm run test:e2e -- --project=parallel`. * `serial`: specs that break the site for other tests. They run one at a time on one worker. Run it with `npm run test:e2e -- --project=serial`. CI runs `parallel` and `serial` in separate jobs. New specs go in `parallel` by default. Add a spec to `SERIAL_SPECS` only if it makes the site unusable for other tests, or if it changes content or settings that other tests use, for example with `requestUtils.deleteAllPosts()`. Before you open a pull request with a new spec, run its project a few times with `--shuffle`. Playwright then runs the spec files in a random order, so a spec that needs another spec to run first, or that breaks a spec that runs after it, fails: ``` npm run test:e2e -- --project=parallel --shuffle ``` Playwright prints the seed of each run, for example `shuffle seed 317753389`. To run the same order again, give the seed: `--shuffle 317753389`. ## Documentation * Block Editor Handbook end to end testing overview: https://developer.wordpress.org/block-editor/contributors/code/testing-overview/#end-to-end-testing * Gutenberg e2e-test-utils-playwright package API docs: https://github.com/WordPress/gutenberg/tree/trunk/packages/e2e-test-utils-playwright * Playwright API docs: https://playwright.dev/docs (the version we are using is indicated in the `@wordpress/scripts` package: https://github.com/WordPress/gutenberg/blob/trunk/packages/scripts/package.json) \ No newline at end of file diff --git a/tests/e2e/playwright.config.js b/tests/e2e/playwright.config.js index 0de694e324c24..d9441043d4049 100644 --- a/tests/e2e/playwright.config.js +++ b/tests/e2e/playwright.config.js @@ -2,7 +2,7 @@ * External dependencies */ import path from 'node:path'; -import { defineConfig } from '@playwright/test'; +import { defineConfig, devices } from '@playwright/test'; /** * WordPress dependencies @@ -15,13 +15,55 @@ process.env.STORAGE_STATE_PATH ??= path.join( 'storage-states/admin.json' ); +const SERIAL_SPECS = [ + 'install.test.js', + 'maintenance-mode.test.js', + 'fatal-error-handler.test.js', + 'login-localize-script.test.js', + 'gutenberg-plugin.test.js', + 'plugins.test.js', + 'themes.test.js', + // These delete all posts or media, including other workers' content. + 'cache-control-headers-directives.test.js', + 'dashboard.test.js', + 'edit-posts.test.js', + 'empty-trash-restore-trashed-posts.test.js', + 'media-library-client-side-upload.test.js', + 'media-modal-client-side-upload.test.js', + 'media-new-client-side-upload.test.js', +]; + const config = defineConfig( { ...baseConfig, globalSetup: require.resolve( './config/global-setup.js' ), + workers: '50%', webServer: { ...baseConfig.webServer, command: 'npm run env:start', }, + projects: [ + { + name: 'all', + fullyParallel: false, + workers: 1, + use: { ...devices[ 'Desktop Chrome' ] }, + }, + { + name: 'parallel', + default: false, + testIgnore: SERIAL_SPECS, + fullyParallel: false, + use: { ...devices[ 'Desktop Chrome' ] }, + }, + { + name: 'serial', + default: false, + testMatch: SERIAL_SPECS, + fullyParallel: false, + workers: 1, + use: { ...devices[ 'Desktop Chrome' ] }, + }, + ], } ); export default config;