From 6870d1f022f078dd57c4c8a477c87bb2865bfe47 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 20 Aug 2026 01:34:05 -0700 Subject: [PATCH 1/9] Build/Test Tools: Add plugin compatibility testing workflow. Core CI covers core itself, but nothing checks that a new version of WordPress can still boot with popular plugins active. When a plugin's assumptions about core stop holding the result is a fatal error on every request, which is a white screen for real sites and is only discovered after release. Add a workflow that fetches the most popular plugins from the WordPress.org API at run time, then installs and activates each one on its own against the version of WordPress under test. A fatal is caught whether it happens on activation, while WP-CLI loads WordPress, on a front end or login request, or in the debug log, so a white screen with error display turned off is still detected. Failures are reported per plugin in the workflow summary and one broken plugin never stops the rest of the shard from being tested. Plugins that cannot be downloaded are reported as skipped rather than failed so that a network flake does not turn the run red. The run is manual or weekly rather than part of every commit, since a third party plugin breaking should be a signal to release leads, not a red check on unrelated work. --- .github/workflows/plugin-compatibility.yml | 205 ++++++++++++ .../reusable-plugin-compatibility.yml | 316 ++++++++++++++++++ 2 files changed, 521 insertions(+) create mode 100644 .github/workflows/plugin-compatibility.yml create mode 100644 .github/workflows/reusable-plugin-compatibility.yml diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml new file mode 100644 index 0000000000000..dc8d2f709c576 --- /dev/null +++ b/.github/workflows/plugin-compatibility.yml @@ -0,0 +1,205 @@ +## +# Confirms that the most popular plugins in the WordPress.org directory can be activated against a version of +# WordPress without fataling. +# +# Core's test suites cover core itself, but nothing checks that a new version of WordPress can still boot with +# popular plugins active. When a plugin's assumptions about core stop holding, the result is a fatal error on +# every request and a white screen for real sites. This workflow is a smoke test for that class of failure, so +# that it can be found while there is still time to fix core or notify the plugin author. +# +# The plugin list is fetched from the WordPress.org API at run time and is sharded across a small matrix. Each +# plugin is installed and activated on its own, so one broken plugin cannot hide another. +# +# This workflow is not meant to test wordpress-develop checkouts, but rather versions officially available on +# WordPress.org. +## +name: Plugin Compatibility Tests + +on: + push: + branches: + - trunk + # Always test the workflow after it's updated. + paths: + - '.github/workflows/plugin-compatibility.yml' + - '.github/workflows/reusable-plugin-compatibility.yml' + pull_request: + # This workflow is only meant to run from trunk. Pull requests changing this file with different BASE branches should be ignored. + branches: + - trunk + # Always test the workflow when changes are suggested. + paths: + - '.github/workflows/plugin-compatibility.yml' + - '.github/workflows/reusable-plugin-compatibility.yml' + schedule: + - cron: '0 2 * * 1' + workflow_dispatch: + inputs: + wp-version: + description: 'The version of WordPress to test plugins against. Accepts "latest", "nightly", or a specific version number.' + type: string + default: 'nightly' + plugin-count: + description: 'How many of the most popular plugins to test.' + type: string + default: '100' + +# Cancels all previous workflow runs for pull requests that have not completed. +concurrency: + # The concurrency group contains the workflow name and the branch name for pull requests + # or the commit hash for any other events. + group: ${{ github.workflow }}-${{ inputs.wp-version || github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Builds the list of plugins to test and splits it into shards for the test matrix. + # + # The list is fetched at run time so that it never goes stale, and it is ordered by popularity so that a + # smaller count still tests the plugins with the widest reach. + # + # Performs the following steps: + # - Queries the WordPress.org plugin directory API for the most popular plugins. + # - Splits the resulting slugs into shards and returns them as a job output. + build-plugin-matrix: + name: Build plugin matrix + permissions: + contents: read + runs-on: ubuntu-24.04 + if: ${{ github.repository == 'WordPress/wordpress-develop' }} + timeout-minutes: 5 + outputs: + shards: ${{ steps.plugin-shards.outputs.shards }} + + steps: + - name: Fetch the most popular plugins + id: plugin-shards + env: + # A pull request or scheduled run has no inputs, so fall back to the default count. + PLUGIN_COUNT: ${{ inputs.plugin-count || '100' }} + SHARD_COUNT: '5' + run: | + set -euo pipefail + + # Guard against a non-numeric value being passed to the API. + if ! printf '%s' "${PLUGIN_COUNT}" | grep -Eq '^[0-9]+$'; then + printf 'The plugin-count input must be a positive integer.\n' + exit 1 + fi + + # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. + # The unneeded response fields are turned off to keep the payload small. + curl -sS --fail --retry 3 --retry-delay 5 \ + --get 'https://api.wordpress.org/plugins/info/1.2/' \ + --data-urlencode 'action=query_plugins' \ + --data-urlencode 'request[browse]=popular' \ + --data-urlencode "request[per_page]=${PLUGIN_COUNT}" \ + --data-urlencode 'request[fields][short_description]=0' \ + --data-urlencode 'request[fields][sections]=0' \ + --data-urlencode 'request[fields][icons]=0' \ + --data-urlencode 'request[fields][banners]=0' \ + --data-urlencode 'request[fields][ratings]=0' \ + --data-urlencode 'request[fields][tags]=0' \ + --data-urlencode 'request[fields][compatibility]=0' \ + --data-urlencode 'request[fields][screenshots]=0' \ + -o "${RUNNER_TEMP}/plugins.json" + + SLUGS="$( jq -c '[ .plugins[].slug ] | map( select( . != null and . != "" ) )' "${RUNNER_TEMP}/plugins.json" )" + TOTAL="$( printf '%s' "${SLUGS}" | jq 'length' )" + + if [ "${TOTAL}" -lt 1 ]; then + printf 'The WordPress.org API did not return any plugins.\n' + exit 1 + fi + + # Split the slugs into evenly sized shards, dropping any shard that ends up empty because fewer + # plugins were requested than there are shards. Each shard's slugs are passed to the reusable + # workflow as a JSON string. + SHARDS="$( printf '%s' "${SLUGS}" | jq -c --argjson shard_count "${SHARD_COUNT}" ' + . as $slugs + | ( ( length + $shard_count - 1 ) / $shard_count | floor ) as $size + | [ + range( 0; $shard_count ) + | { index: ( . + 1 ), slugs: $slugs[ ( . * $size ) : ( ( . + 1 ) * $size ) ] } + ] + | map( select( .slugs | length > 0 ) ) + | map( { index: .index, slugs: ( .slugs | @json ) } ) + ' )" + + printf 'Testing %s plugins across %s shards.\n' "${TOTAL}" "$( printf '%s' "${SHARDS}" | jq 'length' )" + printf '%s\n' "${SHARDS}" | jq -r '.[] | "Shard \(.index): \(.slugs)"' + + printf 'shards=%s\n' "${SHARDS}" >> "${GITHUB_OUTPUT}" + + # Tests each shard of plugins against the version of WordPress being tested. + plugin-compatibility-tests: + name: WP ${{ inputs.wp-version || 'nightly' }} / Shard ${{ matrix.shard.index }} + uses: ./.github/workflows/reusable-plugin-compatibility.yml + permissions: + contents: read + needs: [ build-plugin-matrix ] + if: ${{ github.repository == 'WordPress/wordpress-develop' }} + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON( needs.build-plugin-matrix.outputs.shards ) }} + with: + os: 'ubuntu-24.04' + wp-version: ${{ inputs.wp-version || 'nightly' }} + php-version: '8.3' + plugin-slugs: ${{ matrix.shard.slugs }} + db-type: 'mysql' + db-version: '8.4' + + slack-notifications: + name: Slack Notifications + uses: ./.github/workflows/slack-notifications.yml + permissions: + actions: read + contents: read + needs: [ build-plugin-matrix, plugin-compatibility-tests ] + if: ${{ github.repository == 'WordPress/wordpress-develop' && github.event_name != 'pull_request' && always() }} + with: + calling_status: ${{ contains( needs.*.result, 'cancelled' ) && 'cancelled' || contains( needs.*.result, 'failure' ) && 'failure' || 'success' }} + secrets: + SLACK_GHA_SUCCESS_WEBHOOK: ${{ secrets.SLACK_GHA_SUCCESS_WEBHOOK }} + SLACK_GHA_CANCELLED_WEBHOOK: ${{ secrets.SLACK_GHA_CANCELLED_WEBHOOK }} + SLACK_GHA_FIXED_WEBHOOK: ${{ secrets.SLACK_GHA_FIXED_WEBHOOK }} + SLACK_GHA_FAILURE_WEBHOOK: ${{ secrets.SLACK_GHA_FAILURE_WEBHOOK }} + SLACK_GHA_TIMEOUT_WEBHOOK: ${{ secrets.SLACK_GHA_TIMEOUT_WEBHOOK }} + + failed-workflow: + name: Failed workflow tasks + runs-on: ubuntu-24.04 + permissions: + actions: write + needs: [ slack-notifications ] + if: | + always() && + github.repository == 'WordPress/wordpress-develop' && + github.event_name != 'pull_request' && + github.run_attempt < 2 && + ( + contains( needs.*.result, 'cancelled' ) || + contains( needs.*.result, 'failure' ) + ) + + steps: + - name: Dispatch workflow run + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + retries: 2 + retry-exempt-status-codes: 418 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'failed-workflow.yml', + ref: 'trunk', + inputs: { + run_id: `${context.runId}`, + } + }); diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml new file mode 100644 index 0000000000000..8147b6b5ec4b1 --- /dev/null +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -0,0 +1,316 @@ +## +# A reusable workflow that installs a version of WordPress and then checks that a list of plugins can be +# activated against it without fataling. +# +# Each plugin in the `plugin-slugs` shard is tested on its own: it is installed, activated, exercised, and then +# removed before the next one is installed. This keeps one broken plugin from masking (or breaking) the next. +## +name: Plugin Compatibility Tests + +on: + workflow_call: + inputs: + os: + description: 'Operating system to run tests on.' + required: false + type: 'string' + default: 'ubuntu-24.04' + wp-version: + description: 'The version of WordPress to test against. Accepts a version number, "latest", or "nightly".' + required: false + type: 'string' + default: 'nightly' + php-version: + description: 'The version of PHP to use. Expected format: X.Y.' + required: false + type: 'string' + default: '8.3' + plugin-slugs: + description: 'A JSON array of WordPress.org plugin slugs to test in this shard.' + required: true + type: 'string' + db-type: + description: 'Database type. Valid types are mysql and mariadb.' + required: false + type: 'string' + default: 'mysql' + db-version: + description: 'Database version.' + required: false + type: 'string' + default: '8.4' + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Tests that each plugin in the shard can be activated against the given version of WordPress. + # + # Performs the following steps: + # - Sets up PHP. + # - Downloads the specified version of WordPress. + # - Creates a `wp-config.php` file with debugging and error logging enabled. + # - Installs WordPress. + # - Starts the PHP built-in web server so HTTP requests can be made against the site. + # - Installs, activates, exercises, and removes each plugin in the shard, one at a time. + # - Writes a results table to the workflow summary and fails the job if any plugin fataled. + plugin-compatibility-tests: + name: PHP ${{ inputs.php-version }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }} + permissions: + contents: read + runs-on: ${{ inputs.os }} + timeout-minutes: 30 + + services: + database: + image: ${{ inputs.db-type }}:${{ inputs.db-version }} + ports: + - 3306 + options: >- + --health-cmd="mysqladmin ping" + --health-interval="30s" + --health-timeout="10s" + --health-retries="5" + -e MYSQL_ROOT_PASSWORD="root" + -e MYSQL_DATABASE="test_db" + + steps: + - name: Set up PHP ${{ inputs.php-version }} + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 + with: + php-version: '${{ inputs.php-version }}' + coverage: none + tools: wp-cli + + - name: Download WordPress ${{ inputs.wp-version }} + run: wp core download --version="${WP_VERSION}" + env: + WP_VERSION: ${{ inputs.wp-version }} + + - name: Create wp-config.php file + run: wp config create --dbname=test_db --dbuser=root --dbpass=root --dbhost="127.0.0.1:${DB_PORT}" + env: + DB_PORT: ${{ job.services.database.ports['3306'] }} + + # Errors need to reach `wp-content/debug.log` so that a white screen of death is still detectable. + # + # `WP_DEBUG_DISPLAY` is left off on purpose: this should behave the way a production site does, where a + # fatal error is an empty page and an HTTP 500 rather than a printed stack trace. + # + # The fatal error handler is disabled so that a fatal is reported as a fatal instead of being swallowed by + # recovery mode, which would also deactivate the plugin mid-test. + - name: Enable debugging and error logging + run: | + wp config set WP_DEBUG true --raw + wp config set WP_DEBUG_LOG true --raw + wp config set WP_DEBUG_DISPLAY false --raw + wp config set WP_DISABLE_FATAL_ERROR_HANDLER true --raw + + - name: Install WordPress + run: | + wp core install --url="${SITE_URL}" --title="Plugin Compatibility Test" --admin_user=admin \ + --admin_password=password --admin_email=me@example.org --skip-email + env: + SITE_URL: http://127.0.0.1:8889 + + # The site needs to answer real requests so that fatals which only happen on a front end or admin page + # load are caught. The built-in server is enough for that and needs nothing installed. + - name: Start the PHP built-in web server + run: | + set -uo pipefail + + nohup php -S 127.0.0.1:8889 -t "$( pwd )" > "${RUNNER_TEMP}/php-server.log" 2>&1 & + + # Wait for the server to start answering before any plugin is installed. + for _ in $( seq 1 30 ); do + if curl -sSf -o /dev/null "http://127.0.0.1:8889/wp-login.php"; then + printf 'The PHP built-in server is ready.\n' + exit 0 + fi + sleep 1 + done + + printf 'The PHP built-in server did not start.\n' + cat "${RUNNER_TEMP}/php-server.log" + exit 1 + + - name: Test each plugin in isolation + env: + PLUGIN_SLUGS: ${{ inputs.plugin-slugs }} + PHP_VERSION: ${{ inputs.php-version }} + SITE_URL: http://127.0.0.1:8889 + WP_VERSION: ${{ inputs.wp-version }} + run: | + # `set -e` is deliberately not used here: a plugin that fatals must not stop the remaining + # plugins in the shard from being tested. + set -uo pipefail + + RESULTS="${RUNNER_TEMP}/plugin-results.tsv" + RESPONSE_BODY="${RUNNER_TEMP}/response.html" + : > "${RESULTS}" + + # record + # + # Appends one tab separated row to the results file. The reason is flattened so that it cannot + # break the markdown table that is generated from these rows later on. + record() { + SAFE_REASON="$( printf '%s' "${4:--}" | tr '\n\t|' ' ' )" + printf '%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${SAFE_REASON}" >> "${RESULTS}" + } + + # check_url + # + # Requests a path on the test site and prints a reason to stdout when the response looks broken. + # Prints nothing when the request looks healthy. + check_url() { + HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" || printf '000' )" + HTTP_CODE="${HTTP_CODE:-000}" + + if [ "${HTTP_CODE}" = "000" ]; then + printf 'The request to %s did not complete' "${1}" + return + fi + + if [ "${HTTP_CODE}" -ge 500 ]; then + printf 'The request to %s returned HTTP %s' "${1}" "${HTTP_CODE}" + return + fi + + # Belt and braces. Errors are not displayed by default, but a plugin can turn display_errors + # back on for itself. + if grep -qi 'Fatal error' "${RESPONSE_BODY}"; then + printf 'The response from %s contained a fatal error' "${1}" + fi + } + + # The plugins directory itself must never be removed, only directories inside it. + PLUGINS_ROOT="$( wp plugin path )" + + # cleanup_plugin + # + # Returns the site to a clean slate. A plugin that fatals can take WP-CLI down with it, so every + # command here is allowed to fail and the plugin directory is removed directly as a fallback. + # `--skip-plugins` keeps WP-CLI from loading the broken plugin while cleaning up after it. + cleanup_plugin() { + wp plugin deactivate "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 || true + wp plugin delete "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 || true + + if [ -n "${2:-}" ] && [ -d "${2}" ] && [ "${2}" != "${PLUGINS_ROOT}" ]; then + rm -rf "${2}" + fi + + rm -rf "wp-content/plugins/${1}" + + # Make sure nothing is left behind in `active_plugins` pointing at a plugin that is now gone. + wp option update active_plugins '[]' --format=json --skip-plugins --skip-themes > /dev/null 2>&1 || true + } + + while IFS= read -r SLUG; do + [ -n "${SLUG}" ] || continue + + printf '::group::%s\n' "${SLUG}" + + STATUS="PASS" + REASON="-" + VERSION="unknown" + PLUGIN_DIR="" + + # Start every plugin with an empty log so that anything found in it belongs to this plugin. + rm -f wp-content/debug.log + + # Step 1: download the plugin. A download failure is recorded as skipped rather than failed, + # since it generally means a network flake or a plugin that is no longer in the directory. + if ! wp plugin install "${SLUG}" --skip-plugins --skip-themes; then + record "${SLUG}" "unknown" "SKIPPED" "The plugin could not be downloaded from WordPress.org" + cleanup_plugin "${SLUG}" "" + printf '::endgroup::\n' + continue + fi + + VERSION="$( wp plugin get "${SLUG}" --field=version --skip-plugins --skip-themes 2>/dev/null || printf 'unknown' )" + PLUGIN_DIR="$( wp plugin path "${SLUG}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" + + # Step 2: activation. Activation runs the plugin's activation hooks and loads its main file. + if ! wp plugin activate "${SLUG}"; then + STATUS="FAIL" + REASON="The plugin could not be activated" + fi + + # Step 3: boot all of core plus the active plugin in a CLI context. + if [ "${STATUS}" = "PASS" ]; then + EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" + EVAL_EXIT_CODE=$? + + if [ "${EVAL_EXIT_CODE}" -ne 0 ] || [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then + printf '%s\n' "${EVAL_OUTPUT}" + STATUS="FAIL" + REASON="WordPress could not be loaded by WP-CLI with the plugin active" + fi + fi + + # Step 4: request the front page and the login screen through the PHP built-in server. + if [ "${STATUS}" = "PASS" ]; then + for URL_PATH in "/" "/wp-login.php"; do + HTTP_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${HTTP_REASON}" ]; then + STATUS="FAIL" + REASON="${HTTP_REASON}" + break + fi + done + fi + + # Step 5: a fatal can be logged without changing the HTTP status, for example during a shutdown + # hook, so the debug log is checked separately. + if [ "${STATUS}" = "PASS" ] && [ -f wp-content/debug.log ] && grep -q 'PHP Fatal' wp-content/debug.log; then + grep 'PHP Fatal' wp-content/debug.log + STATUS="FAIL" + REASON="A fatal error was logged: $( grep -m 1 'PHP Fatal' wp-content/debug.log | cut -c 1-200 )" + fi + + # Step 6: record the outcome and put the site back the way it was found. + record "${SLUG}" "${VERSION}" "${STATUS}" "${REASON}" + cleanup_plugin "${SLUG}" "${PLUGIN_DIR}" + + printf '%s: %s\n' "${SLUG}" "${STATUS}" + printf '::endgroup::\n' + done < <( printf '%s' "${PLUGIN_SLUGS}" | jq -r '.[]' ) + + PASS_COUNT="$( awk -F '\t' '$3 == "PASS" { count++ } END { print count + 0 }' "${RESULTS}" )" + FAIL_COUNT="$( awk -F '\t' '$3 == "FAIL" { count++ } END { print count + 0 }' "${RESULTS}" )" + SKIP_COUNT="$( awk -F '\t' '$3 == "SKIPPED" { count++ } END { print count + 0 }' "${RESULTS}" )" + + { + printf '### WordPress %s / PHP %s\n\n' "${WP_VERSION}" "${PHP_VERSION}" + printf '%s passed, %s failed, %s skipped.\n\n' "${PASS_COUNT}" "${FAIL_COUNT}" "${SKIP_COUNT}" + printf '| Plugin | Version | Result | Details |\n' + printf '| --- | --- | --- | --- |\n' + + while IFS=$'\t' read -r ROW_SLUG ROW_VERSION ROW_STATUS ROW_REASON; do + case "${ROW_STATUS}" in + PASS ) ICON=':white_check_mark:' ;; + FAIL ) ICON=':x:' ;; + * ) ICON=':warning:' ;; + esac + + printf '| [%s](https://wordpress.org/plugins/%s/) | %s | %s %s | %s |\n' \ + "${ROW_SLUG}" "${ROW_SLUG}" "${ROW_VERSION}" "${ICON}" "${ROW_STATUS}" "${ROW_REASON}" + done < "${RESULTS}" + + printf '\n' + } >> "${GITHUB_STEP_SUMMARY}" + + # Plugins that could not be downloaded are reported but do not fail the run. + if [ "${FAIL_COUNT}" -gt 0 ]; then + printf 'The following plugins failed against WordPress %s:\n' "${WP_VERSION}" + awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $4 }' "${RESULTS}" + exit 1 + fi + + printf 'No plugins fataled against WordPress %s.\n' "${WP_VERSION}" + + - name: Show the web server log + if: ${{ failure() }} + run: cat "${RUNNER_TEMP}/php-server.log" From 14340c7a4bde129331a33bfbbb54bd284f8c1925 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 20 Aug 2026 09:02:36 -0700 Subject: [PATCH 2/9] Build/Test Tools: Correct plugin compatibility failure classification. The first run of this workflow reported four failures that were not fatals. Core refuses to activate a plugin whose `Requires Plugins` dependency is missing, which every WooCommerce extension hits when plugins are tested one at a time, and WP-CLI exits non-zero when a plugin redirects while loading. Both are correct behaviour, so record them as skipped and reserve a failure for an actual fatal. A front end request also reported the nonsense status "200000", because the curl fallback appended to output curl had already written. Capture the exit code separately so a stalled transfer is reported as what it is. The stall itself came from WordPress spawning WP-Cron as a loopback request that the single threaded built-in server could not answer while still serving the request that spawned it. Disable WP-Cron and give the server workers so plugin loopback requests cannot deadlock it. Ignore the zizmor unpinned image finding on the database service, which cannot be pinned to a digest while the version is an input. --- .../reusable-plugin-compatibility.yml | 77 +++++++++++++++---- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml index 8147b6b5ec4b1..4ca645f3c7aa9 100644 --- a/.github/workflows/reusable-plugin-compatibility.yml +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -64,7 +64,10 @@ jobs: services: database: - image: ${{ inputs.db-type }}:${{ inputs.db-version }} + # The database type and version are inputs so that this workflow can be pointed at any supported + # combination, which means the image cannot be pinned to a digest. This matches how the database + # service is declared in install-testing.yml and reusable-upgrade-testing.yml. + image: ${{ inputs.db-type }}:${{ inputs.db-version }} # zizmor: ignore[unpinned-images] ports: - 3306 options: >- @@ -100,12 +103,18 @@ jobs: # # The fatal error handler is disabled so that a fatal is reported as a fatal instead of being swallowed by # recovery mode, which would also deactivate the plugin mid-test. + # + # WP-Cron is disabled because WordPress spawns it as a loopback request during a front end request. The + # loopback lands back on the same PHP built-in server that is still busy serving the request that spawned + # it, and the two deadlock until curl gives up. Catching fatals that only happen on a scheduled event is + # worth doing, but it needs to run through WP-CLI rather than a loopback, which is follow up work. - name: Enable debugging and error logging run: | wp config set WP_DEBUG true --raw wp config set WP_DEBUG_LOG true --raw wp config set WP_DEBUG_DISPLAY false --raw wp config set WP_DISABLE_FATAL_ERROR_HANDLER true --raw + wp config set DISABLE_WP_CRON true --raw - name: Install WordPress run: | @@ -116,7 +125,13 @@ jobs: # The site needs to answer real requests so that fatals which only happen on a front end or admin page # load are caught. The built-in server is enough for that and needs nothing installed. + # + # `PHP_CLI_SERVER_WORKERS` is set because the built-in server is single threaded by default. Plenty of + # plugins make a loopback request to the site they are running on, and a single threaded server cannot + # answer one while it is still serving the request that made it. - name: Start the PHP built-in web server + env: + PHP_CLI_SERVER_WORKERS: '4' run: | set -uo pipefail @@ -164,11 +179,14 @@ jobs: # Requests a path on the test site and prints a reason to stdout when the response looks broken. # Prints nothing when the request looks healthy. check_url() { - HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" || printf '000' )" - HTTP_CODE="${HTTP_CODE:-000}" - - if [ "${HTTP_CODE}" = "000" ]; then - printf 'The request to %s did not complete' "${1}" + CURL_EXIT_CODE=0 + # The exit code is captured separately rather than falling back to a literal inside the command + # substitution, which would append to whatever curl had already written and produce a nonsense + # status like "200000" when a request returned headers and then stalled. + HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" )" || CURL_EXIT_CODE=$? + + if [ -z "${HTTP_CODE}" ] || [ "${HTTP_CODE}" = "000" ]; then + printf 'The request to %s did not complete, curl exit code %s' "${1}" "${CURL_EXIT_CODE}" return fi @@ -177,6 +195,11 @@ jobs: return fi + if [ "${CURL_EXIT_CODE}" -ne 0 ]; then + printf 'The request to %s returned HTTP %s but the response did not finish, curl exit code %s' "${1}" "${HTTP_CODE}" "${CURL_EXIT_CODE}" + return + fi + # Belt and braces. Errors are not displayed by default, but a plugin can turn display_errors # back on for itself. if grep -qi 'Fatal error' "${RESPONSE_BODY}"; then @@ -232,20 +255,46 @@ jobs: PLUGIN_DIR="$( wp plugin path "${SLUG}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" # Step 2: activation. Activation runs the plugin's activation hooks and loads its main file. - if ! wp plugin activate "${SLUG}"; then - STATUS="FAIL" - REASON="The plugin could not be activated" + ACTIVATE_EXIT_CODE=0 + ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? + printf '%s\n' "${ACTIVATE_OUTPUT}" + + if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then + case "${ACTIVATE_OUTPUT}" in + # Core refuses to activate a plugin whose declared requirements are not met, most often a + # `Requires Plugins` dependency that is not installed. Testing each plugin on its own means + # every WooCommerce extension lands here. That is core working as designed rather than a + # fatal, so it is recorded as skipped. + *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) + STATUS="SKIPPED" + REASON="Core declined to activate the plugin because its declared requirements are not met" + ;; + * ) + STATUS="FAIL" + REASON="The plugin could not be activated" + ;; + esac fi # Step 3: boot all of core plus the active plugin in a CLI context. if [ "${STATUS}" = "PASS" ]; then - EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" - EVAL_EXIT_CODE=$? + EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" || true - if [ "${EVAL_EXIT_CODE}" -ne 0 ] || [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then + if [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then printf '%s\n' "${EVAL_OUTPUT}" - STATUS="FAIL" - REASON="WordPress could not be loaded by WP-CLI with the plugin active" + + case "${EVAL_OUTPUT}" in + *"Fatal error"* | *"PHP Fatal"* | *"Uncaught"* ) + STATUS="FAIL" + REASON="A fatal error occurred while WP-CLI loaded WordPress with the plugin active" + ;; + # Some plugins redirect or exit while loading, which stops WP-CLI without anything being + # broken. Only a fatal counts as a failure here. Everything else is left to the HTTP and + # debug log checks below, which see the same code in a real request. + * ) + printf 'WP-CLI did not finish loading WordPress, but no fatal error was reported.\n' + ;; + esac fi fi From 915017705dcd307e748b4e62a30762410f3d789c Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 20 Aug 2026 09:02:36 -0700 Subject: [PATCH 3/9] Build/Test Tools: Support any plugin count in compatibility testing. The WordPress.org API caps `per_page` at 250 and quietly returns 250 for anything larger, so asking for more than that silently tested fewer plugins than requested. Page through the API instead and trim to the requested count, de-duplicating across pages because popularity ordering can shift between two requests. Reject a count above 1000 with a clear message rather than truncating without saying so. Size the shards to the plugin count rather than always splitting into five, so a run of 10 does not spin up five near empty jobs and a run of 250 is not squeezed into the same five. Point pull request and push runs at the latest stable release with a small count. Those runs exist to check that this workflow still works, and a genuine ecosystem fatal against nightly should not sit as a red check on every later change to these files. --- .github/workflows/plugin-compatibility.yml | 128 ++++++++++++++++----- 1 file changed, 100 insertions(+), 28 deletions(-) diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml index dc8d2f709c576..3ae64ce5abeaf 100644 --- a/.github/workflows/plugin-compatibility.yml +++ b/.github/workflows/plugin-compatibility.yml @@ -7,8 +7,13 @@ # every request and a white screen for real sites. This workflow is a smoke test for that class of failure, so # that it can be found while there is still time to fix core or notify the plugin author. # -# The plugin list is fetched from the WordPress.org API at run time and is sharded across a small matrix. Each -# plugin is installed and activated on its own, so one broken plugin cannot hide another. +# The plugin list is fetched from the WordPress.org API at run time and is sharded across a matrix sized to the +# number of plugins being tested. Each plugin is installed and activated on its own, so one broken plugin cannot +# hide another. +# +# It runs weekly against nightly, and can be dispatched manually against any version with any number of plugins, +# which is the intended way to use it as part of the pre-release checklist: point it at the beta or RC and give +# it a count. # # This workflow is not meant to test wordpress-develop checkouts, but rather versions officially available on # WordPress.org. @@ -36,11 +41,11 @@ on: workflow_dispatch: inputs: wp-version: - description: 'The version of WordPress to test plugins against. Accepts "latest", "nightly", or a specific version number.' + description: 'The version of WordPress to test plugins against. Accepts "latest", "nightly", or a specific version number such as a beta or RC, for a pre-release check.' type: string default: 'nightly' plugin-count: - description: 'How many of the most popular plugins to test.' + description: 'Number of most popular plugins to test. Accepts any number from 1 to 1000, eg. 10, 50, or 250.' type: string default: '100' @@ -78,36 +83,84 @@ jobs: - name: Fetch the most popular plugins id: plugin-shards env: - # A pull request or scheduled run has no inputs, so fall back to the default count. - PLUGIN_COUNT: ${{ inputs.plugin-count || '100' }} - SHARD_COUNT: '5' + # Runs that carry no inputs fall back to the defaults. Pull request and push runs exist to test + # this workflow rather than the ecosystem, so they use a small count. See the note on the + # plugin-compatibility-tests job below. + PLUGIN_COUNT: ${{ inputs.plugin-count || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && '10' ) || '100' }} + # The API caps `per_page` at 250 and returns 250 without complaint for anything larger, so counts + # above that have to be paged. + PAGE_SIZE: '250' + # Anything higher would take longer than the 20 minute wall time this is meant to fit inside. The + # directory holds roughly 66,000 plugins, so this is a guard against a typo, not a real limit. + MAX_PLUGIN_COUNT: '1000' + # Shards are sized rather than counted, so that a run of 10 does not spin up 5 near empty jobs and a + # run of 250 is not squeezed into the same 5. + TARGET_PER_SHARD: '25' + MAX_SHARDS: '10' run: | set -euo pipefail # Guard against a non-numeric value being passed to the API. if ! printf '%s' "${PLUGIN_COUNT}" | grep -Eq '^[0-9]+$'; then - printf 'The plugin-count input must be a positive integer.\n' + printf 'The plugin-count input must be a positive integer, got "%s".\n' "${PLUGIN_COUNT}" + exit 1 + fi + + if [ "${PLUGIN_COUNT}" -lt 1 ] || [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then + printf 'The plugin-count input must be between 1 and %s, got %s.\n' "${MAX_PLUGIN_COUNT}" "${PLUGIN_COUNT}" + printf 'Testing more than %s plugins will not fit in the wall time this workflow targets.\n' "${MAX_PLUGIN_COUNT}" exit 1 fi - # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. - # The unneeded response fields are turned off to keep the payload small. - curl -sS --fail --retry 3 --retry-delay 5 \ - --get 'https://api.wordpress.org/plugins/info/1.2/' \ - --data-urlencode 'action=query_plugins' \ - --data-urlencode 'request[browse]=popular' \ - --data-urlencode "request[per_page]=${PLUGIN_COUNT}" \ - --data-urlencode 'request[fields][short_description]=0' \ - --data-urlencode 'request[fields][sections]=0' \ - --data-urlencode 'request[fields][icons]=0' \ - --data-urlencode 'request[fields][banners]=0' \ - --data-urlencode 'request[fields][ratings]=0' \ - --data-urlencode 'request[fields][tags]=0' \ - --data-urlencode 'request[fields][compatibility]=0' \ - --data-urlencode 'request[fields][screenshots]=0' \ - -o "${RUNNER_TEMP}/plugins.json" - - SLUGS="$( jq -c '[ .plugins[].slug ] | map( select( . != null and . != "" ) )' "${RUNNER_TEMP}/plugins.json" )" + RAW_SLUGS="${RUNNER_TEMP}/slugs-raw.txt" + DEDUPED_SLUGS="${RUNNER_TEMP}/slugs.txt" + : > "${RAW_SLUGS}" + PAGE=1 + UNIQUE_COUNT=0 + + while : ; do + # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. + # The unneeded response fields are turned off to keep the payload small. + curl -sS --fail --retry 3 --retry-delay 5 \ + --get 'https://api.wordpress.org/plugins/info/1.2/' \ + --data-urlencode 'action=query_plugins' \ + --data-urlencode 'request[browse]=popular' \ + --data-urlencode "request[per_page]=${PAGE_SIZE}" \ + --data-urlencode "request[page]=${PAGE}" \ + --data-urlencode 'request[fields][short_description]=0' \ + --data-urlencode 'request[fields][sections]=0' \ + --data-urlencode 'request[fields][icons]=0' \ + --data-urlencode 'request[fields][banners]=0' \ + --data-urlencode 'request[fields][ratings]=0' \ + --data-urlencode 'request[fields][tags]=0' \ + --data-urlencode 'request[fields][compatibility]=0' \ + --data-urlencode 'request[fields][screenshots]=0' \ + -o "${RUNNER_TEMP}/plugins-page.json" + + PAGE_COUNT="$( jq '.plugins | length' "${RUNNER_TEMP}/plugins-page.json" )" + jq -r '.plugins[] | .slug // empty' "${RUNNER_TEMP}/plugins-page.json" >> "${RAW_SLUGS}" + + # Popularity ordering can shift between two requests, so the same slug can turn up on more than + # one page. Duplicates are dropped while the first occurrence keeps its position. + awk '!seen[$0]++ && NF > 0' "${RAW_SLUGS}" > "${DEDUPED_SLUGS}" + UNIQUE_COUNT="$( wc -l < "${DEDUPED_SLUGS}" | tr -d ' ' )" + + printf 'Page %s returned %s plugins, %s unique slugs collected so far.\n' "${PAGE}" "${PAGE_COUNT}" "${UNIQUE_COUNT}" + + if [ "${UNIQUE_COUNT}" -ge "${PLUGIN_COUNT}" ]; then + break + fi + + # A short page means the directory has nothing left to give. + if [ "${PAGE_COUNT}" -lt "${PAGE_SIZE}" ]; then + printf 'The API returned fewer than %s plugins on page %s, so %s is everything available.\n' "${PAGE_SIZE}" "${PAGE}" "${UNIQUE_COUNT}" + break + fi + + PAGE=$(( PAGE + 1 )) + done + + SLUGS="$( head -n "${PLUGIN_COUNT}" "${DEDUPED_SLUGS}" | jq -R -s -c 'split( "\n" ) | map( select( . != "" ) )' )" TOTAL="$( printf '%s' "${SLUGS}" | jq 'length' )" if [ "${TOTAL}" -lt 1 ]; then @@ -115,6 +168,19 @@ jobs: exit 1 fi + if [ "${TOTAL}" -lt "${PLUGIN_COUNT}" ]; then + printf 'Only %s plugins were available, fewer than the %s requested.\n' "${TOTAL}" "${PLUGIN_COUNT}" + fi + + # Aim for TARGET_PER_SHARD plugins in each shard, up to MAX_SHARDS shards. Past that point the + # shards get longer instead of more numerous. + SHARD_COUNT="$( awk -v total="${TOTAL}" -v per="${TARGET_PER_SHARD}" -v max="${MAX_SHARDS}" 'BEGIN { + count = int( ( total + per - 1 ) / per ); + if ( count < 1 ) { count = 1 } + if ( count > max ) { count = max } + print count + }' )" + # Split the slugs into evenly sized shards, dropping any shard that ends up empty because fewer # plugins were requested than there are shards. Each shard's slugs are passed to the reusable # workflow as a JSON string. @@ -135,8 +201,14 @@ jobs: printf 'shards=%s\n' "${SHARDS}" >> "${GITHUB_OUTPUT}" # Tests each shard of plugins against the version of WordPress being tested. + # + # Pull request and push runs are here to check that this workflow itself still works, not to report on the + # health of the ecosystem. They use the latest stable release and a small number of plugins, because a genuine + # fatal against nightly is a true result that should not sit as a red check on every future change to these + # two files. Scheduled and manually dispatched runs are the ones that carry the ecosystem signal, and they + # default to nightly and the full count. plugin-compatibility-tests: - name: WP ${{ inputs.wp-version || 'nightly' }} / Shard ${{ matrix.shard.index }} + name: WP ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} / Shard ${{ matrix.shard.index }} uses: ./.github/workflows/reusable-plugin-compatibility.yml permissions: contents: read @@ -148,7 +220,7 @@ jobs: shard: ${{ fromJSON( needs.build-plugin-matrix.outputs.shards ) }} with: os: 'ubuntu-24.04' - wp-version: ${{ inputs.wp-version || 'nightly' }} + wp-version: ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} php-version: '8.3' plugin-slugs: ${{ matrix.shard.slugs }} db-type: 'mysql' From cf18ee6356faabb66a7705db9daf7d300bb8d960 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Thu, 20 Aug 2026 09:05:30 -0700 Subject: [PATCH 4/9] Build/Test Tools: Fix shard count wording in plugin compatibility log. A run that builds a single shard logged "across 1 shards". --- .github/workflows/plugin-compatibility.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml index 3ae64ce5abeaf..b2a9c12d2d6a6 100644 --- a/.github/workflows/plugin-compatibility.yml +++ b/.github/workflows/plugin-compatibility.yml @@ -195,7 +195,7 @@ jobs: | map( { index: .index, slugs: ( .slugs | @json ) } ) ' )" - printf 'Testing %s plugins across %s shards.\n' "${TOTAL}" "$( printf '%s' "${SHARDS}" | jq 'length' )" + printf 'Testing %s plugins across %s shard(s).\n' "${TOTAL}" "$( printf '%s' "${SHARDS}" | jq 'length' )" printf '%s\n' "${SHARDS}" | jq -r '.[] | "Shard \(.index): \(.slugs)"' printf 'shards=%s\n' "${SHARDS}" >> "${GITHUB_OUTPUT}" From 67005f527ba98d30586aa98c1ad89f25fb1867f3 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Tue, 25 Aug 2026 11:58:18 -0700 Subject: [PATCH 5/9] Build/Test Tools: Install plugin dependencies before compatibility testing. Testing each plugin on its own meant core refused to activate anything with a `Requires Plugins` header, so every WooCommerce extension was recorded as skipped without ever being loaded. Those extensions are a large slice of the most popular plugins, which left a gap in exactly the part of the ecosystem the workflow exists to watch. Read the `Requires Plugins` header with WordPress' own parser, install and activate what it names, and only then activate the plugin under test. Chains are followed a level at a time so a dependency declaring its own is installed too, and anything already present is left alone, which terminates a circular declaration. Dependencies mean the plugin under test is no longer alone on the site, so the front page and login screen are checked with only the dependencies active first. A site already broken at that point is the dependency's doing, and the plugin is skipped rather than blamed for it. The debug log is cleared at the same point for the same reason. The results table gains a column naming what else was active so a failure can be read in context. Installing and activating a dependency the size of WooCommerce is not quick, so the job timeout goes from 30 to 45 minutes. --- .../reusable-plugin-compatibility.yml | 251 ++++++++++++++---- 1 file changed, 200 insertions(+), 51 deletions(-) diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml index 4ca645f3c7aa9..d32d2156b9b16 100644 --- a/.github/workflows/reusable-plugin-compatibility.yml +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -4,6 +4,11 @@ # # Each plugin in the `plugin-slugs` shard is tested on its own: it is installed, activated, exercised, and then # removed before the next one is installed. This keeps one broken plugin from masking (or breaking) the next. +# +# The exception is a plugin that declares a `Requires Plugins` dependency, which core will not activate until +# that dependency is installed and active. Those dependencies are installed and activated alongside it, and the +# site is checked with only the dependencies active first so that a dependency's own breakage is not reported +# against the plugin under test. ## name: Plugin Compatibility Tests @@ -53,14 +58,17 @@ jobs: # - Creates a `wp-config.php` file with debugging and error logging enabled. # - Installs WordPress. # - Starts the PHP built-in web server so HTTP requests can be made against the site. - # - Installs, activates, exercises, and removes each plugin in the shard, one at a time. + # - Installs, activates, exercises, and removes each plugin in the shard, one at a time, together with + # anything it declares in `Requires Plugins`. # - Writes a results table to the workflow summary and fails the job if any plugin fataled. plugin-compatibility-tests: name: PHP ${{ inputs.php-version }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }} permissions: contents: read runs-on: ${{ inputs.os }} - timeout-minutes: 30 + # Installing a plugin's dependencies before testing it adds a download and an activation for each + # one, and a dependency the size of WooCommerce is not quick to activate. + timeout-minutes: 45 services: database: @@ -165,13 +173,24 @@ jobs: RESPONSE_BODY="${RUNNER_TEMP}/response.html" : > "${RESULTS}" - # record + # How many levels of `Requires Plugins` to follow. Chains in the directory are one level + # deep in practice, so the bound is only here to stop a circular declaration from looping + # forever. + MAX_DEPENDENCY_DEPTH=3 + + # Everything installed for the plugin currently under test, that plugin included. The + # directories are tracked alongside the slugs because a plugin does not always unpack into + # a directory named after its slug. + INSTALLED_SLUGS=() + INSTALLED_DIRS=() + + # record # # Appends one tab separated row to the results file. The reason is flattened so that it cannot # break the markdown table that is generated from these rows later on. record() { - SAFE_REASON="$( printf '%s' "${4:--}" | tr '\n\t|' ' ' )" - printf '%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${SAFE_REASON}" >> "${RESULTS}" + SAFE_REASON="$( printf '%s' "${5:--}" | tr '\n\t|' ' ' )" + printf '%s\t%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${4:--}" "${SAFE_REASON}" >> "${RESULTS}" } # check_url @@ -210,20 +229,70 @@ jobs: # The plugins directory itself must never be removed, only directories inside it. PLUGINS_ROOT="$( wp plugin path )" - # cleanup_plugin + # install_plugin # - # Returns the site to a clean slate. A plugin that fatals can take WP-CLI down with it, so every + # Installs a plugin from WordPress.org and remembers it so that it is removed again once the + # plugin under test has been checked. The slug is remembered before the download is attempted + # so that a partial download is still cleaned up. Returns non zero when the download failed. + install_plugin() { + INSTALLED_SLUGS+=( "${1}" ) + INSTALLED_DIRS+=( "" ) + + wp plugin install "${1}" --skip-plugins --skip-themes || return 1 + + INSTALLED_DIRS[-1]="$( wp plugin path "${1}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" + } + + # plugin_dependencies + # + # Prints the slugs named in the plugin's `Requires Plugins` header, one per line. WordPress' + # own header parser is used so that the same rules apply here as when core decides whether a + # plugin's dependencies are met. + plugin_dependencies() { + PLUGIN_FILE="$( wp plugin path "${1}" --skip-plugins --skip-themes 2>/dev/null || printf '' )" + + [ -n "${PLUGIN_FILE}" ] || return 0 + + # The single quotes are what keep the shell out of the PHP below. The path is handed over as an + # environment variable rather than interpolated so that it never reaches PHP as source code. + # shellcheck disable=SC2016 + PLUGIN_FILE="${PLUGIN_FILE}" wp eval --skip-plugins --skip-themes ' + require_once ABSPATH . "wp-admin/includes/plugin.php"; + + $plugin_data = get_plugin_data( getenv( "PLUGIN_FILE" ), false, false ); + + foreach ( explode( ",", $plugin_data["RequiresPlugins"] ?? "" ) as $dependency ) { + // Core only accepts a WordPress.org slug here, so anything else is ignored. + if ( preg_match( "/^[a-z0-9-]+$/", trim( $dependency ) ) ) { + echo trim( $dependency ), "\n"; + } + } + ' 2>/dev/null + } + + # cleanup_plugins + # + # Returns the site to a clean slate by removing everything installed for the current plugin, + # its dependencies included. A plugin that fatals can take WP-CLI down with it, so every # command here is allowed to fail and the plugin directory is removed directly as a fallback. # `--skip-plugins` keeps WP-CLI from loading the broken plugin while cleaning up after it. - cleanup_plugin() { - wp plugin deactivate "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 || true - wp plugin delete "${1}" --skip-plugins --skip-themes > /dev/null 2>&1 || true + cleanup_plugins() { + for INDEX in "${!INSTALLED_SLUGS[@]}"; do + CLEANUP_SLUG="${INSTALLED_SLUGS[${INDEX}]}" + CLEANUP_DIR="${INSTALLED_DIRS[${INDEX}]}" - if [ -n "${2:-}" ] && [ -d "${2}" ] && [ "${2}" != "${PLUGINS_ROOT}" ]; then - rm -rf "${2}" - fi + wp plugin deactivate "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 || true + wp plugin delete "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 || true - rm -rf "wp-content/plugins/${1}" + if [ -n "${CLEANUP_DIR}" ] && [ -d "${CLEANUP_DIR}" ] && [ "${CLEANUP_DIR}" != "${PLUGINS_ROOT}" ]; then + rm -rf "${CLEANUP_DIR}" + fi + + rm -rf "wp-content/plugins/${CLEANUP_SLUG}" + done + + INSTALLED_SLUGS=() + INSTALLED_DIRS=() # Make sure nothing is left behind in `active_plugins` pointing at a plugin that is now gone. wp option update active_plugins '[]' --format=json --skip-plugins --skip-themes > /dev/null 2>&1 || true @@ -237,46 +306,119 @@ jobs: STATUS="PASS" REASON="-" VERSION="unknown" - PLUGIN_DIR="" + DEPENDENCIES=() # Start every plugin with an empty log so that anything found in it belongs to this plugin. rm -f wp-content/debug.log # Step 1: download the plugin. A download failure is recorded as skipped rather than failed, # since it generally means a network flake or a plugin that is no longer in the directory. - if ! wp plugin install "${SLUG}" --skip-plugins --skip-themes; then - record "${SLUG}" "unknown" "SKIPPED" "The plugin could not be downloaded from WordPress.org" - cleanup_plugin "${SLUG}" "" + if ! install_plugin "${SLUG}"; then + record "${SLUG}" "unknown" "SKIPPED" "-" "The plugin could not be downloaded from WordPress.org" + cleanup_plugins printf '::endgroup::\n' continue fi VERSION="$( wp plugin get "${SLUG}" --field=version --skip-plugins --skip-themes 2>/dev/null || printf 'unknown' )" - PLUGIN_DIR="$( wp plugin path "${SLUG}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" - - # Step 2: activation. Activation runs the plugin's activation hooks and loads its main file. - ACTIVATE_EXIT_CODE=0 - ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? - printf '%s\n' "${ACTIVATE_OUTPUT}" - - if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then - case "${ACTIVATE_OUTPUT}" in - # Core refuses to activate a plugin whose declared requirements are not met, most often a - # `Requires Plugins` dependency that is not installed. Testing each plugin on its own means - # every WooCommerce extension lands here. That is core working as designed rather than a - # fatal, so it is recorded as skipped. - *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) + + # Step 2: install whatever the plugin names in `Requires Plugins`. Core refuses to activate a + # plugin whose dependencies are missing, so without this every WooCommerce extension - a large + # slice of the most popular plugins - would go untested. + # + # Dependencies are resolved a level at a time so that a dependency declaring its own is + # installed too. Anything already installed is left alone, which covers a plugin named twice + # in the same tree as well as a declaration pointing back at the plugin under test. + mapfile -t PENDING < <( plugin_dependencies "${SLUG}" ) + DEPTH=0 + + while [ "${#PENDING[@]}" -gt 0 ] && [ "${DEPTH}" -lt "${MAX_DEPENDENCY_DEPTH}" ]; do + NEXT=() + + for DEPENDENCY in "${PENDING[@]}"; do + if wp plugin is-installed "${DEPENDENCY}" --skip-plugins --skip-themes > /dev/null 2>&1; then + continue + fi + + printf 'Installing %s, which %s requires.\n' "${DEPENDENCY}" "${SLUG}" + + if ! install_plugin "${DEPENDENCY}"; then STATUS="SKIPPED" - REASON="Core declined to activate the plugin because its declared requirements are not met" - ;; - * ) - STATUS="FAIL" - REASON="The plugin could not be activated" - ;; - esac + REASON="The required plugin ${DEPENDENCY} is not available from WordPress.org" + break 2 + fi + + DEPENDENCIES+=( "${DEPENDENCY}" ) + mapfile -t -O "${#NEXT[@]}" NEXT < <( plugin_dependencies "${DEPENDENCY}" ) + done + + PENDING=( "${NEXT[@]}" ) + DEPTH=$(( DEPTH + 1 )) + done + + # Step 3: activate the dependencies, deepest first. That is the reverse of the order they + # were discovered in, and it matters because core will not activate a plugin ahead of its own + # requirements either. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + for (( INDEX = ${#DEPENDENCIES[@]} - 1; INDEX >= 0; INDEX-- )); do + DEPENDENCY="${DEPENDENCIES[${INDEX}]}" + DEPENDENCY_EXIT_CODE=0 + DEPENDENCY_OUTPUT="$( wp plugin activate "${DEPENDENCY}" 2>&1 )" || DEPENDENCY_EXIT_CODE=$? + printf '%s\n' "${DEPENDENCY_OUTPUT}" + + if [ "${DEPENDENCY_EXIT_CODE}" -ne 0 ]; then + STATUS="SKIPPED" + REASON="The required plugin ${DEPENDENCY} could not be activated" + break + fi + done + fi + + # Step 4: with dependencies active the plugin under test is no longer alone on the site, so + # the baseline is checked before it is activated. A site that is already broken says something + # about the dependency rather than about the plugin being tested, and blaming the plugin for + # it would be the masking this workflow is built to avoid. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + for URL_PATH in "/" "/wp-login.php"; do + BASELINE_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${BASELINE_REASON}" ]; then + STATUS="SKIPPED" + REASON="The required plugins are not healthy on their own: ${BASELINE_REASON}" + break + fi + done + + # Anything the dependencies logged on their way up is not the responsibility of the plugin + # under test, so the log starts empty again here. + rm -f wp-content/debug.log fi - # Step 3: boot all of core plus the active plugin in a CLI context. + # Step 5: activation. Activation runs the plugin's activation hooks and loads its main file. + if [ "${STATUS}" = "PASS" ]; then + ACTIVATE_EXIT_CODE=0 + ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? + printf '%s\n' "${ACTIVATE_OUTPUT}" + + if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then + case "${ACTIVATE_OUTPUT}" in + # Core refuses to activate a plugin whose declared requirements are not met. Dependencies + # are installed above, so what is left here is a plugin asking for a version of PHP or + # WordPress this job is not running, or for a dependency that is not on WordPress.org. + # That is core working as designed rather than a fatal, so it is recorded as skipped. + *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) + STATUS="SKIPPED" + REASON="Core declined to activate the plugin because its declared requirements are not met" + ;; + * ) + STATUS="FAIL" + REASON="The plugin could not be activated" + ;; + esac + fi + fi + + # Step 6: boot all of core plus the active plugin in a CLI context. if [ "${STATUS}" = "PASS" ]; then EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" || true @@ -298,7 +440,7 @@ jobs: fi fi - # Step 4: request the front page and the login screen through the PHP built-in server. + # Step 7: request the front page and the login screen through the PHP built-in server. if [ "${STATUS}" = "PASS" ]; then for URL_PATH in "/" "/wp-login.php"; do HTTP_REASON="$( check_url "${URL_PATH}" )" @@ -311,7 +453,7 @@ jobs: done fi - # Step 5: a fatal can be logged without changing the HTTP status, for example during a shutdown + # Step 8: a fatal can be logged without changing the HTTP status, for example during a shutdown # hook, so the debug log is checked separately. if [ "${STATUS}" = "PASS" ] && [ -f wp-content/debug.log ] && grep -q 'PHP Fatal' wp-content/debug.log; then grep 'PHP Fatal' wp-content/debug.log @@ -319,9 +461,16 @@ jobs: REASON="A fatal error was logged: $( grep -m 1 'PHP Fatal' wp-content/debug.log | cut -c 1-200 )" fi - # Step 6: record the outcome and put the site back the way it was found. - record "${SLUG}" "${VERSION}" "${STATUS}" "${REASON}" - cleanup_plugin "${SLUG}" "${PLUGIN_DIR}" + # Step 9: record the outcome and put the site back the way it was found. + DEPENDENCY_LIST="-" + + if [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + DEPENDENCY_LIST="$( printf '%s, ' "${DEPENDENCIES[@]}" )" + DEPENDENCY_LIST="${DEPENDENCY_LIST%, }" + fi + + record "${SLUG}" "${VERSION}" "${STATUS}" "${DEPENDENCY_LIST}" "${REASON}" + cleanup_plugins printf '%s: %s\n' "${SLUG}" "${STATUS}" printf '::endgroup::\n' @@ -334,18 +483,18 @@ jobs: { printf '### WordPress %s / PHP %s\n\n' "${WP_VERSION}" "${PHP_VERSION}" printf '%s passed, %s failed, %s skipped.\n\n' "${PASS_COUNT}" "${FAIL_COUNT}" "${SKIP_COUNT}" - printf '| Plugin | Version | Result | Details |\n' - printf '| --- | --- | --- | --- |\n' + printf '| Plugin | Version | Result | Also active | Details |\n' + printf '| --- | --- | --- | --- | --- |\n' - while IFS=$'\t' read -r ROW_SLUG ROW_VERSION ROW_STATUS ROW_REASON; do + while IFS=$'\t' read -r ROW_SLUG ROW_VERSION ROW_STATUS ROW_DEPENDENCIES ROW_REASON; do case "${ROW_STATUS}" in PASS ) ICON=':white_check_mark:' ;; FAIL ) ICON=':x:' ;; * ) ICON=':warning:' ;; esac - printf '| [%s](https://wordpress.org/plugins/%s/) | %s | %s %s | %s |\n' \ - "${ROW_SLUG}" "${ROW_SLUG}" "${ROW_VERSION}" "${ICON}" "${ROW_STATUS}" "${ROW_REASON}" + printf '| [%s](https://wordpress.org/plugins/%s/) | %s | %s %s | %s | %s |\n' \ + "${ROW_SLUG}" "${ROW_SLUG}" "${ROW_VERSION}" "${ICON}" "${ROW_STATUS}" "${ROW_DEPENDENCIES}" "${ROW_REASON}" done < "${RESULTS}" printf '\n' @@ -354,7 +503,7 @@ jobs: # Plugins that could not be downloaded are reported but do not fail the run. if [ "${FAIL_COUNT}" -gt 0 ]; then printf 'The following plugins failed against WordPress %s:\n' "${WP_VERSION}" - awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $4 }' "${RESULTS}" + awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $5 }' "${RESULTS}" exit 1 fi From 06f801d8d660f6df80e409aa45477487a9d49a61 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 31 Aug 2026 16:22:12 -0700 Subject: [PATCH 6/9] Build/Test Tools: Allow plugin compatibility runs to name plugins. Trying this workflow out meant taking whatever the popularity query returned, so anyone wanting to check one plugin against a release candidate, or to reproduce a failure from an earlier run, had to sit through a full run and read past everything else in the shard. Reviewers could not run it at all: the repository guard skips both jobs everywhere but this repository, and dispatching it here needs write access. Add an optional `plugin-slugs` input that replaces the directory query with a list written by hand. Commas and whitespace both separate slugs so a list can be pasted in however it was written down, and a value outside the character set the directory uses stops the run immediately rather than being reported as an impossible download several minutes later. The list goes through the same sharding as a fetched one, so naming forty plugins still splits across two jobs. Let a dispatched run through the repository guard wherever it is triggered, which is what makes a fork useful for trying this out. Two runs dispatched against the same version of WordPress used to land in the same concurrency group and cancel each other, which is wrong once the plugins being tested can differ between them. Dispatched runs are now grouped by run id. --- .github/workflows/plugin-compatibility.yml | 158 +++++++++++++-------- 1 file changed, 102 insertions(+), 56 deletions(-) diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml index b2a9c12d2d6a6..5f0a7d4b55d39 100644 --- a/.github/workflows/plugin-compatibility.yml +++ b/.github/workflows/plugin-compatibility.yml @@ -15,6 +15,10 @@ # which is the intended way to use it as part of the pre-release checklist: point it at the beta or RC and give # it a count. # +# A dispatched run can also name the plugins to test instead of taking them from the directory, which is how to +# check a single plugin against a release candidate, or to reproduce a failure from an earlier run without +# waiting for the other plugins in its shard. +# # This workflow is not meant to test wordpress-develop checkouts, but rather versions officially available on # WordPress.org. ## @@ -45,15 +49,22 @@ on: type: string default: 'nightly' plugin-count: - description: 'Number of most popular plugins to test. Accepts any number from 1 to 1000, eg. 10, 50, or 250.' + description: 'Number of most popular plugins to test. Accepts any number from 1 to 1000, eg. 10, 50, or 250. Ignored when plugin-slugs is given.' type: string default: '100' + plugin-slugs: + description: 'Optional. Test these WordPress.org plugin slugs instead of the most popular ones. Accepts a comma or space separated list, eg. "woocommerce, classic-editor".' + type: string + default: '' # Cancels all previous workflow runs for pull requests that have not completed. concurrency: # The concurrency group contains the workflow name and the branch name for pull requests # or the commit hash for any other events. - group: ${{ github.workflow }}-${{ inputs.wp-version || github.event_name == 'pull_request' && github.head_ref || github.sha }} + # + # Dispatched runs are grouped by run instead, so that testing one plugin list does not cancel a run someone + # else started with a different one against the same version of WordPress. + group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.event_name == 'pull_request' && github.head_ref || github.sha }} cancel-in-progress: true # Disable permissions for all available scopes by default. @@ -64,25 +75,31 @@ jobs: # Builds the list of plugins to test and splits it into shards for the test matrix. # # The list is fetched at run time so that it never goes stale, and it is ordered by popularity so that a - # smaller count still tests the plugins with the widest reach. + # smaller count still tests the plugins with the widest reach. A dispatched run can name the plugins to test + # instead, in which case the directory is not queried at all. # # Performs the following steps: - # - Queries the WordPress.org plugin directory API for the most popular plugins. + # - Queries the WordPress.org plugin directory API for the most popular plugins, or reads the plugin-slugs input. # - Splits the resulting slugs into shards and returns them as a job output. build-plugin-matrix: name: Build plugin matrix permissions: contents: read runs-on: ubuntu-24.04 - if: ${{ github.repository == 'WordPress/wordpress-develop' }} + # Dispatched runs are allowed anywhere so that this can be run from a fork, which is the only way for + # someone without write access to the repository to try it against a plugin of their choosing. + if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'workflow_dispatch' }} timeout-minutes: 5 outputs: shards: ${{ steps.plugin-shards.outputs.shards }} steps: - - name: Fetch the most popular plugins + - name: Build the list of plugins to test id: plugin-shards env: + # An explicit list of slugs to test in place of the most popular ones. Empty for everything but a + # dispatched run that filled the input in. + PLUGIN_SLUGS: ${{ inputs.plugin-slugs }} # Runs that carry no inputs fall back to the defaults. Pull request and push runs exist to test # this workflow rather than the ecosystem, so they use a small count. See the note on the # plugin-compatibility-tests job below. @@ -100,65 +117,94 @@ jobs: run: | set -euo pipefail - # Guard against a non-numeric value being passed to the API. - if ! printf '%s' "${PLUGIN_COUNT}" | grep -Eq '^[0-9]+$'; then - printf 'The plugin-count input must be a positive integer, got "%s".\n' "${PLUGIN_COUNT}" - exit 1 - fi - - if [ "${PLUGIN_COUNT}" -lt 1 ] || [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then - printf 'The plugin-count input must be between 1 and %s, got %s.\n' "${MAX_PLUGIN_COUNT}" "${PLUGIN_COUNT}" - printf 'Testing more than %s plugins will not fit in the wall time this workflow targets.\n' "${MAX_PLUGIN_COUNT}" - exit 1 - fi - RAW_SLUGS="${RUNNER_TEMP}/slugs-raw.txt" DEDUPED_SLUGS="${RUNNER_TEMP}/slugs.txt" : > "${RAW_SLUGS}" - PAGE=1 - UNIQUE_COUNT=0 - - while : ; do - # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. - # The unneeded response fields are turned off to keep the payload small. - curl -sS --fail --retry 3 --retry-delay 5 \ - --get 'https://api.wordpress.org/plugins/info/1.2/' \ - --data-urlencode 'action=query_plugins' \ - --data-urlencode 'request[browse]=popular' \ - --data-urlencode "request[per_page]=${PAGE_SIZE}" \ - --data-urlencode "request[page]=${PAGE}" \ - --data-urlencode 'request[fields][short_description]=0' \ - --data-urlencode 'request[fields][sections]=0' \ - --data-urlencode 'request[fields][icons]=0' \ - --data-urlencode 'request[fields][banners]=0' \ - --data-urlencode 'request[fields][ratings]=0' \ - --data-urlencode 'request[fields][tags]=0' \ - --data-urlencode 'request[fields][compatibility]=0' \ - --data-urlencode 'request[fields][screenshots]=0' \ - -o "${RUNNER_TEMP}/plugins-page.json" - - PAGE_COUNT="$( jq '.plugins | length' "${RUNNER_TEMP}/plugins-page.json" )" - jq -r '.plugins[] | .slug // empty' "${RUNNER_TEMP}/plugins-page.json" >> "${RAW_SLUGS}" - - # Popularity ordering can shift between two requests, so the same slug can turn up on more than - # one page. Duplicates are dropped while the first occurrence keeps its position. + + if [ -n "${PLUGIN_SLUGS}" ]; then + # A named list replaces the popularity query entirely. Commas and whitespace both separate slugs, + # so a list can be pasted in however it was written down. + printf '%s' "${PLUGIN_SLUGS}" | tr ',[:space:]' '\n' > "${RAW_SLUGS}" awk '!seen[$0]++ && NF > 0' "${RAW_SLUGS}" > "${DEDUPED_SLUGS}" - UNIQUE_COUNT="$( wc -l < "${DEDUPED_SLUGS}" | tr -d ' ' )" - printf 'Page %s returned %s plugins, %s unique slugs collected so far.\n' "${PAGE}" "${PAGE_COUNT}" "${UNIQUE_COUNT}" + # A value outside the character set the directory uses is a typo worth stopping for now, rather + # than a plugin reported as impossible to download several minutes from now. + if INVALID_SLUGS="$( grep -Ev '^[a-z0-9-]+$' "${DEDUPED_SLUGS}" )"; then + printf 'The plugin-slugs input contains values that are not WordPress.org plugin slugs:\n%s\n' "${INVALID_SLUGS}" + exit 1 + fi + + PLUGIN_COUNT="$( wc -l < "${DEDUPED_SLUGS}" | tr -d ' ' )" - if [ "${UNIQUE_COUNT}" -ge "${PLUGIN_COUNT}" ]; then - break + if [ "${PLUGIN_COUNT}" -lt 1 ]; then + printf 'The plugin-slugs input did not contain any plugin slugs.\n' + exit 1 fi - # A short page means the directory has nothing left to give. - if [ "${PAGE_COUNT}" -lt "${PAGE_SIZE}" ]; then - printf 'The API returned fewer than %s plugins on page %s, so %s is everything available.\n' "${PAGE_SIZE}" "${PAGE}" "${UNIQUE_COUNT}" - break + if [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then + printf 'The plugin-slugs input names %s plugins, more than the %s this workflow will test in one run.\n' "${PLUGIN_COUNT}" "${MAX_PLUGIN_COUNT}" + exit 1 fi - PAGE=$(( PAGE + 1 )) - done + printf 'Testing the %s plugin(s) named in the plugin-slugs input.\n' "${PLUGIN_COUNT}" + else + # Guard against a non-numeric value being passed to the API. + if ! printf '%s' "${PLUGIN_COUNT}" | grep -Eq '^[0-9]+$'; then + printf 'The plugin-count input must be a positive integer, got "%s".\n' "${PLUGIN_COUNT}" + exit 1 + fi + + if [ "${PLUGIN_COUNT}" -lt 1 ] || [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then + printf 'The plugin-count input must be between 1 and %s, got %s.\n' "${MAX_PLUGIN_COUNT}" "${PLUGIN_COUNT}" + printf 'Testing more than %s plugins will not fit in the wall time this workflow targets.\n' "${MAX_PLUGIN_COUNT}" + exit 1 + fi + + PAGE=1 + UNIQUE_COUNT=0 + + while : ; do + # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. + # The unneeded response fields are turned off to keep the payload small. + curl -sS --fail --retry 3 --retry-delay 5 \ + --get 'https://api.wordpress.org/plugins/info/1.2/' \ + --data-urlencode 'action=query_plugins' \ + --data-urlencode 'request[browse]=popular' \ + --data-urlencode "request[per_page]=${PAGE_SIZE}" \ + --data-urlencode "request[page]=${PAGE}" \ + --data-urlencode 'request[fields][short_description]=0' \ + --data-urlencode 'request[fields][sections]=0' \ + --data-urlencode 'request[fields][icons]=0' \ + --data-urlencode 'request[fields][banners]=0' \ + --data-urlencode 'request[fields][ratings]=0' \ + --data-urlencode 'request[fields][tags]=0' \ + --data-urlencode 'request[fields][compatibility]=0' \ + --data-urlencode 'request[fields][screenshots]=0' \ + -o "${RUNNER_TEMP}/plugins-page.json" + + PAGE_COUNT="$( jq '.plugins | length' "${RUNNER_TEMP}/plugins-page.json" )" + jq -r '.plugins[] | .slug // empty' "${RUNNER_TEMP}/plugins-page.json" >> "${RAW_SLUGS}" + + # Popularity ordering can shift between two requests, so the same slug can turn up on more than + # one page. Duplicates are dropped while the first occurrence keeps its position. + awk '!seen[$0]++ && NF > 0' "${RAW_SLUGS}" > "${DEDUPED_SLUGS}" + UNIQUE_COUNT="$( wc -l < "${DEDUPED_SLUGS}" | tr -d ' ' )" + + printf 'Page %s returned %s plugins, %s unique slugs collected so far.\n' "${PAGE}" "${PAGE_COUNT}" "${UNIQUE_COUNT}" + + if [ "${UNIQUE_COUNT}" -ge "${PLUGIN_COUNT}" ]; then + break + fi + + # A short page means the directory has nothing left to give. + if [ "${PAGE_COUNT}" -lt "${PAGE_SIZE}" ]; then + printf 'The API returned fewer than %s plugins on page %s, so %s is everything available.\n' "${PAGE_SIZE}" "${PAGE}" "${UNIQUE_COUNT}" + break + fi + + PAGE=$(( PAGE + 1 )) + done + fi SLUGS="$( head -n "${PLUGIN_COUNT}" "${DEDUPED_SLUGS}" | jq -R -s -c 'split( "\n" ) | map( select( . != "" ) )' )" TOTAL="$( printf '%s' "${SLUGS}" | jq 'length' )" @@ -213,7 +259,7 @@ jobs: permissions: contents: read needs: [ build-plugin-matrix ] - if: ${{ github.repository == 'WordPress/wordpress-develop' }} + if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'workflow_dispatch' }} strategy: fail-fast: false matrix: From 2896efe087596ddad0b4a1d26f108775172dd185 Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Mon, 31 Aug 2026 16:44:31 -0700 Subject: [PATCH 7/9] Build/Test Tools: Drop the removed Slack timeout webhook secret. `slack-notifications.yml` no longer accepts `SLACK_GHA_TIMEOUT_WEBHOOK`, and every other caller stopped passing it, so actionlint failed on this workflow once trunk was merged in. --- .github/workflows/plugin-compatibility.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml index 5f0a7d4b55d39..4a0655438904d 100644 --- a/.github/workflows/plugin-compatibility.yml +++ b/.github/workflows/plugin-compatibility.yml @@ -287,7 +287,6 @@ jobs: SLACK_GHA_CANCELLED_WEBHOOK: ${{ secrets.SLACK_GHA_CANCELLED_WEBHOOK }} SLACK_GHA_FIXED_WEBHOOK: ${{ secrets.SLACK_GHA_FIXED_WEBHOOK }} SLACK_GHA_FAILURE_WEBHOOK: ${{ secrets.SLACK_GHA_FAILURE_WEBHOOK }} - SLACK_GHA_TIMEOUT_WEBHOOK: ${{ secrets.SLACK_GHA_TIMEOUT_WEBHOOK }} failed-workflow: name: Failed workflow tasks From 69d0d379134ce40cf72b80ef12018dbb0394a48e Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Tue, 1 Sep 2026 08:56:01 -0700 Subject: [PATCH 8/9] Build/Test Tools: Make the plugin compatibility checks runnable from a terminal. Move the per-plugin checks out of the reusable workflow and into tools/plugin-compatibility/test-plugins.sh, so that reproducing a result from a run no longer means dispatching a workflow and waiting for it. The script provisions a database and a PHP with WP-CLI in throwaway containers by default, so nothing but Docker is needed locally. The workflow passes --no-docker and runs the same code against the runner's own PHP and database service, which keeps one copy of the logic that decides whether a plugin passed. --- .github/workflows/plugin-compatibility.yml | 6 + .../reusable-plugin-compatibility.yml | 447 +---------- tools/plugin-compatibility/test-plugins.sh | 709 ++++++++++++++++++ 3 files changed, 744 insertions(+), 418 deletions(-) create mode 100755 tools/plugin-compatibility/test-plugins.sh diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml index 4a0655438904d..a496df94681bd 100644 --- a/.github/workflows/plugin-compatibility.yml +++ b/.github/workflows/plugin-compatibility.yml @@ -19,6 +19,10 @@ # check a single plugin against a release candidate, or to reproduce a failure from an earlier run without # waiting for the other plugins in its shard. # +# The checks each shard runs live in `tools/plugin-compatibility/test-plugins.sh`. Running that script from a +# terminal runs exactly what a shard here runs, which is the quickest way to reproduce a result from a run +# without waiting on Actions. +# # This workflow is not meant to test wordpress-develop checkouts, but rather versions officially available on # WordPress.org. ## @@ -32,6 +36,7 @@ on: paths: - '.github/workflows/plugin-compatibility.yml' - '.github/workflows/reusable-plugin-compatibility.yml' + - 'tools/plugin-compatibility/**' pull_request: # This workflow is only meant to run from trunk. Pull requests changing this file with different BASE branches should be ignored. branches: @@ -40,6 +45,7 @@ on: paths: - '.github/workflows/plugin-compatibility.yml' - '.github/workflows/reusable-plugin-compatibility.yml' + - 'tools/plugin-compatibility/**' schedule: - cron: '0 2 * * 1' workflow_dispatch: diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml index d32d2156b9b16..003146860563c 100644 --- a/.github/workflows/reusable-plugin-compatibility.yml +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -9,6 +9,10 @@ # that dependency is installed and active. Those dependencies are installed and activated alongside it, and the # site is checked with only the dependencies active first so that a dependency's own breakage is not reported # against the plugin under test. +# +# The checks themselves live in `tools/plugin-compatibility/test-plugins.sh` rather than in this file, so that +# the same code can be run from a terminal to reproduce a result from a run here. See the comments at the top +# of that script. ## name: Plugin Compatibility Tests @@ -53,14 +57,9 @@ jobs: # Tests that each plugin in the shard can be activated against the given version of WordPress. # # Performs the following steps: - # - Sets up PHP. - # - Downloads the specified version of WordPress. - # - Creates a `wp-config.php` file with debugging and error logging enabled. - # - Installs WordPress. - # - Starts the PHP built-in web server so HTTP requests can be made against the site. - # - Installs, activates, exercises, and removes each plugin in the shard, one at a time, together with - # anything it declares in `Requires Plugins`. - # - Writes a results table to the workflow summary and fails the job if any plugin fataled. + # - Checks out the repository, for the script that runs the checks. + # - Sets up PHP and WP-CLI. + # - Runs the checks against every plugin in the shard. plugin-compatibility-tests: name: PHP ${{ inputs.php-version }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }} permissions: @@ -87,6 +86,14 @@ jobs: -e MYSQL_DATABASE="test_db" steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} + persist-credentials: false + sparse-checkout: tools/plugin-compatibility + sparse-checkout-cone-mode: false + - name: Set up PHP ${{ inputs.php-version }} uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 with: @@ -94,421 +101,25 @@ jobs: coverage: none tools: wp-cli - - name: Download WordPress ${{ inputs.wp-version }} - run: wp core download --version="${WP_VERSION}" - env: - WP_VERSION: ${{ inputs.wp-version }} - - - name: Create wp-config.php file - run: wp config create --dbname=test_db --dbuser=root --dbpass=root --dbhost="127.0.0.1:${DB_PORT}" - env: - DB_PORT: ${{ job.services.database.ports['3306'] }} - - # Errors need to reach `wp-content/debug.log` so that a white screen of death is still detectable. - # - # `WP_DEBUG_DISPLAY` is left off on purpose: this should behave the way a production site does, where a - # fatal error is an empty page and an HTTP 500 rather than a printed stack trace. - # - # The fatal error handler is disabled so that a fatal is reported as a fatal instead of being swallowed by - # recovery mode, which would also deactivate the plugin mid-test. - # - # WP-Cron is disabled because WordPress spawns it as a loopback request during a front end request. The - # loopback lands back on the same PHP built-in server that is still busy serving the request that spawned - # it, and the two deadlock until curl gives up. Catching fatals that only happen on a scheduled event is - # worth doing, but it needs to run through WP-CLI rather than a loopback, which is follow up work. - - name: Enable debugging and error logging - run: | - wp config set WP_DEBUG true --raw - wp config set WP_DEBUG_LOG true --raw - wp config set WP_DEBUG_DISPLAY false --raw - wp config set WP_DISABLE_FATAL_ERROR_HANDLER true --raw - wp config set DISABLE_WP_CRON true --raw - - - name: Install WordPress - run: | - wp core install --url="${SITE_URL}" --title="Plugin Compatibility Test" --admin_user=admin \ - --admin_password=password --admin_email=me@example.org --skip-email - env: - SITE_URL: http://127.0.0.1:8889 - - # The site needs to answer real requests so that fatals which only happen on a front end or admin page - # load are caught. The built-in server is enough for that and needs nothing installed. + # The runner already provides PHP, WP-CLI and a database, so the checks run here rather than in the + # containers the script provisions when it is run from a terminal. # - # `PHP_CLI_SERVER_WORKERS` is set because the built-in server is single threaded by default. Plenty of - # plugins make a loopback request to the site they are running on, and a single threaded server cannot - # answer one while it is still serving the request that made it. - - name: Start the PHP built-in web server - env: - PHP_CLI_SERVER_WORKERS: '4' - run: | - set -uo pipefail - - nohup php -S 127.0.0.1:8889 -t "$( pwd )" > "${RUNNER_TEMP}/php-server.log" 2>&1 & - - # Wait for the server to start answering before any plugin is installed. - for _ in $( seq 1 30 ); do - if curl -sSf -o /dev/null "http://127.0.0.1:8889/wp-login.php"; then - printf 'The PHP built-in server is ready.\n' - exit 0 - fi - sleep 1 - done - - printf 'The PHP built-in server did not start.\n' - cat "${RUNNER_TEMP}/php-server.log" - exit 1 - + # WordPress is installed outside the workspace so that the checkout above is not part of the site under + # test. - name: Test each plugin in isolation env: PLUGIN_SLUGS: ${{ inputs.plugin-slugs }} - PHP_VERSION: ${{ inputs.php-version }} - SITE_URL: http://127.0.0.1:8889 WP_VERSION: ${{ inputs.wp-version }} + DB_PORT: ${{ job.services.database.ports['3306'] }} run: | - # `set -e` is deliberately not used here: a plugin that fatals must not stop the remaining - # plugins in the shard from being tested. set -uo pipefail - RESULTS="${RUNNER_TEMP}/plugin-results.tsv" - RESPONSE_BODY="${RUNNER_TEMP}/response.html" - : > "${RESULTS}" - - # How many levels of `Requires Plugins` to follow. Chains in the directory are one level - # deep in practice, so the bound is only here to stop a circular declaration from looping - # forever. - MAX_DEPENDENCY_DEPTH=3 - - # Everything installed for the plugin currently under test, that plugin included. The - # directories are tracked alongside the slugs because a plugin does not always unpack into - # a directory named after its slug. - INSTALLED_SLUGS=() - INSTALLED_DIRS=() - - # record - # - # Appends one tab separated row to the results file. The reason is flattened so that it cannot - # break the markdown table that is generated from these rows later on. - record() { - SAFE_REASON="$( printf '%s' "${5:--}" | tr '\n\t|' ' ' )" - printf '%s\t%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${4:--}" "${SAFE_REASON}" >> "${RESULTS}" - } - - # check_url - # - # Requests a path on the test site and prints a reason to stdout when the response looks broken. - # Prints nothing when the request looks healthy. - check_url() { - CURL_EXIT_CODE=0 - # The exit code is captured separately rather than falling back to a literal inside the command - # substitution, which would append to whatever curl had already written and produce a nonsense - # status like "200000" when a request returned headers and then stalled. - HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" )" || CURL_EXIT_CODE=$? - - if [ -z "${HTTP_CODE}" ] || [ "${HTTP_CODE}" = "000" ]; then - printf 'The request to %s did not complete, curl exit code %s' "${1}" "${CURL_EXIT_CODE}" - return - fi - - if [ "${HTTP_CODE}" -ge 500 ]; then - printf 'The request to %s returned HTTP %s' "${1}" "${HTTP_CODE}" - return - fi - - if [ "${CURL_EXIT_CODE}" -ne 0 ]; then - printf 'The request to %s returned HTTP %s but the response did not finish, curl exit code %s' "${1}" "${HTTP_CODE}" "${CURL_EXIT_CODE}" - return - fi - - # Belt and braces. Errors are not displayed by default, but a plugin can turn display_errors - # back on for itself. - if grep -qi 'Fatal error' "${RESPONSE_BODY}"; then - printf 'The response from %s contained a fatal error' "${1}" - fi - } - - # The plugins directory itself must never be removed, only directories inside it. - PLUGINS_ROOT="$( wp plugin path )" - - # install_plugin - # - # Installs a plugin from WordPress.org and remembers it so that it is removed again once the - # plugin under test has been checked. The slug is remembered before the download is attempted - # so that a partial download is still cleaned up. Returns non zero when the download failed. - install_plugin() { - INSTALLED_SLUGS+=( "${1}" ) - INSTALLED_DIRS+=( "" ) - - wp plugin install "${1}" --skip-plugins --skip-themes || return 1 - - INSTALLED_DIRS[-1]="$( wp plugin path "${1}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" - } - - # plugin_dependencies - # - # Prints the slugs named in the plugin's `Requires Plugins` header, one per line. WordPress' - # own header parser is used so that the same rules apply here as when core decides whether a - # plugin's dependencies are met. - plugin_dependencies() { - PLUGIN_FILE="$( wp plugin path "${1}" --skip-plugins --skip-themes 2>/dev/null || printf '' )" - - [ -n "${PLUGIN_FILE}" ] || return 0 - - # The single quotes are what keep the shell out of the PHP below. The path is handed over as an - # environment variable rather than interpolated so that it never reaches PHP as source code. - # shellcheck disable=SC2016 - PLUGIN_FILE="${PLUGIN_FILE}" wp eval --skip-plugins --skip-themes ' - require_once ABSPATH . "wp-admin/includes/plugin.php"; - - $plugin_data = get_plugin_data( getenv( "PLUGIN_FILE" ), false, false ); - - foreach ( explode( ",", $plugin_data["RequiresPlugins"] ?? "" ) as $dependency ) { - // Core only accepts a WordPress.org slug here, so anything else is ignored. - if ( preg_match( "/^[a-z0-9-]+$/", trim( $dependency ) ) ) { - echo trim( $dependency ), "\n"; - } - } - ' 2>/dev/null - } - - # cleanup_plugins - # - # Returns the site to a clean slate by removing everything installed for the current plugin, - # its dependencies included. A plugin that fatals can take WP-CLI down with it, so every - # command here is allowed to fail and the plugin directory is removed directly as a fallback. - # `--skip-plugins` keeps WP-CLI from loading the broken plugin while cleaning up after it. - cleanup_plugins() { - for INDEX in "${!INSTALLED_SLUGS[@]}"; do - CLEANUP_SLUG="${INSTALLED_SLUGS[${INDEX}]}" - CLEANUP_DIR="${INSTALLED_DIRS[${INDEX}]}" - - wp plugin deactivate "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 || true - wp plugin delete "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 || true - - if [ -n "${CLEANUP_DIR}" ] && [ -d "${CLEANUP_DIR}" ] && [ "${CLEANUP_DIR}" != "${PLUGINS_ROOT}" ]; then - rm -rf "${CLEANUP_DIR}" - fi - - rm -rf "wp-content/plugins/${CLEANUP_SLUG}" - done - - INSTALLED_SLUGS=() - INSTALLED_DIRS=() - - # Make sure nothing is left behind in `active_plugins` pointing at a plugin that is now gone. - wp option update active_plugins '[]' --format=json --skip-plugins --skip-themes > /dev/null 2>&1 || true - } - - while IFS= read -r SLUG; do - [ -n "${SLUG}" ] || continue - - printf '::group::%s\n' "${SLUG}" - - STATUS="PASS" - REASON="-" - VERSION="unknown" - DEPENDENCIES=() - - # Start every plugin with an empty log so that anything found in it belongs to this plugin. - rm -f wp-content/debug.log - - # Step 1: download the plugin. A download failure is recorded as skipped rather than failed, - # since it generally means a network flake or a plugin that is no longer in the directory. - if ! install_plugin "${SLUG}"; then - record "${SLUG}" "unknown" "SKIPPED" "-" "The plugin could not be downloaded from WordPress.org" - cleanup_plugins - printf '::endgroup::\n' - continue - fi - - VERSION="$( wp plugin get "${SLUG}" --field=version --skip-plugins --skip-themes 2>/dev/null || printf 'unknown' )" - - # Step 2: install whatever the plugin names in `Requires Plugins`. Core refuses to activate a - # plugin whose dependencies are missing, so without this every WooCommerce extension - a large - # slice of the most popular plugins - would go untested. - # - # Dependencies are resolved a level at a time so that a dependency declaring its own is - # installed too. Anything already installed is left alone, which covers a plugin named twice - # in the same tree as well as a declaration pointing back at the plugin under test. - mapfile -t PENDING < <( plugin_dependencies "${SLUG}" ) - DEPTH=0 - - while [ "${#PENDING[@]}" -gt 0 ] && [ "${DEPTH}" -lt "${MAX_DEPENDENCY_DEPTH}" ]; do - NEXT=() - - for DEPENDENCY in "${PENDING[@]}"; do - if wp plugin is-installed "${DEPENDENCY}" --skip-plugins --skip-themes > /dev/null 2>&1; then - continue - fi - - printf 'Installing %s, which %s requires.\n' "${DEPENDENCY}" "${SLUG}" - - if ! install_plugin "${DEPENDENCY}"; then - STATUS="SKIPPED" - REASON="The required plugin ${DEPENDENCY} is not available from WordPress.org" - break 2 - fi - - DEPENDENCIES+=( "${DEPENDENCY}" ) - mapfile -t -O "${#NEXT[@]}" NEXT < <( plugin_dependencies "${DEPENDENCY}" ) - done - - PENDING=( "${NEXT[@]}" ) - DEPTH=$(( DEPTH + 1 )) - done - - # Step 3: activate the dependencies, deepest first. That is the reverse of the order they - # were discovered in, and it matters because core will not activate a plugin ahead of its own - # requirements either. - if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then - for (( INDEX = ${#DEPENDENCIES[@]} - 1; INDEX >= 0; INDEX-- )); do - DEPENDENCY="${DEPENDENCIES[${INDEX}]}" - DEPENDENCY_EXIT_CODE=0 - DEPENDENCY_OUTPUT="$( wp plugin activate "${DEPENDENCY}" 2>&1 )" || DEPENDENCY_EXIT_CODE=$? - printf '%s\n' "${DEPENDENCY_OUTPUT}" - - if [ "${DEPENDENCY_EXIT_CODE}" -ne 0 ]; then - STATUS="SKIPPED" - REASON="The required plugin ${DEPENDENCY} could not be activated" - break - fi - done - fi - - # Step 4: with dependencies active the plugin under test is no longer alone on the site, so - # the baseline is checked before it is activated. A site that is already broken says something - # about the dependency rather than about the plugin being tested, and blaming the plugin for - # it would be the masking this workflow is built to avoid. - if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then - for URL_PATH in "/" "/wp-login.php"; do - BASELINE_REASON="$( check_url "${URL_PATH}" )" - - if [ -n "${BASELINE_REASON}" ]; then - STATUS="SKIPPED" - REASON="The required plugins are not healthy on their own: ${BASELINE_REASON}" - break - fi - done - - # Anything the dependencies logged on their way up is not the responsibility of the plugin - # under test, so the log starts empty again here. - rm -f wp-content/debug.log - fi - - # Step 5: activation. Activation runs the plugin's activation hooks and loads its main file. - if [ "${STATUS}" = "PASS" ]; then - ACTIVATE_EXIT_CODE=0 - ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? - printf '%s\n' "${ACTIVATE_OUTPUT}" - - if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then - case "${ACTIVATE_OUTPUT}" in - # Core refuses to activate a plugin whose declared requirements are not met. Dependencies - # are installed above, so what is left here is a plugin asking for a version of PHP or - # WordPress this job is not running, or for a dependency that is not on WordPress.org. - # That is core working as designed rather than a fatal, so it is recorded as skipped. - *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) - STATUS="SKIPPED" - REASON="Core declined to activate the plugin because its declared requirements are not met" - ;; - * ) - STATUS="FAIL" - REASON="The plugin could not be activated" - ;; - esac - fi - fi - - # Step 6: boot all of core plus the active plugin in a CLI context. - if [ "${STATUS}" = "PASS" ]; then - EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" || true - - if [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then - printf '%s\n' "${EVAL_OUTPUT}" - - case "${EVAL_OUTPUT}" in - *"Fatal error"* | *"PHP Fatal"* | *"Uncaught"* ) - STATUS="FAIL" - REASON="A fatal error occurred while WP-CLI loaded WordPress with the plugin active" - ;; - # Some plugins redirect or exit while loading, which stops WP-CLI without anything being - # broken. Only a fatal counts as a failure here. Everything else is left to the HTTP and - # debug log checks below, which see the same code in a real request. - * ) - printf 'WP-CLI did not finish loading WordPress, but no fatal error was reported.\n' - ;; - esac - fi - fi - - # Step 7: request the front page and the login screen through the PHP built-in server. - if [ "${STATUS}" = "PASS" ]; then - for URL_PATH in "/" "/wp-login.php"; do - HTTP_REASON="$( check_url "${URL_PATH}" )" - - if [ -n "${HTTP_REASON}" ]; then - STATUS="FAIL" - REASON="${HTTP_REASON}" - break - fi - done - fi - - # Step 8: a fatal can be logged without changing the HTTP status, for example during a shutdown - # hook, so the debug log is checked separately. - if [ "${STATUS}" = "PASS" ] && [ -f wp-content/debug.log ] && grep -q 'PHP Fatal' wp-content/debug.log; then - grep 'PHP Fatal' wp-content/debug.log - STATUS="FAIL" - REASON="A fatal error was logged: $( grep -m 1 'PHP Fatal' wp-content/debug.log | cut -c 1-200 )" - fi - - # Step 9: record the outcome and put the site back the way it was found. - DEPENDENCY_LIST="-" - - if [ "${#DEPENDENCIES[@]}" -gt 0 ]; then - DEPENDENCY_LIST="$( printf '%s, ' "${DEPENDENCIES[@]}" )" - DEPENDENCY_LIST="${DEPENDENCY_LIST%, }" - fi - - record "${SLUG}" "${VERSION}" "${STATUS}" "${DEPENDENCY_LIST}" "${REASON}" - cleanup_plugins - - printf '%s: %s\n' "${SLUG}" "${STATUS}" - printf '::endgroup::\n' - done < <( printf '%s' "${PLUGIN_SLUGS}" | jq -r '.[]' ) - - PASS_COUNT="$( awk -F '\t' '$3 == "PASS" { count++ } END { print count + 0 }' "${RESULTS}" )" - FAIL_COUNT="$( awk -F '\t' '$3 == "FAIL" { count++ } END { print count + 0 }' "${RESULTS}" )" - SKIP_COUNT="$( awk -F '\t' '$3 == "SKIPPED" { count++ } END { print count + 0 }' "${RESULTS}" )" - - { - printf '### WordPress %s / PHP %s\n\n' "${WP_VERSION}" "${PHP_VERSION}" - printf '%s passed, %s failed, %s skipped.\n\n' "${PASS_COUNT}" "${FAIL_COUNT}" "${SKIP_COUNT}" - printf '| Plugin | Version | Result | Also active | Details |\n' - printf '| --- | --- | --- | --- | --- |\n' - - while IFS=$'\t' read -r ROW_SLUG ROW_VERSION ROW_STATUS ROW_DEPENDENCIES ROW_REASON; do - case "${ROW_STATUS}" in - PASS ) ICON=':white_check_mark:' ;; - FAIL ) ICON=':x:' ;; - * ) ICON=':warning:' ;; - esac - - printf '| [%s](https://wordpress.org/plugins/%s/) | %s | %s %s | %s | %s |\n' \ - "${ROW_SLUG}" "${ROW_SLUG}" "${ROW_VERSION}" "${ICON}" "${ROW_STATUS}" "${ROW_DEPENDENCIES}" "${ROW_REASON}" - done < "${RESULTS}" - - printf '\n' - } >> "${GITHUB_STEP_SUMMARY}" - - # Plugins that could not be downloaded are reported but do not fail the run. - if [ "${FAIL_COUNT}" -gt 0 ]; then - printf 'The following plugins failed against WordPress %s:\n' "${WP_VERSION}" - awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $5 }' "${RESULTS}" - exit 1 - fi - - printf 'No plugins fataled against WordPress %s.\n' "${WP_VERSION}" - - - name: Show the web server log - if: ${{ failure() }} - run: cat "${RUNNER_TEMP}/php-server.log" + ./tools/plugin-compatibility/test-plugins.sh \ + --no-docker \ + --wp-version="${WP_VERSION}" \ + --dir="${RUNNER_TEMP}/wordpress" \ + --db-host="127.0.0.1:${DB_PORT}" \ + --db-name=test_db \ + --db-user=root \ + --db-pass=root \ + --plugins="$( printf '%s' "${PLUGIN_SLUGS}" | jq -r 'join( " " )' )" diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh new file mode 100755 index 0000000000000..3ecafaaa1f561 --- /dev/null +++ b/tools/plugin-compatibility/test-plugins.sh @@ -0,0 +1,709 @@ +#!/usr/bin/env bash +## +# Checks that plugins from the WordPress.org directory can be activated against a version of WordPress +# without fataling. +# +# Each plugin is installed on its own, activated, exercised over HTTP and through WP-CLI, and then removed +# again before the next one is installed, so that one broken plugin cannot mask (or break) the next. A plugin +# that declares `Requires Plugins` has those dependencies installed and activated alongside it, because core +# refuses to activate it otherwise. +# +# This is the same code the Plugin Compatibility Tests workflow runs. Running it here is how to reproduce a +# failure from a workflow run, or to check a plugin against a release candidate, without waiting on Actions. +# +# By default everything is provisioned in throwaway containers - a database, and a PHP with WP-CLI to run the +# checks in - so nothing but Docker is needed and nothing is left behind: +# +# tools/plugin-compatibility/test-plugins.sh --wp-version=nightly woocommerce classic-editor +# +# Pass `--no-docker` to run the checks in the current environment instead, against a database that is already +# running. That is the path the workflow takes, where the runner already has PHP, WP-CLI and a database +# service. +# +# Usage: test-plugins.sh [options] [slug...] +# +# Run `test-plugins.sh --help` for the options. +## + +# `set -e` is deliberately not used: a plugin that fatals must not stop the remaining plugins from being +# tested. Failures are checked where they matter instead. +set -uo pipefail + +# Written for Bash 3.2 so that it runs on a stock macOS as well as on a runner. + +SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null 2>&1 && pwd )/$( basename "${BASH_SOURCE[0]}" )" + +# Defaults. Every one of these can be overridden with the matching option. +WP_VERSION="nightly" +PLUGIN_LIST="" +USE_DOCKER="yes" +# The official WP-CLI image carries PHP, WP-CLI, `mysqli` and curl, which is everything the checks need. +PHP_IMAGE="wordpress:cli-php8.3" +# Matches the database the workflow runs against by default. +DB_IMAGE="mysql:8.4" +WP_DIR="" +DB_HOST="127.0.0.1" +DB_NAME="test_db" +DB_USER="root" +DB_PASS="password" +SERVER_PORT="8889" +KEEP_INSTALL="no" + +# How many levels of `Requires Plugins` to follow. Chains in the directory are one level deep in practice, so +# the bound is only here to stop a circular declaration from looping forever. +MAX_DEPENDENCY_DEPTH=3 + +usage() { + cat <<'USAGE_EOF' +Usage: test-plugins.sh [options] [slug...] + +Checks that WordPress.org plugins can be activated against a version of WordPress without fataling. + +Options: + --plugins= Plugin slugs to test, separated by commas or spaces. Slugs can also be passed as + positional arguments. + --wp-version= Version of WordPress to test against. Accepts "latest", "nightly", or a version + number such as a beta or RC. Default: nightly. + --php-image= Container image the checks run in. Default: wordpress:cli-php8.3. + --db-image= Container image the database runs in. Default: mysql:8.4. + --no-docker Run the checks in the current environment rather than in a container. Requires PHP, + WP-CLI, curl and a database that is already running. + --dir= Where to install WordPress. Only used with --no-docker. Default: a temporary + directory, which is removed again on exit. + --db-host= Database host, with an optional port. Only used with --no-docker. Default: 127.0.0.1. + --db-name= Database name. Only used with --no-docker. Default: test_db. + --db-user= Database user. Only used with --no-docker. Default: root. + --db-pass= Database password. Only used with --no-docker. Default: password. + --port= Port the PHP built-in server listens on. Default: 8889. + --keep Leave the WordPress install in place on exit rather than removing it. Only used with + --no-docker. + -h, --help Print this message. + +Examples: + # Check two plugins against nightly, provisioning everything in containers. + test-plugins.sh --wp-version=nightly woocommerce classic-editor + + # Reproduce a failure from a workflow run against the release it was seen on. + test-plugins.sh --wp-version=7.1-RC1 eps-301-redirects + + # Run the checks here, against a database that is already running. + test-plugins.sh --no-docker --db-host=127.0.0.1:3306 --db-pass=root hello-dolly + +Exits 0 when no plugin fataled, and 1 when one did. A plugin that could not be downloaded, or that core +declined to activate because its declared requirements are not met, is reported as skipped and does not +fail the run. +USAGE_EOF +} + +# error +# +# Prints a message to stderr and stops. +error() { + printf 'Error: %s\n' "${1}" >&2 + exit 1 +} + +while [ "$#" -gt 0 ]; do + case "${1}" in + --plugins=* ) PLUGIN_LIST="${PLUGIN_LIST} ${1#*=}" ;; + --wp-version=* ) WP_VERSION="${1#*=}" ;; + --php-image=* ) PHP_IMAGE="${1#*=}" ;; + --db-image=* ) DB_IMAGE="${1#*=}" ;; + --no-docker ) USE_DOCKER="no" ;; + --dir=* ) WP_DIR="${1#*=}" ;; + --db-host=* ) DB_HOST="${1#*=}" ;; + --db-name=* ) DB_NAME="${1#*=}" ;; + --db-user=* ) DB_USER="${1#*=}" ;; + --db-pass=* ) DB_PASS="${1#*=}" ;; + --port=* ) SERVER_PORT="${1#*=}" ;; + --keep ) KEEP_INSTALL="yes" ;; + -h | --help ) usage; exit 0 ;; + -* ) usage >&2; error "Unknown option ${1}." ;; + * ) PLUGIN_LIST="${PLUGIN_LIST} ${1}" ;; + esac + shift +done + +# Commas, spaces and newlines all separate slugs, so a list can be pasted in however it was written down. +PLUGIN_SLUGS="$( printf '%s' "${PLUGIN_LIST}" | tr ',[:space:]' '\n' | grep -v '^$' | awk '!seen[$0]++' )" + +if [ -z "${PLUGIN_SLUGS}" ]; then + usage >&2 + error "No plugins were named." +fi + +# A value outside the character set the directory uses is a typo worth stopping for, rather than a plugin +# reported as impossible to download several minutes from now. +INVALID_SLUGS="$( printf '%s\n' "${PLUGIN_SLUGS}" | grep -Ev '^[a-z0-9-]+$' )" + +if [ -n "${INVALID_SLUGS}" ]; then + printf 'These values are not WordPress.org plugin slugs:\n%s\n' "${INVALID_SLUGS}" >&2 + exit 1 +fi + +## +# Provisions a database and a PHP, and runs this same script inside them. +# +# The checks themselves are identical either way: the container is handed this file and runs it with +# `--no-docker`, so there is only one copy of the logic that decides whether a plugin passed. +## +run_in_docker() { + command -v docker > /dev/null 2>&1 || error "Docker is required, or pass --no-docker to run the checks here." + + # Everything is named after the process so that two runs at once do not collide. + DOCKER_NETWORK="plugin-compatibility-$$" + DB_CONTAINER="plugin-compatibility-db-$$" + + # Both are removed however this exits, an interrupt included. + trap 'docker rm --force "${DB_CONTAINER}" > /dev/null 2>&1; docker network rm "${DOCKER_NETWORK}" > /dev/null 2>&1' EXIT INT TERM + + printf 'Starting %s.\n' "${DB_IMAGE}" + + docker network create "${DOCKER_NETWORK}" > /dev/null || error "The Docker network could not be created." + + docker run --detach --name "${DB_CONTAINER}" --network "${DOCKER_NETWORK}" \ + --env MYSQL_ROOT_PASSWORD="${DB_PASS}" \ + --env MYSQL_DATABASE="${DB_NAME}" \ + "${DB_IMAGE}" > /dev/null || error "The database container could not be started." + + # A first run has to initialise the data directory, which takes appreciably longer than a later one. + DB_READY="no" + + for _ in $( seq 1 90 ); do + if docker exec "${DB_CONTAINER}" mysqladmin ping --silent --user=root --password="${DB_PASS}" > /dev/null 2>&1; then + DB_READY="yes" + break + fi + sleep 2 + done + + if [ "${DB_READY}" != "yes" ]; then + docker logs "${DB_CONTAINER}" 2>&1 | tail -n 20 + error "The database did not become available." + fi + + printf 'The database is ready. Running the checks in %s.\n\n' "${PHP_IMAGE}" + + # WordPress is installed inside the container rather than in a mounted directory, so a plugin cannot + # leave anything behind on the host and file ownership never comes into it. `HOME` is pointed at a + # writable directory because WP-CLI caches its downloads there. + docker run --rm --network "${DOCKER_NETWORK}" \ + --volume "${SCRIPT_PATH}:/usr/local/bin/test-plugins.sh:ro" \ + --env HOME=/tmp \ + --entrypoint bash \ + "${PHP_IMAGE}" /usr/local/bin/test-plugins.sh \ + --no-docker \ + --dir=/tmp/wordpress \ + --wp-version="${WP_VERSION}" \ + --db-host="${DB_CONTAINER}" \ + --db-name="${DB_NAME}" \ + --db-user=root \ + --db-pass="${DB_PASS}" \ + --port="${SERVER_PORT}" \ + --plugins="$( printf '%s' "${PLUGIN_SLUGS}" | tr '\n' ' ' )" +} + +## +# Installs WordPress, starts a web server, and checks each plugin against it. +## +run_checks() { + for REQUIRED_COMMAND in php curl; do + command -v "${REQUIRED_COMMAND}" > /dev/null 2>&1 || error "${REQUIRED_COMMAND} is required to run the checks without Docker." + done + + # `type -P` searches the path rather than resolving to the `wp` wrapper function defined below. + WP_CLI_BIN="$( type -P wp )" + + [ -n "${WP_CLI_BIN}" ] || error "WP-CLI is required to run the checks without Docker." + + WORK_DIR="$( mktemp -d )" + RESULTS="${WORK_DIR}/results.tsv" + RESPONSE_BODY="${WORK_DIR}/response.html" + SERVER_LOG="${WORK_DIR}/php-server.log" + SERVER_PID="" + : > "${RESULTS}" + + if [ -z "${WP_DIR}" ]; then + WP_DIR="${WORK_DIR}/wordpress" + fi + + mkdir -p "${WP_DIR}" || error "The WordPress directory could not be created." + + # The server is stopped and the temporary files are removed however this exits. The WordPress install is + # only removed when this script created it, so that `--dir` never deletes a directory it was handed. + cleanup_environment() { + if [ -n "${SERVER_PID}" ]; then + kill "${SERVER_PID}" > /dev/null 2>&1 + fi + + if [ "${KEEP_INSTALL}" = "yes" ]; then + printf '\nThe WordPress install was left in %s.\n' "${WP_DIR}" + else + rm -rf "${WORK_DIR}" + fi + } + + trap cleanup_environment EXIT INT TERM + + cd "${WP_DIR}" || error "The WordPress directory could not be entered." + + SITE_URL="http://127.0.0.1:${SERVER_PORT}" + + printf 'Downloading WordPress %s.\n' "${WP_VERSION}" + wp core download --version="${WP_VERSION}" || error "WordPress ${WP_VERSION} could not be downloaded." + + wp config create --dbname="${DB_NAME}" --dbuser="${DB_USER}" --dbpass="${DB_PASS}" --dbhost="${DB_HOST}" \ + || error "wp-config.php could not be written." + + # Errors need to reach `wp-content/debug.log` so that a white screen of death is still detectable. + # + # `WP_DEBUG_DISPLAY` is left off on purpose: this should behave the way a production site does, where a + # fatal error is an empty page and an HTTP 500 rather than a printed stack trace. + # + # The fatal error handler is disabled so that a fatal is reported as a fatal instead of being swallowed + # by recovery mode, which would also deactivate the plugin mid-test. + # + # WP-Cron is disabled because WordPress spawns it as a loopback request during a front end request. The + # loopback lands back on the same PHP built-in server that is still busy serving the request that + # spawned it, and the two deadlock until curl gives up. + wp config set WP_DEBUG true --raw + wp config set WP_DEBUG_LOG true --raw + wp config set WP_DEBUG_DISPLAY false --raw + wp config set WP_DISABLE_FATAL_ERROR_HANDLER true --raw + wp config set DISABLE_WP_CRON true --raw + + wp core install --url="${SITE_URL}" --title="Plugin Compatibility Test" --admin_user=admin \ + --admin_password=password --admin_email=me@example.org --skip-email \ + || error "WordPress could not be installed. Check that the database is reachable at ${DB_HOST}." + + # The site needs to answer real requests so that fatals which only happen on a front end or admin page + # load are caught. The built-in server is enough for that and needs nothing installed. + # + # `PHP_CLI_SERVER_WORKERS` is set because the built-in server is single threaded by default. Plenty of + # plugins make a loopback request to the site they are running on, and a single threaded server cannot + # answer one while it is still serving the request that made it. + PHP_CLI_SERVER_WORKERS=4 php -S "127.0.0.1:${SERVER_PORT}" -t "${WP_DIR}" > "${SERVER_LOG}" 2>&1 & + SERVER_PID=$! + + SERVER_READY="no" + + for _ in $( seq 1 30 ); do + if curl -sSf -o /dev/null "${SITE_URL}/wp-login.php"; then + SERVER_READY="yes" + break + fi + sleep 1 + done + + if [ "${SERVER_READY}" != "yes" ]; then + cat "${SERVER_LOG}" + error "The PHP built-in server did not start on port ${SERVER_PORT}." + fi + + printf 'The PHP built-in server is ready on %s.\n' "${SITE_URL}" + + test_plugins + report_results +} + +# wp +# +# Runs WP-CLI with the memory limit lifted. Unpacking a WordPress zip needs more than the 128M a stock +# php.ini allows, and WP-CLI reads its own `WP_CLI_PHP_ARGS` only when it is installed as a wrapper script +# rather than as the phar most environments carry. +wp() { + php -d memory_limit=-1 "${WP_CLI_BIN}" "$@" +} + +# record +# +# Appends one tab separated row to the results file. The reason is flattened so that it cannot break the +# markdown table that is generated from these rows later on. +record() { + SAFE_REASON="$( printf '%s' "${5:--}" | tr '\n\t|' ' ' )" + printf '%s\t%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${4:--}" "${SAFE_REASON}" >> "${RESULTS}" +} + +# check_url +# +# Requests a path on the test site and prints a reason to stdout when the response looks broken. Prints +# nothing when the request looks healthy. +check_url() { + CURL_EXIT_CODE=0 + # The exit code is captured separately rather than falling back to a literal inside the command + # substitution, which would append to whatever curl had already written and produce a nonsense status + # like "200000" when a request returned headers and then stalled. + HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" )" || CURL_EXIT_CODE=$? + + if [ -z "${HTTP_CODE}" ] || [ "${HTTP_CODE}" = "000" ]; then + printf 'The request to %s did not complete, curl exit code %s' "${1}" "${CURL_EXIT_CODE}" + return + fi + + if [ "${HTTP_CODE}" -ge 500 ]; then + printf 'The request to %s returned HTTP %s' "${1}" "${HTTP_CODE}" + return + fi + + if [ "${CURL_EXIT_CODE}" -ne 0 ]; then + printf 'The request to %s returned HTTP %s but the response did not finish, curl exit code %s' "${1}" "${HTTP_CODE}" "${CURL_EXIT_CODE}" + return + fi + + # Belt and braces. Errors are not displayed by default, but a plugin can turn display_errors back on for + # itself. + if grep -qi 'Fatal error' "${RESPONSE_BODY}"; then + printf 'The response from %s contained a fatal error' "${1}" + fi +} + +# install_plugin +# +# Installs a plugin from WordPress.org and remembers it so that it is removed again once the plugin under +# test has been checked. The slug is remembered before the download is attempted so that a partial download +# is still cleaned up. Returns non zero when the download failed. +install_plugin() { + INSTALLED_SLUGS[${#INSTALLED_SLUGS[@]}]="${1}" + INSTALLED_DIRS[${#INSTALLED_DIRS[@]}]="" + + wp plugin install "${1}" --skip-plugins --skip-themes || return 1 + + INSTALLED_DIRS[${#INSTALLED_DIRS[@]} - 1]="$( wp plugin path "${1}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" +} + +# plugin_dependencies +# +# Prints the slugs named in the plugin's `Requires Plugins` header, one per line. WordPress' own header +# parser is used so that the same rules apply here as when core decides whether a plugin's dependencies are +# met. +plugin_dependencies() { + PLUGIN_FILE="$( wp plugin path "${1}" --skip-plugins --skip-themes 2>/dev/null || printf '' )" + + [ -n "${PLUGIN_FILE}" ] || return 0 + + # The single quotes are what keep the shell out of the PHP below. The path is handed over as an + # environment variable rather than interpolated so that it never reaches PHP as source code. + export PLUGIN_FILE + + # shellcheck disable=SC2016 + wp eval --skip-plugins --skip-themes ' + require_once ABSPATH . "wp-admin/includes/plugin.php"; + + $plugin_data = get_plugin_data( getenv( "PLUGIN_FILE" ), false, false ); + + foreach ( explode( ",", $plugin_data["RequiresPlugins"] ?? "" ) as $dependency ) { + // Core only accepts a WordPress.org slug here, so anything else is ignored. + if ( preg_match( "/^[a-z0-9-]+$/", trim( $dependency ) ) ) { + echo trim( $dependency ), "\n"; + } + } + ' 2>/dev/null +} + +# cleanup_plugins +# +# Returns the site to a clean slate by removing everything installed for the current plugin, its +# dependencies included. A plugin that fatals can take WP-CLI down with it, so every command here is allowed +# to fail and the plugin directory is removed directly as a fallback. `--skip-plugins` keeps WP-CLI from +# loading the broken plugin while cleaning up after it. +cleanup_plugins() { + if [ "${#INSTALLED_SLUGS[@]}" -gt 0 ]; then + for INDEX in "${!INSTALLED_SLUGS[@]}"; do + CLEANUP_SLUG="${INSTALLED_SLUGS[${INDEX}]}" + CLEANUP_DIR="${INSTALLED_DIRS[${INDEX}]}" + + wp plugin deactivate "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 + wp plugin delete "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 + + if [ -n "${CLEANUP_DIR}" ] && [ -d "${CLEANUP_DIR}" ] && [ "${CLEANUP_DIR}" != "${PLUGINS_ROOT}" ]; then + rm -rf "${CLEANUP_DIR}" + fi + + rm -rf "${WP_DIR}/wp-content/plugins/${CLEANUP_SLUG}" + done + fi + + INSTALLED_SLUGS=() + INSTALLED_DIRS=() + + # Make sure nothing is left behind in `active_plugins` pointing at a plugin that is now gone. + wp option update active_plugins '[]' --format=json --skip-plugins --skip-themes > /dev/null 2>&1 +} + +# group_start / group_end +# +# Collapses a plugin's output in the workflow log. The markers mean nothing to a terminal, so they are only +# printed when this is running in Actions. +group_start() { + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + printf '::group::%s\n' "${1}" + else + printf '\n----- %s -----\n' "${1}" + fi +} + +group_end() { + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + printf '::endgroup::\n' + fi +} + +## +# Tests each plugin in turn, writing a row per plugin to the results file. +## +test_plugins() { + # The plugins directory itself must never be removed, only directories inside it. + PLUGINS_ROOT="$( wp plugin path )" + + # Everything installed for the plugin currently under test, that plugin included. The directories are + # tracked alongside the slugs because a plugin does not always unpack into a directory named after its + # slug. + INSTALLED_SLUGS=() + INSTALLED_DIRS=() + + while IFS= read -r SLUG; do + [ -n "${SLUG}" ] || continue + + group_start "${SLUG}" + + STATUS="PASS" + REASON="-" + VERSION="unknown" + DEPENDENCIES=() + + # Start every plugin with an empty log so that anything found in it belongs to this plugin. + rm -f "${WP_DIR}/wp-content/debug.log" + + # Step 1: download the plugin. A download failure is recorded as skipped rather than failed, since + # it generally means a network flake or a plugin that is no longer in the directory. + if ! install_plugin "${SLUG}"; then + record "${SLUG}" "unknown" "SKIPPED" "-" "The plugin could not be downloaded from WordPress.org" + cleanup_plugins + group_end + continue + fi + + VERSION="$( wp plugin get "${SLUG}" --field=version --skip-plugins --skip-themes 2>/dev/null || printf 'unknown' )" + + # Step 2: install whatever the plugin names in `Requires Plugins`. Core refuses to activate a plugin + # whose dependencies are missing, so without this every WooCommerce extension - a large slice of the + # most popular plugins - would go untested. + # + # Dependencies are resolved a level at a time so that a dependency declaring its own is installed + # too. Anything already installed is left alone, which covers a plugin named twice in the same tree + # as well as a declaration pointing back at the plugin under test. + PENDING=() + + while IFS= read -r DEPENDENCY; do + [ -n "${DEPENDENCY}" ] && PENDING[${#PENDING[@]}]="${DEPENDENCY}" + done < <( plugin_dependencies "${SLUG}" ) + + DEPTH=0 + + while [ "${#PENDING[@]}" -gt 0 ] && [ "${DEPTH}" -lt "${MAX_DEPENDENCY_DEPTH}" ]; do + NEXT=() + + for DEPENDENCY in "${PENDING[@]}"; do + if wp plugin is-installed "${DEPENDENCY}" --skip-plugins --skip-themes > /dev/null 2>&1; then + continue + fi + + printf 'Installing %s, which %s requires.\n' "${DEPENDENCY}" "${SLUG}" + + if ! install_plugin "${DEPENDENCY}"; then + STATUS="SKIPPED" + REASON="The required plugin ${DEPENDENCY} is not available from WordPress.org" + break 2 + fi + + DEPENDENCIES[${#DEPENDENCIES[@]}]="${DEPENDENCY}" + + while IFS= read -r NESTED_DEPENDENCY; do + [ -n "${NESTED_DEPENDENCY}" ] && NEXT[${#NEXT[@]}]="${NESTED_DEPENDENCY}" + done < <( plugin_dependencies "${DEPENDENCY}" ) + done + + PENDING=( ${NEXT[@]+"${NEXT[@]}"} ) + DEPTH=$(( DEPTH + 1 )) + done + + # Step 3: activate the dependencies, deepest first. That is the reverse of the order they were + # discovered in, and it matters because core will not activate a plugin ahead of its own + # requirements either. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + for (( INDEX = ${#DEPENDENCIES[@]} - 1; INDEX >= 0; INDEX-- )); do + DEPENDENCY="${DEPENDENCIES[${INDEX}]}" + DEPENDENCY_EXIT_CODE=0 + DEPENDENCY_OUTPUT="$( wp plugin activate "${DEPENDENCY}" 2>&1 )" || DEPENDENCY_EXIT_CODE=$? + printf '%s\n' "${DEPENDENCY_OUTPUT}" + + if [ "${DEPENDENCY_EXIT_CODE}" -ne 0 ]; then + STATUS="SKIPPED" + REASON="The required plugin ${DEPENDENCY} could not be activated" + break + fi + done + fi + + # Step 4: with dependencies active the plugin under test is no longer alone on the site, so the + # baseline is checked before it is activated. A site that is already broken says something about the + # dependency rather than about the plugin being tested, and blaming the plugin for it would be the + # masking these checks are built to avoid. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + for URL_PATH in "/" "/wp-login.php"; do + BASELINE_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${BASELINE_REASON}" ]; then + STATUS="SKIPPED" + REASON="The required plugins are not healthy on their own: ${BASELINE_REASON}" + break + fi + done + + # Anything the dependencies logged on their way up is not the responsibility of the plugin under + # test, so the log starts empty again here. + rm -f "${WP_DIR}/wp-content/debug.log" + fi + + # Step 5: activation. Activation runs the plugin's activation hooks and loads its main file. + if [ "${STATUS}" = "PASS" ]; then + ACTIVATE_EXIT_CODE=0 + ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? + printf '%s\n' "${ACTIVATE_OUTPUT}" + + if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then + case "${ACTIVATE_OUTPUT}" in + # Core refuses to activate a plugin whose declared requirements are not met. + # Dependencies are installed above, so what is left here is a plugin asking for a + # version of PHP or WordPress this run is not using, or for a dependency that is not on + # WordPress.org. That is core working as designed rather than a fatal, so it is recorded + # as skipped. + *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) + STATUS="SKIPPED" + REASON="Core declined to activate the plugin because its declared requirements are not met" + ;; + * ) + STATUS="FAIL" + REASON="The plugin could not be activated" + ;; + esac + fi + fi + + # Step 6: boot all of core plus the active plugin in a CLI context. + if [ "${STATUS}" = "PASS" ]; then + EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" + + if [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then + printf '%s\n' "${EVAL_OUTPUT}" + + case "${EVAL_OUTPUT}" in + *"Fatal error"* | *"PHP Fatal"* | *"Uncaught"* ) + STATUS="FAIL" + REASON="A fatal error occurred while WP-CLI loaded WordPress with the plugin active" + ;; + # Some plugins redirect or exit while loading, which stops WP-CLI without anything being + # broken. Only a fatal counts as a failure here. Everything else is left to the HTTP and + # debug log checks below, which see the same code in a real request. + * ) + printf 'WP-CLI did not finish loading WordPress, but no fatal error was reported.\n' + ;; + esac + fi + fi + + # Step 7: request the front page and the login screen through the PHP built-in server. + if [ "${STATUS}" = "PASS" ]; then + for URL_PATH in "/" "/wp-login.php"; do + HTTP_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${HTTP_REASON}" ]; then + STATUS="FAIL" + REASON="${HTTP_REASON}" + break + fi + done + fi + + # Step 8: a fatal can be logged without changing the HTTP status, for example during a shutdown + # hook, so the debug log is checked separately. + if [ "${STATUS}" = "PASS" ] && [ -f "${WP_DIR}/wp-content/debug.log" ] && grep -q 'PHP Fatal' "${WP_DIR}/wp-content/debug.log"; then + grep 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" + STATUS="FAIL" + REASON="A fatal error was logged: $( grep -m 1 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" | cut -c 1-200 )" + fi + + # Step 9: record the outcome and put the site back the way it was found. + DEPENDENCY_LIST="-" + + if [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + DEPENDENCY_LIST="$( printf '%s, ' "${DEPENDENCIES[@]}" )" + DEPENDENCY_LIST="${DEPENDENCY_LIST%, }" + fi + + record "${SLUG}" "${VERSION}" "${STATUS}" "${DEPENDENCY_LIST}" "${REASON}" + cleanup_plugins + + printf '%s: %s\n' "${SLUG}" "${STATUS}" + group_end + done <> "${GITHUB_STEP_SUMMARY}" + fi + + # Plugins that could not be downloaded are reported but do not fail the run. + if [ "${FAIL_COUNT}" -gt 0 ]; then + printf '\nThe following plugins failed against WordPress %s:\n' "${WP_VERSION}" + awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $5 }' "${RESULTS}" + return 1 + fi + + printf '\nNo plugins fataled against WordPress %s.\n' "${WP_VERSION}" +} + +if [ "${USE_DOCKER}" = "yes" ]; then + run_in_docker + exit $? +fi + +run_checks From a8abc6e757b98fad550947770d03a5cc32254e2e Mon Sep 17 00:00:00 2001 From: adamsilverstein Date: Tue, 1 Sep 2026 09:37:16 -0700 Subject: [PATCH 9/9] Build/Test Tools: Let a real request decide whether a plugin passed. WP-CLI requires wp-settings.php from inside a method, so a plugin that assigns a variable at file scope and reads it back with global later finds nothing there. eps-301-redirects does exactly that and fatals under WP-CLI on a fresh install, while the front page, the login screen and the debug log are all clean. Failing it reported breakage that no visitor would ever see. Move the WP-CLI boot check after the HTTP and debug log checks and stop letting it fail a plugin on its own. A plugin that is healthy over HTTP now passes with the WP-CLI fatal recorded as a note. Anything that genuinely fatals on load still fails, because it fails the HTTP checks. The reorder also matters because the WP-CLI fatal is written to the debug log, which the log check would otherwise pick up as a failure. --- tools/plugin-compatibility/test-plugins.sh | 60 +++++++++++++--------- 1 file changed, 36 insertions(+), 24 deletions(-) diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh index 3ecafaaa1f561..a225ce799a752 100755 --- a/tools/plugin-compatibility/test-plugins.sh +++ b/tools/plugin-compatibility/test-plugins.sh @@ -8,6 +8,9 @@ # that declares `Requires Plugins` has those dependencies installed and activated alongside it, because core # refuses to activate it otherwise. # +# What a real request does is what decides whether a plugin passed. The WP-CLI check runs last and only ever +# adds a note, because WP-CLI loads WordPress in a way a visitor never does. +# # This is the same code the Plugin Compatibility Tests workflow runs. Running it here is how to reproduce a # failure from a workflow run, or to check a plugin against a release candidate, without waiting on Actions. # @@ -590,29 +593,8 @@ test_plugins() { fi fi - # Step 6: boot all of core plus the active plugin in a CLI context. - if [ "${STATUS}" = "PASS" ]; then - EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" - - if [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then - printf '%s\n' "${EVAL_OUTPUT}" - - case "${EVAL_OUTPUT}" in - *"Fatal error"* | *"PHP Fatal"* | *"Uncaught"* ) - STATUS="FAIL" - REASON="A fatal error occurred while WP-CLI loaded WordPress with the plugin active" - ;; - # Some plugins redirect or exit while loading, which stops WP-CLI without anything being - # broken. Only a fatal counts as a failure here. Everything else is left to the HTTP and - # debug log checks below, which see the same code in a real request. - * ) - printf 'WP-CLI did not finish loading WordPress, but no fatal error was reported.\n' - ;; - esac - fi - fi - - # Step 7: request the front page and the login screen through the PHP built-in server. + # Step 6: request the front page and the login screen through the PHP built-in server. A real request + # is what decides whether a plugin passed, because it is what a visitor gets. if [ "${STATUS}" = "PASS" ]; then for URL_PATH in "/" "/wp-login.php"; do HTTP_REASON="$( check_url "${URL_PATH}" )" @@ -625,7 +607,7 @@ test_plugins() { done fi - # Step 8: a fatal can be logged without changing the HTTP status, for example during a shutdown + # Step 7: a fatal can be logged without changing the HTTP status, for example during a shutdown # hook, so the debug log is checked separately. if [ "${STATUS}" = "PASS" ] && [ -f "${WP_DIR}/wp-content/debug.log" ] && grep -q 'PHP Fatal' "${WP_DIR}/wp-content/debug.log"; then grep 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" @@ -633,6 +615,36 @@ test_plugins() { REASON="A fatal error was logged: $( grep -m 1 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" | cut -c 1-200 )" fi + # Step 8: boot all of core plus the active plugin through WP-CLI. This runs last and cannot fail a + # plugin on its own. + # + # WP-CLI requires `wp-settings.php` from inside a method, so a plugin that assigns a variable at file + # scope and reads it back with `global` later finds nothing there. Plenty of plugins do exactly that, + # and the resulting fatal happens only under WP-CLI - a visitor never sees it. Failing a plugin for + # it would report breakage that does not exist on a real site, so it is recorded as a note against a + # plugin that is otherwise healthy. Anything that genuinely fatals on load fails the HTTP checks + # above. + # + # The fatal is written to the debug log too, which is why this comes after the log has been checked. + if [ "${STATUS}" = "PASS" ]; then + EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" + + if [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then + printf '%s\n' "${EVAL_OUTPUT}" + + case "${EVAL_OUTPUT}" in + *"Fatal error"* | *"PHP Fatal"* | *"Uncaught"* ) + REASON="The site is healthy over HTTP, but WP-CLI fatals when it loads WordPress with the plugin active" + ;; + # Some plugins redirect or exit while loading, which stops WP-CLI without anything being + # broken. + * ) + printf 'WP-CLI did not finish loading WordPress, but no fatal error was reported.\n' + ;; + esac + fi + fi + # Step 9: record the outcome and put the site back the way it was found. DEPENDENCY_LIST="-"