diff --git a/.github/workflows/plugin-compatibility.yml b/.github/workflows/plugin-compatibility.yml new file mode 100644 index 0000000000000..a496df94681bd --- /dev/null +++ b/.github/workflows/plugin-compatibility.yml @@ -0,0 +1,328 @@ +## +# Confirms that the most popular plugins in the WordPress.org directory can be activated against a version of +# WordPress without fataling. +# +# Core's test suites cover core itself, but nothing checks that a new version of WordPress can still boot with +# popular plugins active. When a plugin's assumptions about core stop holding, the result is a fatal error on +# every request and a white screen for real sites. This workflow is a smoke test for that class of failure, so +# that it can be found while there is still time to fix core or notify the plugin author. +# +# The plugin list is fetched from the WordPress.org API at run time and is sharded across a matrix sized to the +# number of plugins being tested. Each plugin is installed and activated on its own, so one broken plugin cannot +# hide another. +# +# It runs weekly against nightly, and can be dispatched manually against any version with any number of plugins, +# which is the intended way to use it as part of the pre-release checklist: point it at the beta or RC and give +# it a count. +# +# A dispatched run can also name the plugins to test instead of taking them from the directory, which is how to +# check a single plugin against a release candidate, or to reproduce a failure from an earlier run without +# waiting for the other plugins in its shard. +# +# The checks each shard runs live in `tools/plugin-compatibility/test-plugins.sh`. Running that script from a +# terminal runs exactly what a shard here runs, which is the quickest way to reproduce a result from a run +# without waiting on Actions. +# +# This workflow is not meant to test wordpress-develop checkouts, but rather versions officially available on +# WordPress.org. +## +name: Plugin Compatibility Tests + +on: + push: + branches: + - trunk + # Always test the workflow after it's updated. + paths: + - '.github/workflows/plugin-compatibility.yml' + - '.github/workflows/reusable-plugin-compatibility.yml' + - 'tools/plugin-compatibility/**' + pull_request: + # This workflow is only meant to run from trunk. Pull requests changing this file with different BASE branches should be ignored. + branches: + - trunk + # Always test the workflow when changes are suggested. + paths: + - '.github/workflows/plugin-compatibility.yml' + - '.github/workflows/reusable-plugin-compatibility.yml' + - 'tools/plugin-compatibility/**' + schedule: + - cron: '0 2 * * 1' + workflow_dispatch: + inputs: + wp-version: + description: 'The version of WordPress to test plugins against. Accepts "latest", "nightly", or a specific version number such as a beta or RC, for a pre-release check.' + type: string + default: 'nightly' + plugin-count: + description: 'Number of most popular plugins to test. Accepts any number from 1 to 1000, eg. 10, 50, or 250. Ignored when plugin-slugs is given.' + type: string + default: '100' + plugin-slugs: + description: 'Optional. Test these WordPress.org plugin slugs instead of the most popular ones. Accepts a comma or space separated list, eg. "woocommerce, classic-editor".' + type: string + default: '' + +# Cancels all previous workflow runs for pull requests that have not completed. +concurrency: + # The concurrency group contains the workflow name and the branch name for pull requests + # or the commit hash for any other events. + # + # Dispatched runs are grouped by run instead, so that testing one plugin list does not cancel a run someone + # else started with a different one against the same version of WordPress. + group: ${{ github.workflow }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Builds the list of plugins to test and splits it into shards for the test matrix. + # + # The list is fetched at run time so that it never goes stale, and it is ordered by popularity so that a + # smaller count still tests the plugins with the widest reach. A dispatched run can name the plugins to test + # instead, in which case the directory is not queried at all. + # + # Performs the following steps: + # - Queries the WordPress.org plugin directory API for the most popular plugins, or reads the plugin-slugs input. + # - Splits the resulting slugs into shards and returns them as a job output. + build-plugin-matrix: + name: Build plugin matrix + permissions: + contents: read + runs-on: ubuntu-24.04 + # Dispatched runs are allowed anywhere so that this can be run from a fork, which is the only way for + # someone without write access to the repository to try it against a plugin of their choosing. + if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'workflow_dispatch' }} + timeout-minutes: 5 + outputs: + shards: ${{ steps.plugin-shards.outputs.shards }} + + steps: + - name: Build the list of plugins to test + id: plugin-shards + env: + # An explicit list of slugs to test in place of the most popular ones. Empty for everything but a + # dispatched run that filled the input in. + PLUGIN_SLUGS: ${{ inputs.plugin-slugs }} + # Runs that carry no inputs fall back to the defaults. Pull request and push runs exist to test + # this workflow rather than the ecosystem, so they use a small count. See the note on the + # plugin-compatibility-tests job below. + PLUGIN_COUNT: ${{ inputs.plugin-count || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && '10' ) || '100' }} + # The API caps `per_page` at 250 and returns 250 without complaint for anything larger, so counts + # above that have to be paged. + PAGE_SIZE: '250' + # Anything higher would take longer than the 20 minute wall time this is meant to fit inside. The + # directory holds roughly 66,000 plugins, so this is a guard against a typo, not a real limit. + MAX_PLUGIN_COUNT: '1000' + # Shards are sized rather than counted, so that a run of 10 does not spin up 5 near empty jobs and a + # run of 250 is not squeezed into the same 5. + TARGET_PER_SHARD: '25' + MAX_SHARDS: '10' + run: | + set -euo pipefail + + RAW_SLUGS="${RUNNER_TEMP}/slugs-raw.txt" + DEDUPED_SLUGS="${RUNNER_TEMP}/slugs.txt" + : > "${RAW_SLUGS}" + + if [ -n "${PLUGIN_SLUGS}" ]; then + # A named list replaces the popularity query entirely. Commas and whitespace both separate slugs, + # so a list can be pasted in however it was written down. + printf '%s' "${PLUGIN_SLUGS}" | tr ',[:space:]' '\n' > "${RAW_SLUGS}" + awk '!seen[$0]++ && NF > 0' "${RAW_SLUGS}" > "${DEDUPED_SLUGS}" + + # A value outside the character set the directory uses is a typo worth stopping for now, rather + # than a plugin reported as impossible to download several minutes from now. + if INVALID_SLUGS="$( grep -Ev '^[a-z0-9-]+$' "${DEDUPED_SLUGS}" )"; then + printf 'The plugin-slugs input contains values that are not WordPress.org plugin slugs:\n%s\n' "${INVALID_SLUGS}" + exit 1 + fi + + PLUGIN_COUNT="$( wc -l < "${DEDUPED_SLUGS}" | tr -d ' ' )" + + if [ "${PLUGIN_COUNT}" -lt 1 ]; then + printf 'The plugin-slugs input did not contain any plugin slugs.\n' + exit 1 + fi + + if [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then + printf 'The plugin-slugs input names %s plugins, more than the %s this workflow will test in one run.\n' "${PLUGIN_COUNT}" "${MAX_PLUGIN_COUNT}" + exit 1 + fi + + printf 'Testing the %s plugin(s) named in the plugin-slugs input.\n' "${PLUGIN_COUNT}" + else + # Guard against a non-numeric value being passed to the API. + if ! printf '%s' "${PLUGIN_COUNT}" | grep -Eq '^[0-9]+$'; then + printf 'The plugin-count input must be a positive integer, got "%s".\n' "${PLUGIN_COUNT}" + exit 1 + fi + + if [ "${PLUGIN_COUNT}" -lt 1 ] || [ "${PLUGIN_COUNT}" -gt "${MAX_PLUGIN_COUNT}" ]; then + printf 'The plugin-count input must be between 1 and %s, got %s.\n' "${MAX_PLUGIN_COUNT}" "${PLUGIN_COUNT}" + printf 'Testing more than %s plugins will not fit in the wall time this workflow targets.\n' "${MAX_PLUGIN_COUNT}" + exit 1 + fi + + PAGE=1 + UNIQUE_COUNT=0 + + while : ; do + # The bracketed `request[...]` parameters are URL encoded by curl rather than written inline. + # The unneeded response fields are turned off to keep the payload small. + curl -sS --fail --retry 3 --retry-delay 5 \ + --get 'https://api.wordpress.org/plugins/info/1.2/' \ + --data-urlencode 'action=query_plugins' \ + --data-urlencode 'request[browse]=popular' \ + --data-urlencode "request[per_page]=${PAGE_SIZE}" \ + --data-urlencode "request[page]=${PAGE}" \ + --data-urlencode 'request[fields][short_description]=0' \ + --data-urlencode 'request[fields][sections]=0' \ + --data-urlencode 'request[fields][icons]=0' \ + --data-urlencode 'request[fields][banners]=0' \ + --data-urlencode 'request[fields][ratings]=0' \ + --data-urlencode 'request[fields][tags]=0' \ + --data-urlencode 'request[fields][compatibility]=0' \ + --data-urlencode 'request[fields][screenshots]=0' \ + -o "${RUNNER_TEMP}/plugins-page.json" + + PAGE_COUNT="$( jq '.plugins | length' "${RUNNER_TEMP}/plugins-page.json" )" + jq -r '.plugins[] | .slug // empty' "${RUNNER_TEMP}/plugins-page.json" >> "${RAW_SLUGS}" + + # Popularity ordering can shift between two requests, so the same slug can turn up on more than + # one page. Duplicates are dropped while the first occurrence keeps its position. + awk '!seen[$0]++ && NF > 0' "${RAW_SLUGS}" > "${DEDUPED_SLUGS}" + UNIQUE_COUNT="$( wc -l < "${DEDUPED_SLUGS}" | tr -d ' ' )" + + printf 'Page %s returned %s plugins, %s unique slugs collected so far.\n' "${PAGE}" "${PAGE_COUNT}" "${UNIQUE_COUNT}" + + if [ "${UNIQUE_COUNT}" -ge "${PLUGIN_COUNT}" ]; then + break + fi + + # A short page means the directory has nothing left to give. + if [ "${PAGE_COUNT}" -lt "${PAGE_SIZE}" ]; then + printf 'The API returned fewer than %s plugins on page %s, so %s is everything available.\n' "${PAGE_SIZE}" "${PAGE}" "${UNIQUE_COUNT}" + break + fi + + PAGE=$(( PAGE + 1 )) + done + fi + + SLUGS="$( head -n "${PLUGIN_COUNT}" "${DEDUPED_SLUGS}" | jq -R -s -c 'split( "\n" ) | map( select( . != "" ) )' )" + TOTAL="$( printf '%s' "${SLUGS}" | jq 'length' )" + + if [ "${TOTAL}" -lt 1 ]; then + printf 'The WordPress.org API did not return any plugins.\n' + exit 1 + fi + + if [ "${TOTAL}" -lt "${PLUGIN_COUNT}" ]; then + printf 'Only %s plugins were available, fewer than the %s requested.\n' "${TOTAL}" "${PLUGIN_COUNT}" + fi + + # Aim for TARGET_PER_SHARD plugins in each shard, up to MAX_SHARDS shards. Past that point the + # shards get longer instead of more numerous. + SHARD_COUNT="$( awk -v total="${TOTAL}" -v per="${TARGET_PER_SHARD}" -v max="${MAX_SHARDS}" 'BEGIN { + count = int( ( total + per - 1 ) / per ); + if ( count < 1 ) { count = 1 } + if ( count > max ) { count = max } + print count + }' )" + + # Split the slugs into evenly sized shards, dropping any shard that ends up empty because fewer + # plugins were requested than there are shards. Each shard's slugs are passed to the reusable + # workflow as a JSON string. + SHARDS="$( printf '%s' "${SLUGS}" | jq -c --argjson shard_count "${SHARD_COUNT}" ' + . as $slugs + | ( ( length + $shard_count - 1 ) / $shard_count | floor ) as $size + | [ + range( 0; $shard_count ) + | { index: ( . + 1 ), slugs: $slugs[ ( . * $size ) : ( ( . + 1 ) * $size ) ] } + ] + | map( select( .slugs | length > 0 ) ) + | map( { index: .index, slugs: ( .slugs | @json ) } ) + ' )" + + printf 'Testing %s plugins across %s shard(s).\n' "${TOTAL}" "$( printf '%s' "${SHARDS}" | jq 'length' )" + printf '%s\n' "${SHARDS}" | jq -r '.[] | "Shard \(.index): \(.slugs)"' + + printf 'shards=%s\n' "${SHARDS}" >> "${GITHUB_OUTPUT}" + + # Tests each shard of plugins against the version of WordPress being tested. + # + # Pull request and push runs are here to check that this workflow itself still works, not to report on the + # health of the ecosystem. They use the latest stable release and a small number of plugins, because a genuine + # fatal against nightly is a true result that should not sit as a red check on every future change to these + # two files. Scheduled and manually dispatched runs are the ones that carry the ecosystem signal, and they + # default to nightly and the full count. + plugin-compatibility-tests: + name: WP ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} / Shard ${{ matrix.shard.index }} + uses: ./.github/workflows/reusable-plugin-compatibility.yml + permissions: + contents: read + needs: [ build-plugin-matrix ] + if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'workflow_dispatch' }} + strategy: + fail-fast: false + matrix: + shard: ${{ fromJSON( needs.build-plugin-matrix.outputs.shards ) }} + with: + os: 'ubuntu-24.04' + wp-version: ${{ inputs.wp-version || ( contains( fromJSON('["pull_request", "push"]'), github.event_name ) && 'latest' ) || 'nightly' }} + php-version: '8.3' + plugin-slugs: ${{ matrix.shard.slugs }} + db-type: 'mysql' + db-version: '8.4' + + slack-notifications: + name: Slack Notifications + uses: ./.github/workflows/slack-notifications.yml + permissions: + actions: read + contents: read + needs: [ build-plugin-matrix, plugin-compatibility-tests ] + if: ${{ github.repository == 'WordPress/wordpress-develop' && github.event_name != 'pull_request' && always() }} + with: + calling_status: ${{ contains( needs.*.result, 'cancelled' ) && 'cancelled' || contains( needs.*.result, 'failure' ) && 'failure' || 'success' }} + secrets: + SLACK_GHA_SUCCESS_WEBHOOK: ${{ secrets.SLACK_GHA_SUCCESS_WEBHOOK }} + SLACK_GHA_CANCELLED_WEBHOOK: ${{ secrets.SLACK_GHA_CANCELLED_WEBHOOK }} + SLACK_GHA_FIXED_WEBHOOK: ${{ secrets.SLACK_GHA_FIXED_WEBHOOK }} + SLACK_GHA_FAILURE_WEBHOOK: ${{ secrets.SLACK_GHA_FAILURE_WEBHOOK }} + + failed-workflow: + name: Failed workflow tasks + runs-on: ubuntu-24.04 + permissions: + actions: write + needs: [ slack-notifications ] + if: | + always() && + github.repository == 'WordPress/wordpress-develop' && + github.event_name != 'pull_request' && + github.run_attempt < 2 && + ( + contains( needs.*.result, 'cancelled' ) || + contains( needs.*.result, 'failure' ) + ) + + steps: + - name: Dispatch workflow run + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + retries: 2 + retry-exempt-status-codes: 418 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'failed-workflow.yml', + ref: 'trunk', + inputs: { + run_id: `${context.runId}`, + } + }); diff --git a/.github/workflows/reusable-plugin-compatibility.yml b/.github/workflows/reusable-plugin-compatibility.yml new file mode 100644 index 0000000000000..003146860563c --- /dev/null +++ b/.github/workflows/reusable-plugin-compatibility.yml @@ -0,0 +1,125 @@ +## +# A reusable workflow that installs a version of WordPress and then checks that a list of plugins can be +# activated against it without fataling. +# +# Each plugin in the `plugin-slugs` shard is tested on its own: it is installed, activated, exercised, and then +# removed before the next one is installed. This keeps one broken plugin from masking (or breaking) the next. +# +# The exception is a plugin that declares a `Requires Plugins` dependency, which core will not activate until +# that dependency is installed and active. Those dependencies are installed and activated alongside it, and the +# site is checked with only the dependencies active first so that a dependency's own breakage is not reported +# against the plugin under test. +# +# The checks themselves live in `tools/plugin-compatibility/test-plugins.sh` rather than in this file, so that +# the same code can be run from a terminal to reproduce a result from a run here. See the comments at the top +# of that script. +## +name: Plugin Compatibility Tests + +on: + workflow_call: + inputs: + os: + description: 'Operating system to run tests on.' + required: false + type: 'string' + default: 'ubuntu-24.04' + wp-version: + description: 'The version of WordPress to test against. Accepts a version number, "latest", or "nightly".' + required: false + type: 'string' + default: 'nightly' + php-version: + description: 'The version of PHP to use. Expected format: X.Y.' + required: false + type: 'string' + default: '8.3' + plugin-slugs: + description: 'A JSON array of WordPress.org plugin slugs to test in this shard.' + required: true + type: 'string' + db-type: + description: 'Database type. Valid types are mysql and mariadb.' + required: false + type: 'string' + default: 'mysql' + db-version: + description: 'Database version.' + required: false + type: 'string' + default: '8.4' + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Tests that each plugin in the shard can be activated against the given version of WordPress. + # + # Performs the following steps: + # - Checks out the repository, for the script that runs the checks. + # - Sets up PHP and WP-CLI. + # - Runs the checks against every plugin in the shard. + plugin-compatibility-tests: + name: PHP ${{ inputs.php-version }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }} + permissions: + contents: read + runs-on: ${{ inputs.os }} + # Installing a plugin's dependencies before testing it adds a download and an activation for each + # one, and a dependency the size of WooCommerce is not quick to activate. + timeout-minutes: 45 + + services: + database: + # The database type and version are inputs so that this workflow can be pointed at any supported + # combination, which means the image cannot be pinned to a digest. This matches how the database + # service is declared in install-testing.yml and reusable-upgrade-testing.yml. + image: ${{ inputs.db-type }}:${{ inputs.db-version }} # zizmor: ignore[unpinned-images] + ports: + - 3306 + options: >- + --health-cmd="mysqladmin ping" + --health-interval="30s" + --health-timeout="10s" + --health-retries="5" + -e MYSQL_ROOT_PASSWORD="root" + -e MYSQL_DATABASE="test_db" + + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} + persist-credentials: false + sparse-checkout: tools/plugin-compatibility + sparse-checkout-cone-mode: false + + - name: Set up PHP ${{ inputs.php-version }} + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 + with: + php-version: '${{ inputs.php-version }}' + coverage: none + tools: wp-cli + + # The runner already provides PHP, WP-CLI and a database, so the checks run here rather than in the + # containers the script provisions when it is run from a terminal. + # + # WordPress is installed outside the workspace so that the checkout above is not part of the site under + # test. + - name: Test each plugin in isolation + env: + PLUGIN_SLUGS: ${{ inputs.plugin-slugs }} + WP_VERSION: ${{ inputs.wp-version }} + DB_PORT: ${{ job.services.database.ports['3306'] }} + run: | + set -uo pipefail + + ./tools/plugin-compatibility/test-plugins.sh \ + --no-docker \ + --wp-version="${WP_VERSION}" \ + --dir="${RUNNER_TEMP}/wordpress" \ + --db-host="127.0.0.1:${DB_PORT}" \ + --db-name=test_db \ + --db-user=root \ + --db-pass=root \ + --plugins="$( printf '%s' "${PLUGIN_SLUGS}" | jq -r 'join( " " )' )" diff --git a/tools/plugin-compatibility/test-plugins.sh b/tools/plugin-compatibility/test-plugins.sh new file mode 100755 index 0000000000000..a225ce799a752 --- /dev/null +++ b/tools/plugin-compatibility/test-plugins.sh @@ -0,0 +1,721 @@ +#!/usr/bin/env bash +## +# Checks that plugins from the WordPress.org directory can be activated against a version of WordPress +# without fataling. +# +# Each plugin is installed on its own, activated, exercised over HTTP and through WP-CLI, and then removed +# again before the next one is installed, so that one broken plugin cannot mask (or break) the next. A plugin +# that declares `Requires Plugins` has those dependencies installed and activated alongside it, because core +# refuses to activate it otherwise. +# +# What a real request does is what decides whether a plugin passed. The WP-CLI check runs last and only ever +# adds a note, because WP-CLI loads WordPress in a way a visitor never does. +# +# This is the same code the Plugin Compatibility Tests workflow runs. Running it here is how to reproduce a +# failure from a workflow run, or to check a plugin against a release candidate, without waiting on Actions. +# +# By default everything is provisioned in throwaway containers - a database, and a PHP with WP-CLI to run the +# checks in - so nothing but Docker is needed and nothing is left behind: +# +# tools/plugin-compatibility/test-plugins.sh --wp-version=nightly woocommerce classic-editor +# +# Pass `--no-docker` to run the checks in the current environment instead, against a database that is already +# running. That is the path the workflow takes, where the runner already has PHP, WP-CLI and a database +# service. +# +# Usage: test-plugins.sh [options] [slug...] +# +# Run `test-plugins.sh --help` for the options. +## + +# `set -e` is deliberately not used: a plugin that fatals must not stop the remaining plugins from being +# tested. Failures are checked where they matter instead. +set -uo pipefail + +# Written for Bash 3.2 so that it runs on a stock macOS as well as on a runner. + +SCRIPT_PATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null 2>&1 && pwd )/$( basename "${BASH_SOURCE[0]}" )" + +# Defaults. Every one of these can be overridden with the matching option. +WP_VERSION="nightly" +PLUGIN_LIST="" +USE_DOCKER="yes" +# The official WP-CLI image carries PHP, WP-CLI, `mysqli` and curl, which is everything the checks need. +PHP_IMAGE="wordpress:cli-php8.3" +# Matches the database the workflow runs against by default. +DB_IMAGE="mysql:8.4" +WP_DIR="" +DB_HOST="127.0.0.1" +DB_NAME="test_db" +DB_USER="root" +DB_PASS="password" +SERVER_PORT="8889" +KEEP_INSTALL="no" + +# How many levels of `Requires Plugins` to follow. Chains in the directory are one level deep in practice, so +# the bound is only here to stop a circular declaration from looping forever. +MAX_DEPENDENCY_DEPTH=3 + +usage() { + cat <<'USAGE_EOF' +Usage: test-plugins.sh [options] [slug...] + +Checks that WordPress.org plugins can be activated against a version of WordPress without fataling. + +Options: + --plugins= Plugin slugs to test, separated by commas or spaces. Slugs can also be passed as + positional arguments. + --wp-version= Version of WordPress to test against. Accepts "latest", "nightly", or a version + number such as a beta or RC. Default: nightly. + --php-image= Container image the checks run in. Default: wordpress:cli-php8.3. + --db-image= Container image the database runs in. Default: mysql:8.4. + --no-docker Run the checks in the current environment rather than in a container. Requires PHP, + WP-CLI, curl and a database that is already running. + --dir= Where to install WordPress. Only used with --no-docker. Default: a temporary + directory, which is removed again on exit. + --db-host= Database host, with an optional port. Only used with --no-docker. Default: 127.0.0.1. + --db-name= Database name. Only used with --no-docker. Default: test_db. + --db-user= Database user. Only used with --no-docker. Default: root. + --db-pass= Database password. Only used with --no-docker. Default: password. + --port= Port the PHP built-in server listens on. Default: 8889. + --keep Leave the WordPress install in place on exit rather than removing it. Only used with + --no-docker. + -h, --help Print this message. + +Examples: + # Check two plugins against nightly, provisioning everything in containers. + test-plugins.sh --wp-version=nightly woocommerce classic-editor + + # Reproduce a failure from a workflow run against the release it was seen on. + test-plugins.sh --wp-version=7.1-RC1 eps-301-redirects + + # Run the checks here, against a database that is already running. + test-plugins.sh --no-docker --db-host=127.0.0.1:3306 --db-pass=root hello-dolly + +Exits 0 when no plugin fataled, and 1 when one did. A plugin that could not be downloaded, or that core +declined to activate because its declared requirements are not met, is reported as skipped and does not +fail the run. +USAGE_EOF +} + +# error +# +# Prints a message to stderr and stops. +error() { + printf 'Error: %s\n' "${1}" >&2 + exit 1 +} + +while [ "$#" -gt 0 ]; do + case "${1}" in + --plugins=* ) PLUGIN_LIST="${PLUGIN_LIST} ${1#*=}" ;; + --wp-version=* ) WP_VERSION="${1#*=}" ;; + --php-image=* ) PHP_IMAGE="${1#*=}" ;; + --db-image=* ) DB_IMAGE="${1#*=}" ;; + --no-docker ) USE_DOCKER="no" ;; + --dir=* ) WP_DIR="${1#*=}" ;; + --db-host=* ) DB_HOST="${1#*=}" ;; + --db-name=* ) DB_NAME="${1#*=}" ;; + --db-user=* ) DB_USER="${1#*=}" ;; + --db-pass=* ) DB_PASS="${1#*=}" ;; + --port=* ) SERVER_PORT="${1#*=}" ;; + --keep ) KEEP_INSTALL="yes" ;; + -h | --help ) usage; exit 0 ;; + -* ) usage >&2; error "Unknown option ${1}." ;; + * ) PLUGIN_LIST="${PLUGIN_LIST} ${1}" ;; + esac + shift +done + +# Commas, spaces and newlines all separate slugs, so a list can be pasted in however it was written down. +PLUGIN_SLUGS="$( printf '%s' "${PLUGIN_LIST}" | tr ',[:space:]' '\n' | grep -v '^$' | awk '!seen[$0]++' )" + +if [ -z "${PLUGIN_SLUGS}" ]; then + usage >&2 + error "No plugins were named." +fi + +# A value outside the character set the directory uses is a typo worth stopping for, rather than a plugin +# reported as impossible to download several minutes from now. +INVALID_SLUGS="$( printf '%s\n' "${PLUGIN_SLUGS}" | grep -Ev '^[a-z0-9-]+$' )" + +if [ -n "${INVALID_SLUGS}" ]; then + printf 'These values are not WordPress.org plugin slugs:\n%s\n' "${INVALID_SLUGS}" >&2 + exit 1 +fi + +## +# Provisions a database and a PHP, and runs this same script inside them. +# +# The checks themselves are identical either way: the container is handed this file and runs it with +# `--no-docker`, so there is only one copy of the logic that decides whether a plugin passed. +## +run_in_docker() { + command -v docker > /dev/null 2>&1 || error "Docker is required, or pass --no-docker to run the checks here." + + # Everything is named after the process so that two runs at once do not collide. + DOCKER_NETWORK="plugin-compatibility-$$" + DB_CONTAINER="plugin-compatibility-db-$$" + + # Both are removed however this exits, an interrupt included. + trap 'docker rm --force "${DB_CONTAINER}" > /dev/null 2>&1; docker network rm "${DOCKER_NETWORK}" > /dev/null 2>&1' EXIT INT TERM + + printf 'Starting %s.\n' "${DB_IMAGE}" + + docker network create "${DOCKER_NETWORK}" > /dev/null || error "The Docker network could not be created." + + docker run --detach --name "${DB_CONTAINER}" --network "${DOCKER_NETWORK}" \ + --env MYSQL_ROOT_PASSWORD="${DB_PASS}" \ + --env MYSQL_DATABASE="${DB_NAME}" \ + "${DB_IMAGE}" > /dev/null || error "The database container could not be started." + + # A first run has to initialise the data directory, which takes appreciably longer than a later one. + DB_READY="no" + + for _ in $( seq 1 90 ); do + if docker exec "${DB_CONTAINER}" mysqladmin ping --silent --user=root --password="${DB_PASS}" > /dev/null 2>&1; then + DB_READY="yes" + break + fi + sleep 2 + done + + if [ "${DB_READY}" != "yes" ]; then + docker logs "${DB_CONTAINER}" 2>&1 | tail -n 20 + error "The database did not become available." + fi + + printf 'The database is ready. Running the checks in %s.\n\n' "${PHP_IMAGE}" + + # WordPress is installed inside the container rather than in a mounted directory, so a plugin cannot + # leave anything behind on the host and file ownership never comes into it. `HOME` is pointed at a + # writable directory because WP-CLI caches its downloads there. + docker run --rm --network "${DOCKER_NETWORK}" \ + --volume "${SCRIPT_PATH}:/usr/local/bin/test-plugins.sh:ro" \ + --env HOME=/tmp \ + --entrypoint bash \ + "${PHP_IMAGE}" /usr/local/bin/test-plugins.sh \ + --no-docker \ + --dir=/tmp/wordpress \ + --wp-version="${WP_VERSION}" \ + --db-host="${DB_CONTAINER}" \ + --db-name="${DB_NAME}" \ + --db-user=root \ + --db-pass="${DB_PASS}" \ + --port="${SERVER_PORT}" \ + --plugins="$( printf '%s' "${PLUGIN_SLUGS}" | tr '\n' ' ' )" +} + +## +# Installs WordPress, starts a web server, and checks each plugin against it. +## +run_checks() { + for REQUIRED_COMMAND in php curl; do + command -v "${REQUIRED_COMMAND}" > /dev/null 2>&1 || error "${REQUIRED_COMMAND} is required to run the checks without Docker." + done + + # `type -P` searches the path rather than resolving to the `wp` wrapper function defined below. + WP_CLI_BIN="$( type -P wp )" + + [ -n "${WP_CLI_BIN}" ] || error "WP-CLI is required to run the checks without Docker." + + WORK_DIR="$( mktemp -d )" + RESULTS="${WORK_DIR}/results.tsv" + RESPONSE_BODY="${WORK_DIR}/response.html" + SERVER_LOG="${WORK_DIR}/php-server.log" + SERVER_PID="" + : > "${RESULTS}" + + if [ -z "${WP_DIR}" ]; then + WP_DIR="${WORK_DIR}/wordpress" + fi + + mkdir -p "${WP_DIR}" || error "The WordPress directory could not be created." + + # The server is stopped and the temporary files are removed however this exits. The WordPress install is + # only removed when this script created it, so that `--dir` never deletes a directory it was handed. + cleanup_environment() { + if [ -n "${SERVER_PID}" ]; then + kill "${SERVER_PID}" > /dev/null 2>&1 + fi + + if [ "${KEEP_INSTALL}" = "yes" ]; then + printf '\nThe WordPress install was left in %s.\n' "${WP_DIR}" + else + rm -rf "${WORK_DIR}" + fi + } + + trap cleanup_environment EXIT INT TERM + + cd "${WP_DIR}" || error "The WordPress directory could not be entered." + + SITE_URL="http://127.0.0.1:${SERVER_PORT}" + + printf 'Downloading WordPress %s.\n' "${WP_VERSION}" + wp core download --version="${WP_VERSION}" || error "WordPress ${WP_VERSION} could not be downloaded." + + wp config create --dbname="${DB_NAME}" --dbuser="${DB_USER}" --dbpass="${DB_PASS}" --dbhost="${DB_HOST}" \ + || error "wp-config.php could not be written." + + # Errors need to reach `wp-content/debug.log` so that a white screen of death is still detectable. + # + # `WP_DEBUG_DISPLAY` is left off on purpose: this should behave the way a production site does, where a + # fatal error is an empty page and an HTTP 500 rather than a printed stack trace. + # + # The fatal error handler is disabled so that a fatal is reported as a fatal instead of being swallowed + # by recovery mode, which would also deactivate the plugin mid-test. + # + # WP-Cron is disabled because WordPress spawns it as a loopback request during a front end request. The + # loopback lands back on the same PHP built-in server that is still busy serving the request that + # spawned it, and the two deadlock until curl gives up. + wp config set WP_DEBUG true --raw + wp config set WP_DEBUG_LOG true --raw + wp config set WP_DEBUG_DISPLAY false --raw + wp config set WP_DISABLE_FATAL_ERROR_HANDLER true --raw + wp config set DISABLE_WP_CRON true --raw + + wp core install --url="${SITE_URL}" --title="Plugin Compatibility Test" --admin_user=admin \ + --admin_password=password --admin_email=me@example.org --skip-email \ + || error "WordPress could not be installed. Check that the database is reachable at ${DB_HOST}." + + # The site needs to answer real requests so that fatals which only happen on a front end or admin page + # load are caught. The built-in server is enough for that and needs nothing installed. + # + # `PHP_CLI_SERVER_WORKERS` is set because the built-in server is single threaded by default. Plenty of + # plugins make a loopback request to the site they are running on, and a single threaded server cannot + # answer one while it is still serving the request that made it. + PHP_CLI_SERVER_WORKERS=4 php -S "127.0.0.1:${SERVER_PORT}" -t "${WP_DIR}" > "${SERVER_LOG}" 2>&1 & + SERVER_PID=$! + + SERVER_READY="no" + + for _ in $( seq 1 30 ); do + if curl -sSf -o /dev/null "${SITE_URL}/wp-login.php"; then + SERVER_READY="yes" + break + fi + sleep 1 + done + + if [ "${SERVER_READY}" != "yes" ]; then + cat "${SERVER_LOG}" + error "The PHP built-in server did not start on port ${SERVER_PORT}." + fi + + printf 'The PHP built-in server is ready on %s.\n' "${SITE_URL}" + + test_plugins + report_results +} + +# wp +# +# Runs WP-CLI with the memory limit lifted. Unpacking a WordPress zip needs more than the 128M a stock +# php.ini allows, and WP-CLI reads its own `WP_CLI_PHP_ARGS` only when it is installed as a wrapper script +# rather than as the phar most environments carry. +wp() { + php -d memory_limit=-1 "${WP_CLI_BIN}" "$@" +} + +# record +# +# Appends one tab separated row to the results file. The reason is flattened so that it cannot break the +# markdown table that is generated from these rows later on. +record() { + SAFE_REASON="$( printf '%s' "${5:--}" | tr '\n\t|' ' ' )" + printf '%s\t%s\t%s\t%s\t%s\n' "${1}" "${2:-unknown}" "${3}" "${4:--}" "${SAFE_REASON}" >> "${RESULTS}" +} + +# check_url +# +# Requests a path on the test site and prints a reason to stdout when the response looks broken. Prints +# nothing when the request looks healthy. +check_url() { + CURL_EXIT_CODE=0 + # The exit code is captured separately rather than falling back to a literal inside the command + # substitution, which would append to whatever curl had already written and produce a nonsense status + # like "200000" when a request returned headers and then stalled. + HTTP_CODE="$( curl -s -o "${RESPONSE_BODY}" -w '%{http_code}' --max-time 60 "${SITE_URL}${1}" )" || CURL_EXIT_CODE=$? + + if [ -z "${HTTP_CODE}" ] || [ "${HTTP_CODE}" = "000" ]; then + printf 'The request to %s did not complete, curl exit code %s' "${1}" "${CURL_EXIT_CODE}" + return + fi + + if [ "${HTTP_CODE}" -ge 500 ]; then + printf 'The request to %s returned HTTP %s' "${1}" "${HTTP_CODE}" + return + fi + + if [ "${CURL_EXIT_CODE}" -ne 0 ]; then + printf 'The request to %s returned HTTP %s but the response did not finish, curl exit code %s' "${1}" "${HTTP_CODE}" "${CURL_EXIT_CODE}" + return + fi + + # Belt and braces. Errors are not displayed by default, but a plugin can turn display_errors back on for + # itself. + if grep -qi 'Fatal error' "${RESPONSE_BODY}"; then + printf 'The response from %s contained a fatal error' "${1}" + fi +} + +# install_plugin +# +# Installs a plugin from WordPress.org and remembers it so that it is removed again once the plugin under +# test has been checked. The slug is remembered before the download is attempted so that a partial download +# is still cleaned up. Returns non zero when the download failed. +install_plugin() { + INSTALLED_SLUGS[${#INSTALLED_SLUGS[@]}]="${1}" + INSTALLED_DIRS[${#INSTALLED_DIRS[@]}]="" + + wp plugin install "${1}" --skip-plugins --skip-themes || return 1 + + INSTALLED_DIRS[${#INSTALLED_DIRS[@]} - 1]="$( wp plugin path "${1}" --dir --skip-plugins --skip-themes 2>/dev/null || printf '' )" +} + +# plugin_dependencies +# +# Prints the slugs named in the plugin's `Requires Plugins` header, one per line. WordPress' own header +# parser is used so that the same rules apply here as when core decides whether a plugin's dependencies are +# met. +plugin_dependencies() { + PLUGIN_FILE="$( wp plugin path "${1}" --skip-plugins --skip-themes 2>/dev/null || printf '' )" + + [ -n "${PLUGIN_FILE}" ] || return 0 + + # The single quotes are what keep the shell out of the PHP below. The path is handed over as an + # environment variable rather than interpolated so that it never reaches PHP as source code. + export PLUGIN_FILE + + # shellcheck disable=SC2016 + wp eval --skip-plugins --skip-themes ' + require_once ABSPATH . "wp-admin/includes/plugin.php"; + + $plugin_data = get_plugin_data( getenv( "PLUGIN_FILE" ), false, false ); + + foreach ( explode( ",", $plugin_data["RequiresPlugins"] ?? "" ) as $dependency ) { + // Core only accepts a WordPress.org slug here, so anything else is ignored. + if ( preg_match( "/^[a-z0-9-]+$/", trim( $dependency ) ) ) { + echo trim( $dependency ), "\n"; + } + } + ' 2>/dev/null +} + +# cleanup_plugins +# +# Returns the site to a clean slate by removing everything installed for the current plugin, its +# dependencies included. A plugin that fatals can take WP-CLI down with it, so every command here is allowed +# to fail and the plugin directory is removed directly as a fallback. `--skip-plugins` keeps WP-CLI from +# loading the broken plugin while cleaning up after it. +cleanup_plugins() { + if [ "${#INSTALLED_SLUGS[@]}" -gt 0 ]; then + for INDEX in "${!INSTALLED_SLUGS[@]}"; do + CLEANUP_SLUG="${INSTALLED_SLUGS[${INDEX}]}" + CLEANUP_DIR="${INSTALLED_DIRS[${INDEX}]}" + + wp plugin deactivate "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 + wp plugin delete "${CLEANUP_SLUG}" --skip-plugins --skip-themes > /dev/null 2>&1 + + if [ -n "${CLEANUP_DIR}" ] && [ -d "${CLEANUP_DIR}" ] && [ "${CLEANUP_DIR}" != "${PLUGINS_ROOT}" ]; then + rm -rf "${CLEANUP_DIR}" + fi + + rm -rf "${WP_DIR}/wp-content/plugins/${CLEANUP_SLUG}" + done + fi + + INSTALLED_SLUGS=() + INSTALLED_DIRS=() + + # Make sure nothing is left behind in `active_plugins` pointing at a plugin that is now gone. + wp option update active_plugins '[]' --format=json --skip-plugins --skip-themes > /dev/null 2>&1 +} + +# group_start / group_end +# +# Collapses a plugin's output in the workflow log. The markers mean nothing to a terminal, so they are only +# printed when this is running in Actions. +group_start() { + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + printf '::group::%s\n' "${1}" + else + printf '\n----- %s -----\n' "${1}" + fi +} + +group_end() { + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + printf '::endgroup::\n' + fi +} + +## +# Tests each plugin in turn, writing a row per plugin to the results file. +## +test_plugins() { + # The plugins directory itself must never be removed, only directories inside it. + PLUGINS_ROOT="$( wp plugin path )" + + # Everything installed for the plugin currently under test, that plugin included. The directories are + # tracked alongside the slugs because a plugin does not always unpack into a directory named after its + # slug. + INSTALLED_SLUGS=() + INSTALLED_DIRS=() + + while IFS= read -r SLUG; do + [ -n "${SLUG}" ] || continue + + group_start "${SLUG}" + + STATUS="PASS" + REASON="-" + VERSION="unknown" + DEPENDENCIES=() + + # Start every plugin with an empty log so that anything found in it belongs to this plugin. + rm -f "${WP_DIR}/wp-content/debug.log" + + # Step 1: download the plugin. A download failure is recorded as skipped rather than failed, since + # it generally means a network flake or a plugin that is no longer in the directory. + if ! install_plugin "${SLUG}"; then + record "${SLUG}" "unknown" "SKIPPED" "-" "The plugin could not be downloaded from WordPress.org" + cleanup_plugins + group_end + continue + fi + + VERSION="$( wp plugin get "${SLUG}" --field=version --skip-plugins --skip-themes 2>/dev/null || printf 'unknown' )" + + # Step 2: install whatever the plugin names in `Requires Plugins`. Core refuses to activate a plugin + # whose dependencies are missing, so without this every WooCommerce extension - a large slice of the + # most popular plugins - would go untested. + # + # Dependencies are resolved a level at a time so that a dependency declaring its own is installed + # too. Anything already installed is left alone, which covers a plugin named twice in the same tree + # as well as a declaration pointing back at the plugin under test. + PENDING=() + + while IFS= read -r DEPENDENCY; do + [ -n "${DEPENDENCY}" ] && PENDING[${#PENDING[@]}]="${DEPENDENCY}" + done < <( plugin_dependencies "${SLUG}" ) + + DEPTH=0 + + while [ "${#PENDING[@]}" -gt 0 ] && [ "${DEPTH}" -lt "${MAX_DEPENDENCY_DEPTH}" ]; do + NEXT=() + + for DEPENDENCY in "${PENDING[@]}"; do + if wp plugin is-installed "${DEPENDENCY}" --skip-plugins --skip-themes > /dev/null 2>&1; then + continue + fi + + printf 'Installing %s, which %s requires.\n' "${DEPENDENCY}" "${SLUG}" + + if ! install_plugin "${DEPENDENCY}"; then + STATUS="SKIPPED" + REASON="The required plugin ${DEPENDENCY} is not available from WordPress.org" + break 2 + fi + + DEPENDENCIES[${#DEPENDENCIES[@]}]="${DEPENDENCY}" + + while IFS= read -r NESTED_DEPENDENCY; do + [ -n "${NESTED_DEPENDENCY}" ] && NEXT[${#NEXT[@]}]="${NESTED_DEPENDENCY}" + done < <( plugin_dependencies "${DEPENDENCY}" ) + done + + PENDING=( ${NEXT[@]+"${NEXT[@]}"} ) + DEPTH=$(( DEPTH + 1 )) + done + + # Step 3: activate the dependencies, deepest first. That is the reverse of the order they were + # discovered in, and it matters because core will not activate a plugin ahead of its own + # requirements either. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + for (( INDEX = ${#DEPENDENCIES[@]} - 1; INDEX >= 0; INDEX-- )); do + DEPENDENCY="${DEPENDENCIES[${INDEX}]}" + DEPENDENCY_EXIT_CODE=0 + DEPENDENCY_OUTPUT="$( wp plugin activate "${DEPENDENCY}" 2>&1 )" || DEPENDENCY_EXIT_CODE=$? + printf '%s\n' "${DEPENDENCY_OUTPUT}" + + if [ "${DEPENDENCY_EXIT_CODE}" -ne 0 ]; then + STATUS="SKIPPED" + REASON="The required plugin ${DEPENDENCY} could not be activated" + break + fi + done + fi + + # Step 4: with dependencies active the plugin under test is no longer alone on the site, so the + # baseline is checked before it is activated. A site that is already broken says something about the + # dependency rather than about the plugin being tested, and blaming the plugin for it would be the + # masking these checks are built to avoid. + if [ "${STATUS}" = "PASS" ] && [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + for URL_PATH in "/" "/wp-login.php"; do + BASELINE_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${BASELINE_REASON}" ]; then + STATUS="SKIPPED" + REASON="The required plugins are not healthy on their own: ${BASELINE_REASON}" + break + fi + done + + # Anything the dependencies logged on their way up is not the responsibility of the plugin under + # test, so the log starts empty again here. + rm -f "${WP_DIR}/wp-content/debug.log" + fi + + # Step 5: activation. Activation runs the plugin's activation hooks and loads its main file. + if [ "${STATUS}" = "PASS" ]; then + ACTIVATE_EXIT_CODE=0 + ACTIVATE_OUTPUT="$( wp plugin activate "${SLUG}" 2>&1 )" || ACTIVATE_EXIT_CODE=$? + printf '%s\n' "${ACTIVATE_OUTPUT}" + + if [ "${ACTIVATE_EXIT_CODE}" -ne 0 ]; then + case "${ACTIVATE_OUTPUT}" in + # Core refuses to activate a plugin whose declared requirements are not met. + # Dependencies are installed above, so what is left here is a plugin asking for a + # version of PHP or WordPress this run is not using, or for a dependency that is not on + # WordPress.org. That is core working as designed rather than a fatal, so it is recorded + # as skipped. + *"to be installed and activated"* | *"requires PHP version"* | *"requires WordPress version"* ) + STATUS="SKIPPED" + REASON="Core declined to activate the plugin because its declared requirements are not met" + ;; + * ) + STATUS="FAIL" + REASON="The plugin could not be activated" + ;; + esac + fi + fi + + # Step 6: request the front page and the login screen through the PHP built-in server. A real request + # is what decides whether a plugin passed, because it is what a visitor gets. + if [ "${STATUS}" = "PASS" ]; then + for URL_PATH in "/" "/wp-login.php"; do + HTTP_REASON="$( check_url "${URL_PATH}" )" + + if [ -n "${HTTP_REASON}" ]; then + STATUS="FAIL" + REASON="${HTTP_REASON}" + break + fi + done + fi + + # Step 7: a fatal can be logged without changing the HTTP status, for example during a shutdown + # hook, so the debug log is checked separately. + if [ "${STATUS}" = "PASS" ] && [ -f "${WP_DIR}/wp-content/debug.log" ] && grep -q 'PHP Fatal' "${WP_DIR}/wp-content/debug.log"; then + grep 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" + STATUS="FAIL" + REASON="A fatal error was logged: $( grep -m 1 'PHP Fatal' "${WP_DIR}/wp-content/debug.log" | cut -c 1-200 )" + fi + + # Step 8: boot all of core plus the active plugin through WP-CLI. This runs last and cannot fail a + # plugin on its own. + # + # WP-CLI requires `wp-settings.php` from inside a method, so a plugin that assigns a variable at file + # scope and reads it back with `global` later finds nothing there. Plenty of plugins do exactly that, + # and the resulting fatal happens only under WP-CLI - a visitor never sees it. Failing a plugin for + # it would report breakage that does not exist on a real site, so it is recorded as a note against a + # plugin that is otherwise healthy. Anything that genuinely fatals on load fails the HTTP checks + # above. + # + # The fatal is written to the debug log too, which is why this comes after the log has been checked. + if [ "${STATUS}" = "PASS" ]; then + EVAL_OUTPUT="$( wp eval 'echo "loaded-ok";' 2>&1 )" + + if [ "${EVAL_OUTPUT#*loaded-ok}" = "${EVAL_OUTPUT}" ]; then + printf '%s\n' "${EVAL_OUTPUT}" + + case "${EVAL_OUTPUT}" in + *"Fatal error"* | *"PHP Fatal"* | *"Uncaught"* ) + REASON="The site is healthy over HTTP, but WP-CLI fatals when it loads WordPress with the plugin active" + ;; + # Some plugins redirect or exit while loading, which stops WP-CLI without anything being + # broken. + * ) + printf 'WP-CLI did not finish loading WordPress, but no fatal error was reported.\n' + ;; + esac + fi + fi + + # Step 9: record the outcome and put the site back the way it was found. + DEPENDENCY_LIST="-" + + if [ "${#DEPENDENCIES[@]}" -gt 0 ]; then + DEPENDENCY_LIST="$( printf '%s, ' "${DEPENDENCIES[@]}" )" + DEPENDENCY_LIST="${DEPENDENCY_LIST%, }" + fi + + record "${SLUG}" "${VERSION}" "${STATUS}" "${DEPENDENCY_LIST}" "${REASON}" + cleanup_plugins + + printf '%s: %s\n' "${SLUG}" "${STATUS}" + group_end + done <> "${GITHUB_STEP_SUMMARY}" + fi + + # Plugins that could not be downloaded are reported but do not fail the run. + if [ "${FAIL_COUNT}" -gt 0 ]; then + printf '\nThe following plugins failed against WordPress %s:\n' "${WP_VERSION}" + awk -F '\t' '$3 == "FAIL" { print "- " $1 " (" $2 "): " $5 }' "${RESULTS}" + return 1 + fi + + printf '\nNo plugins fataled against WordPress %s.\n' "${WP_VERSION}" +} + +if [ "${USE_DOCKER}" = "yes" ]; then + run_in_docker + exit $? +fi + +run_checks