Make the test runners cheaper to run repeatedly and quieter to read back

.gherkin-lintrc

@@ -0,0 +1,32 @@
+{
+  "file-name": [
+    "on",
+    {
+      "style": "kebab-case"
+    }
+  ],
+  "indentation": [
+    "on",
+    {
+      "Feature": 0,
+      "Background": 2,
+      "Scenario": 2,
+      "Examples": 4,
+      "Step": 4,
+      "given": 4,
+      "example": 6,
+      "and": 4
+    }
+  ],
+  "no-dupe-feature-names": "on",
+  "no-dupe-scenario-names": "off",
+  "no-empty-file": "on",
+  "no-files-without-scenarios": "on",
+  "no-multiple-empty-lines": "off",
+  "no-partially-commented-tag-lines": "on",
+  "no-trailing-spaces": "off",
+  "no-unnamed-features": "on",
+  "no-unnamed-scenarios": "on",
+  "no-scenario-outlines-without-examples": "on",
+  "use-and": "on"
+}

.readme-partials/USING.md

@@ -11,13 +11,15 @@ To make use of the WP-CLI testing framework, you need to complete the following
         "behat": "run-behat-tests",
         "behat-rerun": "rerun-behat-tests",
         "lint": "run-linter-tests",
+        "lint-gherkin": "run-gherkin-lint-tests",
         "phpcs": "run-phpcs-tests",
         "phpcbf": "run-phpcbf-cleanup",
         "phpstan": "run-phpstan-tests",
         "phpunit": "run-php-unit-tests",
         "prepare-tests": "install-package-tests",
         "test": [
             "@lint",
+            "@lint-gherkin",
             "@phpcs",
             "@phpstan",
             "@phpunit",
@@ -96,6 +98,7 @@ You can use the following commands to control the tests:
 * `composer prepare-tests` - Set up the database that is needed for running the functional tests. This is only needed once.
 * `composer test` - Run all test suites.
 * `composer lint` - Run only the linting test suite.
+* `composer lint-gherkin` - Run only the Gherkin linter over the feature files.
 * `composer phpcs` - Run only the code sniffer test suite.
 * `composer phpcbf` - Run only the code sniffer cleanup.
 * `composer phpstan` - Run only the static analysis.
@@ -226,6 +229,53 @@ composer behat -- features/cli-info.feature
 
 Prepending with the double dash is needed because the arguments would otherwise be sent to Composer itself, not the tool that Composer executes.
 
+The same mechanism works for narrowing a run down further, or for bailing out early:
+```bash
+# A single scenario, identified by the line it starts on.
+composer behat -- features/cli-info.feature:12
+
+# Every scenario carrying a given tag.
+composer behat -- --tags=@require-wp-5.0
+
+# Stop at the first failing scenario instead of running the whole suite.
+composer behat -- --stop-on-failure
+
+# Re-run only the scenarios that failed the last time.
+composer behat-rerun
+```
+
+### Linting the feature files
+
+`composer lint-gherkin` checks `features/` with
+[gherkin-lint-plus](https://www.npmjs.com/package/gherkin-lint-plus), against the
+`.gherkin-lintrc` ruleset shipped with this package. A project that needs
+different rules can override it by committing its own `.gherkin-lintrc`.
+
+The linter is a Node package, so it is run through `npx` and needs Node.js 20 or
+later. Where `npx` is not available the check reports that it is skipping, rather
+than failing a suite that is otherwise entirely PHP. Its version is pinned in
+this package's `package.json`, which exists only to hold that pin.
+
+### Controlling the amount of output
+
+Two environment variables make the test tools less chatty. Both are unset by default, which leaves the output exactly as it has always been.
+
+* `NO_COLOR` (the [no-color.org](https://no-color.org/) convention) turns off the ANSI color codes in the output of every runner. Set this when capturing output to a file or a pipe, where the escape sequences are noise.
+* `WP_CLI_TEST_QUIET` switches the reporters to their most compact form: PHP_CodeSniffer reports one `file:line:col` line per violation with no progress ticker, PHPStan reports one `file:line:message` line per error with no progress bar and no result table. This covers the analysis of the PHP files themselves; the checks over the PHP blocks embedded in feature files keep their own reports, which are rewritten to point back at the feature file a block came from. Behat's own output is already minimal, so it is unaffected.
+
+`NO_COLOR` also covers the Gherkin linter, which colors its report unconditionally and has no plain output format of its own.
+
+```bash
+NO_COLOR=1 WP_CLI_TEST_QUIET=1 composer phpstan
+```
+
+This is worth setting permanently in environments that read the output back rather than display it, such as an AI coding agent's shell:
+
+```bash
+export NO_COLOR=1
+export WP_CLI_TEST_QUIET=1
+```
+
 ### Controlling the test environment
 
 #### WordPress Version
@@ -241,6 +291,13 @@ Here's how to run your tests against the latest trunk version of WordPress:
 WP_VERSION=trunk composer behat
 ```
 
+Resolving `latest`, or a `X.Y` version without a patch number, needs the
+WordPress versions data, which is fetched once and cached in the system temp
+directory for a day. Repeated runs do not repeat the request, and a run without
+connectivity falls back to the last known copy.
+`WP_CLI_TEST_WP_VERSION_CACHE_TTL` sets the lifetime of that cache in seconds;
+`0` fetches it every time.
+
 #### WordPress Archive
 
 Instead of downloading WordPress from WordPress.org, you can run the tests against an arbitrary

README.md

@@ -22,13 +22,15 @@ To make use of the WP-CLI testing framework, you need to complete the following
         "behat": "run-behat-tests",
         "behat-rerun": "rerun-behat-tests",
         "lint": "run-linter-tests",
+        "lint-gherkin": "run-gherkin-lint-tests",
         "phpcs": "run-phpcs-tests",
         "phpcbf": "run-phpcbf-cleanup",
         "phpstan": "run-phpstan-tests",
         "phpunit": "run-php-unit-tests",
         "prepare-tests": "install-package-tests",
         "test": [
             "@lint",
+            "@lint-gherkin",
             "@phpcs",
             "@phpstan",
             "@phpunit",
@@ -107,6 +109,7 @@ You can use the following commands to control the tests:
 * `composer prepare-tests` - Set up the database that is needed for running the functional tests. This is only needed once.
 * `composer test` - Run all test suites.
 * `composer lint` - Run only the linting test suite.
+* `composer lint-gherkin` - Run only the Gherkin linter over the feature files.
 * `composer phpcs` - Run only the code sniffer test suite.
 * `composer phpcbf` - Run only the code sniffer cleanup.
 * `composer phpstan` - Run only the static analysis.
@@ -237,6 +240,53 @@ composer behat -- features/cli-info.feature
 
 Prepending with the double dash is needed because the arguments would otherwise be sent to Composer itself, not the tool that Composer executes.
 
+The same mechanism works for narrowing a run down further, or for bailing out early:
+```bash
+# A single scenario, identified by the line it starts on.
+composer behat -- features/cli-info.feature:12
+
+# Every scenario carrying a given tag.
+composer behat -- --tags=@require-wp-5.0
+
+# Stop at the first failing scenario instead of running the whole suite.
+composer behat -- --stop-on-failure
+
+# Re-run only the scenarios that failed the last time.
+composer behat-rerun
+```
+
+### Linting the feature files
+
+`composer lint-gherkin` checks `features/` with
+[gherkin-lint-plus](https://www.npmjs.com/package/gherkin-lint-plus), against the
+`.gherkin-lintrc` ruleset shipped with this package. A project that needs
+different rules can override it by committing its own `.gherkin-lintrc`.
+
+The linter is a Node package, so it is run through `npx` and needs Node.js 20 or
+later. Where `npx` is not available the check reports that it is skipping, rather
+than failing a suite that is otherwise entirely PHP. Its version is pinned in
+this package's `package.json`, which exists only to hold that pin.
+
+### Controlling the amount of output
+
+Two environment variables make the test tools less chatty. Both are unset by default, which leaves the output exactly as it has always been.
+
+* `NO_COLOR` (the [no-color.org](https://no-color.org/) convention) turns off the ANSI color codes in the output of every runner. Set this when capturing output to a file or a pipe, where the escape sequences are noise.
+* `WP_CLI_TEST_QUIET` switches the reporters to their most compact form: PHP_CodeSniffer reports one `file:line:col` line per violation with no progress ticker, PHPStan reports one `file:line:message` line per error with no progress bar and no result table. This covers the analysis of the PHP files themselves; the checks over the PHP blocks embedded in feature files keep their own reports, which are rewritten to point back at the feature file a block came from. Behat's own output is already minimal, so it is unaffected.
+
+`NO_COLOR` also covers the Gherkin linter, which colors its report unconditionally and has no plain output format of its own.
+
+```bash
+NO_COLOR=1 WP_CLI_TEST_QUIET=1 composer phpstan
+```
+
+This is worth setting permanently in environments that read the output back rather than display it, such as an AI coding agent's shell:
+
+```bash
+export NO_COLOR=1
+export WP_CLI_TEST_QUIET=1
+```
+
 ### Controlling the test environment
 
 #### WordPress Version
@@ -252,6 +302,13 @@ Here's how to run your tests against the latest trunk version of WordPress:
 WP_VERSION=trunk composer behat
 ```
 
+Resolving `latest`, or a `X.Y` version without a patch number, needs the
+WordPress versions data, which is fetched once and cached in the system temp
+directory for a day. Repeated runs do not repeat the request, and a run without
+connectivity falls back to the last known copy.
+`WP_CLI_TEST_WP_VERSION_CACHE_TTL` sets the lifetime of that cache in seconds;
+`0` fetches it every time.
+
 #### WordPress Archive
 
 Instead of downloading WordPress from WordPress.org, you can run the tests against an arbitrary

bin/run-behat-tests

@@ -97,22 +97,137 @@ if [ -n "${WP_CLI_TEST_CORE_ZIP-}" ] && [ -z "${WP_VERSION-}" ]; then
 	export WP_VERSION=trunk
 fi
 
+# Everything WP_VERSION resolution needs is in one file: the wp-versions artifact
+# maps every WordPress release to its status, with the current one marked
+# "latest". Cache it, so that re-running a single scenario while iterating does
+# not refetch it every time, and so that a run without connectivity can fall back
+# to the last known answer instead of ending up with no version at all.
+#
+# Set WP_CLI_TEST_WP_VERSION_CACHE_TTL to 0 to always refetch.
+WP_VERSIONS_URL="https://raw.githubusercontent.com/wp-cli/wp-cli-tests/artifacts/wp-versions.json"
+WP_VERSIONS_CACHE_FILE="${TMPDIR:-/tmp}/wp-cli-test-wp-version-cache/wp-versions.json"
+WP_VERSIONS_CACHE_TTL="${WP_CLI_TEST_WP_VERSION_CACHE_TTL:-86400}"
+
+# The versions data is a non-empty object mapping every release to its status.
+# An error page, or a copy that was truncated on its way to disk, is not one.
+is_valid_versions_json() {
+	jq -e 'type == "object" and length > 0' > /dev/null 2>&1
+}
+
+# Print the cached versions file if it is younger than the given number of
+# seconds. A negative TTL accepts it at any age.
+read_versions_cache() {
+	local ttl="$1"
+	local age
+
+	[ -s "${WP_VERSIONS_CACHE_FILE}" ] || return 1
+
+	if [ "${ttl}" -ge 0 ]; then
+		# PHP rather than `find -newermt`, which is not portable across
+		# GNU and BSD userlands. The Behat runner needs PHP anyway.
+		age=$(php -r 'echo time() - filemtime( $argv[1] );' "${WP_VERSIONS_CACHE_FILE}" 2>/dev/null)
+		case ${age} in
+			''|*[!0-9]*) return 1;;
+		esac
+		[ "${age}" -lt "${ttl}" ] || return 1
+	fi
+
+	# Held to the same standard as a fetched copy, so that an unusable cache
+	# counts as a miss and the network gets a chance to replace it, instead of
+	# being served unchecked for the rest of its lifetime.
+	is_valid_versions_json < "${WP_VERSIONS_CACHE_FILE}" || return 1
+
+	cat "${WP_VERSIONS_CACHE_FILE}"
+}
+
+# Store the versions data for the next run. The write goes through a temporary
+# file in the same directory, so that a concurrent runner reading the cache sees
+# either the previous copy or the new one, never a partial write. Caching is
+# best effort throughout: a temp directory that cannot be written to is not a
+# reason to fail the run.
+write_versions_cache() {
+	local dir
+	local tmp
+
+	dir=$( dirname "${WP_VERSIONS_CACHE_FILE}" )
+	mkdir -p "${dir}" 2>/dev/null || return 0
+
+	tmp=$( mktemp "${dir}/wp-versions.XXXXXX" 2>/dev/null ) || return 0
+
+	if printf '%s' "$1" > "${tmp}" 2>/dev/null; then
+		# mktemp creates the file private to its owner; the cache directory is
+		# shared, and the data in it is public.
+		chmod 644 "${tmp}" 2>/dev/null
+		mv -f "${tmp}" "${WP_VERSIONS_CACHE_FILE}" 2>/dev/null || rm -f "${tmp}"
+	else
+		rm -f "${tmp}"
+	fi
+
+	return 0
+}
+
+# Print the WordPress versions data, from the cache where possible. Warnings go
+# to STDERR so that they cannot end up inside the returned JSON.
+get_wp_versions() {
+	local json
+	local ttl="${WP_VERSIONS_CACHE_TTL}"
+
+	# A typo must not turn into a cache that never expires: the age comparison
+	# errors out on a non-numeric TTL, which would then accept the cached copy
+	# at any age.
+	if ! is_numeric "${ttl}"; then
+		echo "Warning: WP_CLI_TEST_WP_VERSION_CACHE_TTL is not a number of seconds, falling back to 86400." >&2
+		ttl=86400
+	fi
+
+	json=$( read_versions_cache "${ttl}" )
+	if [ -n "${json}" ]; then
+		printf '%s' "${json}"
+		return 0
+	fi
+
+	# Bounded, so that an unreachable or unresponsive host falls back to the
+	# cached copy instead of holding up the run indefinitely.
+	json=$( curl -s --connect-timeout 10 --max-time 30 "${WP_VERSIONS_URL}" )
+
+	# Only cache a well-formed response; an error page is not one.
+	if printf '%s' "${json}" | is_valid_versions_json; then
+		write_versions_cache "${json}"
+		printf '%s' "${json}"
+		return 0
+	fi
+
+	# Prefer a stale answer over no answer.
+	json=$( read_versions_cache -1 )
+	if [ -n "${json}" ]; then
+		echo "Warning: Could not fetch the WordPress versions data, falling back to the cached copy." >&2
+		printf '%s' "${json}"
+		return 0
+	fi
+
+	return 1
+}
+
 # Turn WP_VERSION into an actual number to make sure our tags work correctly.
 if [ "${WP_VERSION-latest}" = "latest" ]; then
-	export WP_VERSION=$(curl -s https://api.wordpress.org/core/version-check/1.7/ | jq -r ".offers[0].current")
-fi
+	WP_VERSION=$( get_wp_versions | jq -r 'to_entries | map( select( .value == "latest" ) ) | last | .key // empty' )
 
-# Normalize WP_VERSION=X.Y.0 to X.Y (WordPress uses X.Y for the initial release, not X.Y.0).
-# If WP_VERSION=X.Y (major.minor only), resolve to the latest available patch release.
-if [[ "${WP_VERSION}" =~ ^([0-9]+\.[0-9]+)\.0$ ]]; then
+	if [ -z "${WP_VERSION}" ]; then
+		echo "Warning: Could not determine the latest WordPress version. Version-specific tags will not be filtered." >&2
+	fi
+
+	export WP_VERSION
+# Normalize WP_VERSION=X.Y.0 to X.Y (WordPress uses X.Y for the initial release,
+# not X.Y.0). This asks for that specific release, so it must not fall through to
+# the patch resolution below.
+elif [[ "${WP_VERSION}" =~ ^([0-9]+\.[0-9]+)\.0$ ]]; then
 	export WP_VERSION="${BASH_REMATCH[1]}"
+# If WP_VERSION=X.Y (major.minor only), resolve to the latest available patch release.
 elif [[ "${WP_VERSION}" =~ ^[0-9]+\.[0-9]+$ ]]; then
-	WP_VERSIONS_JSON=$(curl -s https://raw.githubusercontent.com/wp-cli/wp-cli-tests/artifacts/wp-versions.json)
-	if [ -n "${WP_VERSIONS_JSON}" ]; then
-		RESOLVED_VERSION=$(echo "${WP_VERSIONS_JSON}" | jq -r --arg prefix "${WP_VERSION}." 'keys | map(select(startswith($prefix))) | sort_by(split(".") | map(tonumber)) | last // empty')
-		if [ -n "${RESOLVED_VERSION}" ]; then
-			export WP_VERSION="${RESOLVED_VERSION}"
-		fi
+	RESOLVED_VERSION=$( get_wp_versions | jq -r --arg prefix "${WP_VERSION}." 'keys | map( select( startswith( $prefix ) ) ) | sort_by( split(".") | map( tonumber ) ) | last // empty' )
+
+	if [ -n "${RESOLVED_VERSION}" ]; then
+		export WP_VERSION="${RESOLVED_VERSION}"
 	fi
 fi
 
@@ -141,6 +256,11 @@ if [[ "${WP_CLI_TEST_COVERAGE}" == "true" ]] && vendor/bin/behat --help 2>/dev/n
 	BEHAT_EXTRA_ARGS+=('--xdebug')
 fi
 
+# Honor the NO_COLOR convention (https://no-color.org/).
+if [ -n "${NO_COLOR}" ]; then
+	BEHAT_EXTRA_ARGS+=('--no-colors')
+fi
+
 # Run the functional tests.
 FORMAT_ARGS=(--format progress)
 for arg in "$@"; do