From 85dedd3c8b426bfd5d8605ac948ce7a21a3c86bf Mon Sep 17 00:00:00 2001 From: Sal Date: Wed, 2 Sep 2026 20:13:53 +0100 Subject: [PATCH] test(ci): cover the commit-lint policy patterns Three defects have shipped in commit-lint.yml. #575 could not start at all and took 53 runs to notice. The empty-pattern trap found during #586 would have flagged every commit and passed every branch. #587 failed open on a large commit message. Two of the three were silent, and nothing automated caught any of them. Add a suite that extracts every pattern from the workflow rather than restating it, so the test cannot drift into a second source of truth, and assert the constructs as well as the patterns: no grep -q on the trailer pipeline, an in-step fallback for each pattern, and no workflow_call input default that a pull_request run would ignore. Verified by mutation. Six deliberate regressions were introduced one at a time and every one failed the suite: reintroducing grep -q, removing a fallback, loosening the branch pattern to accept feature-0, dropping the copilot and codex prefixes, widening the trailer pattern to ban human co-authors, and raising the subject cap past 72. Closes #592 --- .github/workflows/commit-lint-test.yml | 42 ++++++ scripts/test-commit-lint-policy.sh | 188 +++++++++++++++++++++++++ 2 files changed, 230 insertions(+) create mode 100644 .github/workflows/commit-lint-test.yml create mode 100755 scripts/test-commit-lint-policy.sh diff --git a/.github/workflows/commit-lint-test.yml b/.github/workflows/commit-lint-test.yml new file mode 100644 index 000000000..27cd6aea7 --- /dev/null +++ b/.github/workflows/commit-lint-test.yml @@ -0,0 +1,42 @@ +--- +name: Commit Lint Policy Tests + +# Covers the patterns and constructs inside commit-lint.yml. The suite reads +# that workflow and asserts against what it finds, so it runs whenever either +# file changes. See z-shell/.github#592. + +on: + push: + paths: + - ".github/workflows/commit-lint.yml" + - ".github/workflows/commit-lint-test.yml" + - "scripts/test-commit-lint-policy.sh" + pull_request: + paths: + - ".github/workflows/commit-lint.yml" + - ".github/workflows/commit-lint-test.yml" + - "scripts/test-commit-lint-policy.sh" + workflow_dispatch: {} + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + commit-lint-policy-test: + name: Commit Lint Policy Tests + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check shell syntax + run: bash -n scripts/test-commit-lint-policy.sh + + - name: Test commit-lint policy patterns + run: bash scripts/test-commit-lint-policy.sh diff --git a/scripts/test-commit-lint-policy.sh b/scripts/test-commit-lint-policy.sh new file mode 100755 index 000000000..b956b4e7a --- /dev/null +++ b/scripts/test-commit-lint-policy.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +# Tests the policy patterns in .github/workflows/commit-lint.yml. +# +# Every pattern is EXTRACTED from the workflow rather than restated here. A +# test carrying its own copy of a regex drifts from the thing it claims to +# check and then proves nothing, which is the second source of truth AGENTS.md +# warns against. If an extraction fails, that is a test failure: the workflow +# changed shape and this file has to be re-pointed, not quietly skipped. +# +# Three defects have shipped in that workflow (z-shell/.github#575, the empty +# pattern trap found in #586, and the fail-open in #587). Two of the three +# failed silently. The cases below exist so a fourth does not. +# +# Dialect: Bash, floor 4.0. CI runs it on ubuntu-latest, which ships Bash 5. +# ShellCheck applies with bash selected, per +# .github/instructions/shell.instructions.md. +set -euo pipefail + +ROOT=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd) +WORKFLOW=$ROOT/.github/workflows/commit-lint.yml + +failures=0 +checks=0 + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + failures=$((failures + 1)) +} + +# Pull the single capture of an anchored extraction, failing loudly when the +# workflow no longer has the shape this file assumes. +extract() { + local label=$1 regex=$2 value + value=$(sed -nE "s/$regex/\1/p" "$WORKFLOW") + if [ -z "$value" ]; then + printf 'FAIL: could not extract %s from %s\n' "$label" "$WORKFLOW" >&2 + printf ' the workflow changed shape; re-point this test\n' >&2 + exit 1 + fi + if [ "$(printf '%s\n' "$value" | wc -l)" -ne 1 ]; then + printf 'FAIL: %s matched more than once in %s\n' "$label" "$WORKFLOW" >&2 + exit 1 + fi + printf '%s' "$value" +} + +TRAILER_PATTERN=$(extract "trailer pattern" \ + '^ *: "\$\{DISALLOWED_TRAILER_PATTERN:=(.*)\}"$') +BRANCH_PATTERN=$(extract "branch pattern" \ + '^ *: "\$\{BRANCH_PATTERN:=(.*)\}"$') +CONVENTIONAL_PATTERN=$(extract "conventional pattern" \ + "^ *CONVENTIONAL_PATTERN='(.*)'$") +PREFIX_PATTERN=$(extract "allowed prefixes" \ + "^ *if echo \"\\\$BRANCH\" \| grep -qE '(.*)' \|\| \\\\$") + +# --- guards on the constructs themselves --------------------------------- + +# z-shell/.github#587: grep -q exits on its first match, so under pipefail a +# commit message larger than the pipe buffer gives git a SIGPIPE and the +# pipeline reports no match. grep -c reads to the end and cannot fail open. +check_no_grep_q_on_trailer() { + checks=$((checks + 1)) + if grep -q 'git show .*|[[:space:]]*grep -q' "$WORKFLOW"; then + fail "trailer check pipes git show into grep -q; it fails open on a large message (#587)" + fi +} + +# z-shell/.github#586: workflow_call input defaults do not apply on a +# pull_request run, so each pattern needs an in-step fallback. An empty +# grep -E pattern matches every line, which silently passes every branch and +# flags every commit. +check_fallbacks_present() { + local name + for name in DISALLOWED_TRAILER_PATTERN BRANCH_PATTERN; do + checks=$((checks + 1)) + grep -q ": \"\${$name:=" "$WORKFLOW" || + fail "$name has no in-step fallback; an empty pattern matches everything (#586)" + done +} + +# The same trap in the other direction: a default left on the workflow_call +# input is a second source of truth that the pull_request path never reads. +check_no_input_defaults() { + checks=$((checks + 1)) + if sed -n '/workflow_call:/,/^concurrency:/p' "$WORKFLOW" | grep -q '^ *default:'; then + fail "workflow_call input carries a default that a pull_request run ignores (#586)" + fi +} + +# --- table-driven pattern cases ------------------------------------------ + +# assert_match