test: add shell script tests

.github/workflows/test-scripts.yml

@@ -1,7 +1,18 @@
 name: Test Scripts
 
-# Caller-only: the caller owns the paths: filter (its scripts, data files and tests).
+# Callers own their paths: filter (their scripts, data files and tests). The push/pull_request triggers are
+# tooling's own CI: its tests run the inline logic of every workflow here.
 on:
+  pull_request:
+    paths:
+      - ".github/workflows/**"
+      - "tests/**"
+  push:
+    branches: [main]
+    paths:
+      - ".github/workflows/**"
+      - "tests/**"
+  workflow_dispatch:
   workflow_call:
 
 permissions:

AGENTS.md

@@ -30,6 +30,8 @@ Reusable workflows (`workflow_call`) and the shell test harness shared by every
 | `.github/workflows/actionlint.yml`, `lint.yml` | actionlint, ShellCheck, PSScriptAnalyzer |
 | `.github/workflows/test-scripts.yml` | Runs the caller's `tests/*.test.sh` on ubuntu, macOS and Windows |
 | `tests/lib.sh` | Test harness, consumed as the `.tooling` git submodule (tests only, never at runtime) |
+| `tests/workflow.sh` | Tooling-only helpers: extract a step's `run:` block (`step_script`) or field (`step_field`) |
+| `tests/*.test.sh` | Tooling's own tests: release, check-refs, test-scripts, linters (Docker) and the harness itself |
 
 ## Project Rules
 
@@ -57,9 +59,10 @@ Reusable workflows (`workflow_call`) and the shell test harness shared by every
 
 ## Testing
 
-- `tests/lib.sh` must stay bash 3.2 compatible (macOS runners). Verify a change by running a consumer's tests against it, e.g. from `../flutter`: `bash tests/resolve-matrix.test.sh` with `.tooling` pointing at your working copy.
-- Inline `run:` logic you change (check-refs, release rewrite): extract it to a temp script and run it against a scratch git repo with sample refs before finishing.
+- Every inline `run:` block with logic has a test in `tests/<workflow>.test.sh` that runs the exact block via `step_script` (no copies, so tests can't drift). Add cases for every branch you change; refer to steps by `name:`, so renaming a step means updating its test.
+- `tests/lib.test.sh` covers the harness on all three OSes. A `lib.sh` change also gets a run of a consumer's tests, e.g. from `../flutter` with `.tooling` pointing at your working copy.
+- `test-scripts.yml` runs all of them on push/PR. Locally: `for t in tests/*.test.sh; do bash "$t"; done` (needs `yq` or python3 + PyYAML; `linters.test.sh` needs Docker and skips without it).
 - Local checks before finishing (run whichever apply):
-    - `shellcheck tests/lib.sh`
+    - `for t in tests/*.test.sh; do bash "$t"; done`
     - `docker run --rm -v "$PWD:/repo" -w /repo rhysd/actionlint:1.7.12` for workflow changes
 - Report honestly which checks ran and which could not (e.g. multi-OS builds only run in CI).

README.md

@@ -76,6 +76,10 @@ finish
 
 `REPO_ROOT` is the consuming repo (derived from the test file's path). Functions: `new_case`, `run`, `stub`, `only_on`, `output`, `env_var`, `stub_log`, `expect`, `expect_ok`, `expect_fail`, `expect_file`, `finish`. Bump the submodule with Dependabot (`gitsubmodule`).
 
+## Tooling's own tests
+
+`tests/*.test.sh` run the exact inline `run:` blocks of these workflows (extracted with `yq`) against scratch repos: release versioning, rewrite, tagging and floating tags; both check-refs rules; the test loop; actionlint and ShellCheck through the pinned image (needs Docker); and the harness itself on all three OSes. `test-scripts.yml` runs them on every push and PR here.
+
 ## Org setup (once)
 
 Releases authenticate as a GitHub App, because `GITHUB_TOKEN` can't push tags to commits that change `.github/workflows/`.

tests/check-refs.test.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# shellcheck source=tests/lib.sh
+source "$(dirname "$0")/lib.sh"
+# shellcheck source=tests/workflow.sh
+source "$(dirname "$0")/workflow.sh"
+only_on linux
+require_yaml_tool
+
+CHECK=$(step_script "$REPO_ROOT/.github/workflows/check-refs.yml" check-refs "Check references to dotactions repos")
+SHA=0123456789abcdef0123456789abcdef01234567
+GITVARS=(GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1)
+
+# check <file> <content>: a repo for dotactions/flutter with known-good refs, plus <content> written to <file>.
+check() {
+  new_case
+  mkdir -p "$WORK/.github/workflows" "$WORK/build"
+  cat > "$WORK/.github/workflows/ci.yml" << EOF
+uses: dotactions/flutter/build@main
+uses: dotactions/node/setup@$SHA # v0.6.0
+uses: dotactions/sarif-reporter@$SHA # v1.0.0
+uses: actions/checkout@v4
+EOF
+  echo 'uses: dotactions/flutter/matrix@main' > "$WORK/build/action.yml"
+  if [ -n "$1" ]; then
+    mkdir -p "$WORK/$(dirname "$1")"
+    echo "$2" >> "$WORK/$1"
+  fi
+  env "${GITVARS[@]}" git -C "$WORK" init -q
+  env "${GITVARS[@]}" git -C "$WORK" add -A
+  run "$CHECK" "${GITVARS[@]}" GITHUB_REPOSITORY=dotactions/flutter GITHUB_REPOSITORY_OWNER=dotactions
+}
+
+check "" ""
+expect_ok "self @main, other repos by sha + version, third party untouched"
+
+check action.yml "uses: dotactions/flutter@main"
+expect_ok "root action self-reference @main"
+
+check README.md "uses: dotactions/node/setup@v1"
+expect_ok "README is not checked"
+
+check build/action.yml "uses: dotactions/flutter/setup@v1"
+expect_fail "self-reference by tag" "must use @main"
+
+check .github/workflows/ci.yml "uses: dotactions/flutter/setup@$SHA # v1.0.0"
+expect_fail "self-reference by sha" "must use @main"
+
+check .github/workflows/ci.yml "uses: dotactions/node/setup@main"
+expect_fail "other repo @main" "must be pinned"
+
+check .github/workflows/ci.yml "uses: dotactions/node/setup@v0.6"
+expect_fail "other repo by floating tag" "must be pinned"
+
+check .github/workflows/ci.yml "uses: dotactions/node/setup@$SHA"
+expect_fail "other repo sha without version comment" "must be pinned"
+
+check .github/workflows/ci.yml "uses: dotactions/flutter-extra/x@main"
+expect_fail "lookalike repo is another repo" "must be pinned"
+
+check .github/workflows/x.yaml "uses: dotactions/node/setup@v1"
+expect_fail ".yaml workflows are checked" "must be pinned"
+
+check action.yaml "uses: dotactions/flutter/build@v1"
+expect_fail "root action.yaml is checked" "must use @main"
+
+finish

tests/lib.test.sh

@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+# Tests the harness by running small test files that use it, so every consuming repo can trust a pass or fail.
+# shellcheck source=tests/lib.sh
+source "$(dirname "$0")/lib.sh"
+only_on linux macos windows
+
+LIB="$REPO_ROOT/tests/lib.sh"
+
+# harness <body>: runs a test file at $WORK/tests/inner.test.sh that sources lib.sh, then <body>.
+harness() {
+  new_case
+  mkdir -p "$WORK/tests"
+  printf 'source "%s"\n%s\n' "$LIB" "$1" > "$WORK/tests/inner.test.sh"
+  run "$WORK/tests/inner.test.sh"
+}
+says() { grep -qF -- "$1" "$CASE_DIR/stdout" && echo yes; }
+
+harness 'finish'
+expect "a test file without assertions fails" "1" "$STATUS"
+expect "and says why" "yes" "$(says "no assertions ran")"
+
+harness 'expect same a a
+finish'
+expect_ok "a passing assertion passes"
+expect "summary counts it" "yes" "$(says "1/1 passed in inner.test.sh")"
+
+harness 'expect differs a b
+finish'
+expect "a failing assertion fails the file" "1" "$STATUS"
+expect "failure shows expected and actual" "yes" "$(says "FAIL differs: expected 'a', got 'b'")"
+
+harness 'only_on nowhere
+expect never a b
+finish'
+expect_ok "only_on skips other OSes before any assertion"
+expect "skip is reported" "yes" "$(says "skip inner.test.sh")"
+
+# shellcheck disable=SC2016 # expands inside the inner test.
+harness 'echo "$REPO_ROOT"
+finish'
+expect "REPO_ROOT is the repo of the test file, not of lib.sh" "yes" "$(says "$(cd "$WORK" && pwd)")"
+
+# shellcheck disable=SC2016 # expands inside the inner test.
+harness 'export LEAK=outer
+printf "echo \"\${LEAK:-unset} \${GIVEN:-}\"\n" > "$TEST_TMP/s.sh"
+new_case
+run "$TEST_TMP/s.sh" GIVEN=passed
+expect env "unset passed" "$(cat "$CASE_DIR/stdout")"
+finish'
+expect_ok "run hides the outer environment and passes VAR=val"
+
+# shellcheck disable=SC2016 # expands inside the inner test.
+harness 'printf "echo a=1 >> \"\$GITHUB_OUTPUT\"\necho a=2 >> \"\$GITHUB_OUTPUT\"\necho E=x >> \"\$GITHUB_ENV\"\n" > "$TEST_TMP/s.sh"
+new_case
+run "$TEST_TMP/s.sh"
+expect last "2" "$(output a)"
+expect env "x" "$(env_var E)"
+finish'
+expect_ok "output takes the last value; env_var reads GITHUB_ENV"
+
+# shellcheck disable=SC2016 # expands inside the inner test.
+harness 'printf "echo \"::error::bad thing\"\nexit 1\n" > "$TEST_TMP/s.sh"
+new_case
+run "$TEST_TMP/s.sh"
+expect_fail matches "bad thing"
+finish'
+expect_ok "expect_fail passes on a matching ::error::"
+
+# shellcheck disable=SC2016 # expands inside the inner test.
+harness 'printf "echo \"::error::bad thing\"\nexit 1\n" > "$TEST_TMP/s.sh"
+new_case
+run "$TEST_TMP/s.sh"
+expect_fail other "something else"
+finish'
+expect "expect_fail fails on a different ::error::" "1" "$STATUS"
+
+# shellcheck disable=SC2016 # expands inside the inner test.
+harness 'printf "exit 0\n" > "$TEST_TMP/s.sh"
+new_case
+run "$TEST_TMP/s.sh"
+expect_fail exits0
+finish'
+expect "expect_fail fails when the script succeeds" "1" "$STATUS"
+
+# shellcheck disable=SC2016 # expands inside the inner test.
+harness 'stub tool "echo stubbed"
+printf "tool a b\n" > "$TEST_TMP/s.sh"
+new_case
+run "$TEST_TMP/s.sh"
+expect log "tool a b" "$(stub_log)"
+expect body "stubbed" "$(cat "$CASE_DIR/stdout")"
+finish'
+expect_ok "stub logs its calls and runs its body"
+
+# shellcheck disable=SC2016 # expands inside the inner test.
+harness 'new_case
+touch "$WORK/made"
+expect_file there made
+finish'
+expect_ok "expect_file finds a file in WORK"
+
+finish

tests/linters.test.sh

@@ -0,0 +1,93 @@
+#!/usr/bin/env bash
+# shellcheck source=tests/lib.sh
+source "$(dirname "$0")/lib.sh"
+# shellcheck source=tests/workflow.sh
+source "$(dirname "$0")/workflow.sh"
+only_on linux
+require_yaml_tool
+if ! docker info > /dev/null 2>&1; then
+  echo "skip $(basename "$0"): docker not available (it is on ubuntu runners)"
+  exit 0
+fi
+
+ACTIONLINT_WF="$REPO_ROOT/.github/workflows/actionlint.yml"
+LINT_WF="$REPO_ROOT/.github/workflows/lint.yml"
+ACTIONLINT=$(step_script "$ACTIONLINT_WF" actionlint "Run actionlint")
+SHELLCHECK=$(step_script "$LINT_WF" shellcheck "Run ShellCheck")
+ACTIONLINT_IMAGE=$(step_field "$ACTIONLINT_WF" actionlint "Run actionlint" env.ACTIONLINT_IMAGE)
+SHELLCHECK_IMAGE=$(step_field "$LINT_WF" shellcheck "Run ShellCheck" env.ACTIONLINT_IMAGE)
+
+expect "both workflows pin the same image" "$ACTIONLINT_IMAGE" "$SHELLCHECK_IMAGE"
+
+GITVARS=(GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1)
+DOCKERVARS=()
+if [ -n "${DOCKER_HOST:-}" ]; then DOCKERVARS=(DOCKER_HOST="$DOCKER_HOST"); fi
+
+# put <path> <content>: a tracked file in $WORK, since both linters only see git-tracked files.
+put() {
+  mkdir -p "$WORK/$(dirname "$1")"
+  printf '%s\n' "$2" > "$WORK/$1"
+  env "${GITVARS[@]}" git -C "$WORK" init -q
+  env "${GITVARS[@]}" git -C "$WORK" add "$1"
+}
+lint() { run "$1" "${GITVARS[@]}" "${DOCKERVARS[@]}" ACTIONLINT_IMAGE="$ACTIONLINT_IMAGE"; }
+stdout_has() { grep -qE -- "$1" "$CASE_DIR/stdout" && echo yes; }
+
+# --- actionlint
+new_case
+put .github/workflows/ok.yml 'on: push
+jobs:
+  a:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo ok'
+lint "$ACTIONLINT"
+expect_ok "clean workflow passes"
+
+new_case
+# shellcheck disable=SC2016 # the ${{ }} is the workflow under test, not bash.
+put .github/workflows/bad.yml 'on: push
+jobs:
+  a:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo ${{ github.nope }}'
+lint "$ACTIONLINT"
+expect "broken workflow fails" "yes" "$([ "$STATUS" -ne 0 ] && echo yes)"
+expect "error is a file/line annotation" "yes" "$(stdout_has '^::error file=\.github/workflows/bad\.yml,line=6,col=[0-9]+::')"
+
+# --- ShellCheck
+new_case
+mkdir -p "$WORK/lib"
+# Like .tooling/tests/lib.sh in a consumer: on disk but untracked, so only -x makes ShellCheck follow it.
+# shellcheck disable=SC2016 # the script under test, not this one.
+printf '%s\n' '# shellcheck shell=bash' 'greet() { echo "hi $1"; }' > "$WORK/lib/helper.sh"
+# shellcheck disable=SC2016 # the script under test, not this one.
+put main.sh '#!/usr/bin/env bash
+# shellcheck source=lib/helper.sh
+source "$(dirname "$0")/lib/helper.sh"
+greet "$@"'
+lint "$SHELLCHECK"
+expect_ok "untracked sourced file is followed (-x)"
+
+new_case
+# shellcheck disable=SC2016 # the script under test, not this one.
+put bad.sh '#!/usr/bin/env bash
+echo $1'
+lint "$SHELLCHECK"
+expect "unquoted variable fails" "yes" "$([ "$STATUS" -ne 0 ] && echo yes)"
+expect "finding names the rule" "yes" "$(stdout_has 'SC2086')"
+
+new_case
+# shellcheck disable=SC2016 # the script under test, not this one.
+put tests/fixtures/bad.sh '#!/usr/bin/env bash
+echo $1'
+lint "$SHELLCHECK"
+expect_ok "tests/fixtures is skipped"
+
+new_case
+put README.md 'no scripts here'
+lint "$SHELLCHECK"
+expect_ok "repo without scripts passes"
+
+finish