From 214d6f6735a6dd58de0e3e7bd5d031b899b49bc9 Mon Sep 17 00:00:00 2001 From: Sem Van Broekhoven <144097969+dotsem@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:53:07 +0100 Subject: [PATCH] test: add shell script tests --- .github/workflows/test-scripts.yml | 13 ++- AGENTS.md | 9 +- README.md | 4 + tests/check-refs.test.sh | 66 ++++++++++++++ tests/lib.test.sh | 102 +++++++++++++++++++++ tests/linters.test.sh | 93 +++++++++++++++++++ tests/release.test.sh | 138 +++++++++++++++++++++++++++++ tests/test-scripts.test.sh | 33 +++++++ tests/workflow.sh | 43 +++++++++ 9 files changed, 497 insertions(+), 4 deletions(-) create mode 100644 tests/check-refs.test.sh create mode 100644 tests/lib.test.sh create mode 100644 tests/linters.test.sh create mode 100644 tests/release.test.sh create mode 100644 tests/test-scripts.test.sh create mode 100644 tests/workflow.sh diff --git a/.github/workflows/test-scripts.yml b/.github/workflows/test-scripts.yml index 41c9243..5e23e52 100644 --- a/.github/workflows/test-scripts.yml +++ b/.github/workflows/test-scripts.yml @@ -1,7 +1,18 @@ name: Test Scripts -# Caller-only: the caller owns the paths: filter (its scripts, data files and tests). +# Callers own their paths: filter (their scripts, data files and tests). The push/pull_request triggers are +# tooling's own CI: its tests run the inline logic of every workflow here. on: + pull_request: + paths: + - ".github/workflows/**" + - "tests/**" + push: + branches: [main] + paths: + - ".github/workflows/**" + - "tests/**" + workflow_dispatch: workflow_call: permissions: diff --git a/AGENTS.md b/AGENTS.md index e8bacc8..43bc26c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,6 +30,8 @@ Reusable workflows (`workflow_call`) and the shell test harness shared by every | `.github/workflows/actionlint.yml`, `lint.yml` | actionlint, ShellCheck, PSScriptAnalyzer | | `.github/workflows/test-scripts.yml` | Runs the caller's `tests/*.test.sh` on ubuntu, macOS and Windows | | `tests/lib.sh` | Test harness, consumed as the `.tooling` git submodule (tests only, never at runtime) | +| `tests/workflow.sh` | Tooling-only helpers: extract a step's `run:` block (`step_script`) or field (`step_field`) | +| `tests/*.test.sh` | Tooling's own tests: release, check-refs, test-scripts, linters (Docker) and the harness itself | ## Project Rules @@ -57,9 +59,10 @@ Reusable workflows (`workflow_call`) and the shell test harness shared by every ## Testing -- `tests/lib.sh` must stay bash 3.2 compatible (macOS runners). Verify a change by running a consumer's tests against it, e.g. from `../flutter`: `bash tests/resolve-matrix.test.sh` with `.tooling` pointing at your working copy. -- Inline `run:` logic you change (check-refs, release rewrite): extract it to a temp script and run it against a scratch git repo with sample refs before finishing. +- Every inline `run:` block with logic has a test in `tests/.test.sh` that runs the exact block via `step_script` (no copies, so tests can't drift). Add cases for every branch you change; refer to steps by `name:`, so renaming a step means updating its test. +- `tests/lib.test.sh` covers the harness on all three OSes. A `lib.sh` change also gets a run of a consumer's tests, e.g. from `../flutter` with `.tooling` pointing at your working copy. +- `test-scripts.yml` runs all of them on push/PR. Locally: `for t in tests/*.test.sh; do bash "$t"; done` (needs `yq` or python3 + PyYAML; `linters.test.sh` needs Docker and skips without it). - Local checks before finishing (run whichever apply): - - `shellcheck tests/lib.sh` + - `for t in tests/*.test.sh; do bash "$t"; done` - `docker run --rm -v "$PWD:/repo" -w /repo rhysd/actionlint:1.7.12` for workflow changes - Report honestly which checks ran and which could not (e.g. multi-OS builds only run in CI). diff --git a/README.md b/README.md index 85d006a..f5db51b 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,10 @@ finish `REPO_ROOT` is the consuming repo (derived from the test file's path). Functions: `new_case`, `run`, `stub`, `only_on`, `output`, `env_var`, `stub_log`, `expect`, `expect_ok`, `expect_fail`, `expect_file`, `finish`. Bump the submodule with Dependabot (`gitsubmodule`). +## Tooling's own tests + +`tests/*.test.sh` run the exact inline `run:` blocks of these workflows (extracted with `yq`) against scratch repos: release versioning, rewrite, tagging and floating tags; both check-refs rules; the test loop; actionlint and ShellCheck through the pinned image (needs Docker); and the harness itself on all three OSes. `test-scripts.yml` runs them on every push and PR here. + ## Org setup (once) Releases authenticate as a GitHub App, because `GITHUB_TOKEN` can't push tags to commits that change `.github/workflows/`. diff --git a/tests/check-refs.test.sh b/tests/check-refs.test.sh new file mode 100644 index 0000000..e57cfc9 --- /dev/null +++ b/tests/check-refs.test.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# shellcheck source=tests/lib.sh +source "$(dirname "$0")/lib.sh" +# shellcheck source=tests/workflow.sh +source "$(dirname "$0")/workflow.sh" +only_on linux +require_yaml_tool + +CHECK=$(step_script "$REPO_ROOT/.github/workflows/check-refs.yml" check-refs "Check references to dotactions repos") +SHA=0123456789abcdef0123456789abcdef01234567 +GITVARS=(GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1) + +# check : a repo for dotactions/flutter with known-good refs, plus written to . +check() { + new_case + mkdir -p "$WORK/.github/workflows" "$WORK/build" + cat > "$WORK/.github/workflows/ci.yml" << EOF +uses: dotactions/flutter/build@main +uses: dotactions/node/setup@$SHA # v0.6.0 +uses: dotactions/sarif-reporter@$SHA # v1.0.0 +uses: actions/checkout@v4 +EOF + echo 'uses: dotactions/flutter/matrix@main' > "$WORK/build/action.yml" + if [ -n "$1" ]; then + mkdir -p "$WORK/$(dirname "$1")" + echo "$2" >> "$WORK/$1" + fi + env "${GITVARS[@]}" git -C "$WORK" init -q + env "${GITVARS[@]}" git -C "$WORK" add -A + run "$CHECK" "${GITVARS[@]}" GITHUB_REPOSITORY=dotactions/flutter GITHUB_REPOSITORY_OWNER=dotactions +} + +check "" "" +expect_ok "self @main, other repos by sha + version, third party untouched" + +check action.yml "uses: dotactions/flutter@main" +expect_ok "root action self-reference @main" + +check README.md "uses: dotactions/node/setup@v1" +expect_ok "README is not checked" + +check build/action.yml "uses: dotactions/flutter/setup@v1" +expect_fail "self-reference by tag" "must use @main" + +check .github/workflows/ci.yml "uses: dotactions/flutter/setup@$SHA # v1.0.0" +expect_fail "self-reference by sha" "must use @main" + +check .github/workflows/ci.yml "uses: dotactions/node/setup@main" +expect_fail "other repo @main" "must be pinned" + +check .github/workflows/ci.yml "uses: dotactions/node/setup@v0.6" +expect_fail "other repo by floating tag" "must be pinned" + +check .github/workflows/ci.yml "uses: dotactions/node/setup@$SHA" +expect_fail "other repo sha without version comment" "must be pinned" + +check .github/workflows/ci.yml "uses: dotactions/flutter-extra/x@main" +expect_fail "lookalike repo is another repo" "must be pinned" + +check .github/workflows/x.yaml "uses: dotactions/node/setup@v1" +expect_fail ".yaml workflows are checked" "must be pinned" + +check action.yaml "uses: dotactions/flutter/build@v1" +expect_fail "root action.yaml is checked" "must use @main" + +finish diff --git a/tests/lib.test.sh b/tests/lib.test.sh new file mode 100644 index 0000000..fac3b64 --- /dev/null +++ b/tests/lib.test.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Tests the harness by running small test files that use it, so every consuming repo can trust a pass or fail. +# shellcheck source=tests/lib.sh +source "$(dirname "$0")/lib.sh" +only_on linux macos windows + +LIB="$REPO_ROOT/tests/lib.sh" + +# harness : runs a test file at $WORK/tests/inner.test.sh that sources lib.sh, then . +harness() { + new_case + mkdir -p "$WORK/tests" + printf 'source "%s"\n%s\n' "$LIB" "$1" > "$WORK/tests/inner.test.sh" + run "$WORK/tests/inner.test.sh" +} +says() { grep -qF -- "$1" "$CASE_DIR/stdout" && echo yes; } + +harness 'finish' +expect "a test file without assertions fails" "1" "$STATUS" +expect "and says why" "yes" "$(says "no assertions ran")" + +harness 'expect same a a +finish' +expect_ok "a passing assertion passes" +expect "summary counts it" "yes" "$(says "1/1 passed in inner.test.sh")" + +harness 'expect differs a b +finish' +expect "a failing assertion fails the file" "1" "$STATUS" +expect "failure shows expected and actual" "yes" "$(says "FAIL differs: expected 'a', got 'b'")" + +harness 'only_on nowhere +expect never a b +finish' +expect_ok "only_on skips other OSes before any assertion" +expect "skip is reported" "yes" "$(says "skip inner.test.sh")" + +# shellcheck disable=SC2016 # expands inside the inner test. +harness 'echo "$REPO_ROOT" +finish' +expect "REPO_ROOT is the repo of the test file, not of lib.sh" "yes" "$(says "$(cd "$WORK" && pwd)")" + +# shellcheck disable=SC2016 # expands inside the inner test. +harness 'export LEAK=outer +printf "echo \"\${LEAK:-unset} \${GIVEN:-}\"\n" > "$TEST_TMP/s.sh" +new_case +run "$TEST_TMP/s.sh" GIVEN=passed +expect env "unset passed" "$(cat "$CASE_DIR/stdout")" +finish' +expect_ok "run hides the outer environment and passes VAR=val" + +# shellcheck disable=SC2016 # expands inside the inner test. +harness 'printf "echo a=1 >> \"\$GITHUB_OUTPUT\"\necho a=2 >> \"\$GITHUB_OUTPUT\"\necho E=x >> \"\$GITHUB_ENV\"\n" > "$TEST_TMP/s.sh" +new_case +run "$TEST_TMP/s.sh" +expect last "2" "$(output a)" +expect env "x" "$(env_var E)" +finish' +expect_ok "output takes the last value; env_var reads GITHUB_ENV" + +# shellcheck disable=SC2016 # expands inside the inner test. +harness 'printf "echo \"::error::bad thing\"\nexit 1\n" > "$TEST_TMP/s.sh" +new_case +run "$TEST_TMP/s.sh" +expect_fail matches "bad thing" +finish' +expect_ok "expect_fail passes on a matching ::error::" + +# shellcheck disable=SC2016 # expands inside the inner test. +harness 'printf "echo \"::error::bad thing\"\nexit 1\n" > "$TEST_TMP/s.sh" +new_case +run "$TEST_TMP/s.sh" +expect_fail other "something else" +finish' +expect "expect_fail fails on a different ::error::" "1" "$STATUS" + +# shellcheck disable=SC2016 # expands inside the inner test. +harness 'printf "exit 0\n" > "$TEST_TMP/s.sh" +new_case +run "$TEST_TMP/s.sh" +expect_fail exits0 +finish' +expect "expect_fail fails when the script succeeds" "1" "$STATUS" + +# shellcheck disable=SC2016 # expands inside the inner test. +harness 'stub tool "echo stubbed" +printf "tool a b\n" > "$TEST_TMP/s.sh" +new_case +run "$TEST_TMP/s.sh" +expect log "tool a b" "$(stub_log)" +expect body "stubbed" "$(cat "$CASE_DIR/stdout")" +finish' +expect_ok "stub logs its calls and runs its body" + +# shellcheck disable=SC2016 # expands inside the inner test. +harness 'new_case +touch "$WORK/made" +expect_file there made +finish' +expect_ok "expect_file finds a file in WORK" + +finish diff --git a/tests/linters.test.sh b/tests/linters.test.sh new file mode 100644 index 0000000..a06526b --- /dev/null +++ b/tests/linters.test.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# shellcheck source=tests/lib.sh +source "$(dirname "$0")/lib.sh" +# shellcheck source=tests/workflow.sh +source "$(dirname "$0")/workflow.sh" +only_on linux +require_yaml_tool +if ! docker info > /dev/null 2>&1; then + echo "skip $(basename "$0"): docker not available (it is on ubuntu runners)" + exit 0 +fi + +ACTIONLINT_WF="$REPO_ROOT/.github/workflows/actionlint.yml" +LINT_WF="$REPO_ROOT/.github/workflows/lint.yml" +ACTIONLINT=$(step_script "$ACTIONLINT_WF" actionlint "Run actionlint") +SHELLCHECK=$(step_script "$LINT_WF" shellcheck "Run ShellCheck") +ACTIONLINT_IMAGE=$(step_field "$ACTIONLINT_WF" actionlint "Run actionlint" env.ACTIONLINT_IMAGE) +SHELLCHECK_IMAGE=$(step_field "$LINT_WF" shellcheck "Run ShellCheck" env.ACTIONLINT_IMAGE) + +expect "both workflows pin the same image" "$ACTIONLINT_IMAGE" "$SHELLCHECK_IMAGE" + +GITVARS=(GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1) +DOCKERVARS=() +if [ -n "${DOCKER_HOST:-}" ]; then DOCKERVARS=(DOCKER_HOST="$DOCKER_HOST"); fi + +# put : a tracked file in $WORK, since both linters only see git-tracked files. +put() { + mkdir -p "$WORK/$(dirname "$1")" + printf '%s\n' "$2" > "$WORK/$1" + env "${GITVARS[@]}" git -C "$WORK" init -q + env "${GITVARS[@]}" git -C "$WORK" add "$1" +} +lint() { run "$1" "${GITVARS[@]}" "${DOCKERVARS[@]}" ACTIONLINT_IMAGE="$ACTIONLINT_IMAGE"; } +stdout_has() { grep -qE -- "$1" "$CASE_DIR/stdout" && echo yes; } + +# --- actionlint +new_case +put .github/workflows/ok.yml 'on: push +jobs: + a: + runs-on: ubuntu-latest + steps: + - run: echo ok' +lint "$ACTIONLINT" +expect_ok "clean workflow passes" + +new_case +# shellcheck disable=SC2016 # the ${{ }} is the workflow under test, not bash. +put .github/workflows/bad.yml 'on: push +jobs: + a: + runs-on: ubuntu-latest + steps: + - run: echo ${{ github.nope }}' +lint "$ACTIONLINT" +expect "broken workflow fails" "yes" "$([ "$STATUS" -ne 0 ] && echo yes)" +expect "error is a file/line annotation" "yes" "$(stdout_has '^::error file=\.github/workflows/bad\.yml,line=6,col=[0-9]+::')" + +# --- ShellCheck +new_case +mkdir -p "$WORK/lib" +# Like .tooling/tests/lib.sh in a consumer: on disk but untracked, so only -x makes ShellCheck follow it. +# shellcheck disable=SC2016 # the script under test, not this one. +printf '%s\n' '# shellcheck shell=bash' 'greet() { echo "hi $1"; }' > "$WORK/lib/helper.sh" +# shellcheck disable=SC2016 # the script under test, not this one. +put main.sh '#!/usr/bin/env bash +# shellcheck source=lib/helper.sh +source "$(dirname "$0")/lib/helper.sh" +greet "$@"' +lint "$SHELLCHECK" +expect_ok "untracked sourced file is followed (-x)" + +new_case +# shellcheck disable=SC2016 # the script under test, not this one. +put bad.sh '#!/usr/bin/env bash +echo $1' +lint "$SHELLCHECK" +expect "unquoted variable fails" "yes" "$([ "$STATUS" -ne 0 ] && echo yes)" +expect "finding names the rule" "yes" "$(stdout_has 'SC2086')" + +new_case +# shellcheck disable=SC2016 # the script under test, not this one. +put tests/fixtures/bad.sh '#!/usr/bin/env bash +echo $1' +lint "$SHELLCHECK" +expect_ok "tests/fixtures is skipped" + +new_case +put README.md 'no scripts here' +lint "$SHELLCHECK" +expect_ok "repo without scripts passes" + +finish diff --git a/tests/release.test.sh b/tests/release.test.sh new file mode 100644 index 0000000..eb2fedd --- /dev/null +++ b/tests/release.test.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# shellcheck source=tests/lib.sh +source "$(dirname "$0")/lib.sh" +# shellcheck source=tests/workflow.sh +source "$(dirname "$0")/workflow.sh" +only_on linux +require_yaml_tool + +WF="$REPO_ROOT/.github/workflows/release.yml" +VALIDATE=$(step_script "$WF" release "Validate inputs") +EXISTS=$(step_script "$WF" release "Ensure tag does not exist") +PIN=$(step_script "$WF" release "Pin self-references to the release tag") +COMMIT=$(step_script "$WF" release "Commit, tag and push") +RELEASE=$(step_script "$WF" release "Create GitHub release") +FLOAT=$(step_script "$WF" release "Update floating tags") + +# The developer's own git config (signing, hooks) must not leak into the cases. +GITVARS=(GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1) +REAL_GIT=$(command -v git) +# Pushes land in the stub log and never leave the machine; every other git command is real. +stub git "if [ \"\$1\" = push ]; then exit 0; fi +exec \"$REAL_GIT\" \"\$@\"" +stub gh + +g() { env "${GITVARS[@]}" "$REAL_GIT" -C "$WORK" "$@"; } +repo() { + g init -q -b main + g config user.name test + g config user.email test@example.com + g commit -q --allow-empty -m init +} +at() { g rev-parse -q --verify "$1^{commit}" || true; } + +# --- Validate inputs +validate() { + new_case + run "$VALIDATE" GITHUB_REF=refs/heads/main VERSION="$1" +} +for v in 1.0.0:1.0.0 v1.0.0:1.0.0 v0.1.0-rc.1:0.1.0-rc.1; do + validate "${v%%:*}" + expect_ok "accepts ${v%%:*}" + expect "${v%%:*} becomes ${v#*:}" "${v#*:}" "$(output version)" +done +for bad in vv1.0.0 V1.0.0 1.0 1.0.0.0 ""; do + validate "$bad" + expect_fail "rejects '$bad'" "is not a valid semver version" +done +new_case +run "$VALIDATE" GITHUB_REF=refs/heads/feature VERSION=1.0.0 +expect_fail "rejects a branch other than main" "Releases must be cut from main" + +# --- Ensure tag does not exist +new_case +repo +g tag -a v1.0.0 -m v1.0.0 +run "$EXISTS" "${GITVARS[@]}" VERSION=1.0.0 +expect_fail "existing tag" "Tag v1.0.0 already exists" +run "$EXISTS" "${GITVARS[@]}" VERSION=1.0.1 +expect_ok "new tag" + +# --- Pin self-references +SHA=0123456789abcdef0123456789abcdef01234567 +new_case +repo +mkdir -p "$WORK/.github/workflows" "$WORK/build" +cat > "$WORK/.github/workflows/ci.yml" << EOF +uses: dotactions/flutter/build@main +uses: dotactions/flutter/.github/workflows/cd.yml@main +uses: dotactions/flutter-extra/x@main +uses: dotactions/node/setup@$SHA # v0.6.0 +EOF +echo 'uses: dotactions/flutter/setup@main' > "$WORK/.github/workflows/other.yaml" +echo 'uses: dotactions/flutter/matrix@main' > "$WORK/build/action.yml" +echo 'uses: dotactions/flutter@main' > "$WORK/action.yaml" +echo 'uses: dotactions/flutter/build@main' > "$WORK/README.md" +g add -A +g commit -q -m files +run "$PIN" "${GITVARS[@]}" VERSION=1.2.3 GITHUB_REPOSITORY=dotactions/flutter +expect_ok "pin exits 0" +expect "workflow self-references pinned, lookalike and other repos untouched" "uses: dotactions/flutter/build@v1.2.3 +uses: dotactions/flutter/.github/workflows/cd.yml@v1.2.3 +uses: dotactions/flutter-extra/x@main +uses: dotactions/node/setup@$SHA # v0.6.0" "$(cat "$WORK/.github/workflows/ci.yml")" +expect ".yaml workflow pinned" "uses: dotactions/flutter/setup@v1.2.3" "$(cat "$WORK/.github/workflows/other.yaml")" +expect "nested action.yml pinned" "uses: dotactions/flutter/matrix@v1.2.3" "$(cat "$WORK/build/action.yml")" +expect "root action.yaml pinned" "uses: dotactions/flutter@v1.2.3" "$(cat "$WORK/action.yaml")" +expect "README is never rewritten" "uses: dotactions/flutter/build@main" "$(cat "$WORK/README.md")" + +# --- Commit, tag and push +new_case +repo +echo before > "$WORK/file" +g add file +g commit -q -m file +echo after > "$WORK/file" +run "$COMMIT" "${GITVARS[@]}" VERSION=1.2.3 +expect_ok "commit exits 0" +expect "release commit message" "chore(release): v1.2.3" "$(g log -1 --format=%s v1.2.3)" +expect "release commit holds the rewrite" "after" "$(g show v1.2.3:file)" +expect "tag is annotated" "tag" "$(g cat-file -t v1.2.3)" +expect "main is untouched" "file" "$(g log -1 --format=%s main)" +expect "only the tag is pushed" "git push origin v1.2.3" "$(stub_log | grep '^git push')" + +# --- Create GitHub release +new_case +run "$RELEASE" VERSION=1.0.0-rc.1 GH_TOKEN=x +expect "pre-release is marked" "gh release create v1.0.0-rc.1 --verify-tag --generate-notes --title v1.0.0-rc.1 --prerelease" "$(stub_log)" +new_case +run "$RELEASE" VERSION=1.0.0 GH_TOKEN=x +expect "stable release is not marked" "gh release create v1.0.0 --verify-tag --generate-notes --title v1.0.0" "$(stub_log)" + +# --- Update floating tags: one repo, a sequence of releases on separate commits +new_case +repo +release() { + g commit -q --allow-empty -m "$1" + g tag -a "v$1" -m "v$1" + run "$FLOAT" "${GITVARS[@]}" VERSION="$1" +} +release 0.5.0 +expect "0.5.0 moves v0" "$(at v0.5.0)" "$(at v0)" +expect "0.5.0 moves v0.5" "$(at v0.5.0)" "$(at v0.5)" +release 0.6.0 +expect "0.6.0 moves v0" "$(at v0.6.0)" "$(at v0)" +release 0.5.1 +expect_ok "older release exits 0" +expect "0.5.1 leaves v0 on 0.6.0" "$(at v0.6.0)" "$(at v0)" +expect "0.5.1 moves v0.5" "$(at v0.5.1)" "$(at v0.5)" +expect "skipped tag is explained" "yes" "$(grep -q '::notice::Not moving v0' "$CASE_DIR/stdout" && echo yes)" +release 1.0.0 +expect "1.0.0 moves v1" "$(at v1.0.0)" "$(at v1)" +expect "1.0.0 leaves v0 alone" "$(at v0.6.0)" "$(at v0)" +expect "no minor tag floats after 0.x" "" "$(at v1.0)" +release 1.1.0-rc.1 +expect "pre-release moves nothing" "$(at v1.0.0)" "$(at v1)" +expect "floating tags are force-pushed" "git push origin v1 --force" "$(stub_log | grep '^git push' | tail -n 1)" + +finish diff --git a/tests/test-scripts.test.sh b/tests/test-scripts.test.sh new file mode 100644 index 0000000..0484e37 --- /dev/null +++ b/tests/test-scripts.test.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# shellcheck source=tests/lib.sh +source "$(dirname "$0")/lib.sh" +# shellcheck source=tests/workflow.sh +source "$(dirname "$0")/workflow.sh" +only_on linux +require_yaml_tool + +LOOP=$(step_script "$REPO_ROOT/.github/workflows/test-scripts.yml" test-scripts "Run tests/*.test.sh") + +# with_tests ...: a repo whose tests/.test.sh exit with the given codes. +with_tests() { + new_case + mkdir -p "$WORK/tests" + local t + for t in "$@"; do + printf 'echo "ran %s"\nexit %s\n' "${t%%:*}" "${t#*:}" > "$WORK/tests/${t%%:*}.test.sh" + done + run "$LOOP" +} + +with_tests +expect_fail "no tests is an error, not a silent pass" "No tests/*.test.sh found" + +with_tests a:0 b:0 +expect_ok "all passing" + +with_tests a:1 b:0 +expect "one failure fails the run" "1" "$STATUS" +expect "tests after a failure still run" "yes" "$(grep -q '^ran b$' "$CASE_DIR/stdout" && echo yes)" +expect "each test gets a log group" "2" "$(grep -c '^::group::tests/' "$CASE_DIR/stdout")" + +finish diff --git a/tests/workflow.sh b/tests/workflow.sh new file mode 100644 index 0000000..9147827 --- /dev/null +++ b/tests/workflow.sh @@ -0,0 +1,43 @@ +# shellcheck shell=bash +# Tooling's own test helpers: run a workflow step's inline run: block as a script, so tests exercise the shipped code. +# Sourced after lib.sh. Needs yq (preinstalled on ubuntu runners) or python3 with PyYAML. + +require_yaml_tool() { + if ! command -v yq > /dev/null && ! python3 -c 'import yaml' 2> /dev/null; then + echo "skip $(basename "$0"): needs yq or python3 with PyYAML" + exit 0 + fi +} + +# step_field +step_field() { + local value + if command -v yq > /dev/null; then + value=$(yq -r ".jobs.\"$2\".steps[] | select(.name == \"$3\") | .$4" "$1") + else + value=$(python3 - "$@" << 'PY' +import sys, yaml +wf, job, step, field = sys.argv[1:] +for s in yaml.safe_load(open(wf))["jobs"][job]["steps"]: + if s.get("name") == step: + for key in field.split("."): + s = s[key] + print(s) +PY +) + fi + if [ -z "$value" ] || [ "$value" = null ]; then + echo "FAIL $(basename "$0"): no '$4' in step '$3' of job '$2' in $1" >&2 + return 1 + fi + printf '%s\n' "$value" +} + +# step_script : prints the path of a script holding that step's run: block, +# with the flags GitHub uses for `shell: bash`. +step_script() { + local script + script=$(mktemp "$TEST_TMP/step.XXXXXX") + { echo "set -eo pipefail"; step_field "$1" "$2" "$3" run; } > "$script" || exit 1 + echo "$script" +}