diff --git a/.github/workflows/reusable-ci.yml b/.github/workflows/reusable-ci.yml index 2254a442..e6f6ab99 100644 --- a/.github/workflows/reusable-ci.yml +++ b/.github/workflows/reusable-ci.yml @@ -1,6 +1,7 @@ # file: .github/workflows/reusable-ci.yml -# version: 1.15.0 +# version: 1.16.0 # guid: reusable-ci-2025-09-24-core-workflow +# last-edited: 2026-10-05 name: Reusable CI Workflow @@ -62,6 +63,18 @@ on: required: false type: string default: '' + force-all: + description: >- + Run every CI job whose language the repository contains, instead of only + the jobs whose files changed. 'auto' (default) forces on schedule and + workflow_dispatch events, where there is no meaningful diff to inspect + (a scheduled run compares the commit with itself and finds nothing); + 'true' always forces; 'false' never does. A forced category is reported + as changed only if the repository tracks at least one file matching its + filter, so forcing never starts rust-ci in a repo with no Rust. + required: false + type: string + default: 'auto' outputs: go-files: description: 'Whether Go files were detected' @@ -141,17 +154,78 @@ jobs: permissions: contents: read needs: load-config + env: + # Single source of truth for the change categories. dorny/paths-filter + # reads it on a normal diff run; the inventory step parses it on a forced + # run. Keep the format to `key:` lines and `- 'glob'` lines -- the + # inventory parser fails the job on anything else rather than guess. + CHANGE_FILTERS: | + go_files: + - '**/*.go' + - 'go.mod' + - 'go.sum' + python_files: + - '**/*.py' + - 'requirements*.txt' + - 'pyproject.toml' + - 'setup.py' + - 'setup.cfg' + rust_files: + - '**/*.rs' + - 'Cargo.toml' + - 'Cargo.lock' + frontend_files: + - '**/*.js' + - '**/*.ts' + - '**/*.jsx' + - '**/*.tsx' + - '**/*.vue' + - '**/*.svelte' + - '**/*.html' + - '**/*.css' + - '**/*.scss' + - '**/*.sass' + - '**/*.less' + - 'package*.json' + - 'yarn.lock' + - 'pnpm-lock.yaml' + docker_files: + - 'Dockerfile*' + - 'docker-compose*.yml' + - 'docker-compose*.yaml' + - '.dockerignore' + docs_files: + - '**/*.md' + - 'docs/**' + - '*.md' + workflow_yaml: + - '.github/workflows/**' + workflow_scripts: + - '.github/actions/**' + - '.github/scripts/**' + lint_files: + - 'super-linter-*.env' + - '.markdownlint.json' + - '.yaml-lint.yml' + - 'clippy.toml' + - 'rustfmt.toml' + - '.eslintrc.*' + - '.pylintrc' + - '.python-black' + - '.prettierrc*' + - '.flake8' outputs: - go-files: ${{ steps.changes.outputs.go_files }} - python-files: ${{ steps.changes.outputs.python_files }} - rust-files: ${{ steps.changes.outputs.rust_files }} - frontend-files: ${{ steps.changes.outputs.frontend_files }} - docker-files: ${{ steps.changes.outputs.docker_files }} - docs-files: ${{ steps.changes.outputs.docs_files }} - workflow-files: ${{ steps.changes.outputs.workflow_yaml }} - workflows-files: ${{ steps.changes.outputs.workflow_yaml }} - workflows-scripts: ${{ steps.changes.outputs.workflow_scripts }} - lint-files: ${{ steps.changes.outputs.lint_files }} + force-all: ${{ steps.mode.outputs.force }} + go-files: ${{ steps.changes.outputs.go_files || steps.inventory.outputs.go_files }} + python-files: ${{ steps.changes.outputs.python_files || steps.inventory.outputs.python_files }} + rust-files: ${{ steps.changes.outputs.rust_files || steps.inventory.outputs.rust_files }} + frontend-files: ${{ steps.changes.outputs.frontend_files || steps.inventory.outputs.frontend_files }} + docker-files: ${{ steps.changes.outputs.docker_files || steps.inventory.outputs.docker_files }} + docs-files: ${{ steps.changes.outputs.docs_files || steps.inventory.outputs.docs_files }} + workflow-files: ${{ steps.changes.outputs.workflow_yaml || steps.inventory.outputs.workflow_yaml }} + workflows-files: ${{ steps.changes.outputs.workflow_yaml || steps.inventory.outputs.workflow_yaml }} + workflows-scripts: ${{ steps.changes.outputs.workflow_scripts || steps.inventory.outputs.workflow_scripts }} + lint-files: ${{ steps.changes.outputs.lint_files || steps.inventory.outputs.lint_files }} steps: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -159,83 +233,162 @@ jobs: submodules: recursive fetch-depth: 2 + # A schedule or workflow_dispatch run has no diff worth inspecting: + # github.event.before is unset, so base falls back to github.sha and + # paths-filter compares the commit with itself, reporting every category + # unchanged. Every job then skipped, so a nightly caller never ran its + # tests. Resolve whether this run diffs or inventories. + - name: Resolve change-detection mode + id: mode + env: + FORCE_ALL: ${{ inputs.force-all }} + EVENT_NAME: ${{ github.event_name }} + run: | + case "$FORCE_ALL" in + true) force=true ;; + false) force=false ;; + auto) + case "$EVENT_NAME" in + schedule|workflow_dispatch) force=true ;; + *) force=false ;; + esac + ;; + *) + echo "::error::force-all must be 'auto', 'true' or 'false', got '$FORCE_ALL'" + exit 1 + ;; + esac + echo "force=$force" >> "$GITHUB_OUTPUT" + echo "force-all=$FORCE_ALL event=$EVENT_NAME -> force=$force" + - name: Detect file changes + if: steps.mode.outputs.force != 'true' uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 id: changes with: base: ${{ github.event.pull_request.base.sha || github.event.before || github.sha }} ref: ${{ github.event.pull_request.head.sha || github.sha }} - filters: | - go_files: - - '**/*.go' - - 'go.mod' - - 'go.sum' - python_files: - - '**/*.py' - - 'requirements*.txt' - - 'pyproject.toml' - - 'setup.py' - - 'setup.cfg' - rust_files: - - '**/*.rs' - - 'Cargo.toml' - - 'Cargo.lock' - frontend_files: - - '**/*.js' - - '**/*.ts' - - '**/*.jsx' - - '**/*.tsx' - - '**/*.vue' - - '**/*.svelte' - - '**/*.html' - - '**/*.css' - - '**/*.scss' - - '**/*.sass' - - '**/*.less' - - 'package*.json' - - 'yarn.lock' - - 'pnpm-lock.yaml' - docker_files: - - 'Dockerfile*' - - 'docker-compose*.yml' - - 'docker-compose*.yaml' - - '.dockerignore' - docs_files: - - '**/*.md' - - 'docs/**' - - '*.md' - workflow_yaml: - - '.github/workflows/**' - workflow_scripts: - - '.github/actions/**' - - '.github/scripts/**' - lint_files: - - 'super-linter-*.env' - - '.markdownlint.json' - - '.yaml-lint.yml' - - 'clippy.toml' - - 'rustfmt.toml' - - '.eslintrc.*' - - '.pylintrc' - - '.python-black' - - '.prettierrc*' - - '.flake8' + filters: ${{ env.CHANGE_FILTERS }} + + # frontend-ci runs in the directory this action resolves, so a forced + # run asks the same question with the same pin and config. + - name: Get frontend working directory + if: steps.mode.outputs.force == 'true' + id: frontend-dir + uses: falkcorp/gha-get-frontend-config@97f9fea2ba59e35fa23a74f0733b7a07005c5809 # v1.1.3 + with: + repository-config: ${{ needs.load-config.outputs.config }} + + # Forced run: a category counts as changed when the repository tracks any + # file matching its filter. `:(glob)` pathspec magic gives `**/` the same + # zero-or-more-directories meaning paths-filter's picomatch gives it. + # + # Three categories gate on the job's entry point instead, because a + # matching file is not a buildable project: + # rust - rust-ci runs `cargo build` at the repo root. + # frontend - frontend-ci installs from the resolved frontend directory. + # python - python-ci treats the repo root as the package (`pip install + # -e .` whenever pyproject.toml exists, then pytest at the + # root). A pyproject.toml holding only tool config (black, + # ruff) is not a package and that install fails. + # Measured on forced runs: github-common (testdata/rust/Cargo.toml, a root + # package.json with no web/) failed rust-ci and frontend-ci; + # audiobook-organizer (config-only pyproject.toml, helper scripts) failed + # python-ci at `pip install -e .`. + - name: Inventory tracked files + if: steps.mode.outputs.force == 'true' + id: inventory + env: + FRONTEND_DIR: ${{ steps.frontend-dir.outputs.dir }} + run: | + key_re='^([a-z_]+):[[:space:]]*$' + pat_re="^[[:space:]]+-[[:space:]]+'([^']+)'[[:space:]]*$" + # Parallel arrays (keys[i] -> hits[i]); the last element is the + # category currently being read. + keys=() + hits=() + while IFS= read -r line; do + [[ -z "${line//[[:space:]]/}" ]] && continue + if [[ $line =~ $key_re ]]; then + keys+=("${BASH_REMATCH[1]}") + hits+=(false) + elif [[ $line =~ $pat_re ]] && [[ ${#keys[@]} -gt 0 ]]; then + pat="${BASH_REMATCH[1]}" + last=$(( ${#keys[@]} - 1 )) + if [[ "${hits[last]}" == false ]]; then + files="$(git ls-files -- ":(glob)$pat")" + if [[ -n "$files" ]]; then + hits[last]=true + fi + fi + else + echo "::error::Unparseable CHANGE_FILTERS line: '$line'" + exit 1 + fi + done <<< "$CHANGE_FILTERS" + if [[ ${#keys[@]} -eq 0 ]]; then + echo "::error::CHANGE_FILTERS defined no categories" + exit 1 + fi + tracked() { + [[ -n "$(git ls-files -- "$1")" ]] + } + python_project() { + if tracked setup.py || tracked setup.cfg || tracked requirements.txt; then + return 0 + fi + tracked pyproject.toml && grep -qE '^\[(project|build-system)\]' pyproject.toml + } + frontend_manifest="${FRONTEND_DIR:-.}" + frontend_manifest="${frontend_manifest%/}/package.json" + frontend_manifest="${frontend_manifest#./}" + for i in "${!keys[@]}"; do + case "${keys[$i]}" in + rust_files) + if [[ "${hits[$i]}" == true ]] && ! tracked Cargo.toml; then + echo "rust_files: no root Cargo.toml, not forcing rust-ci" + hits[i]=false + fi + ;; + python_files) + if [[ "${hits[$i]}" == true ]] && ! python_project; then + echo "python_files: no root Python package (pyproject.toml with [project]/[build-system], setup.py, setup.cfg or requirements.txt), not forcing python-ci" + hits[i]=false + fi + ;; + frontend_files) + if [[ "${hits[$i]}" == true ]] && ! tracked "$frontend_manifest"; then + echo "frontend_files: no $frontend_manifest, not forcing frontend-ci" + hits[i]=false + fi + ;; + esac + done + for i in "${!keys[@]}"; do + echo "${keys[$i]}=${hits[$i]}" >> "$GITHUB_OUTPUT" + echo "${keys[$i]}=${hits[$i]}" + done - name: Debug change outputs env: - CI_GO_FILES: ${{ steps.changes.outputs.go_files }} - CI_PYTHON_FILES: ${{ steps.changes.outputs.python_files }} - CI_RUST_FILES: ${{ steps.changes.outputs.rust_files }} - CI_FRONTEND_FILES: ${{ steps.changes.outputs.frontend_files }} - CI_DOCKER_FILES: ${{ steps.changes.outputs.docker_files }} - CI_DOCS_FILES: ${{ steps.changes.outputs.docs_files }} - CI_WORKFLOW_FILES: ${{ steps.changes.outputs.workflow_yaml }} - CI_WORKFLOW_YAML_FILES: ${{ steps.changes.outputs.workflow_yaml }} - CI_WORKFLOW_SCRIPT_FILES: ${{ steps.changes.outputs.workflow_scripts }} - CI_LINT_FILES: ${{ steps.changes.outputs.lint_files }} + CI_FORCE_ALL: ${{ steps.mode.outputs.force }} + CI_GO_FILES: ${{ steps.changes.outputs.go_files || steps.inventory.outputs.go_files }} + CI_PYTHON_FILES: ${{ steps.changes.outputs.python_files || steps.inventory.outputs.python_files }} + CI_RUST_FILES: ${{ steps.changes.outputs.rust_files || steps.inventory.outputs.rust_files }} + CI_FRONTEND_FILES: ${{ steps.changes.outputs.frontend_files || steps.inventory.outputs.frontend_files }} + CI_DOCKER_FILES: ${{ steps.changes.outputs.docker_files || steps.inventory.outputs.docker_files }} + CI_DOCS_FILES: ${{ steps.changes.outputs.docs_files || steps.inventory.outputs.docs_files }} + CI_WORKFLOW_FILES: ${{ steps.changes.outputs.workflow_yaml || steps.inventory.outputs.workflow_yaml }} + CI_WORKFLOW_SCRIPT_FILES: ${{ steps.changes.outputs.workflow_scripts || steps.inventory.outputs.workflow_scripts }} + CI_LINT_FILES: ${{ steps.changes.outputs.lint_files || steps.inventory.outputs.lint_files }} run: | { printf '## Change Detection\n\n' + if [ "$CI_FORCE_ALL" = "true" ]; then + printf -- '- Mode: forced (every category the repository contains)\n' + else + printf -- '- Mode: diff\n' + fi printf -- '- Go files: %s\n' "$CI_GO_FILES" printf -- '- Python files: %s\n' "$CI_PYTHON_FILES" printf -- '- Rust files: %s\n' "$CI_RUST_FILES" @@ -250,7 +403,7 @@ jobs: workflow-lint: name: Workflow Lint needs: detect-changes - if: needs.detect-changes.outputs.workflow-files == 'true' || github.event_name == 'workflow_dispatch' + if: needs.detect-changes.outputs.workflow-files == 'true' runs-on: ubuntu-latest timeout-minutes: 15 permissions: @@ -310,7 +463,7 @@ jobs: workflow-scripts-npm-cache: name: Cache npm (Workflow Scripts) needs: detect-changes - if: needs.detect-changes.outputs.workflows-scripts == 'true' || github.event_name == 'workflow_dispatch' + if: needs.detect-changes.outputs.workflows-scripts == 'true' uses: falkcorp/github-common/.github/workflows/reusable-advanced-cache.yml@d0c3326b96557c8ea9117c1c196b628e5e028186 # v1.10.3+ with: language: 'node' @@ -320,7 +473,7 @@ jobs: workflow-scripts: name: Workflow Scripts needs: [detect-changes, workflow-scripts-npm-cache] - if: needs.detect-changes.outputs.workflows-scripts == 'true' || github.event_name == 'workflow_dispatch' + if: needs.detect-changes.outputs.workflows-scripts == 'true' runs-on: ubuntu-latest timeout-minutes: 20 permissions: diff --git a/changelog.d/20261005-reusable-ci-force-all.md b/changelog.d/20261005-reusable-ci-force-all.md new file mode 100644 index 00000000..5130e983 --- /dev/null +++ b/changelog.d/20261005-reusable-ci-force-all.md @@ -0,0 +1,53 @@ +### Fixed + +#### Scheduled and manually dispatched `reusable-ci.yml` runs now run the jobs + +`detect-changes` diffed `github.event.before || github.sha` against +`github.sha`. A `schedule` or `workflow_dispatch` event has no `before`, so the +base fell back to the head commit and the run compared the commit with itself: +every category reported unchanged and every language job skipped. In +`falkcorp/audiobook-organizer` the nightly caller skipped `Go CI` on 5 of 5 +scheduled runs checked, so the full non-short race suite never ran nightly. + +New input `force-all` (string, default `'auto'`): + +- `'auto'` forces on `schedule` and `workflow_dispatch`, diffs otherwise. +- `'true'` always forces; `'false'` always diffs. + +A forced run reports a category as changed when the repository tracks at least +one file matching that category's filter (`git ls-files` with `:(glob)` +pathspecs), so forcing never starts `rust-ci` in a repo with no Rust. Three +categories gate on the job's entry point instead of any matching file: + +- `rust_files` needs a tracked root `Cargo.toml` (rust-ci runs `cargo build` + at the root). +- `frontend_files` needs `package.json` in the directory + `gha-get-frontend-config` resolves (where frontend-ci installs). +- `python_files` needs a root Python package: `pyproject.toml` with a + `[project]` or `[build-system]` table, `setup.py`, `setup.cfg` or + `requirements.txt` (python-ci runs `pip install -e .` at the root whenever + `pyproject.toml` exists, which fails on a config-only file). + +Without these, forced test runs failed on fixtures and tool config: this repo +(`testdata/rust/Cargo.toml`, a root `package.json` with no `web/`) failed +rust-ci and frontend-ci, and audiobook-organizer (a black-config-only +`pyproject.toml`) failed python-ci at `pip install -e .`. The +filter list is now defined once (`CHANGE_FILTERS`) and read by both paths. The +Change Detection summary prints which mode ran, and `detect-changes` exposes a +`force-all` output. + +`workflow-lint` and `workflow-scripts` drop their separate +`|| github.event_name == 'workflow_dispatch'` bypass; the forced inventory now +covers that case. The only behavior change there is for a caller that sets +`force-all: 'false'` and dispatches manually — those two jobs then follow the +diff like everything else. + +Callers affected by the `'auto'` default: this repo's own weekly `ci.yml` +schedule and any caller on `@main` that dispatches manually +(`cockroach-rollout-agent`). SHA-pinned callers change only when they bump. + +Known, not fixed here: on a forced run this repo's own `Workflow Scripts` job +fails `ruff check .github/scripts` (hundreds of findings, mostly `T201` +`print`, plus docstring and `subprocess` rules; `ruff` is installed unpinned). +The job already ran on every manual dispatch through the removed bypass; the +weekly schedule now exposes it too.