ci: weekly CI timing review and per-test timings from the unit legs (#1538)

.github/workflows/ci-timing.yml

@@ -0,0 +1,100 @@
+name: CI Timing
+
+# The periodic CI-duration review #1538 asked for. util/ci_timing.py reads the
+# last week of Unit Tests, Compat, Lint, CodeQL and Pages runs through the API,
+# writes a report to the job summary and to an artifact, and fails the run when a
+# HARD threshold fires (status 1; 2 and 3 are an unreadable API and a crash, each
+# annotated as such). GitHub emails a scheduled workflow's failures to whoever
+# last edited its cron line, so the owner hears about it with no `issues: write`.
+# docs/source/contributing/workflows.rst has the thresholds and why each sits
+# where it does; conventions/process.rst has what to do when one fires.
+#
+# Deliberately not a `workflow_run` on Unit Tests: that would add a job per run
+# to a runner pool already measured saturated at 20 concurrent jobs.
+on:
+  schedule:
+    # Mondays 07:23 UTC: off the hour, where GitHub delays scheduled runs most,
+    # and clear of Saturday's 02:00-10:00 crons and Project Status at 00:37.
+    - cron: '23 7 * * 1'
+  workflow_dispatch:
+    inputs:
+      days:
+        description: Window in days.
+        type: number
+        default: 7
+      concurrency_hours:
+        description: Also measure the runner pool over this many hours, at one API call per run (0 = off).
+        type: number
+        default: 0
+
+# Read-only: the runs, jobs and artifacts need `actions: read`, the workflow
+# files at each run's sha need `contents: read`. GITHUB_TOKEN allows 1,000 API
+# requests an hour per repository; a default run uses about 600.
+permissions:
+  actions: read
+  contents: read
+
+concurrency:
+  group: ci-timing
+  cancel-in-progress: false
+
+jobs:
+  report:
+    name: CI timing report
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    steps:
+      - uses: actions/checkout@v7
+
+      # Nothing is installed: the script is standard library only.
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.14"
+
+      # The newest earlier run that uploaded a report, so a week whose run failed
+      # on a HARD threshold is still next week's baseline. A first run, or one
+      # after 90 days without a report, has none and says so.
+      - name: Fetch the previous report as the baseline
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          RUN_ID: ${{ github.run_id }}
+        run: |
+          set -u
+          mkdir -p prev
+          for id in $(gh run list -R "$REPO" --workflow ci-timing.yml --limit 10 --json databaseId -q '.[].databaseId'); do
+            [ "$id" = "$RUN_ID" ] && continue
+            if gh run download "$id" -R "$REPO" -n ci-timing-report -D prev 2>/dev/null; then
+              echo "Baseline: run $id"
+              exit 0
+            fi
+          done
+          echo "::notice title=No baseline::No earlier ci-timing-report artifact was found, so the week-over-week checks are skipped."
+
+      - name: Report CI timing
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          DAYS: ${{ inputs.days || 7 }}
+          CONCURRENCY_HOURS: ${{ inputs.concurrency_hours || 0 }}
+        run: |
+          status=0
+          python util/ci_timing.py --repo "$REPO" --days "$DAYS" \
+            --concurrency-hours "$CONCURRENCY_HOURS" --baseline prev/report.json \
+            --out report.json --summary report.md || status=$?
+          if [ -f report.md ]; then
+            cat report.md >> "$GITHUB_STEP_SUMMARY"
+          fi
+          exit "$status"
+
+      # 90 days, so the trend is a series of weekly reports rather than one.
+      - name: Upload the report
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: ci-timing-report
+          path: |
+            report.json
+            report.md
+          retention-days: 90

.github/workflows/unit-tests.yml

@@ -136,11 +136,14 @@ jobs:
         # `--dist load` is pinned explicitly rather than left to default: a
         # future change to xdist's own default, or someone "optimising" to
         # `loadfile`/`loadscope`, cannot silently change how this suite is
-        # distributed. The coverage leg runs the identical selection.
+        # distributed. The coverage leg runs the identical selection. The
+        # durations table and JUnit file are #1538's per-test timings, uploaded
+        # by the last step of this job.
         env:
           COVERAGE: ${{ matrix.coverage }}
         run: |
           set -- -q -n auto --dist load \
+            --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" \
             --ignore=tests/integration \
             --ignore-glob='*_runtime.py' \
             --ignore-glob='*_regression.py'
@@ -268,6 +271,19 @@ jobs:
           path: coverage-comment/coverage-summary.json
           retention-days: 7
 
+      # #1538: the weekly CI Timing review (ci-timing.yml) reads this from the
+      # newest push runs. One artifact per leg, so the name carries the version.
+      # `continue-on-error` because the file is only diagnostic: an artifact
+      # service outage must not fail a leg that `Required checks passed` needs.
+      - name: Upload test timings
+        if: ${{ !cancelled() }}
+        continue-on-error: true
+        uses: actions/upload-artifact@v7
+        with:
+          name: junit-unit-${{ matrix.python-version }}
+          path: ${{ runner.temp }}/junit.xml
+          retention-days: 14
+
   integration:
     name: Integration Python ${{ matrix.python-version }}
     if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }}
@@ -380,7 +396,18 @@ jobs:
             exit 1
           fi
           echo "Fixture-dependent selection: $selection"
-          python -m pytest -q -n auto --dist load $selection
+          python -m pytest -q -n auto --dist load \
+            --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" $selection
+
+      # See the `test` job's step of the same name.
+      - name: Upload test timings
+        if: ${{ !cancelled() }}
+        continue-on-error: true
+        uses: actions/upload-artifact@v7
+        with:
+          name: junit-integration-${{ matrix.python-version }}
+          path: ${{ runner.temp }}/junit.xml
+          retention-days: 14
 
   # Per-engine coverage for the third-party capture engines (#845): every
   # (Python version, engine) pair is checked, and a failure -- or a correctly

docs/source/contributing/conventions/process.rst

@@ -390,3 +390,25 @@ metadata that changes as work moves, and a link to them goes stale the way a lin
 number does; the changelog's ``:issue:`` and ``:pr:`` roles are the durable trail.
 The repository wiki is disabled -- contributor documentation lives on these pages.
 
+CI Timing Review
+~~~~~~~~~~~~~~~~
+
+CI time is reviewed on a schedule rather than when it starts to hurt, at the owner's
+request on :issue:`1538`, because the suite's run time grows faster than its test count:
+measured there, the tests grew by about a third while test time grew by about 90%.
+:file:`.github/workflows/ci-timing.yml` reports every Monday; what it measures and the
+thresholds are on :doc:`../workflows`.
+
+*  **The owner reads the Monday summary.** Warnings are annotations on it. A run fails,
+   which is what sends the email, on a HARD threshold or when it could not read the API
+   or crashed; its annotation says which.
+*  **A HARD failure gets an issue** labelled ``ci``, with the run's ``ci-timing-report``
+   artifact attached and the summary's slowest-module table pasted in. The cap
+   thresholds are set to fire with about two weeks to spare.
+*  **Re-baseline after a shape change.** When a merged ``ci`` pull request changes a
+   job's shape or selection -- a leg added, split or moved -- dispatch the workflow, so
+   the next week compares like with like.
+*  **Monthly, refresh the measured figures** quoted in :file:`unit-tests.yml`'s
+   comments that the report shows to be stale -- the coverage leg's headroom, the
+   ordering cells' timings and their step cap's rationale.
+

docs/source/contributing/releasing.rst

@@ -46,7 +46,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s
 (``:494-514``) asserts the two agree, and ``create-release.yml``'s
 ``unit-tests`` job (``:362-376``) calls ``unit-tests.yml`` with
 ``gate-only: true``, which runs the **full** suite rather than the tiered
-subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1402-1471``).
+subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1429-1498``).
 That gate runs whenever a release will -- a moved ``__version__`` is exactly
 what makes ``version_check``'s evidence read "not yet published" -- and every
 publishing job requires it to have succeeded (``:392,493,573,674``), so a
@@ -56,7 +56,7 @@ Either run the script, or move both fields by hand in the same commit.
 This is the **only** edit a person makes to get a release started -- the tag,
 the Release, and every upload are the workflow's job from here. Two other
 checks run unconditionally on the same path, though neither is a step in *this*
-process: the ``changelog`` job (``unit-tests.yml:1262-1283``) fails outright if
+process: the ``changelog`` job (``unit-tests.yml:1289-1310``) fails outright if
 ``CHANGELOG.md`` has drifted from its source entry under
 :file:`docs/source/changelog/`, and the release-body step warns, without
 failing, if that entry's heading still reads "unreleased"