diff --git a/.github/workflows/ci-timing.yml b/.github/workflows/ci-timing.yml new file mode 100644 index 0000000000..053a235ae2 --- /dev/null +++ b/.github/workflows/ci-timing.yml @@ -0,0 +1,100 @@ +name: CI Timing + +# The periodic CI-duration review #1538 asked for. util/ci_timing.py reads the +# last week of Unit Tests, Compat, Lint, CodeQL and Pages runs through the API, +# writes a report to the job summary and to an artifact, and fails the run when a +# HARD threshold fires (status 1; 2 and 3 are an unreadable API and a crash, each +# annotated as such). GitHub emails a scheduled workflow's failures to whoever +# last edited its cron line, so the owner hears about it with no `issues: write`. +# docs/source/contributing/workflows.rst has the thresholds and why each sits +# where it does; conventions/process.rst has what to do when one fires. +# +# Deliberately not a `workflow_run` on Unit Tests: that would add a job per run +# to a runner pool already measured saturated at 20 concurrent jobs. +on: + schedule: + # Mondays 07:23 UTC: off the hour, where GitHub delays scheduled runs most, + # and clear of Saturday's 02:00-10:00 crons and Project Status at 00:37. + - cron: '23 7 * * 1' + workflow_dispatch: + inputs: + days: + description: Window in days. + type: number + default: 7 + concurrency_hours: + description: Also measure the runner pool over this many hours, at one API call per run (0 = off). + type: number + default: 0 + +# Read-only: the runs, jobs and artifacts need `actions: read`, the workflow +# files at each run's sha need `contents: read`. GITHUB_TOKEN allows 1,000 API +# requests an hour per repository; a default run uses about 600. +permissions: + actions: read + contents: read + +concurrency: + group: ci-timing + cancel-in-progress: false + +jobs: + report: + name: CI timing report + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - uses: actions/checkout@v7 + + # Nothing is installed: the script is standard library only. + - uses: actions/setup-python@v7 + with: + python-version: "3.14" + + # The newest earlier run that uploaded a report, so a week whose run failed + # on a HARD threshold is still next week's baseline. A first run, or one + # after 90 days without a report, has none and says so. + - name: Fetch the previous report as the baseline + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + run: | + set -u + mkdir -p prev + for id in $(gh run list -R "$REPO" --workflow ci-timing.yml --limit 10 --json databaseId -q '.[].databaseId'); do + [ "$id" = "$RUN_ID" ] && continue + if gh run download "$id" -R "$REPO" -n ci-timing-report -D prev 2>/dev/null; then + echo "Baseline: run $id" + exit 0 + fi + done + echo "::notice title=No baseline::No earlier ci-timing-report artifact was found, so the week-over-week checks are skipped." + + - name: Report CI timing + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + DAYS: ${{ inputs.days || 7 }} + CONCURRENCY_HOURS: ${{ inputs.concurrency_hours || 0 }} + run: | + status=0 + python util/ci_timing.py --repo "$REPO" --days "$DAYS" \ + --concurrency-hours "$CONCURRENCY_HOURS" --baseline prev/report.json \ + --out report.json --summary report.md || status=$? + if [ -f report.md ]; then + cat report.md >> "$GITHUB_STEP_SUMMARY" + fi + exit "$status" + + # 90 days, so the trend is a series of weekly reports rather than one. + - name: Upload the report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v7 + with: + name: ci-timing-report + path: | + report.json + report.md + retention-days: 90 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e6215b5b6d..d9def589f5 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -136,11 +136,14 @@ jobs: # `--dist load` is pinned explicitly rather than left to default: a # future change to xdist's own default, or someone "optimising" to # `loadfile`/`loadscope`, cannot silently change how this suite is - # distributed. The coverage leg runs the identical selection. + # distributed. The coverage leg runs the identical selection. The + # durations table and JUnit file are #1538's per-test timings, uploaded + # by the last step of this job. env: COVERAGE: ${{ matrix.coverage }} run: | set -- -q -n auto --dist load \ + --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" \ --ignore=tests/integration \ --ignore-glob='*_runtime.py' \ --ignore-glob='*_regression.py' @@ -268,6 +271,19 @@ jobs: path: coverage-comment/coverage-summary.json retention-days: 7 + # #1538: the weekly CI Timing review (ci-timing.yml) reads this from the + # newest push runs. One artifact per leg, so the name carries the version. + # `continue-on-error` because the file is only diagnostic: an artifact + # service outage must not fail a leg that `Required checks passed` needs. + - name: Upload test timings + if: ${{ !cancelled() }} + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + name: junit-unit-${{ matrix.python-version }} + path: ${{ runner.temp }}/junit.xml + retention-days: 14 + integration: name: Integration Python ${{ matrix.python-version }} if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }} @@ -380,7 +396,18 @@ jobs: exit 1 fi echo "Fixture-dependent selection: $selection" - python -m pytest -q -n auto --dist load $selection + python -m pytest -q -n auto --dist load \ + --durations=30 --durations-min=1.0 --junitxml="$RUNNER_TEMP/junit.xml" $selection + + # See the `test` job's step of the same name. + - name: Upload test timings + if: ${{ !cancelled() }} + continue-on-error: true + uses: actions/upload-artifact@v7 + with: + name: junit-integration-${{ matrix.python-version }} + path: ${{ runner.temp }}/junit.xml + retention-days: 14 # Per-engine coverage for the third-party capture engines (#845): every # (Python version, engine) pair is checked, and a failure -- or a correctly diff --git a/docs/source/contributing/conventions/process.rst b/docs/source/contributing/conventions/process.rst index 7873987846..f4c0902ad1 100644 --- a/docs/source/contributing/conventions/process.rst +++ b/docs/source/contributing/conventions/process.rst @@ -390,3 +390,25 @@ metadata that changes as work moves, and a link to them goes stale the way a lin number does; the changelog's ``:issue:`` and ``:pr:`` roles are the durable trail. The repository wiki is disabled -- contributor documentation lives on these pages. +CI Timing Review +~~~~~~~~~~~~~~~~ + +CI time is reviewed on a schedule rather than when it starts to hurt, at the owner's +request on :issue:`1538`, because the suite's run time grows faster than its test count: +measured there, the tests grew by about a third while test time grew by about 90%. +:file:`.github/workflows/ci-timing.yml` reports every Monday; what it measures and the +thresholds are on :doc:`../workflows`. + +* **The owner reads the Monday summary.** Warnings are annotations on it. A run fails, + which is what sends the email, on a HARD threshold or when it could not read the API + or crashed; its annotation says which. +* **A HARD failure gets an issue** labelled ``ci``, with the run's ``ci-timing-report`` + artifact attached and the summary's slowest-module table pasted in. The cap + thresholds are set to fire with about two weeks to spare. +* **Re-baseline after a shape change.** When a merged ``ci`` pull request changes a + job's shape or selection -- a leg added, split or moved -- dispatch the workflow, so + the next week compares like with like. +* **Monthly, refresh the measured figures** quoted in :file:`unit-tests.yml`'s + comments that the report shows to be stale -- the coverage leg's headroom, the + ordering cells' timings and their step cap's rationale. + diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index fbcdd0a65c..0c184dc39d 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -46,7 +46,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:362-376``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1402-1471``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1429-1498``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:392,493,573,674``), so a @@ -56,7 +56,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Two other checks run unconditionally on the same path, though neither is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1262-1283``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1289-1310``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index b0932671b8..33298d14ae 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -3,7 +3,7 @@ GitHub Actions Workflows .. important:: - The ten workflows under :file:`.github/workflows/` trigger each other, and + The eleven workflows under :file:`.github/workflows/` trigger each other, and the chain that results is not visible from any single file -- reading one ``on:`` block never shows what a *different* workflow's completion goes on to start. This page is the repository-wide graph. The release *pipeline* @@ -93,6 +93,13 @@ At a Glance - -- - **Unit Tests** - -- + * - CI Timing + - :file:`ci-timing.yml` + - -- + - -- + - Mon 07:23 + - -- + - ``workflow_dispatch`` .. note:: @@ -119,7 +126,7 @@ than starting a new one. PUSH["push: main"] PR["pull_request: main"] TAG["push: tags v*"] - SCHED["schedule (Saturday, or daily)"] + SCHED["schedule (Saturday, Monday, or daily)"] ISSUES["issues"] PRT["pull_request_target"] DISPATCH["workflow_dispatch"] @@ -134,6 +141,7 @@ than starting a new one. CR["Create Release
create-release.yml"] PS["Project Status
project-status.yml"] CC["Coverage Comment
coverage-comment.yml"] + CT["CI Timing
ci-timing.yml"] PUSH --> CQ PR --> CQ @@ -163,6 +171,9 @@ than starting a new one. SCHED -->|daily 00:37| PS DISPATCH --> PS + SCHED -->|Mon 07:23| CT + DISPATCH --> CT + UT ==>|workflow_run: completed| VU UT ==>|workflow_run: completed| CU UT ==>|workflow_run: completed| GP @@ -245,7 +256,7 @@ and ``project-tests``, and ``required-checks`` (see `Required Status Checks`_ below), gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1250-1256``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1277-1283``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -373,6 +384,102 @@ sha, and renders its own Markdown from validated numbers. It edits one comment, found by a hidden marker, on each push. The first line says when the tests failed. Neither workflow's coverage step is a required check. +CI Timing Review (`#1538 `__) +--------------------------------------------------------------------------------- + +**CI Timing** (:file:`ci-timing.yml`) runs :file:`util/ci_timing.py` every Monday, +or on dispatch with a ``days`` window and an optional ``concurrency_hours`` runner-pool +measurement. Through the API it lists every completed run in the window of Unit Tests, +Python Compatibility, Lint, CodeQL and GitHub Pages, a day per query because a +filtered listing stops at 1,000 runs (Pages completed 1,154 in the week to +2026-10-09). It then reads the jobs of an evenly spaced sample -- 150 push and 150 +pull-request runs of Unit Tests, 60 runs of each other workflow -- and the summary +says how many runs each sample holds and which span they cover. That is about 625 +calls (estimate) of ``GITHUB_TOKEN``'s 1,000 an hour, on a read-only token +(``actions: read``, ``contents: read``). A job is measured by its own outcome: one +cancelled at 95% of its cap or later timed out and counts, even in a run that reads +``cancelled`` for that reason; one cancelled earlier was superseded and does not. +The report goes to the job summary and to the ``ci-timing-report`` artifact (90 +days), and the newest earlier report is the next run's baseline. It is not a +``workflow_run`` on Unit Tests, which would add a job per run to a runner pool +measured saturated at 20 concurrent jobs. + +Per-test timings come from the ``test`` and ``integration`` legs, which run pytest +with ``--durations=30 --durations-min=1.0`` and ``--junitxml`` and upload the file as +``junit-unit-`` or ``junit-integration-`` (14 days). Each upload +step is ``continue-on-error``, so an artifact-service outage cannot fail a leg that +``Required checks passed`` needs. pytest writes one ```` per test however +many subtests it runs -- about 0.6 MB per unit leg (estimate) -- but the suite's +``tests=`` count includes subtests, so the review reads each testcase's ``time``. The +ordering, ``gate``, ``engine-tests`` and ``pypcap-parity`` legs write no file, and nor +does a leg killed at its cap, since pytest writes it at session end. + +A HARD threshold fails the run with status 1, and GitHub emails a scheduled +workflow's failure to whoever last edited its cron line; a WARN is an annotation. A +run that cannot read the API ends with 2 and one that crashes with 3, each with an +annotation saying no threshold was evaluated. Each threshold is anchored to a figure +measured on the issue: + +.. list-table:: + :header-rows: 1 + :widths: 8 64 28 + + * - ID + - Fires when + - Measured + * - ``H1`` + - a job's p90 reaches 50% of its ``timeout-minutes`` + - ``Python 3.14``: 12.1 of 45 min + * - ``H2`` + - a step with a cap of its own reaches 50% of it at p90 (the ordering cells' 20) + - ``protocols (rest)``: 319 s + * - ``H3`` + - ``W9``'s p90 is at or over 30 min and 25% or more above the baseline's (needs a + baseline) + - -- + * - ``H4`` + - a job's or capped step's linear fit reaches its cap within 14 days + - -- + * - ``W1`` + - a job's median rises 20% on the baseline, with 20 samples each side + - -- + * - ``W2`` + - pull-request time to mergeable passes 18 min at the median + - 14.4 min + * - ``W3`` + - the coverage leg's tests step passes 1.8 times its paired legs' (median) + - 1.57 + * - ``W4`` + - ms per test on 3.13 rises 15% on the baseline + - 110.8 ms + * - ``W5`` + - run queue p90 passes 5 min on pull requests or 15 min on pushes + - 3.1 and 9.6 min + * - ``W6`` + - more than 35% of pull-request runs are cancelled, i.e. superseded + - 29% + * - ``W7`` + - a module passes 30 s summed or a test 10 s on 3.13, a module enters the top + 10, or a class's every test costs about what its first does + - -- + * - ``W8`` + - a different job finishes last before ``Required checks passed`` + - ``Python 3.14`` + * - ``W9`` + - pull-request time to ``Required checks passed`` reaches 30 min at p90 + - 19.6 min; 36.7 in the two days to 2026-10-10 + +The cap line sits at 50% because at the fastest growth measured, +1.27 min/day on +``Python 3.14``, crossing it leaves 12-18 days before the cap: one review and one +fixing pull request. Time to ``Required checks passed`` is HARD only as a regression +because its p90 measures runner-queue load as much as test time: it was 19.6 min when +the line was set and 36.7 min in the two days to 2026-10-10 with the pool saturated, +so an absolute HARD line would email about the queue every week. The 25% rise is an +initial value, to tune once weekly reports exist. ``W9`` is therefore expected to fire +while the pool stays saturated, and the first runs with JUnit will show whether +``W7``'s 10-second test line needs raising. What to do when one fires is on +:doc:`conventions/process`. + Required Status Checks ---------------------- @@ -389,7 +496,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:1559`` + - ``unit-tests.yml:1586`` * - ``Compat Python 3.10`` - job ``compatibility``, matrix leg ``3.10`` - ``python-compatibility.yml:31,41`` @@ -407,20 +514,20 @@ grepping every workflow file for its name: - ``python-compatibility.yml:31,45`` ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:1559``). ``Compat Python`` is defined by two in +(``unit-tests.yml:1586``). ``Compat Python`` is defined by two in ``python-compatibility.yml``: the required ``compatibility`` job (``:31``, ``Compat Python ${{ matrix.python-version }}``, expanding to the five required legs at ``:41-45``) and the non-required ``compatibility-nightly`` job (``:69``, ``Compat Python 3.15 (scheduled)``, see below). A comment at -``unit-tests.yml:1474`` mentions the string without defining it. +``unit-tests.yml:1501`` mentions the string without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the five ``Compat Python 3.10``-``3.14`` legs, nothing else, as -``unit-tests.yml``'s own comment (``unit-tests.yml:1473-1483``) records. It +``unit-tests.yml``'s own comment (``unit-tests.yml:1500-1510``) records. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- ``test``, ``integration``, ``engine-tests`` and ``pypcap-parity``, plus ``changes`` and ``project-tests`` for the docs-only path -- and checks each result explicitly -(``unit-tests.yml:1558-1612``). Rulesets match check names literally, with no +(``unit-tests.yml:1585-1639``). Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. It accepts the four legs as ``skipped`` only on a docs-only pull request, where @@ -429,13 +536,13 @@ instead. The ``Compat`` legs stay required separately because they come from ``python-compatibility.yml``, and a job cannot ``needs:`` a job in another workflow file. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1561``), +``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1588``), so it is never produced -- and never expected -- on those paths. ``Compat Python 3.10``-``3.14`` are five ordinary matrix legs, not an aggregate: ``python-compatibility.yml``'s own comment (``:37-40``) and ``unit-tests.yml``'s matching one for its own, differently-matrixed -``test``/``integration`` jobs (``unit-tests.yml:59-62``, ``unit-tests.yml:281``) both say why 3.15 is +``test``/``integration`` jobs (``unit-tests.yml:59-62``, ``unit-tests.yml:297``) both say why 3.15 is excluded from every *required* matrix in this repository -- the ruleset's required-checks list stops at 3.14, so a 3.15 leg cannot gate a merge and is kept advisory (``continue-on-error: true`` in ``python-compatibility.yml``'s diff --git a/tests/project/ci_timing_fixtures/artifacts-37998639834.json b/tests/project/ci_timing_fixtures/artifacts-37998639834.json new file mode 100644 index 0000000000..750816a151 --- /dev/null +++ b/tests/project/ci_timing_fixtures/artifacts-37998639834.json @@ -0,0 +1,32 @@ +{ + "total_count": 3, + "artifacts": [ + { + "id": 11648726855, + "name": "coverage-summary", + "size_in_bytes": 442, + "expired": false, + "created_at": "2026-10-09T22:39:57Z", + "expires_at": "2026-10-16T22:39:56Z", + "archive_download_url": "https://api.github.com/repos/JarryShaw/PyPCAPKit/actions/artifacts/11648726855/zip" + }, + { + "id": 11648726901, + "name": "junit-unit-3.13", + "size_in_bytes": 1650, + "expired": false, + "created_at": "2026-10-09T22:31:02Z", + "expires_at": "2026-10-23T22:31:01Z", + "archive_download_url": "https://api.github.com/repos/JarryShaw/PyPCAPKit/actions/artifacts/11648726901/zip" + }, + { + "id": 11648726902, + "name": "junit-unit-3.14", + "size_in_bytes": 1702, + "expired": true, + "created_at": "2026-10-09T22:39:41Z", + "expires_at": "2026-10-09T22:39:41Z", + "archive_download_url": "https://api.github.com/repos/JarryShaw/PyPCAPKit/actions/artifacts/11648726902/zip" + } + ] +} diff --git a/tests/project/ci_timing_fixtures/jobs-37323913186.json b/tests/project/ci_timing_fixtures/jobs-37323913186.json new file mode 100644 index 0000000000..4e6420fc84 --- /dev/null +++ b/tests/project/ci_timing_fixtures/jobs-37323913186.json @@ -0,0 +1,120 @@ +{ + "total_count": 3, + "jobs": [ + { + "id": 111809650222, + "run_id": 37323913186, + "run_attempt": 1, + "name": "Python 3.14", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-05T14:20:31Z", + "started_at": "2026-10-05T14:20:33Z", + "completed_at": "2026-10-05T14:36:31Z", + "runner_name": "GitHub Actions 1000067272", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-05T14:20:34Z", + "completed_at": "2026-10-05T14:20:35Z" + }, + { + "number": 4, + "name": "Install package and test dependencies", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-05T14:20:39Z", + "completed_at": "2026-10-05T14:21:12Z" + }, + { + "number": 6, + "name": "Run unit tests", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-05T14:21:12Z", + "completed_at": "2026-10-05T14:36:28Z" + } + ] + }, + { + "id": 111809650330, + "run_id": 37323913186, + "run_attempt": 1, + "name": "Plain unittest ordering (protocols)", + "status": "completed", + "conclusion": "cancelled", + "created_at": "2026-10-05T14:20:31Z", + "started_at": "2026-10-05T14:20:33Z", + "completed_at": "2026-10-05T15:06:08Z", + "runner_name": "GitHub Actions 1000067284", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-05T14:20:33Z", + "completed_at": "2026-10-05T14:20:34Z" + }, + { + "number": 4, + "name": "Install package and test dependencies", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-05T14:20:40Z", + "completed_at": "2026-10-05T14:21:07Z" + }, + { + "number": 6, + "name": "Run tests/protocols and the root-level modules under plain unittest", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-10-05T14:21:07Z", + "completed_at": "2026-10-05T15:06:06Z" + } + ] + }, + { + "id": 111819495539, + "run_id": 37323913186, + "run_attempt": 1, + "name": "Required checks passed", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-05T14:41:55Z", + "started_at": "2026-10-05T14:55:17Z", + "completed_at": "2026-10-05T14:55:20Z", + "runner_name": "GitHub Actions 1000067534", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-05T14:55:17Z", + "completed_at": "2026-10-05T14:55:18Z" + }, + { + "number": 2, + "name": "Check every required job succeeded", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-05T14:55:18Z", + "completed_at": "2026-10-05T14:55:18Z" + } + ] + } + ] +} diff --git a/tests/project/ci_timing_fixtures/jobs-37998639834.json b/tests/project/ci_timing_fixtures/jobs-37998639834.json new file mode 100644 index 0000000000..1749fc96ed --- /dev/null +++ b/tests/project/ci_timing_fixtures/jobs-37998639834.json @@ -0,0 +1,341 @@ +{ + "total_count": 8, + "jobs": [ + { + "id": 114050978598, + "run_id": 37998639834, + "run_attempt": 1, + "name": "Classify changed files", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:20:10Z", + "started_at": "2026-10-09T22:21:07Z", + "completed_at": "2026-10-09T22:21:11Z", + "runner_name": "GitHub Actions 1000093015", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:21:08Z", + "completed_at": "2026-10-09T22:21:09Z" + }, + { + "number": 3, + "name": "Classify the diff", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:21:09Z", + "completed_at": "2026-10-09T22:21:09Z" + } + ] + }, + { + "id": 114050979061, + "run_id": 37998639834, + "run_attempt": 1, + "name": "Plain unittest ordering (protocols (rest))", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:20:10Z", + "started_at": "2026-10-09T22:26:52Z", + "completed_at": "2026-10-09T22:31:41Z", + "runner_name": "GitHub Actions 1000093065", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:26:52Z", + "completed_at": "2026-10-09T22:26:53Z" + }, + { + "number": 4, + "name": "Install package and test dependencies", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:26:57Z", + "completed_at": "2026-10-09T22:27:23Z" + }, + { + "number": 6, + "name": "Run tests/protocols and the root-level modules under plain unittest", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:27:23Z", + "completed_at": "2026-10-09T22:31:39Z" + } + ] + }, + { + "id": 114051305952, + "run_id": 37998639834, + "run_attempt": 1, + "name": "Python 3.13", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:21:12Z", + "started_at": "2026-10-09T22:29:24Z", + "completed_at": "2026-10-09T22:35:53Z", + "runner_name": "GitHub Actions 1000093080", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:29:25Z", + "completed_at": "2026-10-09T22:29:26Z" + }, + { + "number": 4, + "name": "Install package and test dependencies", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:29:29Z", + "completed_at": "2026-10-09T22:29:51Z" + }, + { + "number": 7, + "name": "Run unit tests", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:29:51Z", + "completed_at": "2026-10-09T22:35:49Z" + }, + { + "number": 8, + "name": "Report coverage", + "status": "completed", + "conclusion": "skipped", + "started_at": "2026-10-09T22:35:49Z", + "completed_at": "2026-10-09T22:35:49Z" + } + ] + }, + { + "id": 114051305974, + "run_id": 37998639834, + "run_attempt": 1, + "name": "Integration Python 3.14", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:21:12Z", + "started_at": "2026-10-09T22:27:08Z", + "completed_at": "2026-10-09T22:29:57Z", + "runner_name": "GitHub Actions 1000093070", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:27:09Z", + "completed_at": "2026-10-09T22:27:10Z" + }, + { + "number": 7, + "name": "Run full test suite", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:27:34Z", + "completed_at": "2026-10-09T22:29:53Z" + } + ] + }, + { + "id": 114051305976, + "run_id": 37998639834, + "run_attempt": 1, + "name": "Python 3.11", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:21:12Z", + "started_at": "2026-10-09T22:29:02Z", + "completed_at": "2026-10-09T22:35:25Z", + "runner_name": "GitHub Actions 1000093079", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:29:03Z", + "completed_at": "2026-10-09T22:29:04Z" + }, + { + "number": 4, + "name": "Install package and test dependencies", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:29:08Z", + "completed_at": "2026-10-09T22:29:35Z" + }, + { + "number": 7, + "name": "Run unit tests", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:29:35Z", + "completed_at": "2026-10-09T22:35:22Z" + }, + { + "number": 8, + "name": "Report coverage", + "status": "completed", + "conclusion": "skipped", + "started_at": "2026-10-09T22:35:22Z", + "completed_at": "2026-10-09T22:35:22Z" + } + ] + }, + { + "id": 114051305985, + "run_id": 37998639834, + "run_attempt": 1, + "name": "Python 3.14", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:21:12Z", + "started_at": "2026-10-09T22:30:12Z", + "completed_at": "2026-10-09T22:40:00Z", + "runner_name": "GitHub Actions 1000093085", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:30:13Z", + "completed_at": "2026-10-09T22:30:14Z" + }, + { + "number": 4, + "name": "Install package and test dependencies", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:30:18Z", + "completed_at": "2026-10-09T22:30:50Z" + }, + { + "number": 7, + "name": "Run unit tests", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:30:51Z", + "completed_at": "2026-10-09T22:39:31Z" + }, + { + "number": 8, + "name": "Report coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:39:31Z", + "completed_at": "2026-10-09T22:39:52Z" + } + ] + }, + { + "id": 114051305990, + "run_id": 37998639834, + "run_attempt": 1, + "name": "Python 3.10", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:21:12Z", + "started_at": "2026-10-09T22:30:01Z", + "completed_at": "2026-10-09T22:37:06Z", + "runner_name": "GitHub Actions 1000093082", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:30:02Z", + "completed_at": "2026-10-09T22:30:03Z" + }, + { + "number": 4, + "name": "Install package and test dependencies", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:30:05Z", + "completed_at": "2026-10-09T22:30:34Z" + }, + { + "number": 7, + "name": "Run unit tests", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:30:34Z", + "completed_at": "2026-10-09T22:37:04Z" + }, + { + "number": 8, + "name": "Report coverage", + "status": "completed", + "conclusion": "skipped", + "started_at": "2026-10-09T22:37:04Z", + "completed_at": "2026-10-09T22:37:04Z" + } + ] + }, + { + "id": 114056915189, + "run_id": 37998639834, + "run_attempt": 1, + "name": "Required checks passed", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:40:00Z", + "started_at": "2026-10-09T22:41:54Z", + "completed_at": "2026-10-09T22:41:56Z", + "runner_name": "GitHub Actions 1000093170", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:41:54Z", + "completed_at": "2026-10-09T22:41:55Z" + }, + { + "number": 2, + "name": "Check every required job succeeded", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:41:55Z", + "completed_at": "2026-10-09T22:41:55Z" + } + ] + } + ] +} diff --git a/tests/project/ci_timing_fixtures/jobs-38001768735.json b/tests/project/ci_timing_fixtures/jobs-38001768735.json new file mode 100644 index 0000000000..b4fed14f37 --- /dev/null +++ b/tests/project/ci_timing_fixtures/jobs-38001768735.json @@ -0,0 +1,316 @@ +{ + "total_count": 8, + "jobs": [ + { + "id": 114061264509, + "run_id": 38001768735, + "run_attempt": 1, + "name": "Classify changed files", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:55:12Z", + "started_at": "2026-10-09T22:57:40Z", + "completed_at": "2026-10-09T22:57:46Z", + "runner_name": "GitHub Actions 1000093333", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:57:41Z", + "completed_at": "2026-10-09T22:57:42Z" + }, + { + "number": 3, + "name": "Classify the diff", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:57:43Z", + "completed_at": "2026-10-09T22:57:44Z" + } + ] + }, + { + "id": 114061987271, + "run_id": 38001768735, + "run_attempt": 1, + "name": "Integration Python 3.14", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:57:46Z", + "started_at": "2026-10-09T23:05:19Z", + "completed_at": "2026-10-09T23:07:59Z", + "runner_name": "GitHub Actions 1000093397", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:05:20Z", + "completed_at": "2026-10-09T23:05:20Z" + }, + { + "number": 7, + "name": "Run full test suite", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:05:38Z", + "completed_at": "2026-10-09T23:07:57Z" + } + ] + }, + { + "id": 114061987279, + "run_id": 38001768735, + "run_attempt": 1, + "name": "Python 3.11", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:57:46Z", + "started_at": "2026-10-09T22:58:44Z", + "completed_at": "2026-10-09T23:05:39Z", + "runner_name": "GitHub Actions 1000093343", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:58:45Z", + "completed_at": "2026-10-09T22:58:46Z" + }, + { + "number": 4, + "name": "Install package and test dependencies", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:58:51Z", + "completed_at": "2026-10-09T22:59:20Z" + }, + { + "number": 7, + "name": "Run unit tests", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:59:20Z", + "completed_at": "2026-10-09T23:05:37Z" + }, + { + "number": 8, + "name": "Report coverage", + "status": "completed", + "conclusion": "skipped", + "started_at": "2026-10-09T23:05:37Z", + "completed_at": "2026-10-09T23:05:37Z" + } + ] + }, + { + "id": 114061987291, + "run_id": 38001768735, + "run_attempt": 1, + "name": "Python 3.10", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:57:46Z", + "started_at": "2026-10-09T23:00:06Z", + "completed_at": "2026-10-09T23:07:11Z", + "runner_name": "GitHub Actions 1000093358", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:00:07Z", + "completed_at": "2026-10-09T23:00:08Z" + }, + { + "number": 4, + "name": "Install package and test dependencies", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:00:13Z", + "completed_at": "2026-10-09T23:00:42Z" + }, + { + "number": 7, + "name": "Run unit tests", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:00:42Z", + "completed_at": "2026-10-09T23:07:09Z" + }, + { + "number": 8, + "name": "Report coverage", + "status": "completed", + "conclusion": "skipped", + "started_at": "2026-10-09T23:07:09Z", + "completed_at": "2026-10-09T23:07:09Z" + } + ] + }, + { + "id": 114061987374, + "run_id": 38001768735, + "run_attempt": 1, + "name": "Python 3.13", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:57:46Z", + "started_at": "2026-10-09T23:12:19Z", + "completed_at": "2026-10-09T23:19:01Z", + "runner_name": "GitHub Actions 1000093467", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:12:19Z", + "completed_at": "2026-10-09T23:12:20Z" + }, + { + "number": 4, + "name": "Install package and test dependencies", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:12:22Z", + "completed_at": "2026-10-09T23:12:52Z" + }, + { + "number": 7, + "name": "Run unit tests", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:12:52Z", + "completed_at": "2026-10-09T23:18:59Z" + }, + { + "number": 8, + "name": "Report coverage", + "status": "completed", + "conclusion": "skipped", + "started_at": "2026-10-09T23:18:59Z", + "completed_at": "2026-10-09T23:18:59Z" + } + ] + }, + { + "id": 114061987436, + "run_id": 38001768735, + "run_attempt": 1, + "name": "Python 3.14", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:57:46Z", + "started_at": "2026-10-09T23:17:22Z", + "completed_at": "2026-10-09T23:29:50Z", + "runner_name": "GitHub Actions 1000093514", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:17:23Z", + "completed_at": "2026-10-09T23:17:24Z" + }, + { + "number": 4, + "name": "Install package and test dependencies", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:17:26Z", + "completed_at": "2026-10-09T23:18:04Z" + }, + { + "number": 7, + "name": "Run unit tests", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:18:05Z", + "completed_at": "2026-10-09T23:29:19Z" + }, + { + "number": 8, + "name": "Report coverage", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:29:19Z", + "completed_at": "2026-10-09T23:29:46Z" + } + ] + }, + { + "id": 114061988445, + "run_id": 38001768735, + "run_attempt": 1, + "name": "Project tests (docs-only)", + "status": "completed", + "conclusion": "skipped", + "created_at": "2026-10-09T22:57:46Z", + "started_at": "2026-10-09T22:57:46Z", + "completed_at": "2026-10-09T22:57:46Z", + "runner_name": null, + "labels": [ + "ubuntu-latest" + ], + "steps": [] + }, + { + "id": 114070699437, + "run_id": 38001768735, + "run_attempt": 1, + "name": "Required checks passed", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T23:29:51Z", + "started_at": "2026-10-09T23:34:56Z", + "completed_at": "2026-10-09T23:35:00Z", + "runner_name": "GitHub Actions 1000093648", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:34:57Z", + "completed_at": "2026-10-09T23:34:57Z" + }, + { + "number": 2, + "name": "Check every required job succeeded", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:34:57Z", + "completed_at": "2026-10-09T23:34:57Z" + } + ] + } + ] +} diff --git a/tests/project/ci_timing_fixtures/jobs-38002005234.json b/tests/project/ci_timing_fixtures/jobs-38002005234.json new file mode 100644 index 0000000000..b9f61b00f4 --- /dev/null +++ b/tests/project/ci_timing_fixtures/jobs-38002005234.json @@ -0,0 +1,169 @@ +{ + "total_count": 5, + "jobs": [ + { + "id": 114062040386, + "run_id": 38002005234, + "run_attempt": 1, + "name": "Classify changed files", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:57:58Z", + "started_at": "2026-10-09T23:02:23Z", + "completed_at": "2026-10-09T23:02:29Z", + "runner_name": "GitHub Actions 1000093373", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:02:24Z", + "completed_at": "2026-10-09T23:02:25Z" + }, + { + "number": 3, + "name": "Classify the diff", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:02:26Z", + "completed_at": "2026-10-09T23:02:26Z" + } + ] + }, + { + "id": 114063308717, + "run_id": 38002005234, + "run_attempt": 1, + "name": "Integration Python 3.14", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T23:02:29Z", + "started_at": "2026-10-09T23:04:01Z", + "completed_at": "2026-10-09T23:06:43Z", + "runner_name": "GitHub Actions 1000093387", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:04:03Z", + "completed_at": "2026-10-09T23:04:03Z" + }, + { + "number": 7, + "name": "Run full test suite", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:04:24Z", + "completed_at": "2026-10-09T23:06:39Z" + } + ] + }, + { + "id": 114063308849, + "run_id": 38002005234, + "run_attempt": 1, + "name": "Python 3.14", + "status": "completed", + "conclusion": "cancelled", + "created_at": "2026-10-09T23:02:30Z", + "started_at": "2026-10-09T23:21:48Z", + "completed_at": "2026-10-09T23:28:11Z", + "runner_name": "GitHub Actions 1000093554", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:21:49Z", + "completed_at": "2026-10-09T23:21:51Z" + }, + { + "number": 4, + "name": "Install package and test dependencies", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:21:54Z", + "completed_at": "2026-10-09T23:22:26Z" + }, + { + "number": 7, + "name": "Run unit tests", + "status": "completed", + "conclusion": "cancelled", + "started_at": "2026-10-09T23:22:27Z", + "completed_at": "2026-10-09T23:28:07Z" + }, + { + "number": 8, + "name": "Report coverage", + "status": "completed", + "conclusion": "skipped", + "started_at": "2026-10-09T23:28:07Z", + "completed_at": "2026-10-09T23:28:07Z" + } + ] + }, + { + "id": 114063308937, + "run_id": 38002005234, + "run_attempt": 1, + "name": "Python 3.10", + "status": "completed", + "conclusion": "cancelled", + "created_at": "2026-10-09T23:02:30Z", + "started_at": "2026-10-09T23:02:30Z", + "completed_at": "2026-10-09T23:27:54Z", + "runner_name": "", + "labels": [ + "ubuntu-latest" + ], + "steps": [] + }, + { + "id": 114070278597, + "run_id": 38002005234, + "run_attempt": 1, + "name": "Required checks passed", + "status": "completed", + "conclusion": "failure", + "created_at": "2026-10-09T23:28:11Z", + "started_at": "2026-10-09T23:31:10Z", + "completed_at": "2026-10-09T23:31:12Z", + "runner_name": "GitHub Actions 1000093623", + "labels": [ + "ubuntu-latest" + ], + "steps": [ + { + "number": 1, + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T23:31:11Z", + "completed_at": "2026-10-09T23:31:11Z" + }, + { + "number": 2, + "name": "Check every required job succeeded", + "status": "completed", + "conclusion": "failure", + "started_at": "2026-10-09T23:31:11Z", + "completed_at": "2026-10-09T23:31:11Z" + } + ] + } + ] +} diff --git a/tests/project/ci_timing_fixtures/junit-unit-3.13.xml b/tests/project/ci_timing_fixtures/junit-unit-3.13.xml new file mode 100644 index 0000000000..a30d4215be --- /dev/null +++ b/tests/project/ci_timing_fixtures/junit-unit-3.13.xml @@ -0,0 +1 @@ +git is not availableAssertionError: 1 != 2 diff --git a/tests/project/ci_timing_fixtures/runs-unit-tests.json b/tests/project/ci_timing_fixtures/runs-unit-tests.json new file mode 100644 index 0000000000..91343db300 --- /dev/null +++ b/tests/project/ci_timing_fixtures/runs-unit-tests.json @@ -0,0 +1,57 @@ +{ + "total_count": 4, + "workflow_runs": [ + { + "id": 38002005234, + "name": "Unit Tests", + "event": "pull_request", + "status": "completed", + "conclusion": "cancelled", + "run_attempt": 1, + "created_at": "2026-10-09T22:57:57Z", + "updated_at": "2026-10-09T23:31:13Z", + "head_sha": "471e3954148ef0b47b7f74879e1861ff177c1836", + "head_branch": "fix/1513-pypcapfile-ipv6-tcp", + "path": ".github/workflows/unit-tests.yml" + }, + { + "id": 38001768735, + "name": "Unit Tests", + "event": "pull_request", + "status": "completed", + "conclusion": "success", + "run_attempt": 1, + "created_at": "2026-10-09T22:55:11Z", + "updated_at": "2026-10-09T23:35:01Z", + "head_sha": "e549c91623ddd847562c8ec3d5e963d5cea5a9f5", + "head_branch": "fix/1504-registrar-restore-descriptor", + "path": ".github/workflows/unit-tests.yml" + }, + { + "id": 37998639834, + "name": "Unit Tests", + "event": "push", + "status": "completed", + "conclusion": "success", + "run_attempt": 1, + "created_at": "2026-10-09T22:20:09Z", + "updated_at": "2026-10-09T22:41:57Z", + "head_sha": "ffca64637774cc10e227ee3ebe77800f7ae5f6fd", + "head_branch": "main", + "path": ".github/workflows/unit-tests.yml" + }, + { + "id": 37323913186, + "name": "Unit Tests", + "event": "push", + "status": "completed", + "conclusion": "cancelled", + "run_attempt": 1, + "created_at": "2026-10-05T14:20:30Z", + "updated_at": "2026-10-05T15:06:09Z", + "head_sha": "1007b00f9d18ecb87ed311907419b9cb8466e18f", + "head_branch": "main", + "path": ".github/workflows/unit-tests.yml" + } + ] +} diff --git a/tests/project/ci_timing_fixtures/workflow-unit-tests.yml b/tests/project/ci_timing_fixtures/workflow-unit-tests.yml new file mode 100644 index 0000000000..0fb80461ca --- /dev/null +++ b/tests/project/ci_timing_fixtures/workflow-unit-tests.yml @@ -0,0 +1,69 @@ +# An excerpt of .github/workflows/unit-tests.yml at ffca64637, cut down to the +# keys util/ci_timing.py reads: job names, caps, needs and the one step cap. +name: Unit Tests + +on: + push: + branches: [main] + +jobs: + test: + name: Python ${{ matrix.python-version }} + if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }} + needs: [changes] + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v7 + + - name: Run unit tests + id: tests + run: | + set -- -q -n auto --dist load + + integration: + name: Integration Python ${{ matrix.python-version }} + needs: [changes] + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Run full test suite + shell: bash + run: echo + + unittest-ordering: + name: Plain unittest ordering (${{ matrix.label || matrix.leg }}) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Run tests/${{ matrix.leg }} and the root-level modules under plain unittest + timeout-minutes: 20 + run: python util/run_unittest_leg.py ${{ matrix.leg }} ${{ matrix.exclude }} --verbose + + changes: + name: Classify changed files + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Classify the diff + run: echo + + project-tests: + name: Project tests (docs-only) + needs: [changes] + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Run tests/project and the root-level modules under plain unittest + timeout-minutes: 15 + run: python util/run_unittest_leg.py project --verbose + + required-checks: + name: Required checks passed + needs: [changes, test, integration, project-tests] + if: ${{ always() && inputs.gate-only != true }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Check every required job succeeded + run: echo diff --git a/tests/project/test_ci_timing.py b/tests/project/test_ci_timing.py new file mode 100644 index 0000000000..65580a1850 --- /dev/null +++ b/tests/project/test_ci_timing.py @@ -0,0 +1,758 @@ +# -*- coding: utf-8 -*- +"""Tests for :file:`util/ci_timing.py`, the weekly CI timing review, and its inputs. + +GitHub issue #1538 added the review and the per-test timings it reads. Four things +are checked, offline, since this tier has no network and no token: + +* :class:`TestArithmetic` and :class:`TestWorkflowText` -- the quantiles, the + projection to a cap, and the indentation reader the caps come from, on the real + workflow files as well as on the excerpt below. +* :class:`TestClient`, :class:`TestEndToEnd` and the classes after it -- the script + against a fake API serving the fixtures in :file:`tests/project/ci_timing_fixtures/`. + The run listing and the job listings are real responses from 2026-10-05 and + 2026-10-09, trimmed to the fields the script reads; run 37323913186 is the push + run that reads ``cancelled`` because its ``protocols`` ordering job hit the 45-minute + cap. ``artifacts-37998639834.json`` is that push run's real listing with two + **synthetic** entries added, ``junit-unit-3.13`` and an expired ``junit-unit-3.14``: + the run predates the upload, so it has no JUnit artifact of its own. + ``junit-unit-3.13.xml`` is hand-written in pytest's JUnit format, with times chosen + to trip the test-level checks, and the workflow file is an excerpt. Expected + figures are re-derived from the fixtures here rather than restated. +* :class:`TestThresholds` -- every threshold, at its boundary, on a report held + just short of all of them. +* :class:`TestTimingArtifacts` and :class:`TestReviewWorkflow` -- what the script + relies on in :file:`unit-tests.yml` (artifact names, the coverage leg, upload + steps that cannot fail a required leg) and the review workflow's own triggers and + permissions, which have to stay read-only. + +""" + +from __future__ import annotations + +import contextlib +import datetime +import importlib.util +import io +import json +import os +import pathlib +import re +import statistics +import tempfile +import unittest +import urllib.error +import urllib.parse +import zipfile +from unittest import mock + +ROOT = pathlib.Path(__file__).resolve().parents[2] +FIXTURES = pathlib.Path(__file__).resolve().parent / 'ci_timing_fixtures' +WORKFLOWS = ROOT / '.github' / 'workflows' +UNIT_TESTS = WORKFLOWS / 'unit-tests.yml' +REVIEW = WORKFLOWS / 'ci-timing.yml' +DOC = ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst' + +REPO = 'JarryShaw/PyPCAPKit' +NOW = datetime.datetime(2026, 10, 10, tzinfo=datetime.timezone.utc) +#: Where the real ``Link`` header points a run listing's next page: at the +#: repository by id, not by name. +LINK_ROOT = 'https://api.github.com/repositories/109791841/actions/workflows/unit-tests.yml/runs' +#: Runs per fake page, small so the paging is exercised. +PAGE = 2 +PR_RUN, CANCELLED_RUN, PUSH_RUN, TIMEOUT_RUN = 38001768735, 38002005234, 37998639834, 37323913186 +JUNIT_ARTIFACT_ID = 11648726901 +#: The day slices a 7-day window ending at :data:`NOW` is listed in. +SLICES = [f'2026-10-{day:02d}T00:00:00Z..2026-10-{day:02d}T23:59:59Z' for day in range(9, 2, -1)] + + +def _load_script(): + """Load :file:`util/ci_timing.py`, which is a script and not a package.""" + spec = importlib.util.spec_from_file_location('ci_timing', ROOT / 'util' / 'ci_timing.py') + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +ci_timing = _load_script() + + +def fixture(name: 'str') -> 'bytes': + return (FIXTURES / name).read_bytes() + + +def jobs_of(run_id: 'int') -> 'list[dict]': + return json.loads(fixture(f'jobs-{run_id}.json'))['jobs'] + + +def all_runs() -> 'list[dict]': + return json.loads(fixture('runs-unit-tests.json'))['workflow_runs'] + + +def stamp(value: 'str') -> 'datetime.datetime': + """An API timestamp, parsed here independently of the script's own parser.""" + return datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ') + + +def span(start: 'str', end: 'str') -> 'float': + return (stamp(end) - stamp(start)).total_seconds() / 60 + + +class FakeAPI: + """Serves the fixtures by URL, recording every request. + + Args: + inflate: Added to every listing's ``total_count``, to model the API's + 1,000-run cap on a filtered listing. + broken: Request paths whose body is replaced by bytes no parser accepts. + + """ + + def __init__(self, inflate: 'int' = 0, broken: 'tuple[str, ...]' = ()) -> 'None': + self.requests = [] # type: list[tuple[str, str]] + self.inflate = inflate + self.broken = broken + + def __call__(self, url: 'str', accept: 'str') -> 'tuple[bytes, dict[str, str]]': + self.requests.append((url, accept)) + parts = urllib.parse.urlsplit(url) + path, query = parts.path, urllib.parse.parse_qs(parts.query) + if any(path.endswith(end) for end in self.broken): + return b'\x00 neither JSON nor a zip archive', {} + if path.endswith('/actions/workflows/unit-tests.yml/runs'): + return self.listing(all_runs(), query) + if path.endswith('/runs') and '/actions/workflows/' in path: + return self.listing([], query) + match = re.search(r'/actions/runs/(\d+)/(attempts/1/jobs|artifacts)$', path) + if match: + kind = 'jobs' if match.group(2).endswith('jobs') else 'artifacts' + return fixture(f'{kind}-{match.group(1)}.json'), {'x-ratelimit-remaining': '4321'} + if path.endswith(f'/actions/artifacts/{JUNIT_ARTIFACT_ID}/zip'): + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, 'w') as archive: + archive.writestr('junit.xml', fixture('junit-unit-3.13.xml')) + return buffer.getvalue(), {} + if '/contents/.github/workflows/unit-tests.yml' in path: + assert accept == 'application/vnd.github.raw+json', accept + return fixture('workflow-unit-tests.yml'), {} + raise urllib.error.HTTPError(url, 404, 'Not Found', None, None) # type: ignore[arg-type] + + def listing(self, runs: 'list[dict]', query: 'dict[str, list[str]]') -> 'tuple[bytes, dict[str, str]]': + """The runs created inside the query's inclusive ``A..B`` range, a page at a time.""" + if 'created' in query: + low, high = query['created'][0].split('..') + runs = [run for run in runs if low <= run['created_at'] <= high] + page = int(query.get('page', ['1'])[0]) + headers = {} + if page * PAGE < len(runs): + rest = {key: value[0] for key, value in query.items() if key != 'page'} + headers['link'] = f'<{LINK_ROOT}?{urllib.parse.urlencode({**rest, "page": page + 1})}>; rel="next"' + body = {'total_count': len(runs) + self.inflate, 'workflow_runs': runs[(page - 1) * PAGE:page * PAGE]} + return json.dumps(body).encode(), headers + + +def run_main(*argv: 'str', api: 'FakeAPI', + baseline: 'Optional[dict]' = None) -> 'tuple[int, str, dict, str]': + """``main`` against ``api``; returns the status, stdout, report.json and report.md.""" + with tempfile.TemporaryDirectory() as tmp: + out, md = os.path.join(tmp, 'report.json'), os.path.join(tmp, 'report.md') + before = os.path.join(tmp, 'baseline.json') + if baseline is not None: + pathlib.Path(before).write_text(json.dumps(baseline), encoding='utf-8') + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(io.StringIO()): + status = ci_timing.main(['--repo', REPO, '--out', out, '--summary', md, + '--baseline', before, *argv], transport=api, now=NOW) + report = json.loads(pathlib.Path(out).read_text(encoding='utf-8')) if os.path.exists(out) else {} + text = pathlib.Path(md).read_text(encoding='utf-8') if os.path.exists(md) else '' + return status, stdout.getvalue(), report, text + + +class TestArithmetic(unittest.TestCase): + """The statistics every threshold is judged on.""" + + def test_quantile_interpolates_between_ranks(self) -> 'None': + values = [float(v) for v in range(1, 11)] + self.assertAlmostEqual(ci_timing.quantile(values, 0.9), 9.1) + self.assertAlmostEqual(ci_timing.quantile(values, 0.5), 5.5) + self.assertEqual(ci_timing.quantile([3.0], 0.9), 3.0) + self.assertIsNone(ci_timing.quantile([], 0.5)) + + def test_projection_reaches_the_cap_on_the_fitted_line(self) -> 'None': + # 1 min/day from 10 min at day 0 to 12 min at day 2: 33 days to 45. + points = [(day / 5, 10 + day / 5) for day in range(11)] + fit = ci_timing.projection(points, 45) + self.assertAlmostEqual(fit['slope_per_day'], 1.0) + self.assertAlmostEqual(fit['days_to_cap'], 33.0) + self.assertAlmostEqual(fit['days_to_half_cap'], 10.5) + + def test_projection_refuses_thin_or_flat_data(self) -> 'None': + rising = [(day / 5, 10 + day / 5) for day in range(11)] + self.assertIsNone(ci_timing.projection(rising[:ci_timing.MIN_FIT_SAMPLES - 1], 45)['days_to_cap']) + narrow = [(day / 100, 10 + day) for day in range(20)] + self.assertIsNone(ci_timing.projection(narrow, 45)['slope_per_day']) + falling = [(day / 5, 20 - day / 5) for day in range(11)] + self.assertIsNone(ci_timing.projection(falling, 45)['days_to_cap']) + over = [(day / 5, 50 - day / 5) for day in range(11)] + self.assertEqual(ci_timing.projection(over, 45)['days_to_cap'], 0.0) + + +class TestWorkflowText(unittest.TestCase): + """The caps and ``needs:`` the script reads off the workflow files.""" + + def test_the_excerpt(self) -> 'None': + specs = ci_timing.parse_workflow(fixture('workflow-unit-tests.yml').decode()) + by_id = {spec['id']: spec for spec in specs} + self.assertEqual(by_id['test']['timeout'], 45) + self.assertEqual(by_id['required-checks']['needs'], + ['changes', 'test', 'integration', 'project-tests']) + ordering = ci_timing.match_job(specs, 'Plain unittest ordering (protocols (rest))') + self.assertEqual(ordering['id'], 'unittest-ordering') + self.assertEqual(ci_timing.step_cap( + ordering, 'Run tests/protocols and the root-level modules under plain unittest'), 20) + self.assertIsNone(ci_timing.match_job(specs, 'Lint')) + + def test_the_real_unit_tests_workflow(self) -> 'None': + """The job names the script keys on exist, so its tables are not empty.""" + specs = ci_timing.parse_workflow(UNIT_TESTS.read_text(encoding='utf-8')) + required = ci_timing.match_job(specs, ci_timing.REQUIRED_JOB) + self.assertIsNotNone(required) + self.assertIn('test', required['needs']) + for name in (ci_timing.COVERAGE_JOB,) + ci_timing.PAIRED_JOBS: + with self.subTest(job=name): + self.assertEqual(ci_timing.match_job(specs, name)['id'], 'test') + test = next(spec for spec in specs if spec['id'] == 'test') + self.assertIn(ci_timing.TESTS_STEP, [step['name'] for step in test['steps']]) + ordering = ci_timing.match_job(specs, 'Plain unittest ordering (protocols (rest))') + self.assertEqual(ci_timing.step_cap( + ordering, 'Run tests/protocols and the root-level modules under plain unittest'), 20) + caps = {spec['id']: spec['timeout'] for spec in specs} + self.assertEqual((caps['test'], caps['integration'], caps['engine-tests']), (45, 45, 30)) + + def test_every_other_workflow_reads(self) -> 'None': + """Including CodeQL's four-space step list, and a literal name over a template.""" + for workflow in ci_timing.OTHER_WORKFLOWS: + with self.subTest(workflow=workflow): + self.assertTrue(ci_timing.parse_workflow((WORKFLOWS / workflow).read_text(encoding='utf-8'))) + codeql = ci_timing.parse_workflow((WORKFLOWS / 'codeql-analysis.yml').read_text(encoding='utf-8')) + self.assertTrue(codeql[0]['steps']) + compat = ci_timing.parse_workflow((WORKFLOWS / 'python-compatibility.yml').read_text(encoding='utf-8')) + self.assertEqual(ci_timing.match_job(compat, 'Compat Python 3.15 (scheduled)')['id'], + 'compatibility-nightly') + self.assertEqual(ci_timing.match_job(compat, 'Compat Python 3.12')['id'], 'compatibility') + + +class TestClient(unittest.TestCase): + """Paging, the call budget, retries, and where the token goes.""" + + def test_paging_follows_the_link_header_verbatim(self) -> 'None': + api = FakeAPI() + client = ci_timing.Client('t', transport=api) + runs = list(client.paginate(f'/repos/{REPO}/actions/workflows/unit-tests.yml/runs', + 'workflow_runs', status='completed')) + self.assertEqual([run['id'] for run in runs], [CANCELLED_RUN, PR_RUN, PUSH_RUN, TIMEOUT_RUN]) + self.assertTrue(api.requests[1][0].startswith(f'{LINK_ROOT}?'), api.requests[1][0]) + self.assertEqual(client.calls, 2) + + def test_the_budget_stops_the_run(self) -> 'None': + client = ci_timing.Client('t', transport=FakeAPI(), max_calls=1) + with self.assertRaises(ci_timing.BudgetExceeded): + list(client.paginate(f'/repos/{REPO}/actions/workflows/unit-tests.yml/runs', 'workflow_runs')) + + def test_a_server_error_is_retried_and_a_client_error_is_not(self) -> 'None': + replies = [urllib.error.HTTPError('u', 502, 'Bad Gateway', None, None), # type: ignore[arg-type] + (b'{"ok": 1}', {})] + + def flaky(url: 'str', accept: 'str') -> 'tuple[bytes, dict[str, str]]': + reply = replies.pop(0) + if isinstance(reply, Exception): + raise reply + return reply + + waits = [] # type: list[float] + client = ci_timing.Client('t', transport=flaky, sleep=waits.append) + self.assertEqual(client.get('/x'), {'ok': 1}) + self.assertEqual((client.calls, waits), (2, [ci_timing.RETRY_WAITS[0]])) + missing = ci_timing.Client('t', transport=FakeAPI(), sleep=waits.append) + with self.assertRaises(urllib.error.HTTPError): + missing.get('/nowhere') + self.assertEqual(missing.calls, 1) + + def test_the_token_is_not_forwarded_on_a_redirect(self) -> 'None': + """An artifact download redirects to blob storage, which must not see it.""" + seen = [] + + class Response(io.BytesIO): + headers = {'X-RateLimit-Remaining': '7'} + + def urlopen(request, timeout): # noqa: ARG001 + seen.append(request) + return Response(b'{}') + + client = ci_timing.Client('secret') + with mock.patch.object(ci_timing.urllib.request, 'urlopen', urlopen): + client.get('/x') + self.assertEqual(seen[0].unredirected_hdrs.get('Authorization'), 'Bearer secret') + self.assertNotIn('Authorization', seen[0].headers) + self.assertEqual(client.rate_remaining, 7) + + +def counted(job: 'dict', cap: 'float') -> 'bool': + """Whether a job is measured, by its own outcome: finished, or killed at its cap.""" + if not job['runner_name']: + return False + if job['conclusion'] in ('success', 'failure'): + return True + return job['conclusion'] == 'cancelled' and span(job['started_at'], job['completed_at']) >= 0.95 * cap + + +class TestEndToEnd(unittest.TestCase): + """``main`` over the recorded fixtures.""" + + @classmethod + def setUpClass(cls) -> 'None': + cls.api = FakeAPI() + # One or two runs per event in the fixtures, so the sample floor comes down. + with mock.patch.object(ci_timing, 'MIN_SAMPLES', 1): + cls.status, cls.stdout, cls.report, cls.summary = run_main(api=cls.api) + + def ids(self) -> 'list[str]': + return [item['id'] for item in self.report['findings']] + + def test_a_timed_out_job_in_a_cancelled_run_is_measured(self) -> 'None': + """Run 37323913186 reads ``cancelled`` only because this job hit its 45-minute cap.""" + run = next(run for run in all_runs() if run['id'] == TIMEOUT_RUN) + job = next(job for job in jobs_of(TIMEOUT_RUN) if job['name'] == 'Plain unittest ordering (protocols)') + self.assertEqual((run['conclusion'], job['conclusion']), ('cancelled', 'cancelled')) + took = span(job['started_at'], job['completed_at']) + self.assertGreaterEqual(took, 0.95 * 45) + row = next(row for row in self.report['jobs'] if row['name'] == job['name']) + self.assertEqual(row['n'], 1) + self.assertAlmostEqual(row['max'], took) + # The step running at the timeout was cut off by the job's cap, so it counts too. + cut = next(s for s in job['steps'] if s['name'].startswith('Run tests/')) + step = next(row for row in self.report['steps'] + if row['job'] == job['name'] and row['name'] == cut['name']) + self.assertEqual(cut['conclusion'], 'cancelled') + self.assertAlmostEqual(step['max'], span(cut['started_at'], cut['completed_at'])) + self.assertIn('H1', self.ids()) + self.assertIn('::error title=CI timing H1::', self.stdout) + self.assertEqual(self.status, ci_timing.EXIT_HARD) + + def test_a_superseded_job_is_not_measured(self) -> 'None': + """The cancelled pull-request run's ``Python 3.14`` stopped at 6 minutes, not at its cap.""" + stopped = next(job for job in jobs_of(CANCELLED_RUN) if job['name'] == 'Python 3.14') + self.assertFalse(counted(stopped, 45)) + legs = [job for run in (PR_RUN, CANCELLED_RUN, PUSH_RUN, TIMEOUT_RUN) for job in jobs_of(run) + if job['name'] == 'Python 3.14' and counted(job, 45)] + row = next(row for row in self.report['jobs'] if row['name'] == 'Python 3.14') + self.assertEqual((row['n'], row['cap']), (len(legs), 45)) + self.assertEqual(row['n'], 3) + self.assertAlmostEqual(row['max'], max(span(job['started_at'], job['completed_at']) for job in legs)) + self.assertAlmostEqual(row['p90_share_of_cap'], row['p90'] / 45) + + def test_without_a_baseline_a_slow_queue_only_warns(self) -> 'None': + """Created 22:55:11, ``Required checks passed`` done 23:35:00: W9, not H3.""" + required = next(job for job in jobs_of(PR_RUN) if job['name'] == ci_timing.REQUIRED_JOB) + want = span('2026-10-09T22:55:11Z', required['completed_at']) + self.assertAlmostEqual(self.report['runs']['pull_request']['mergeable']['median'], want) + self.assertGreaterEqual(want, ci_timing.MERGEABLE_P90) + self.assertIn('W9', self.ids()) + self.assertNotIn('H3', self.ids()) + + def test_a_regression_on_the_baseline_is_hard(self) -> 'None': + before = {'runs': {'pull_request': {'mergeable': {'n': 1, 'p90': 20.0}}}} + with mock.patch.object(ci_timing, 'MIN_SAMPLES', 1): + _, stdout, report, _ = run_main(api=FakeAPI(), baseline=before) + ids = [item['id'] for item in report['findings']] + self.assertIn('H3', ids) + self.assertNotIn('W9', ids) + self.assertIn('::error title=CI timing H3::', stdout) + + def test_counts_come_from_the_whole_listing(self) -> 'None': + runs = self.report['runs'] + self.assertEqual([runs['pull_request'][key] for key in ('listed', 'cancelled', 'runs')], [2, 1, 1]) + # The timed-out push run is measured; only a superseded run is not. + self.assertEqual([runs['push'][key] for key in ('listed', 'cancelled', 'runs')], [2, 1, 2]) + superseded = self.report['superseded'] + self.assertEqual((superseded['runs'], superseded['of'], superseded['read']), (1, 2, 1)) + # The never-started ``Python 3.10`` (empty runner_name, 25 minutes queued) + # is not runner time. + self.assertTrue(any(not job['runner_name'] for job in jobs_of(CANCELLED_RUN))) + self.assertAlmostEqual(superseded['runner_min'], sum( + span(job['started_at'], job['completed_at']) for job in jobs_of(CANCELLED_RUN) if job['runner_name'])) + + def test_skipped_jobs_are_dropped(self) -> 'None': + names = {row['name'] for row in self.report['jobs']} + self.assertNotIn('Project tests (docs-only)', names) + self.assertIn('Plain unittest ordering (protocols (rest))', names) + + def test_the_ordering_step_is_judged_against_its_step_cap(self) -> 'None': + row = next(row for row in self.report['steps'] + if row['job'] == 'Plain unittest ordering (protocols (rest))' + and row['name'].startswith('Run tests/')) + self.assertEqual((row['cap'], row['n']), (20, 1)) + + def test_the_critical_path_is_the_last_needed_job(self) -> 'None': + needed = re.compile(r'(Classify changed files|(Integration )?Python .*|Project tests.*)') + for event, runs in (('pull_request', (PR_RUN,)), ('push', (PUSH_RUN, TIMEOUT_RUN))): + leaders, slack = [], [] + for run in runs: + ended = sorted((job['completed_at'], job['name']) for job in jobs_of(run) + if job['runner_name'] and needed.fullmatch(job['name'])) + leaders.append(ended[-1][1]) + if len(ended) > 1: + slack.append(span(ended[-2][0], ended[-1][0])) + with self.subTest(event=event): + path = self.report['critical_path'][event] + self.assertEqual(path['runs'], len(runs)) + self.assertEqual(path['leader'], max(set(leaders), key=leaders.count)) + self.assertAlmostEqual(path['slack_median'], statistics.median(slack)) + + def test_the_coverage_ratio_pairs_legs_within_a_run(self) -> 'None': + ratios = [] + for run in (PR_RUN, PUSH_RUN, TIMEOUT_RUN): + steps = {job['name']: next(span(s['started_at'], s['completed_at']) for s in job['steps'] + if s['name'] == ci_timing.TESTS_STEP) + for job in jobs_of(run) if re.fullmatch(r'Python 3\.1\d', job['name'])} + paired = [steps[name] for name in ci_timing.PAIRED_JOBS if name in steps] + if len(paired) >= 2: + ratios.append(steps[ci_timing.COVERAGE_JOB] / statistics.median(paired)) + self.assertEqual(len(ratios), 2) + self.assertAlmostEqual(self.report['derived']['coverage_ratio']['median'], statistics.median(ratios)) + + def test_ms_per_test_counts_testcases_not_the_suite_total(self) -> 'None': + job = next(job for job in jobs_of(PUSH_RUN) if job['name'] == 'Python 3.13') + step = next(s for s in job['steps'] if s['name'] == ci_timing.TESTS_STEP) + self.assertAlmostEqual(self.report['derived']['ms_per_test']['median'], + span(step['started_at'], step['completed_at']) * 60000 / 12) + self.assertEqual(self.report['derived']['test_counts'], + [{'run_id': PUSH_RUN, 'tests': 12, 'suite_tests': 25}]) + + def test_the_test_level_checks(self) -> 'None': + """An expired artifact is skipped; the module, slow-test and uniform checks fire.""" + self.assertEqual(list(self.report['tests']), ['3.13']) + leg = self.report['tests']['3.13'] + self.assertEqual(leg['top_modules'][0], ['tests.foundation.test_delta_unit', 32.519]) + self.assertEqual([name for name, _ in leg['slow_tests']], + ['tests.foundation.test_delta_unit.TestDelta::test_slow', + 'tests.foundation.test_delta_unit.TestDelta::test_slower']) + # TestGamma's first test carries its setup and the rest are cheap: not flagged. + self.assertEqual(leg['uniform_classes'], ['tests.protocols.test_beta_unit.TestBeta']) + self.assertEqual(self.ids().count('W7'), 3) + + def test_the_whole_window_is_listed_a_day_at_a_time(self) -> 'None': + self.assertEqual(self.report['api'], {'calls': len(self.api.requests), 'remaining': 4321}) + for url, _ in self.api.requests: + self.assertTrue(url.startswith('https://api.github.com/'), url) + listings = [urllib.parse.urlsplit(url) for url, _ in self.api.requests + if urllib.parse.urlsplit(url).path.endswith('/runs')] + unit = [urllib.parse.parse_qs(part.query) for part in listings if 'unit-tests.yml' in part.path] + # 10-09 holds three runs, two fake pages; every other day is one page. + self.assertEqual(len(unit), len(SLICES) + 1) + self.assertEqual(sorted({query['created'][0] for query in unit}), sorted(SLICES)) + for query in unit: + self.assertEqual(query['status'], ['completed']) + self.assertEqual(len(listings), len(unit) + len(SLICES) * len(ci_timing.OTHER_WORKFLOWS)) + # The caps come from the newest and oldest push runs' shas, one read each. + refs = [urllib.parse.parse_qs(urllib.parse.urlsplit(url).query)['ref'][0] + for url, _ in self.api.requests if '/contents/' in url] + self.assertEqual(refs, [run['head_sha'] for run in all_runs() if run['event'] == 'push']) + + def test_the_summary_says_what_was_read(self) -> 'None': + self.assertIn('- **HARD H1:**', self.summary) + self.assertIn('| pull_request | 2 | 1 | 1 |', self.summary) + self.assertIn('unit-tests.yml 4 of 4 (2026-10-05T14:20Z to 2026-10-09T22:57Z)', self.summary) + self.assertEqual(self.report['window']['read']['unit-tests.yml'], + {'listed': 4, 'read': 4, 'first': '2026-10-05T14:20:30Z', 'last': '2026-10-09T22:57:57Z'}) + self.assertTrue(any('No baseline' in note for note in self.report['notes'])) + self.assertIn(self.summary.strip(), self.stdout) + + def test_a_quiet_window_passes(self) -> 'None': + class Empty(FakeAPI): + def __call__(self, url: 'str', accept: 'str') -> 'tuple[bytes, dict[str, str]]': + return b'{"total_count": 0, "workflow_runs": []}', {} + + status, _, report, summary = run_main(api=Empty()) + self.assertEqual((status, report['findings']), (0, [])) + self.assertIn('No threshold fired.', summary) + self.assertTrue(any('no test-level timings' in note for note in report['notes'])) + + def test_no_token_is_a_usage_error(self) -> 'None': + with mock.patch.dict(os.environ, {'GH_TOKEN': '', 'GITHUB_TOKEN': ''}), \ + contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(ci_timing.main(['--repo', REPO]), ci_timing.EXIT_API) + + def test_a_cap_moving_inside_the_window_is_noted(self) -> 'None': + text = fixture('workflow-unit-tests.yml').decode() + older = text.replace('timeout-minutes: 45', 'timeout-minutes: 40', 1) + data = {'repo': REPO, 'since': '2026-10-03T00:00:00Z', 'until': '2026-10-10T00:00:00Z', + 'days': 7, 'junit': [], 'workflows': {'unit-tests.yml': {'runs': [], 'specs': { + 'newest': ('b', text), 'oldest': ('a', older)}}}} + notes = ci_timing.analyse(data)['notes'] + self.assertEqual(len(notes), 1) + self.assertIn('moved from 40 to 45', notes[0]) + + +class TestSampling(unittest.TestCase): + """Jobs are read for a sample spanning the window; the counts cover all of it.""" + + def test_spread_is_even(self) -> 'None': + self.assertEqual(ci_timing.spread(list(range(10)), 4), [0, 3, 6, 9]) + self.assertEqual(ci_timing.spread([1, 2, 3], 5), [1, 2, 3]) + self.assertEqual(ci_timing.spread([1, 2, 3], 1), [1]) + self.assertEqual(ci_timing.spread([1, 2, 3], 0), []) + + def test_a_smaller_sample_keeps_the_listing_counts(self) -> 'None': + api = FakeAPI() + _, _, report, _ = run_main('--runs', '1', '--junit-runs', '0', api=api) + read = {int(m.group(1)) for url, _ in api.requests + if (m := re.search(r'/runs/(\d+)/attempts/1/jobs', url))} + self.assertEqual(read, {CANCELLED_RUN, PUSH_RUN}) + self.assertEqual(report['window']['read']['unit-tests.yml']['read'], 2) + self.assertEqual(report['window']['read']['unit-tests.yml']['listed'], 4) + self.assertEqual((report['superseded']['runs'], report['superseded']['of']), (1, 2)) + + def test_a_day_past_the_listing_cap_is_reported(self) -> 'None': + _, _, report, _ = run_main(api=FakeAPI(inflate=ci_timing.LISTING_CAP)) + self.assertTrue(any('of which the API returned 3' in note for note in report['notes']), report['notes']) + + +class TestExitStatus(unittest.TestCase): + """1 is a HARD finding; a failure to read or to run says so with a status of its own.""" + + def test_an_api_failure_is_not_reported_as_a_threshold(self) -> 'None': + status, stdout, report, _ = run_main('--max-calls', '3', api=FakeAPI()) + self.assertEqual((status, report), (ci_timing.EXIT_API, {})) + self.assertIn('No threshold was evaluated', stdout) + + def test_a_crash_is_not_reported_as_a_threshold(self) -> 'None': + for broken in ('/attempts/1/jobs', f'/artifacts/{JUNIT_ARTIFACT_ID}/zip'): + with self.subTest(broken=broken): + status, stdout, report, _ = run_main(api=FakeAPI(broken=(broken,))) + self.assertEqual((status, report), (ci_timing.EXIT_CRASH, {})) + self.assertIn('::error title=CI timing crashed::', stdout) + + +class TestRunningJobs(unittest.TestCase): + """A job with no conclusion is still running, whatever its timestamps say.""" + + def test_a_job_without_a_conclusion_is_not_counted(self) -> 'None': + def job(name: 'str', conclusion: 'Optional[str]', end: 'Optional[str]') -> 'dict': + return {'name': name, 'conclusion': conclusion, 'runner_name': 'GitHub Actions 1', + 'run_attempt': 1, 'created_at': '2026-10-09T10:00:30Z', + 'started_at': '2026-10-09T10:01:00Z', 'completed_at': end, 'steps': []} + + run = {'id': 1, 'event': 'pull_request', 'conclusion': 'success', + 'created_at': '2026-10-09T10:00:00Z', 'jobs': [ + job('Python 3.14', 'success', '2026-10-09T10:13:00Z'), + # A check run's empty conclusion, with an end time all the same. + job('Python 3.13', '', '2026-10-09T10:31:00Z'), + job('Python 3.12', None, None)]} + data = {'repo': REPO, 'since': '2026-10-03T00:00:00Z', 'until': '2026-10-10T00:00:00Z', + 'days': 7, 'junit': [], 'workflows': {'unit-tests.yml': {'runs': [run], 'specs': { + 'newest': ('a', fixture('workflow-unit-tests.yml').decode())}}}} + report = ci_timing.analyse(data) + self.assertEqual([row['name'] for row in report['jobs']], ['Python 3.14']) + pr = report['runs']['pull_request'] + self.assertEqual((pr['runner_min']['median'], pr['wall']['median']), (12.0, 13.0)) + + +def quiet() -> 'tuple[dict, dict]': + """A report and baseline held just short of every threshold.""" + modules = [[f'tests.m{i}', 30.0 - i] for i in range(10)] + report = { + 'jobs': [{'workflow': 'unit-tests.yml', 'name': 'Python 3.14', 'n': 20, 'median': 10.0, + 'p90': 22.4, 'cap': 45, 'slope_per_day': 0.5, 'days_to_cap': 14.1}], + 'steps': [{'workflow': 'unit-tests.yml', 'job': 'Plain unittest ordering (protocols (rest))', + 'name': 'Run tests/protocols and the root-level modules under plain unittest', + 'n': 20, 'p90': 9.9, 'cap': 20, 'days_to_cap': 14.1}], + 'runs': {'pull_request': {'mergeable': {'n': 20, 'median': 18.0, 'p90': 29.9}, + 'queue': {'n': 20, 'p90': 5.0}}, + 'push': {'mergeable': {'n': 20, 'median': 20.0, 'p90': 40.0}, + 'queue': {'n': 20, 'p90': 15.0}}}, + 'superseded': {'runs': 35, 'of': 100, 'share': 0.35, 'runner_min': 1000.0}, + 'derived': {'coverage_ratio': {'n': 20, 'median': 1.8}, 'ms_per_test': {'n': 5, 'median': 114.9}}, + 'tests': {'3.13': {'top_modules': modules, 'slow_tests': [], 'uniform_classes': []}}, + 'critical_path': {'pull_request': {'leader': 'Python 3.14'}, 'push': {'leader': 'Python 3.14'}}, + } + baseline = { + 'jobs': [{'workflow': 'unit-tests.yml', 'name': 'Python 3.14', 'n': 20, 'median': 8.4}], + 'runs': {'pull_request': {'mergeable': {'n': 20, 'p90': 25.0}}}, + 'derived': {'ms_per_test': {'median': 100.0}}, + 'tests': {'3.13': {'top_modules': [list(row) for row in modules]}}, + 'critical_path': {'pull_request': {'leader': 'Python 3.14'}, 'push': {'leader': 'Python 3.14'}}, + } + return report, baseline + + +class TestThresholds(unittest.TestCase): + """Each threshold fires at its boundary and not short of it.""" + + def fired(self, change: 'Callable[[dict, dict], None]') -> 'list[str]': + report, baseline = quiet() + change(report, baseline) + return [item['id'] for item in ci_timing.evaluate(report, baseline)] + + def test_nothing_fires_just_short_of_every_threshold(self) -> 'None': + self.assertEqual(self.fired(lambda report, baseline: None), []) + + def test_each_threshold_at_its_boundary(self) -> 'None': + def job(report: 'dict') -> 'dict': + return report['jobs'][0] + + def merge(report: 'dict') -> 'dict': + return report['runs']['pull_request']['mergeable'] + + cases = { + 'H1': lambda r, b: job(r).update(p90=22.5), + 'H2': lambda r, b: r['steps'][0].update(p90=10.0), + # 30.0 is 1.25 times the baseline's 24.0, and at the floor. + 'H3': lambda r, b: (merge(r).update(p90=30.0), merge(b).update(p90=24.0)), + 'W1': lambda r, b: b['jobs'][0].update(median=8.3), + 'W2': lambda r, b: merge(r).update(median=18.1), + 'W3': lambda r, b: r['derived']['coverage_ratio'].update(median=1.81), + 'W4': lambda r, b: r['derived']['ms_per_test'].update(median=115.0), + 'W5': lambda r, b: r['runs']['push']['queue'].update(p90=15.1), + 'W6': lambda r, b: r['superseded'].update(runs=36, share=0.36), + 'W8': lambda r, b: r['critical_path']['push'].update(leader='Python 3.13'), + # At the floor but under 1.25 times the baseline's 25.0: no regression. + 'W9': lambda r, b: merge(r).update(p90=30.0), + } + for code, change in cases.items(): + with self.subTest(threshold=code): + self.assertEqual(self.fired(change), [code]) + self.assertEqual(self.fired(lambda r, b: job(r).update(days_to_cap=14.0)), ['H4']) + self.assertEqual(self.fired(lambda r, b: r['steps'][0].update(days_to_cap=14.0)), ['H4']) + self.assertEqual(self.fired(lambda r, b: r['runs']['pull_request']['queue'].update(p90=5.1)), ['W5']) + # A rise that stays under the floor is not H3 either. + self.assertEqual(self.fired(lambda r, b: merge(b).update(p90=10.0)), []) + + def test_every_w7_signal(self) -> 'None': + leg = lambda r: r['tests']['3.13'] # noqa: E731 + self.assertEqual(self.fired(lambda r, b: leg(r)['top_modules'][0].__setitem__(1, 30.1)), ['W7']) + self.assertEqual(self.fired(lambda r, b: leg(r).update(slow_tests=[['t::x', 10.5]])), ['W7']) + self.assertEqual(self.fired(lambda r, b: leg(r).update(uniform_classes=['tests.m.T'])), ['W7']) + self.assertEqual(self.fired(lambda r, b: leg(r)['top_modules'].insert(0, ['tests.new', 29.5])), + ['W7']) + + def test_a_thin_sample_does_not_fire(self) -> 'None': + def thin(report: 'dict', baseline: 'dict') -> 'None': + report['jobs'][0].update(n=ci_timing.MIN_SAMPLES - 1, p90=40.0) + report['runs']['pull_request']['mergeable'].update(n=ci_timing.MIN_SAMPLES - 1, p90=60.0) + baseline['jobs'][0].update(n=ci_timing.WARN_MEDIAN_SAMPLES - 1, median=1.0) + + self.assertEqual(self.fired(thin), []) + # A thin baseline cannot make a slow week HARD; it stays W9. + self.assertEqual(self.fired(lambda r, b: ( + r['runs']['pull_request']['mergeable'].update(p90=60.0), + b['runs']['pull_request']['mergeable'].update(n=ci_timing.MIN_SAMPLES - 1, p90=10.0))), ['W9']) + + def test_no_baseline_skips_the_week_over_week_checks(self) -> 'None': + report, _ = quiet() + report['critical_path']['push']['leader'] = 'Python 3.13' + self.assertEqual(ci_timing.evaluate(report, None), []) + report['runs']['pull_request']['mergeable']['p90'] = 60.0 + self.assertEqual([item['id'] for item in ci_timing.evaluate(report, None)], ['W9']) + + def test_the_documented_thresholds_are_the_evaluated_ones(self) -> 'None': + source = (ROOT / 'util' / 'ci_timing.py').read_text(encoding='utf-8') + codes = set(re.findall(r"fire\('([HW]\d)'", source)) + self.assertEqual(codes, {f'H{i}' for i in range(1, 5)} | {f'W{i}' for i in range(1, 10)}) + documented = set(re.findall(r'\* - ``([HW]\d)``', DOC.read_text(encoding='utf-8'))) + self.assertEqual(documented, codes) + + +def step_blocks(text: 'str', name: 'str') -> 'list[str]': + """Every step called ``name``, from its ``- name:`` line to the next step or job.""" + return re.findall(rf'(?ms)^ - name: {re.escape(name)}\n(.*?)(?=^ - |^ \S|\Z)', text) + + +class TestTimingArtifacts(unittest.TestCase): + """#1538's P3 in :file:`unit-tests.yml`, as the review and the gates need it.""" + + def setUp(self) -> 'None': + self.text = UNIT_TESTS.read_text(encoding='utf-8') + + def test_every_unit_and_integration_leg_uploads_its_timings(self) -> 'None': + blocks = step_blocks(self.text, 'Upload test timings') + names = [re.search(r'(?m)^ name: (.+)$', block).group(1) for block in blocks] + self.assertEqual(names, ['junit-unit-${{ matrix.python-version }}', + 'junit-integration-${{ matrix.python-version }}']) + self.assertIn(ci_timing.JUNIT_ARTIFACT.format(leg='${{ matrix.python-version }}'), names) + for block in blocks: + with self.subTest(step=block.split('\n', 3)[:3]): + # A failed upload of a diagnostic file must not fail a required leg. + self.assertIn('continue-on-error: true', block) + self.assertIn('if: ${{ !cancelled() }}', block) + self.assertIn('path: ${{ runner.temp }}/junit.xml', block) + # tests/_dependency_gates.py reads exactly one pytest step per job. + self.assertNotIn('python -m pytest', block) + + def test_artifact_names_are_unique_in_a_run(self) -> 'None': + names = re.findall(r'uses: actions/upload-artifact@\S+\n +with:\n +name: ([^\n]+)', self.text) + self.assertGreaterEqual(len(names), 5) + self.assertEqual(len(names), len(set(names)), names) + + def test_both_pytest_runs_write_the_file_the_upload_reads(self) -> 'None': + for job in ('test', 'integration'): + section = re.search(rf'(?ms)^ {job}:\n(.*?)(?=^ [\w-]+:\n)', self.text).group(1) + with self.subTest(job=job): + self.assertIn('--junitxml="$RUNNER_TEMP/junit.xml"', section) + self.assertIn('--durations=', section) + + def test_the_legs_the_review_reads_exist(self) -> 'None': + test = re.search(r'(?ms)^ test:\n(.*?)(?=^ [\w-]+:\n)', self.text).group(1) + for leg in ci_timing.JUNIT_LEGS: + with self.subTest(leg=leg): + self.assertIn(f'- "{leg}"', test) + coverage = re.search(r'include:\n\s+- python-version: "([\d.]+)"\n\s+coverage: true', test) + self.assertEqual(f'Python {coverage.group(1)}', ci_timing.COVERAGE_JOB) + + def test_the_reusable_workflow_asks_for_no_write_permission(self) -> 'None': + """Its four callers grant ``contents: read``, so a write scope would break them.""" + code = '\n'.join(line.split('#', 1)[0] for line in self.text.splitlines()) + self.assertNotRegex(code, r'(?m)^\s+[\w-]+:\s*write\s*$') + + +class TestReviewWorkflow(unittest.TestCase): + """:file:`ci-timing.yml` stays read-only, weekly, and off Unit Tests' runs.""" + + def setUp(self) -> 'None': + self.text = REVIEW.read_text(encoding='utf-8') + self.code = '\n'.join(line.split('#', 1)[0] for line in self.text.splitlines()) + + def test_triggers(self) -> 'None': + self.assertRegex(self.code, r"(?m)^ - cron: '\d+ \d+ \* \* \d'$") + self.assertIn('workflow_dispatch:', self.code) + for trigger in ('workflow_run', 'pull_request', 'push'): + with self.subTest(trigger=trigger): + self.assertNotRegex(self.code, rf'(?m)^ {trigger}:') + + def test_permissions_are_read_only(self) -> 'None': + block = re.search(r'(?ms)^permissions:\n(.*?)(?=^\S)', self.code).group(1) + self.assertEqual(sorted(block.split()), ['actions:', 'contents:', 'read', 'read']) + self.assertNotIn('write', self.code) + + def test_the_job(self) -> 'None': + specs = ci_timing.parse_workflow(self.text) + self.assertEqual([(spec['name'], spec['timeout']) for spec in specs], [('CI timing report', 15)]) + self.assertIn('python util/ci_timing.py', self.code) + # The upload and the baseline download name the same artifact. + self.assertIn('name: ci-timing-report', self.code) + self.assertIn('-n ci-timing-report', self.code) + + def test_yaml_agrees(self) -> 'None': + try: + import yaml # pylint: disable=import-outside-toplevel + except ImportError: # pragma: no cover + self.skipTest('PyYAML, from the test extra, is not installed') + data = yaml.safe_load(self.text) + # PyYAML reads the bare key ``on`` as the boolean True. + self.assertEqual(sorted(data[True]), ['schedule', 'workflow_dispatch']) + self.assertEqual(data['permissions'], {'actions': 'read', 'contents': 'read'}) + self.assertEqual(data['jobs']['report']['timeout-minutes'], 15) + + +if __name__ == '__main__': + unittest.main() diff --git a/util/ci_timing.py b/util/ci_timing.py new file mode 100644 index 0000000000..6bca91d44d --- /dev/null +++ b/util/ci_timing.py @@ -0,0 +1,1181 @@ +# -*- coding: utf-8 -*- +"""Report how long CI takes, and fail when a job is heading for its cap. + +This is the script :file:`.github/workflows/ci-timing.yml` runs every Monday, the +periodic review GitHub issue #1538 asked for. It reads the Actions API for the last +``--days`` days and writes three things: ``report.json`` (the next run's baseline), a +Markdown summary, and one ``::error::`` or ``::warning::`` annotation per threshold +that fired. Exit status: 0 when no HARD threshold fired, 1 when one did, 2 when the +API could not be read or the arguments are wrong, 3 for anything else. A HARD +failure is what emails the owner: GitHub notifies the user who last edited a +scheduled workflow's cron line of its failures, so no ``issues: write`` is needed. + +What is read, and the rules for reading it: + +* **Every completed run in the window is listed**, for Unit Tests and for each of + :data:`OTHER_WORKFLOWS`, one day per query: a filtered listing returns at most + 1,000 runs, and GitHub Pages alone completed 1,154 in the week to 2026-10-09. The + run counts, and so the superseded share, come from the whole listing. +* **Jobs are read for an evenly spaced sample** of those runs -- ``--runs`` per event + for Unit Tests, ``--other-runs`` per other workflow -- so the sample spans the whole + window rather than its newest days. The span read is printed with the report. +* **At job level**, from ``attempts/1/jobs``: a re-run attempt is a second copy of + the same work and is dropped, as is any job with no ``runner_name`` (skipped, or + never started). A job or step whose ``conclusion`` is empty is still running and is + not counted. +* **A job counts by its own outcome, not its run's.** It is measured when it + succeeded or failed, or when it was cancelled at :data:`TIMEOUT_SHARE` of its cap + or later, which is how a timeout reports -- including inside a run that reads + ``cancelled`` for that reason. A job cancelled earlier was superseded and is not. + A run is measured when it succeeded, failed, or holds such a timed-out job; other + cancelled runs are the superseded ones, counted and costed but not timed. +* **Pages are followed to exhaustion** through the ``Link`` header, the equivalent + of ``gh api --paginate``, with no fixed page count. +* **Each cap is read from the workflow file itself**, at the newest push (or other + non-PR) run's head sha, rather than written down here. It is also read at the + oldest such run's sha, and a cap that differs between the two is reported, since a + trend across it is not one trend. One read per run would cost ~300 calls for a + value that rarely moves. +* **Test timings** come from the ``junit-unit-*`` artifacts the unit legs upload, + for the newest :data:`JUNIT_RUNS` successful push runs. Each testcase's own + ``time`` is used; a suite's ``tests=`` count includes subtests and is not. + +A default run makes about 625 API calls (estimate, for the week to 2026-10-09: ~60 +listing pages, 300 + 240 job reads, 15 for artifacts, up to 10 workflow reads), +inside ``GITHUB_TOKEN``'s 1,000 per hour; ``--max-calls`` stops it before it can +take more. ``--concurrency-hours`` adds the +runner-pool measurement, which costs one call per run of every workflow in that +window and is therefore off unless asked for. + +Standard library only, so the workflow runs it on a bare interpreter. The token is +``GH_TOKEN`` (or ``GITHUB_TOKEN``); locally, ``GH_TOKEN=$(gh auth token)`` works. + +""" + +from __future__ import annotations + +import argparse +import collections +import datetime +import io +import json +import math +import os +import re +import statistics +import sys +import time +import traceback +import urllib.error +import urllib.parse +import urllib.request +# Parses JUnit XML that this repository's own CI wrote, never anything untrusted. +import xml.etree.ElementTree as ET # nosec B405 +import zipfile +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Any, Callable, Iterable, Iterator, Optional + + #: Fetches one URL with an ``Accept`` header; returns the body and the headers, + #: names lower-cased. + Transport = Callable[[str, str], tuple[bytes, dict[str, str]]] + +API = 'https://api.github.com' + +#: The workflow whose runs are read job by job. +UNIT_TESTS = 'unit-tests.yml' +#: The other workflows on a pull request's path, read for job durations only. +OTHER_WORKFLOWS = ('python-compatibility.yml', 'lint.yml', 'codeql-analysis.yml', + 'deploy-pages.yml') +#: GitHub's own ``timeout-minutes`` when a job sets none. +DEFAULT_TIMEOUT = 360 + +#: The aggregate job the ruleset requires; its completion is "mergeable". +REQUIRED_JOB = 'Required checks passed' +#: The leg running under coverage, and the legs its tests step is paired against. +COVERAGE_JOB = 'Python 3.14' +PAIRED_JOBS = ('Python 3.10', 'Python 3.11', 'Python 3.12', 'Python 3.13') +TESTS_STEP = 'Run unit tests' +#: The leg ms/test is measured on: the newest version not under coverage. +MS_PER_TEST_LEG = '3.13' +#: The JUnit artifacts read, by leg, and from how many push runs. +JUNIT_LEGS = ('3.13', '3.14') +JUNIT_ARTIFACT = 'junit-unit-{leg}' +JUNIT_RUNS = 5 + +#: Steps reported on their own, by full-match pattern against the step's name. +STEPS_OF_INTEREST = ( + r'Run unit tests', + r'Install package and test dependencies', + r'Report coverage', + r'Run full test suite', + r'Run tests/.* under plain unittest', + r'Run the 3\.15 cell.*', +) + +# Thresholds. Each is anchored to a value measured on 2026-10-05..09 for #1538, +# given in brackets; see the CI Timing Review section of +# docs/source/contributing/workflows.rst for why each sits where it does. + +#: H1, H2: a job's (or a capped step's) p90 reaching this share of its cap. 50% +#: leaves 12-18 days at the fastest growth measured, one review plus one fix. +#: [Python 3.14 p90 12.1 of 45 min, 27%; protocols (rest) step 319 s of 20 min] +HARD_CAP_SHARE = 0.5 +#: W9, and H3's floor: pull-request time to ``Required checks passed``, p90, +#: minutes. [19.6 on 10-05..09; 36.7 in the two days to 10-10, the pool saturated] +MERGEABLE_P90 = 30.0 +#: H3: that p90, over the floor, rising this much against the baseline report's. +#: Absolute, it measures runner-queue load as much as test time, so it pages only on +#: a regression; the absolute breach alone is W9. Initial value, to tune once weekly +#: reports exist -- W1's 20% median rise, loosened for a tail statistic. +HARD_MERGEABLE_RISE = 0.25 +#: H4: a linear projection reaching its cap within this many days. +HARD_PROJECTION_DAYS = 14.0 +#: W1: a job's median rising this much against the baseline report... +WARN_MEDIAN_RISE = 0.20 +#: ...with at least this many samples on each side. +WARN_MEDIAN_SAMPLES = 20 +#: W2: pull-request time to mergeable, median, minutes. [14.4] +WARN_MERGEABLE_MEDIAN = 18.0 +#: W3: the coverage leg's tests step over its paired legs', median. [1.57] +WARN_COVERAGE_RATIO = 1.8 +#: W4: ms per test on the 3.13 leg rising this much week over week. [110.8 ms] +WARN_MS_PER_TEST_RISE = 0.15 +#: W5: run queue p90, minutes, per event. [3.1 and 9.6] +WARN_QUEUE_P90 = {'pull_request': 5.0, 'push': 15.0} +#: W6: share of pull-request runs cancelled, i.e. superseded. [29%] +WARN_SUPERSEDED_SHARE = 0.35 +#: W7: a module's summed test time, and one test's, in seconds. +WARN_MODULE_SECONDS = 30.0 +WARN_TEST_SECONDS = 10.0 +#: W7: a class whose tests cost about the same each, at least this much, flags the +#: per-test re-import the #1538 measurement found: the first test pays setup, and +#: in such a class every later one pays it again. +UNIFORM_MIN_TESTS = 3 +UNIFORM_MIN_MEDIAN = 0.3 +UNIFORM_MEDIAN_TO_MAX = 0.5 + +#: A job or step cancelled at this share of its cap or later timed out. +TIMEOUT_SHARE = 0.95 + +#: Fewest samples a percentile threshold is judged on. +MIN_SAMPLES = 5 +#: A linear fit needs this many points spread over this many days to project. +MIN_FIT_SAMPLES = 10 +MIN_FIT_SPAN_DAYS = 2.0 + +#: How many rows each summary table shows; report.json keeps them all. +TABLE_ROWS = 15 +TOP_TESTS = 30 + +DEFAULT_MAX_CALLS = 900 +#: Transient failures retried, with these waits in seconds. +RETRY_WAITS = (5, 20) +#: A filtered run listing returns at most this many runs, whatever total_count says. +LISTING_CAP = 1000 + +#: Exit statuses; see the module docstring. +EXIT_HARD, EXIT_API, EXIT_CRASH = 1, 2, 3 + + +class BudgetExceeded(RuntimeError): + """The run would make more API calls than ``--max-calls`` allows.""" + + +# --------------------------------------------------------------------- arithmetic + + +def parse_time(value: 'Optional[str]') -> 'Optional[datetime.datetime]': + """An API timestamp as an aware datetime, or ``None`` when it is empty.""" + if not value: + return None + return datetime.datetime.fromisoformat(value.replace('Z', '+00:00')) + + +def minutes(start: 'Optional[str]', end: 'Optional[str]') -> 'Optional[float]': + """Minutes from ``start`` to ``end``, or ``None`` if either is missing.""" + first, last = parse_time(start), parse_time(end) + if first is None or last is None: + return None + return (last - first).total_seconds() / 60 + + +def quantile(values: 'Iterable[float]', q: 'float') -> 'Optional[float]': + """The ``q`` quantile by linear interpolation between ranks.""" + data = sorted(values) + if not data: + return None + position = q * (len(data) - 1) + low, high = math.floor(position), math.ceil(position) + return data[low] + (data[high] - data[low]) * (position - low) + + +def describe(values: 'Iterable[float]') -> 'dict[str, Any]': + """``n``, ``median``, ``p90`` and ``max`` of ``values``.""" + data = list(values) + return {'n': len(data), 'median': quantile(data, 0.5), 'p90': quantile(data, 0.9), + 'max': max(data) if data else None} + + +def projection(points: 'list[tuple[float, float]]', cap: 'float') -> 'dict[str, Any]': + """A least-squares line through ``(day, minutes)`` points, projected to the cap. + + Returns the slope in minutes per day and the days until the fitted value reaches + half the cap and the cap, each ``None`` when there is too little data, or when + the line is not rising and has not reached it. + + """ + empty = {'slope_per_day': None, 'days_to_half_cap': None, 'days_to_cap': None} + if len(points) < MIN_FIT_SAMPLES: + return empty + xs = [x for x, _ in points] + if max(xs) - min(xs) < MIN_FIT_SPAN_DAYS: + return empty + mean_x = statistics.fmean(xs) + mean_y = statistics.fmean(y for _, y in points) + sxx = sum((x - mean_x) ** 2 for x in xs) + slope = sum((x - mean_x) * (y - mean_y) for x, y in points) / sxx + now = mean_y + slope * (max(xs) - mean_x) + + def days_to(target: 'float') -> 'Optional[float]': + if now >= target: + return 0.0 + if slope <= 0: + return None + return (target - now) / slope + + return {'slope_per_day': slope, 'days_to_half_cap': days_to(cap * HARD_CAP_SHARE), + 'days_to_cap': days_to(cap)} + + +# --------------------------------------------------------------------- workflow text + + +def template_pattern(template: 'str') -> 're.Pattern[str]': + """A full-match pattern for a name that may carry ``${{ ... }}`` expressions.""" + parts = re.split(r'\$\{\{.*?\}\}', template) + return re.compile('.+'.join(re.escape(part) for part in parts)) + + +def _scalar(value: 'str') -> 'str': + value = value.split(' #', 1)[0].strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in '"\'': + return value[1:-1] + return value + + +def parse_workflow(text: 'str') -> 'list[dict[str, Any]]': + """Each job's name, ``timeout-minutes``, ``needs:`` and step caps. + + Read off the text by indentation, the way the tests under :file:`tests/project` + read these files, because PyYAML is not in the standard library. Every workflow + here keeps job keys at four spaces, which is all this relies on. + + """ + jobs = [] # type: list[dict[str, Any]] + body = re.split(r'(?m)^jobs:\s*$', text, maxsplit=1) + if len(body) != 2: + return jobs + for match in re.finditer(r'(?ms)^ ([\w-]+):\s*\n(.*?)(?=^ [\w-]+:\s*$|^\S|\Z)', body[1]): + job_id, section = match.group(1), match.group(2) + name = re.search(r'(?m)^ name:\s*(.+)$', section) + timeout = re.search(r'(?m)^ timeout-minutes:\s*(\d+)', section) + needs = re.search(r'(?m)^ needs:\s*(.+)$', section) + needed = [] if needs is None else [ + item.strip() for item in _scalar(needs.group(1)).strip('[]').split(',') if item.strip()] + template = _scalar(name.group(1)) if name else job_id + jobs.append({ + 'id': job_id, + 'name': template, + 'pattern': template_pattern(template), + 'literal': '${{' not in template, + # Unnamed matrix jobs are listed as ``id (values)``. + 'unnamed': name is None, + 'timeout': int(timeout.group(1)) if timeout else DEFAULT_TIMEOUT, + 'needs': needed, + 'steps': _parse_steps(section), + }) + return jobs + + +def _parse_steps(section: 'str') -> 'list[dict[str, Any]]': + """The named steps of one job section, with any step-level cap.""" + found = [] # type: list[dict[str, Any]] + lines = section.splitlines() + try: + start = next(i for i, line in enumerate(lines) if re.match(r'^ steps:\s*$', line)) + except StopIteration: + return found + items = [i for i in range(start + 1, len(lines)) if re.match(r'^ {4,}- ', lines[i])] + if not items: + return found + indent = len(lines[items[0]]) - len(lines[items[0]].lstrip()) + items = [i for i in items if len(lines[i]) - len(lines[i].lstrip()) == indent] + for number, first in enumerate(items): + last = items[number + 1] if number + 1 < len(items) else len(lines) + block = [lines[first][:indent] + ' ' + lines[first][indent + 2:]] + lines[first + 1:last] + keys = {} # type: dict[str, str] + for line in block: + key = re.match(rf'^ {{{indent + 2}}}([\w-]+):\s*(.*)$', line) + if key: + keys.setdefault(key.group(1), key.group(2)) + if 'name' not in keys: + continue + template = _scalar(keys['name']) + timeout = keys.get('timeout-minutes') + found.append({'name': template, 'pattern': template_pattern(template), + 'timeout': int(_scalar(timeout)) if timeout else None}) + return found + + +def match_job(specs: 'list[dict[str, Any]]', name: 'str') -> 'Optional[dict[str, Any]]': + """The spec a job name in the API came from; a literal name wins over a template.""" + for spec in specs: + if spec['literal'] and spec['name'] == name: + return spec + for spec in specs: + if spec['pattern'].fullmatch(name): + return spec + if spec['unnamed'] and re.fullmatch(rf'{re.escape(spec["id"])} \(.*\)', name): + return spec + return None + + +def step_cap(spec: 'Optional[dict[str, Any]]', name: 'str') -> 'Optional[int]': + """A step's own ``timeout-minutes``, if its job declares one for it.""" + for step in (spec or {}).get('steps', ()): + if step['pattern'].fullmatch(name): + return step['timeout'] + return None + + +# --------------------------------------------------------------------- the API + + +def next_link(header: 'str') -> 'Optional[str]': + """The ``rel="next"`` URL of a ``Link`` header.""" + match = re.search(r'<([^>]+)>;\s*rel="next"', header or '') + return match.group(1) if match else None + + +class Client: + """The REST API, counting calls against a budget. + + Args: + token: Bearer token. + transport: Replaces the network, for tests. + max_calls: Refuse the call after this many. + sleep: Replaces :func:`time.sleep` between retries, for tests. + + """ + + def __init__(self, token: 'str', *, transport: 'Optional[Transport]' = None, + max_calls: 'int' = DEFAULT_MAX_CALLS, + sleep: 'Callable[[float], None]' = time.sleep) -> 'None': + self.token = token + self.transport = transport or self._urlopen + self.max_calls = max_calls + self.sleep = sleep + self.calls = 0 + self.rate_remaining = None # type: Optional[int] + + def _urlopen(self, url: 'str', accept: 'str') -> 'tuple[bytes, dict[str, str]]': + request = urllib.request.Request(url, headers={ + 'Accept': accept, 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': 'PyPCAPKit-ci-timing'}) + # Unredirected, so an artifact download's redirect to blob storage does not + # carry the token to a host that rejects it. + request.add_unredirected_header('Authorization', f'Bearer {self.token}') + # Only https URLs reach here: the API root, or a Link/download URL it sent. + with urllib.request.urlopen(request, timeout=60) as response: # nosec B310 + return response.read(), {k.lower(): v for k, v in response.headers.items()} + + def fetch(self, url: 'str', accept: 'str' = 'application/vnd.github+json') -> 'tuple[bytes, dict[str, str]]': + """One request; ``url`` may be a path under the API root.""" + if url.startswith('/'): + url = API + url + if not url.startswith('https://'): + raise ValueError(f'refusing a non-https URL: {url}') + for wait in RETRY_WAITS + (None,): + if self.calls >= self.max_calls: + raise BudgetExceeded(f'stopped at {self.calls} API calls (--max-calls)') + self.calls += 1 + try: + body, headers = self.transport(url, accept) + except urllib.error.HTTPError as error: + if wait is None or error.code < 500: + raise + except OSError: # URLError, and a timeout or reset mid-read + if wait is None: + raise + else: + remaining = headers.get('x-ratelimit-remaining') + if remaining is not None: + self.rate_remaining = int(remaining) + return body, headers + self.sleep(wait) + raise AssertionError('unreachable') # pragma: no cover + + def get(self, path: 'str', **params: 'Any') -> 'Any': + """One JSON document.""" + query = f'?{urllib.parse.urlencode(params)}' if params else '' + return json.loads(self.fetch(path + query)[0]) + + def text(self, path: 'str', **params: 'Any') -> 'str': + """One file's raw content, through the contents API.""" + query = f'?{urllib.parse.urlencode(params)}' if params else '' + return self.fetch(path + query, 'application/vnd.github.raw+json')[0].decode('utf-8') + + def paginate(self, path: 'str', key: 'str', meta: 'Optional[dict[str, Any]]' = None, + **params: 'Any') -> 'Iterator[dict[str, Any]]': + """Every item under ``key``, following ``Link: rel="next"`` to the end. + + A generator, so a caller that stops consuming stops the paging. ``meta``, if + given, receives the first page's ``total_count``. + + """ + params.setdefault('per_page', 100) + url = f'{path}?{urllib.parse.urlencode(params)}' # type: Optional[str] + while url: + body, headers = self.fetch(url) + page = json.loads(body) + if meta is not None and 'total_count' in page: + meta.setdefault('total_count', page['total_count']) + yield from page.get(key) or () + url = next_link(headers.get('link', '')) + + +def iso(moment: 'datetime.datetime') -> 'str': + """``moment`` the way the API writes it.""" + return moment.astimezone(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + + +def list_runs(client: 'Client', path: 'str', since: 'datetime.datetime', + now: 'datetime.datetime', notes: 'list[str]', + **params: 'Any') -> 'list[dict[str, Any]]': + """Every run at ``path`` created in ``[since, now)``, newest first. + + One query per day, because a filtered listing stops at :data:`LISTING_CAP` runs + however many match; a day that still exceeds it is reported in ``notes``. + + """ + found = {} # type: dict[int, dict[str, Any]] + end = now + while end > since: + start = max(since, end - datetime.timedelta(days=1)) + meta = {} # type: dict[str, Any] + last = iso(end - datetime.timedelta(seconds=1)) + day = list(client.paginate(path, 'workflow_runs', meta=meta, + created=f'{iso(start)}..{last}', **params)) + if meta.get('total_count', 0) > len(day): + notes.append(f'{path.rsplit("/", 2)[-2]}: {meta["total_count"]} runs created ' + f'{iso(start)}..{last}, of which the API returned {len(day)}.') + for run in day: + found.setdefault(run['id'], run) + end = start + return sorted(found.values(), key=lambda run: run['created_at'], reverse=True) + + +def spread(items: 'list[Any]', count: 'int') -> 'list[Any]': + """``count`` of ``items``, evenly spaced from the first to the last.""" + if len(items) <= count: + return list(items) + if count < 2: + return list(items[:count]) + return [items[round(i * (len(items) - 1) / (count - 1))] for i in range(count)] + + +def collect(client: 'Client', repo: 'str', *, now: 'datetime.datetime', days: 'float', + runs: 'int' = 150, other_runs: 'int' = 60, junit_runs: 'int' = JUNIT_RUNS, + concurrency_hours: 'float' = 0) -> 'dict[str, Any]': + """Everything the report is computed from, read from the API. + + Args: + client: The API. + repo: ``owner/name``. + now: The end of the window. + days: The window's length. + runs: Unit Tests runs per event whose jobs are read. + other_runs: Runs per other workflow whose jobs are read. + junit_runs: Successful push runs whose JUnit artifacts are read. + concurrency_hours: Hours of every workflow's runs to measure the runner pool + over; 0 skips it. + + """ + since = now - datetime.timedelta(days=days) + base = f'/repos/{repo}/actions' + data = {'repo': repo, 'since': iso(since), 'until': iso(now), 'days': days, + 'workflows': {}, 'junit': [], 'concurrency': None, 'notes': []} # type: dict[str, Any] + + for workflow in (UNIT_TESTS,) + OTHER_WORKFLOWS: + listed = [run for run in list_runs(client, f'{base}/workflows/{workflow}/runs', since, now, + data['notes'], status='completed') + if run.get('conclusion') in ('success', 'failure', 'cancelled')] + if workflow == UNIT_TESTS: + chosen = [] # type: list[dict[str, Any]] + for event in ('pull_request', 'push'): + chosen += spread([run for run in listed if run.get('event') == event], runs) + # The pushes whose JUnit artifacts are read need their jobs read too. + chosen += [run for run in listed if run.get('event') == 'push' + and run['conclusion'] == 'success'][:junit_runs] + chosen = sorted({run['id']: run for run in chosen}.values(), + key=lambda run: run['created_at'], reverse=True) + else: + chosen = spread(listed, other_runs) + for run in chosen: + run['jobs'] = list(client.paginate(f'{base}/runs/{run["id"]}/attempts/1/jobs', 'jobs')) + counts = collections.Counter((run.get('event'), run['conclusion']) for run in listed) + data['workflows'][workflow] = { + 'runs': chosen, 'specs': {}, + 'listed': {event: {'runs': sum(n for (e, _), n in counts.items() if e == event), + 'cancelled': counts[(event, 'cancelled')]} + for event in sorted({e for e, _ in counts})}} + # A pull request's head may carry a cap that never merged, so the caps are + # read at a non-PR run's sha where the window has one. + path = f'/repos/{repo}/contents/.github/workflows/{workflow}' + mainline = [run for run in chosen if run.get('event') != 'pull_request'] or chosen + for label, run in (('newest', mainline[0]), ('oldest', mainline[-1])) if mainline else (): + sha = run['head_sha'] + known = [text for text_sha, text in data['workflows'][workflow]['specs'].values() + if text_sha == sha] + try: + text = known[0] if known else client.text(path, ref=sha) + except urllib.error.HTTPError as error: + if error.code != 404: + raise + data['notes'].append(f'{workflow} is not at {sha[:9]}, so its {label} caps are unknown.') + continue + data['workflows'][workflow]['specs'][label] = (sha, text) + + pushes = [run for run in data['workflows'][UNIT_TESTS]['runs'] + if run.get('event') == 'push' and run.get('conclusion') == 'success'] + for run in pushes[:junit_runs]: + artifacts = {item['name']: item for item in + client.paginate(f'{base}/runs/{run["id"]}/artifacts', 'artifacts') + if not item.get('expired')} + for leg in JUNIT_LEGS: + artifact = artifacts.get(JUNIT_ARTIFACT.format(leg=leg)) + if artifact is None: + continue + archive = zipfile.ZipFile(io.BytesIO(client.fetch(artifact['archive_download_url'])[0])) + members = [name for name in archive.namelist() if name.endswith('.xml')] + if members: + data['junit'].append({'run_id': run['id'], 'created_at': run['created_at'], + 'leg': leg, **parse_junit(archive.read(members[0]))}) + if not data['junit']: + data['notes'].append(f'No {JUNIT_ARTIFACT.format(leg="*")} artifact in the newest ' + f'{min(len(pushes), junit_runs)} successful push runs, so there ' + 'are no test-level timings.') + + if concurrency_hours: + start = now - datetime.timedelta(hours=concurrency_hours) + spans = [] # type: list[tuple[datetime.datetime, datetime.datetime, str]] + for run in list_runs(client, f'{base}/runs', start, now, data['notes']): + for job in client.paginate(f'{base}/runs/{run["id"]}/jobs', 'jobs', filter='all'): + begun = parse_time(job.get('started_at')) + if not job.get('runner_name') or begun is None: + continue + ended = parse_time(job.get('completed_at')) or now + labels = ' '.join(job.get('labels') or ()) + spans.append((begun, ended, 'macos' if 'macos' in labels else 'other')) + data['concurrency'] = {'hours': concurrency_hours, **pool_concurrency(spans)} + return data + + +def parse_junit(raw: 'bytes') -> 'dict[str, Any]': + """The testcases of one pytest JUnit file, and its suite's ``tests=`` total.""" + root = ET.fromstring(raw) # nosec B314 + suites = [root] if root.tag == 'testsuite' else root.findall('testsuite') + cases = [(case.get('classname') or '', case.get('name') or '', float(case.get('time') or 0)) + for suite in suites for case in suite.iter('testcase')] + return {'cases': cases, 'suite_tests': sum(int(suite.get('tests') or 0) for suite in suites)} + + +def pool_concurrency(spans: 'list[tuple[datetime.datetime, datetime.datetime, str]]') -> 'dict[str, Any]': + """The most jobs holding a runner at once, and for how long that peak held.""" + events = sorted([(begun, 1, kind) for begun, _, kind in spans] + + [(ended, -1, kind) for _, ended, kind in spans], + key=lambda item: (item[0], item[1])) + running = collections.Counter() # type: collections.Counter[str] + peak, held, macos_peak = 0, 0.0, 0 + for (moment, delta, kind), following in zip(events, events[1:] + [None]): + running[kind] += delta + total = sum(running.values()) + macos_peak = max(macos_peak, running['macos']) + if total > peak: + peak, held = total, 0.0 + if total == peak and following is not None: + held += (following[0] - moment).total_seconds() + return {'jobs': len(spans), 'peak': peak, 'seconds_at_peak': held, 'macos_peak': macos_peak} + + +# --------------------------------------------------------------------- the report + + +def _ran(job: 'dict[str, Any]') -> 'bool': + """A job that held a runner, finished, and belongs to the first attempt.""" + return bool(job.get('runner_name') and job.get('conclusion') and job.get('started_at') + and job.get('completed_at') and job.get('run_attempt', 1) == 1) + + +def _days(moment: 'Optional[str]', origin: 'datetime.datetime') -> 'float': + stamp = parse_time(moment) + return 0.0 if stamp is None else (stamp - origin).total_seconds() / 86400 + + +def _timed_out(item: 'dict[str, Any]', cap: 'Optional[float]') -> 'bool': + """A job or step cancelled at :data:`TIMEOUT_SHARE` of ``cap`` or later.""" + took = minutes(item.get('started_at'), item.get('completed_at')) + return (item.get('conclusion') == 'cancelled' and bool(cap) and took is not None + and took >= TIMEOUT_SHARE * cap) + + +def _counted(item: 'dict[str, Any]', cap: 'Optional[float]') -> 'bool': + """Measured by its own outcome: it succeeded, failed, or was killed at its cap.""" + return item.get('conclusion') in ('success', 'failure') or _timed_out(item, cap) + + +def _job_cap(specs: 'list[dict[str, Any]]', name: 'str') -> 'int': + spec = match_job(specs, name) + return spec['timeout'] if spec else DEFAULT_TIMEOUT + + +def _measured(run: 'dict[str, Any]', specs: 'list[dict[str, Any]]') -> 'bool': + """A run that finished, or that reads ``cancelled`` because a job timed out. + + Any other cancelled run was superseded, or cancelled by hand, and its timings + describe when it was stopped rather than how long the work takes. + + """ + if run.get('conclusion') in ('success', 'failure'): + return True + return run.get('conclusion') == 'cancelled' and any( + _timed_out(job, _job_cap(specs, job['name'])) for job in run.get('jobs', ()) if _ran(job)) + + +def analyse(data: 'dict[str, Any]') -> 'dict[str, Any]': + """The metrics of one window, from what :func:`collect` read.""" + origin = parse_time(data['since']) + assert origin is not None + report = {'schema': 1, 'repo': data['repo'], 'window': { + 'since': data['since'], 'until': data['until'], 'days': data['days'], 'read': {}}, + 'notes': list(data.get('notes', ())), 'runs': {}, 'jobs': [], 'steps': [], + 'critical_path': {}, 'derived': {}, 'tests': {}} # type: dict[str, Any] + for workflow, info in data['workflows'].items(): + listed = info.get('listed', {}) + report['window']['read'][workflow] = { + 'listed': sum(row['runs'] for row in listed.values()) if listed else len(info['runs']), + 'read': len(info['runs']), + 'first': min((run['created_at'] for run in info['runs']), default=None), + 'last': max((run['created_at'] for run in info['runs']), default=None)} + + specs = {} + for workflow, info in data['workflows'].items(): + newest = parse_workflow(info['specs']['newest'][1]) if 'newest' in info['specs'] else [] + specs[workflow] = newest + if 'oldest' in info['specs']: + old = {spec['name']: spec['timeout'] for spec in parse_workflow(info['specs']['oldest'][1])} + for spec in newest: + if spec['name'] in old and old[spec['name']] != spec['timeout']: + report['notes'].append( + f'{workflow}: `{spec["name"]}` cap moved from {old[spec["name"]]} to ' + f'{spec["timeout"]} min inside the window; its trend spans the change.') + + unit = data['workflows'].get(UNIT_TESTS, {}) + report['runs'], report['superseded'] = _run_level(unit, specs.get(UNIT_TESTS, [])) + jobs = collections.defaultdict(list) # type: dict[tuple[str, str], list[dict[str, Any]]] + steps = collections.defaultdict(list) # type: dict[tuple[str, str, str], list[dict[str, Any]]] + for workflow, info in data['workflows'].items(): + # Every run, whatever its own conclusion: a job is judged by its own. + for run in info['runs']: + for job in filter(_ran, run.get('jobs', ())): + spec = match_job(specs.get(workflow, []), job['name']) + cap = spec['timeout'] if spec else DEFAULT_TIMEOUT + if not _counted(job, cap): + continue + jobs[(workflow, job['name'])].append(job) + for step in job.get('steps') or (): + if not any(re.fullmatch(pattern, step['name']) for pattern in STEPS_OF_INTEREST): + continue + # The step running when its job timed out was cut off by that cap. + if _counted(step, step_cap(spec, step['name'])) or ( + _timed_out(job, cap) and step.get('conclusion') == 'cancelled'): + steps[(workflow, job['name'], step['name'])].append(step) + + unmatched = set() + for (workflow, name), samples in sorted(jobs.items()): + spec = match_job(specs.get(workflow, []), name) + if spec is None: + unmatched.add(f'{workflow}: {name}') + cap = spec['timeout'] if spec else DEFAULT_TIMEOUT + durations = [(_days(job['started_at'], origin), minutes(job['started_at'], job['completed_at'])) + for job in samples] + row = {'workflow': workflow, 'name': name, 'cap': cap, + **describe(value for _, value in durations), + 'queue': describe(minutes(job.get('created_at') or job['started_at'], job['started_at']) + for job in samples)} + row['p90_share_of_cap'] = row['p90'] / cap + row.update(projection(durations, cap)) + report['jobs'].append(row) + if unmatched: + report['notes'].append('Matched no job in the workflow file, so judged against the ' + f'{DEFAULT_TIMEOUT}-minute default cap: ' + ', '.join(sorted(unmatched))) + + for (workflow, job_name, name), samples in sorted(steps.items()): + cap = step_cap(match_job(specs.get(workflow, []), job_name), name) + durations = [(_days(step['started_at'], origin), minutes(step['started_at'], step['completed_at'])) + for step in samples if step.get('started_at') and step.get('completed_at')] + row = {'workflow': workflow, 'job': job_name, 'name': name, 'cap': cap, + **describe(value for _, value in durations)} + if cap: + row['p90_share_of_cap'] = row['p90'] / cap if row['p90'] is not None else None + row.update(projection(durations, cap)) + report['steps'].append(row) + + report['critical_path'] = _critical_path(unit, specs.get(UNIT_TESTS, [])) + report['derived'] = _derived(unit, specs.get(UNIT_TESTS, []), data.get('junit', ())) + report['tests'] = _tests(data.get('junit', ())) + if data.get('concurrency'): + report['concurrency'] = data['concurrency'] + return report + + +def _step_minutes(job: 'Optional[dict[str, Any]]', name: 'str') -> 'Optional[float]': + for step in (job or {}).get('steps') or (): + if step['name'] == name and step.get('conclusion') in ('success', 'failure'): + return minutes(step.get('started_at'), step.get('completed_at')) + return None + + +def _run_level(unit: 'dict[str, Any]', + specs: 'list[dict[str, Any]]') -> 'tuple[dict[str, Any], dict[str, Any]]': + """Queue, time to mergeable, wall and runner time per event; then the superseded. + + The run counts come from the whole listing; the timings from the runs read. The + superseded runs' runner time is the mean of those read, scaled to all of them. + + """ + rows = {} # type: dict[str, Any] + listed = unit.get('listed', {}) + stopped = [] # type: list[float] + for event in ('pull_request', 'push'): + measured = [] # type: list[dict[str, Any]] + sample = [run for run in unit.get('runs', ()) if run.get('event') == event] + for run in sample: + ran = [job for job in run.get('jobs', ()) if _ran(job)] + runner = sum(minutes(job['started_at'], job['completed_at']) or 0 for job in ran) + if not _measured(run, specs): + if event == 'pull_request' and run.get('conclusion') == 'cancelled': + stopped.append(runner) + continue + if not ran: + continue + required = [job for job in ran if job['name'] == REQUIRED_JOB] + measured.append({ + 'queue': minutes(run['created_at'], min(job['started_at'] for job in ran)), + 'wall': minutes(run['created_at'], max(job['completed_at'] for job in ran)), + 'mergeable': minutes(run['created_at'], required[0]['completed_at']) + if required and required[0]['conclusion'] == 'success' else None, + 'runner_min': runner, 'jobs': len(ran)}) + rows[event] = {key: describe(row[key] for row in measured if row[key] is not None) + for key in ('queue', 'mergeable', 'wall', 'runner_min', 'jobs')} + counts = listed.get(event) or { + 'runs': len(sample), 'cancelled': sum(run.get('conclusion') == 'cancelled' for run in sample)} + rows[event]['runs'] = len(measured) + rows[event]['listed'] = counts['runs'] + rows[event]['cancelled'] = counts['cancelled'] + pr = rows['pull_request'] + superseded = {'runs': pr['cancelled'], 'of': pr['listed'], + 'share': pr['cancelled'] / pr['listed'] if pr['listed'] else None, + 'read': len(stopped), + 'runner_min': statistics.fmean(stopped) * pr['cancelled'] if stopped else None} + return rows, superseded + + +def _critical_path(unit: 'dict[str, Any]', specs: 'list[dict[str, Any]]') -> 'dict[str, Any]': + """Which needed job finished last before ``Required checks passed``, per event.""" + required = next((spec for spec in specs if spec['name'] == REQUIRED_JOB), None) + needed = set(required['needs']) if required else set() + paths = {} + for event in ('pull_request', 'push'): + leaders = collections.Counter() # type: collections.Counter[str] + slack = [] + for run in unit.get('runs', ()): + if run.get('event') != event or not _measured(run, specs): + continue + ended = sorted(((job['completed_at'], job['name']) for job in run.get('jobs', ()) + if _ran(job) and job['name'] != REQUIRED_JOB + and (match_job(specs, job['name']) or {}).get('id') in needed), + reverse=True) + if not ended: + continue + leaders[ended[0][1]] += 1 + if len(ended) > 1: + slack.append(minutes(ended[1][0], ended[0][0])) + total = sum(leaders.values()) + paths[event] = { + 'runs': total, 'leader': leaders.most_common(1)[0][0] if leaders else None, + 'shares': {name: count / total for name, count in leaders.most_common()}, + 'slack_median': quantile([value for value in slack if value is not None], 0.5)} + return paths + + +def _derived(unit: 'dict[str, Any]', specs: 'list[dict[str, Any]]', + junit: 'Iterable[dict[str, Any]]') -> 'dict[str, Any]': + """The coverage leg's paired slowdown, and ms per test where JUnit exists.""" + ratios = [] + by_run = {} + for run in unit.get('runs', ()): + if not _measured(run, specs): + continue + named = {job['name']: job for job in run.get('jobs', ()) if _ran(job)} + by_run[run['id']] = named + covered = _step_minutes(named.get(COVERAGE_JOB), TESTS_STEP) + paired = [value for value in (_step_minutes(named.get(name), TESTS_STEP) + for name in PAIRED_JOBS) if value] + if covered and len(paired) >= 2: + ratios.append(covered / statistics.median(paired)) + per_test = [] + counts = [] + for entry in junit: + if entry['leg'] != MS_PER_TEST_LEG or not entry['cases']: + continue + counts.append({'run_id': entry['run_id'], 'tests': len(entry['cases']), + 'suite_tests': entry['suite_tests']}) + step = _step_minutes(by_run.get(entry['run_id'], {}).get(f'Python {entry["leg"]}'), TESTS_STEP) + if step: + per_test.append(step * 60000 / len(entry['cases'])) + return {'coverage_ratio': {**describe(ratios), 'min': min(ratios) if ratios else None}, + 'ms_per_test': describe(per_test), 'test_counts': counts} + + +def _split(classname: 'str') -> 'tuple[str, Optional[str]]': + """``(module, class)`` of a JUnit classname; a module-level test has no class.""" + head, _, tail = classname.rpartition('.') + if head and tail[:1].isupper(): + return head, classname + return classname, None + + +def _tests(junit: 'Iterable[dict[str, Any]]') -> 'dict[str, Any]': + """Per leg: top modules and classes, slow tests, and uniform-cost classes.""" + legs = {} + entries = list(junit) + for leg in JUNIT_LEGS: + runs = [entry for entry in entries if entry['leg'] == leg] + if not runs: + continue + modules = collections.defaultdict(list) # type: dict[str, list[float]] + classes = collections.defaultdict(list) # type: dict[str, list[float]] + slow = {} # type: dict[str, float] + uniform = collections.Counter() # type: collections.Counter[str] + for entry in runs: + module_sum = collections.Counter() # type: collections.Counter[str] + class_times = collections.defaultdict(list) # type: dict[str, list[float]] + for classname, name, seconds in entry['cases']: + module, cls = _split(classname) + module_sum[module] += seconds + if cls: + class_times[cls].append(seconds) + if seconds > WARN_TEST_SECONDS: + test = f'{classname}::{name}' + slow[test] = max(slow.get(test, 0.0), seconds) + for module, total in module_sum.items(): + modules[module].append(total) + for cls, times in class_times.items(): + classes[cls].append(sum(times)) + middle = statistics.median(times) + if (len(times) >= UNIFORM_MIN_TESTS and middle >= UNIFORM_MIN_MEDIAN + and middle >= UNIFORM_MEDIAN_TO_MAX * max(times)): + uniform[cls] += 1 + + def top(table: 'dict[str, list[float]]') -> 'list[list[Any]]': + ranked = sorted(((statistics.median(v), k) for k, v in table.items()), reverse=True) + return [[name, round(seconds, 3)] for seconds, name in ranked[:TOP_TESTS]] + + legs[leg] = {'runs': len(runs), 'top_modules': top(modules), 'top_classes': top(classes), + 'slow_tests': sorted(([k, v] for k, v in slow.items()), key=lambda r: -r[1]), + 'uniform_classes': sorted(k for k, v in uniform.items() if v * 2 > len(runs))} + return legs + + +# --------------------------------------------------------------------- thresholds + + +def _fmt(value: 'Optional[float]', digits: 'int' = 1) -> 'str': + return '-' if value is None else f'{value:.{digits}f}' + + +def evaluate(report: 'dict[str, Any]', baseline: 'Optional[dict[str, Any]]') -> 'list[dict[str, str]]': + """Every HARD (``H*``) and WARN (``W*``) threshold that fires on ``report``.""" + found = [] # type: list[dict[str, str]] + + def fire(code: 'str', message: 'str') -> 'None': + found.append({'id': code, 'level': 'hard' if code[0] == 'H' else 'warn', 'message': message}) + + for row in report['jobs']: + label = f'`{row["name"]}` ({row["workflow"]})' + if row['n'] >= MIN_SAMPLES and row['p90'] >= HARD_CAP_SHARE * row['cap']: + fire('H1', f'{label} p90 is {_fmt(row["p90"])} min, at or above ' + f'{HARD_CAP_SHARE:.0%} of its {row["cap"]}-minute cap.') + if row.get('days_to_cap') is not None and row['days_to_cap'] <= HARD_PROJECTION_DAYS: + fire('H4', f'{label} reaches its {row["cap"]}-minute cap in ' + f'{_fmt(row["days_to_cap"])} days on a linear fit ' + f'(+{_fmt(row["slope_per_day"], 2)} min/day).') + for row in report['steps']: + if not row.get('cap'): + continue + label = f'`{row["job"]}` step `{row["name"]}`' + if row['n'] >= MIN_SAMPLES and row['p90'] >= HARD_CAP_SHARE * row['cap']: + fire('H2', f'{label} p90 is {_fmt(row["p90"])} min, at or above ' + f'{HARD_CAP_SHARE:.0%} of its {row["cap"]}-minute step cap.') + if row.get('days_to_cap') is not None and row['days_to_cap'] <= HARD_PROJECTION_DAYS: + fire('H4', f'{label} reaches its {row["cap"]}-minute step cap in ' + f'{_fmt(row["days_to_cap"])} days on a linear fit.') + + pr = report['runs'].get('pull_request', {}) + mergeable = pr.get('mergeable', {}) + slow = mergeable.get('n', 0) >= MIN_SAMPLES and mergeable['p90'] >= MERGEABLE_P90 + before = ((baseline or {}).get('runs', {}).get('pull_request', {}).get('mergeable') or {}) + if (slow and before.get('n', 0) >= MIN_SAMPLES and before.get('p90') + and mergeable['p90'] >= (1 + HARD_MERGEABLE_RISE) * before['p90']): + fire('H3', f'pull-request time to `{REQUIRED_JOB}` p90 rose from {_fmt(before["p90"])} ' + f'to {_fmt(mergeable["p90"])} min, at least {HARD_MERGEABLE_RISE:.0%} and over ' + f'{MERGEABLE_P90:.0f}.') + elif slow: + fire('W9', f'pull-request time to `{REQUIRED_JOB}` p90 is {_fmt(mergeable["p90"])} min, ' + f'at or above {MERGEABLE_P90:.0f}; HARD (H3) only as a rise on the baseline.') + if mergeable.get('n', 0) >= MIN_SAMPLES and mergeable['median'] > WARN_MERGEABLE_MEDIAN: + fire('W2', f'pull-request time to mergeable median is {_fmt(mergeable["median"])} min, ' + f'above {WARN_MERGEABLE_MEDIAN:.0f}.') + ratio = report['derived'].get('coverage_ratio', {}) + if ratio.get('n', 0) >= MIN_SAMPLES and ratio['median'] > WARN_COVERAGE_RATIO: + fire('W3', f'`{COVERAGE_JOB}`\'s `{TESTS_STEP}` takes {_fmt(ratio["median"], 2)}x its ' + f'paired legs\' (median), above {WARN_COVERAGE_RATIO}.') + for event, limit in WARN_QUEUE_P90.items(): + queue = report['runs'].get(event, {}).get('queue', {}) + if queue.get('n', 0) >= MIN_SAMPLES and queue['p90'] > limit: + fire('W5', f'{event} run queue p90 is {_fmt(queue["p90"])} min, above {limit:.0f}.') + superseded = report.get('superseded', {}) + if superseded.get('of', 0) >= MIN_SAMPLES and superseded['share'] > WARN_SUPERSEDED_SHARE: + fire('W6', f'{superseded["runs"]} of {superseded["of"]} pull-request runs ' + f'({superseded["share"]:.0%}) were cancelled, above ' + f'{WARN_SUPERSEDED_SHARE:.0%}; about {_fmt(superseded.get("runner_min"), 0)} ' + 'runner-min.') + + leg = report['tests'].get(MS_PER_TEST_LEG, {}) + heavy = [name for name, seconds in leg.get('top_modules', ()) if seconds > WARN_MODULE_SECONDS] + if heavy: + fire('W7', f'{len(heavy)} module(s) over {WARN_MODULE_SECONDS:.0f} s summed on ' + f'{MS_PER_TEST_LEG}: ' + ', '.join(f'`{name}`' for name in heavy[:5])) + if leg.get('slow_tests'): + fire('W7', f'{len(leg["slow_tests"])} test(s) over {WARN_TEST_SECONDS:.0f} s on ' + f'{MS_PER_TEST_LEG}: ' + ', '.join(f'`{name}`' for name, _ in leg['slow_tests'][:3])) + if leg.get('uniform_classes'): + fire('W7', f'{len(leg["uniform_classes"])} class(es) whose every test costs about as much ' + 'as the first, the per-test re-import signature: ' + + ', '.join(f'`{name}`' for name in leg['uniform_classes'][:5])) + + if baseline: + before = {(row['workflow'], row['name']): row for row in baseline.get('jobs', ())} + for row in report['jobs']: + old = before.get((row['workflow'], row['name'])) + if (old and min(old['n'], row['n']) >= WARN_MEDIAN_SAMPLES and old['median'] + and row['median'] >= (1 + WARN_MEDIAN_RISE) * old['median']): + fire('W1', f'`{row["name"]}` ({row["workflow"]}) median rose from ' + f'{_fmt(old["median"])} to {_fmt(row["median"])} min.') + old_ms = baseline.get('derived', {}).get('ms_per_test', {}).get('median') + new_ms = report['derived'].get('ms_per_test', {}).get('median') + if old_ms and new_ms and new_ms >= (1 + WARN_MS_PER_TEST_RISE) * old_ms: + fire('W4', f'{MS_PER_TEST_LEG} ms/test rose from {old_ms:.1f} to {new_ms:.1f}.') + old_top = [name for name, _ in baseline.get('tests', {}).get(MS_PER_TEST_LEG, {}) + .get('top_modules', ())[:10]] + new_top = [name for name, _ in leg.get('top_modules', ())[:10]] + entrants = [name for name in new_top if name not in old_top] + if old_top and entrants: + fire('W7', 'new in the top 10 modules: ' + ', '.join(f'`{name}`' for name in entrants)) + for event in ('pull_request', 'push'): + was = baseline.get('critical_path', {}).get(event, {}).get('leader') + now = report['critical_path'].get(event, {}).get('leader') + if was and now and was != now: + fire('W8', f'the {event} critical path now ends on `{now}`, not `{was}`.') + return found + + +# --------------------------------------------------------------------- output + + +def summary(report: 'dict[str, Any]', findings: 'list[dict[str, str]]') -> 'str': + """The Markdown the workflow appends to its job summary.""" + window = report['window'] + out = [f'## CI timing, {window["since"][:16]}Z to {window["until"][:16]}Z', ''] + read = window.get('read', {}) + if read: + out += ['Jobs read for an evenly spaced sample of each workflow\'s completed runs: ' + + '; '.join(f'{name} {row["read"]} of {row["listed"]}' + + (f' ({row["first"][:16]}Z to {row["last"][:16]}Z)' if row['first'] else '') + for name, row in read.items()) + '.', ''] + for level, title in (('hard', 'HARD'), ('warn', 'WARN')): + for item in findings: + if item['level'] == level: + out.append(f'- **{title} {item["id"]}:** {item["message"]}') + if not findings: + out.append('No threshold fired.') + out += ['', '### Unit Tests runs', '', + '| Event | Listed | Cancelled | Measured | Queue p50 / p90 | To mergeable p50 / p90 ' + '| Wall p50 / p90 | Runner-min per run | Jobs per run |', '|---|--:|--:|--:|--:|--:|--:|--:|--:|'] + for event, row in report['runs'].items(): + out.append(f'| {event} | {row["listed"]} | {row["cancelled"]} | {row["runs"]} | ' + f'{_fmt(row["queue"]["median"])} / {_fmt(row["queue"]["p90"])} | ' + f'{_fmt(row["mergeable"]["median"])} / {_fmt(row["mergeable"]["p90"])} | ' + f'{_fmt(row["wall"]["median"])} / {_fmt(row["wall"]["p90"])} | ' + f'{_fmt(row["runner_min"]["median"])} | {_fmt(row["jobs"]["median"], 0)} |') + superseded = report.get('superseded', {}) + if superseded.get('of'): + out += ['', f'Superseded (cancelled) pull-request runs: {superseded["runs"]} of ' + f'{superseded["of"]} ({superseded["share"]:.0%}), about ' + f'{_fmt(superseded["runner_min"], 0)} runner-min, scaled from the ' + f'{superseded["read"]} read.'] + + jobs = sorted(report['jobs'], key=lambda row: -row['p90_share_of_cap'])[:TABLE_ROWS] + out += ['', f'### Jobs, by p90 share of cap (top {TABLE_ROWS})', '', + '| Workflow | Job | n | Median | p90 | Max | Queue p90 | Cap | p90 / cap ' + '| Trend min/day | Days to 50% | Days to cap |', '|---|---|--:|--:|--:|--:|--:|--:|--:|--:|--:|--:|'] + for row in jobs: + out.append(f'| {row["workflow"]} | {row["name"]} | {row["n"]} | {_fmt(row["median"])} | ' + f'{_fmt(row["p90"])} | {_fmt(row["max"])} | {_fmt(row["queue"]["p90"])} | ' + f'{row["cap"]} | {row["p90_share_of_cap"]:.0%} | {_fmt(row["slope_per_day"], 2)} | ' + f'{_fmt(row["days_to_half_cap"])} | {_fmt(row["days_to_cap"])} |') + + steps = sorted(report['steps'], key=lambda row: -(row['p90'] or 0))[:TABLE_ROWS] + out += ['', f'### Steps, by p90 (top {TABLE_ROWS})', '', + '| Job | Step | n | Median | p90 | Step cap | Days to cap |', '|---|---|--:|--:|--:|--:|--:|'] + for row in steps: + out.append(f'| {row["job"]} | {row["name"]} | {row["n"]} | {_fmt(row["median"])} | ' + f'{_fmt(row["p90"])} | {row["cap"] or "-"} | {_fmt(row.get("days_to_cap"))} |') + + out += ['', '### Critical path to `Required checks passed`', '', + '| Event | Runs | Last to finish | Share | Median slack to runner-up |', '|---|--:|---|--:|--:|'] + for event, row in report['critical_path'].items(): + share = row['shares'].get(row['leader'], 0) if row['leader'] else 0 + out.append(f'| {event} | {row["runs"]} | {row["leader"] or "-"} | {share:.0%} | ' + f'{_fmt(row["slack_median"])} min |') + + derived = report['derived'] + ratio, per_test = derived['coverage_ratio'], derived['ms_per_test'] + out += ['', '### Derived', '', + f'- `{COVERAGE_JOB}` `{TESTS_STEP}` over its paired legs: median ' + f'{_fmt(ratio["median"], 2)}x, range {_fmt(ratio["min"], 2)}-{_fmt(ratio["max"], 2)}, ' + f'n={ratio["n"]}.', + f'- ms per test on {MS_PER_TEST_LEG}: median {_fmt(per_test["median"])}, n={per_test["n"]}.'] + for count in derived['test_counts'][:1]: + out.append(f'- Tests on {MS_PER_TEST_LEG} (run {count["run_id"]}): {count["tests"]} testcases; ' + f'the suite\'s `tests=` reads {count["suite_tests"]}, which counts subtests.') + + for leg, row in report['tests'].items(): + out += ['', f'### Tests on {leg}, median of {row["runs"]} push run(s)', '', + '| Module | Seconds |', '|---|--:|'] + out += [f'| `{name}` | {seconds:.1f} |' for name, seconds in row['top_modules'][:10]] + out += ['', '| Class | Seconds |', '|---|--:|'] + out += [f'| `{name}` | {seconds:.1f} |' for name, seconds in row['top_classes'][:10]] + + if report.get('concurrency'): + pool = report['concurrency'] + out += ['', f'Runner pool over the last {pool["hours"]} h: {pool["jobs"]} jobs, peak ' + f'{pool["peak"]} at once for {pool["seconds_at_peak"]:.0f} s ' + f'(macOS peak {pool["macos_peak"]}).'] + out += [''] + [f'> {note}' for note in report['notes']] + calls = report.get('api', {}) + if calls: + out += ['', f'API calls: {calls["calls"]} (rate limit remaining: {calls["remaining"]}).'] + return '\n'.join(out).rstrip() + '\n' + + +def main(argv: 'Optional[list[str]]' = None, *, transport: 'Optional[Transport]' = None, + now: 'Optional[datetime.datetime]' = None) -> 'int': + """Collect, analyse, evaluate, write. Returns the exit status. + + Args: + argv: Command-line arguments. + transport: Replaces the network, for tests. + now: Replaces the current time, for tests. + + """ + parser = argparse.ArgumentParser(description=__doc__.split('\n', 1)[0]) + parser.add_argument('--repo', default=os.environ.get('GITHUB_REPOSITORY'), + help='owner/name (default: $GITHUB_REPOSITORY)') + parser.add_argument('--days', type=float, default=7, help='window length (default: 7)') + parser.add_argument('--runs', type=int, default=150, + help='Unit Tests runs per event whose jobs are read (default: 150)') + parser.add_argument('--other-runs', type=int, default=60, + help='runs per other workflow whose jobs are read (default: 60)') + parser.add_argument('--junit-runs', type=int, default=JUNIT_RUNS, + help=f'push runs whose JUnit artifacts are read (default: {JUNIT_RUNS})') + parser.add_argument('--concurrency-hours', type=float, default=0, + help='measure the runner pool over this many hours (default: 0, off)') + parser.add_argument('--baseline', help='the previous report.json; a missing file means none') + parser.add_argument('--out', help='write report.json here') + parser.add_argument('--summary', help='write the Markdown summary here') + parser.add_argument('--max-calls', type=int, default=DEFAULT_MAX_CALLS, + help=f'API call budget (default: {DEFAULT_MAX_CALLS})') + args = parser.parse_args(argv) + + token = os.environ.get('GH_TOKEN') or os.environ.get('GITHUB_TOKEN') + if not args.repo or not (token or transport): + print('::error title=CI timing::Needs --repo (or $GITHUB_REPOSITORY) and a token in ' + '$GH_TOKEN or $GITHUB_TOKEN.') + return EXIT_API + + client = Client(token or '', transport=transport, max_calls=args.max_calls) + moment = now or datetime.datetime.now(datetime.timezone.utc) + # Neither failure is a threshold, and each says so, so that a failed run is not + # read as a slow CI: 1 is reserved for a HARD finding. + try: + try: + data = collect(client, args.repo, now=moment, days=args.days, runs=args.runs, + other_runs=args.other_runs, junit_runs=args.junit_runs, + concurrency_hours=args.concurrency_hours) + except (BudgetExceeded, OSError) as error: + print(f'::error title=CI timing could not read the API::{error}. ' + 'No threshold was evaluated.') + return EXIT_API + return _report(args, client, data) + except Exception: # pylint: disable=broad-except + traceback.print_exc() + print('::error title=CI timing crashed::See the traceback above. No threshold was evaluated.') + return EXIT_CRASH + + +def _report(args: 'argparse.Namespace', client: 'Client', data: 'dict[str, Any]') -> 'int': + """Analyse, evaluate and write what :func:`collect` read; returns the exit status.""" + report = analyse(data) + report['api'] = {'calls': client.calls, 'remaining': client.rate_remaining} + + baseline = None + if args.baseline and os.path.isfile(args.baseline): + with open(args.baseline, encoding='utf-8') as file: + baseline = json.load(file) + elif args.baseline: + report['notes'].append(f'No baseline at {args.baseline}, so H3, W1, W4, W7\'s new ' + 'entrants and W8 were not evaluated.') + findings = evaluate(report, baseline) + report['findings'] = findings + + text = summary(report, findings) + if args.out: + with open(args.out, 'w', encoding='utf-8') as file: + json.dump(report, file, indent=1, sort_keys=True) + if args.summary: + with open(args.summary, 'w', encoding='utf-8') as file: + file.write(text) + print(text) + for item in findings: + kind = 'error' if item['level'] == 'hard' else 'warning' + message = item['message'].replace('`', '').replace('%', '%25') + print(f'::{kind} title=CI timing {item["id"]}::{message}') + return EXIT_HARD if any(item['level'] == 'hard' for item in findings) else 0 + + +if __name__ == '__main__': + sys.exit(main())