diff --git a/.github/workflows/ci-timing.yml b/.github/workflows/ci-timing.yml index c45f670786..aa571345bd 100644 --- a/.github/workflows/ci-timing.yml +++ b/.github/workflows/ci-timing.yml @@ -1,22 +1,19 @@ name: CI Timing -# The periodic CI-duration review #1538 asked for. util/ci_timing.py reads the -# last week of Unit Tests, Compat, Lint, CodeQL, Pages and Experimental Tests -# runs through the API, writes a report to the job summary and to an artifact, -# and fails the run when a -# HARD threshold fires (status 1; 2 and 3 are an unreadable API and a crash, each -# annotated as such). GitHub emails a scheduled workflow's failures to whoever -# last edited its cron line, so the owner hears about it with no `issues: write`. -# docs/source/contributing/workflows.rst has the thresholds and why each sits -# where it does; conventions/process.rst has what to do when one fires. +# The periodic CI-duration review (#1538). util/ci_timing.py reads the last +# week of runs through the API, writes a report to the job summary and an +# artifact, and fails when a HARD threshold fires (status 1; 2 and 3 are an +# unreadable API and a crash). GitHub emails a scheduled workflow's failures to +# whoever last edited its cron line, so no `issues: write` is needed. The +# thresholds are in docs/source/contributing/workflows.rst, the response to one +# in conventions/process.rst. # -# Deliberately not a `workflow_run` on Unit Tests: that would add a job per run -# to a runner pool already measured saturated at 20 concurrent jobs. +# Not a `workflow_run` on Unit Tests: that adds a job per run to a runner pool +# already measured saturated. on: schedule: # Mondays 07:23 UTC: off the hour, where GitHub delays scheduled runs most, - # and clear of Saturday's crons, 02:00 to 18:00 since #1967, and Project - # Status at 00:37. + # and clear of the Saturday crons (#1967) and Project Status at 00:37. - cron: '23 7 * * 1' workflow_dispatch: inputs: @@ -29,9 +26,8 @@ on: type: number default: 0 -# Read-only: the runs, jobs and artifacts need `actions: read`, the workflow -# files at each run's sha need `contents: read`. GITHUB_TOKEN allows 1,000 API -# requests an hour per repository; a default run made 677 (#1930). +# GITHUB_TOKEN allows 1,000 API requests an hour per repository; a default run +# made 677 (#1930). permissions: actions: read contents: read @@ -44,20 +40,18 @@ jobs: report: name: CI timing report runs-on: ubuntu-latest - # Three nested limits, each under the next, so that a slow week ends in a - # smaller report rather than none (#1930). The script stops reading jobs at - # --budget-minutes and writes what it has. The step's cap is the backstop, and - # sits far enough above to let a request already in flight finish its retries. - # This one leaves room for the setup steps and the uploads after it. Measured - # for #1930, a full sample took 16.6 minutes, nearly all of it job reads, which - # the API answers at up to 2.4 s each and which are made one at a time. + # Three nested limits, so a slow week ends in a smaller report rather than + # none (#1930): the script stops reading jobs at --budget-minutes and writes + # what it has; the step's cap sits far enough above it for an in-flight + # request to finish its retries; this one leaves room for setup and uploads. + # A full sample took 16.6 minutes for #1930, nearly all of it serial job + # reads at up to 2.4 s each. timeout-minutes: 40 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # #1660: `latest` from .github/python-versions.toml, read with the - # runner's own python3. + # #1660: `latest` from .github/python-versions.toml. - name: Read the Python versions id: python-versions run: python3 util/python_versions.py --github-output @@ -66,8 +60,7 @@ jobs: with: python-version: ${{ steps.python-versions.outputs.latest }} - # The script fetches with requests and parses the JUnit artifacts with lxml - # (#1894), and needs nothing else outside the standard library. + # lxml parses the JUnit artifacts (#1894). - name: Install requests and lxml run: python -m pip install requests lxml @@ -106,8 +99,7 @@ jobs: --budget-minutes 30 --out report.json --summary report.md # `always()`, not `!cancelled()`, here and below: a report written before a - # cap stopped the job is still worth having (#1930), and with none written - # there is nothing to add. + # cap stopped the job is still worth having (#1930). - name: Add the report to the job summary if: ${{ always() }} run: | diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 827a4e05f3..5f4660deb6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -4,7 +4,6 @@ on: push: branches: [main] pull_request: - # The branches below must be a subset of the branches above branches: [main] schedule: - cron: '0 2 * * 6' @@ -12,20 +11,16 @@ on: # #1968: no scopes by default. `analyze` declares the ones it needs. permissions: {} -# Only pull-request runs may be superseded -- see the note in unit-tests.yml. A -# ref-keyed group drops a *pending* run when a newer one joins it, which for a -# security scan means a commit reaching main with no analysis on record. +# Only pull-request runs may be superseded (see unit-tests.yml's +# `concurrency`): a ref-keyed group drops a *pending* run when a newer one +# joins, which here would leave a commit on main with no analysis on record. concurrency: group: codeql-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - # This workflow used to call ./.github/workflows/unit-tests.yml and gate - # `analyze` behind it. That re-ran the entire unit and integration matrix - # against a commit the Unit Tests workflow is already testing, and it bought - # nothing: CodeQL reads the source, so its findings do not depend on the test - # suite passing. Worse, it meant a failing test suppressed the security - # analysis for that commit. Static analysis runs on its own. + # Not gated on Unit Tests: CodeQL reads the source, so its findings do not + # depend on the tests passing, and a failing test would suppress the scan. analyze: name: Analyze runs-on: ubuntu-latest @@ -38,28 +33,11 @@ jobs: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 - # Override language selection by uncommenting this and choosing your languages - # with: - # languages: go, javascript, csharp, python, cpp, java - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee # v4.39.0 diff --git a/.github/workflows/coverage-comment.yml b/.github/workflows/coverage-comment.yml index 41437e61d0..16a79bd4e5 100644 --- a/.github/workflows/coverage-comment.yml +++ b/.github/workflows/coverage-comment.yml @@ -1,12 +1,10 @@ name: Coverage Comment -# Posts the coverage the `test` job's `latest` leg measured in -# unit-tests.yml as a pull-request comment. It runs no tests. The comment cannot -# be posted from unit-tests.yml itself: that file is also a reusable workflow, -# and a called workflow can only narrow the token its caller passes, never widen -# it. Every caller grants `contents: read`, so asking for -# `pull-requests: write` there would break every one of them, the release gate -# included. +# Posts the coverage the `test` job's `latest` leg measured in unit-tests.yml +# as a pull-request comment. Not from unit-tests.yml itself: that file is also a +# reusable workflow whose callers grant `contents: read`, and a called workflow +# can only narrow its caller's token, so `pull-requests: write` there would +# break every caller, the release gate included. # # `workflow_run` runs this file as it is on the default branch, with a token of # its own, so fork pull requests get a comment too. The price is that the diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 87112b75ba..860bff1c77 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -1,6 +1,5 @@ on: push: - # Sequence of patterns matched against refs/tags tags: - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 workflow_run: @@ -12,221 +11,130 @@ permissions: {} name: Create Release -# One release run at a time per ref. A ``workflow_run`` runs on ``main``'s tip, -# so two Vendor Update runs completing close together start two runs on one tip -# (78ec96afb: runs 34894022730 and 34894646592, six minutes apart), and both -# would read PCAPKIT_TAG_EXISTS before either had tagged. Unlike cron-conda and -# cron-vendor, no job-level group backs this one up, so it never cancels a run in -# progress, and a run that may release is keyed on its ref, never on itself. +# One release run at a time per ref, never cancelling one in progress: two +# Vendor Update runs completing close together start two runs on one tip (runs +# 34894022730 and 34894646592), and both would read PCAPKIT_TAG_EXISTS before +# either had tagged. # # #1705: a group holds one pending run and cancels it when another joins, so a -# run whose Vendor Update did not conclude ``success`` -- one ``version_check`` -# skips, so it releases nothing -- is keyed on its own ``github.run_id``. Keyed on -# the ref, such a run, 38499295798, displaced the pending 38499150441. So only a -# run that may release can wait in ``main``'s group or displace the one waiting -# there, evaluating ``main``'s newer tip instead. A tag push keeps its tag's group. -# GitHub documents the one pending run, and ``github.run_id`` as unique per run: +# run that cannot release -- its Vendor Update did not conclude ``success``, so +# ``version_check`` skips -- is keyed on its own ``github.run_id``. Keyed on the +# ref, such a run, 38499295798, displaced the pending 38499150441. A tag push +# keeps its tag's group. The one pending run, and ``run_id`` being unique: # https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency concurrency: group: create-release-${{ (github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success') && github.ref || github.run_id }} cancel-in-progress: false -# Every job below with an effect outside this run declares a deployment -# ``environment:``, but only ``github-release`` carries a required reviewer -# (#887). Each such job still gets an environment of its own rather than -# one shared ``release``, for credential and blast-radius separation, not for -# approval count: +# Environments: every job with an effect outside this run declares one, one per +# credential and blast radius, and only ``github-release`` carries a required +# reviewer (#887): # -# github -> github-release the public GitHub Release, and the ``v*`` tag that -# ``action-gh-release``'s ``tag_name`` creates with it +# github -> github-release the GitHub Release, and the ``v*`` tag +# ``action-gh-release``'s ``tag_name`` creates # tag -> conda-tag a commit pushed to ``main``, and the ``conda-*`` tag -# pypi -> pypi an upload to PyPI, and to TestPyPI -# conda -> anaconda an upload to anaconda.org/jarryshaw +# pypi -> pypi PyPI and TestPyPI; trusted publishing matches its +# OIDC claim against this name +# conda -> anaconda anaconda.org/jarryshaw # -# Approving the GitHub Release is the release's *only* approval. -# ``tag`` and ``pypi`` both carry ``needs: [ github, ... ]``, and ``conda`` needs -# ``tag`` and ``github`` too, so nothing downstream of ``github`` can start before -# that approval lands, and rejecting it leaves nothing tagged and nothing -# published. See :file:`docs/source/contributing/releasing.rst` for the full pipeline and the -# reasoning behind one approval being enough. +# That one approval gates the release because ``tag``, ``pypi`` and ``conda`` +# all need ``github``: rejecting it leaves nothing tagged and nothing published. +# ``TestGatedJobsDependOnGithub`` in tests/project/test_release_gates.py checks +# the ``needs:`` closure. Do not add a reviewer elsewhere without re-examining +# it: a reviewer on ``conda-tag`` with ``needs: [ version_check ]`` is the bug +# #887 fixed, which tagged and committed to ``main`` before the Release was +# approved. One shared ``release`` would publish PyPI and Anaconda, different +# credentials and indexes, together; and PyPI cannot afford an unintended +# publish, since an accepted version number cannot be reused (``skip-existing: +# true`` makes the re-upload *succeed* having published nothing). ``github`` +# spends the release App's token, minted from ``github-release``'s secret and +# revoked when the job ends; ``tag`` a long-lived PAT and a deploy key (#1634). +# docs/source/contributing/releasing.rst has the full pipeline. # -# The other environments keep their names -- ``pypi``'s matters because -# PyPI trusted publishing matches its OIDC claim against it -- and keep their -# secret scoping, they just hold no required reviewer of their own. Do -# not add one back to any of them without re-examining the ``needs:`` chain: a -# reviewer on ``conda-tag`` with its old ``needs: [ version_check ]`` is exactly -# the bug #887 fixed, because it let the commit-to-``main`` and ``conda-*`` tag -# run *before* the GitHub Release was ever approved. -# ``tests/project/test_release_gates.py``'s ``TestGatedJobsDependOnGithub`` -# asserts that every gated job's ``needs:`` closure includes ``github``, which -# is the checkable half of this paragraph -- reverting the line above to -# ``needs: [ version_check ]`` fails only that test. +# Repository settings no test can check (#641): # -# One environment per distinct credential and blast radius regardless of who -# gates it: a single ``release`` approved once would release PyPI and Anaconda -# together, and those are different credentials (OIDC trusted publishing versus -# ``ANACONDA_TOKEN``) reaching different indexes. PyPI in particular cannot afford -# an unintended publish -- a version number it has accepted cannot be reused, and -# ``skip-existing: true`` below means a re-upload then *succeeds* having published -# nothing, so the number the intended release wanted is gone regardless. ``github`` -# and ``tag`` were split on the same principle (#1634): ``github`` spends a -# token of the release App, minted in the job from ``github-release``'s secret -# and revoked when it ends (the run's ``GITHUB_TOKEN`` while no App is set up), -# whereas ``tag`` spends a long-lived PAT and a deploy key to write to the -# default branch. +# * An ``environment:`` naming one that does not exist is created implicitly, +# with no protection rules, and the job proceeds unapproved -- as +# ``github-pages`` has, with ``"protection_rules": []``, since 2021. So +# ``github-release`` needs creating under Settings -> Environments with a +# required reviewer. +# * Leave each one's "Deployment branches and tags" at "No restriction": a +# branch policy not listing ``v*`` fails the job with "Branch is not allowed +# to deploy to " rather than pausing it for approval. # -# Some things about this are not self-enforcing, and #641 is the first of them: +# tests/project/test_release_gates.py checks the workflow half: no job here +# publishes without naming an environment. # -# * An ``environment:`` naming an environment that does not exist is created -# implicitly, with no protection rules, and the job proceeds unapproved. So -# the ``github-release`` line above is inert until that environment exists -# under Settings -> Environments with a required reviewer on it. This -# repository already ships the worked example of that failure: -# ``github-pages`` has existed since 2021 with ``"protection_rules": []``. -# * Leave each one's "Deployment branches and tags" as "No restriction". This -# workflow's ``push`` trigger is ``tags: v*``, and a branch policy that does -# not list that tag pattern does not pause the job for approval -- it fails it -# outright with "Branch is not allowed to deploy to ". +# #888: each publisher reads its own evidence, never "the ``v*`` tag exists" -- +# the two diverge on a half-completed release, and the next run then skipped +# every job and reported green. ``github`` still reads ``PCAPKIT_TAG_EXISTS``, +# since that tag is its own artefact; ``tag`` reads whether ``conda-+0`` +# exists, ``pypi`` PyPI's file count, and ``conda`` Anaconda's aggregate count +# (a cost saver) plus, per leg, its own distribution -- the correctness fix, +# since ``anaconda/actions/upload-package`` has no ``skip-existing``, only +# ``force:``, never set here. # -# ``tests/project/test_release_gates.py`` asserts the first bullet's workflow half, -# i.e. that no job here publishes without naming an environment. It cannot assert -# the repository settings, which is why they are written down here. +# GitHub skips a job whose ``needs:`` skipped before evaluating its ``if:``, +# unless the ``if:`` calls a status-check function, and ``github`` skips on +# exactly the retry #888 is for. So ``tag``, ``pypi`` and ``conda`` open with +# ``!cancelled()`` and spell each publishing predecessor (``github``, and for +# ``conda`` ``tag``) as ``(== 'success' || == 'skipped')``, not as the callers +# of unit-tests.yml do, ``!= 'failure'``: that admits ``cancelled``, and a +# cancelled ``tag`` would send ``conda`` into checking a ``conda-+0`` +# tag never pushed, a failed step for a clean skip. ``version_check`` is held to +# ``== 'success'`` alone, since it produces the evidence the gates read, and so +# is ``unit-tests`` (#1052 below). # -# #888: every job below ``version_check`` used to share one guard -- -# ``startsWith(github.ref_name, 'v') || needs.version_check.outputs.PCAPKIT_TAG_EXISTS -# == 'false'`` -- using "the v* tag exists" as a proxy for "this version was -# already published". Those diverge exactly when a release half-completes: the -# ``github`` job tags, a later job fails or is rejected, and the next -# ``workflow_run``-triggered attempt reads a tag that already exists and skips -# every job, reporting green. ``github`` still uses that guard for its own -# artefact -- the tag it checks is the one it is about to create -- but ``tag``, -# ``pypi`` and ``conda`` each read their own evidence instead: ``tag`` checks -# for the ``conda-+0`` tag it creates, ``pypi`` checks PyPI's file count -# for the version against the count expected, and ``conda`` both checks Anaconda's -# aggregate count as a job-level cost saver and, per leg, whether that leg's own -# platform/Python distribution is already there -- the correctness fix, since -# retrying an upload Anaconda already has fails outright rather than skipping -# (``anaconda/actions/upload-package`` has no ``skip-existing`` equivalent, only -# ``force:``, never set below). +# ``release_status`` at the end reports why a run released nothing; a stranded +# partial release, the shape #888 was about, is its loud (``::error``) case. # -# Reading your own evidence instead of the tag is not enough on its own, though: -# ``tag``, ``pypi`` and ``conda`` each depend on ``github`` (and ``conda`` also on -# ``tag``), and GitHub Actions skips a job whose ``needs:`` included a job that -# itself skipped, *before* that job's own ``if:`` is even evaluated -- unless the -# ``if:`` calls a status-check function itself. ``github`` legitimately skips -# whenever its own tag already exists, which is exactly the retry case #888 is -# for, so ``tag``, ``pypi`` and ``conda`` all reach for the same escape hatch -# ``unit-tests``'s callers already use for the identical problem -- -# ``cron-vendor.yml``, ``deploy-pages.yml`` and ``cron-conda.yml`` all open with -# ``!cancelled() && needs.unit-tests.result != 'failure'`` so that a skipped -# gate does not cascade-skip them, while an outright failure or a genuine run -# cancellation still does. ``tag``, ``pypi`` and ``conda`` below are stricter -# about their *publishing* predecessors -- ``github``, and for ``conda`` also -# ``tag`` -- spelling the guard as ``!cancelled() && (needs..result == -# 'success' || needs..result == 'skipped')`` rather than the sibling files' -# ``!= 'failure'``: the two spellings differ only when a predecessor's own -# result is ``cancelled``, which ``!= 'failure'`` still admits and the equality -# pair does not. A cancelled ``tag`` would otherwise send ``conda`` into its -# check that ``conda-+0`` names this run's commit, for a tag ``tag`` -# never pushed, trading a clean skip for a failed step. ``version_check`` is held -# to a stricter bar still: ``tag``, ``pypi`` and ``conda`` all require -# ``needs.version_check.result == 'success'`` with no ``|| 'skipped'``, -# since it produces the evidence the gates below read, and a skipped or -# cancelled ``version_check`` leaves them nothing to decide on. Either -# spelling bypasses the default ``success()`` GitHub Actions would otherwise -# prepend to an ``if:`` that does not itself call a status-check function. -# -# See ``release_status`` at the bottom of this file for the other half: a job -# that runs whenever the run itself was not cancelled (``if: !cancelled()``, -# the same escape hatch) and reports, in one place, why a run released nothing -# -- because the triggering Vendor Update run had nothing to do, because the -# version is already fully out, or because something is stuck only partly -# published. The third is the shape #888 was actually about, and the one this -# job treats as loud (``::error``, non-zero exit) rather than quiet. -# -# #1052: ``version_check`` runs *before* the release test gate, not after it, so -# that the gate -- a ~20-minute full-suite run -- is spent only when something -# is going to be published. Every successful Vendor Update triggers this -# workflow, and most of them leave ``__version__`` alone, so the gate used to -# run on commits Unit Tests had already passed only for every publisher -# below to skip (eight times on 2026-10-05 alone). ``unit-tests`` runs when -# the ref is a ``v*`` tag or *any* of the evidence outputs is not ``'true'``, -# which covers every case in which ``github``, ``tag``, ``pypi`` or ``conda`` -# could run: each of them runs only on a ``v*`` ref or on its own evidence -# reading ``'false'``. ``!= 'true'`` rather than ``== 'false'`` is deliberate -- -# an evidence output nothing set runs the gate rather than skipping it, which -# can cost a run but never a release. -# -# Moving the gate off the front of the chain means it does not gate by -# position, so every publishing job names it instead: ``github`` lists it in -# ``needs:`` and keeps the implicit ``success()`` (its ``if:`` calls no -# status-check function), and ``tag``, ``pypi`` and ``conda`` -- whose -# ``!cancelled()`` opts out of that implicit check -- require -# ``needs.unit-tests.result == 'success'`` explicitly, held to the same strict -# bar as ``version_check``. A skipped gate therefore skips every publisher: -# if the predicate above ever drifts from theirs, the failure is a release -# that did not happen, reported by ``release_status``, never a release that -# went out untested. ``tests/project/test_release_gates.py`` asserts both -# halves. +# #1052: ``version_check`` runs before the release test gate, so the ~20-minute +# gate is spent only when something will be published -- most Vendor Update +# runs leave ``__version__`` alone. ``unit-tests`` runs on a ``v*`` ref or when +# any evidence output is not ``'true'``, which covers every case a publisher can +# run in; ``!= 'true'`` rather than ``== 'false'``, so an unset output costs a +# run, never a release. Every publisher names the gate: ``github`` through +# ``needs:`` and the implicit ``success()``, the others with +# ``needs.unit-tests.result == 'success'``. So if the predicates drift, the +# failure is a release that did not happen, reported by ``release_status``, +# never one that went out untested. tests/project/test_release_gates.py checks +# both halves. # # #1634: every job that checks out, tags, builds or publishes pins -# ``github.sha``, the commit this run was created for, never a ref that moves -# while the run waits on the ``github-release`` approval. On a ``v*`` push that -# is the pushed tag's commit; on ``workflow_run`` it is ``main``'s tip when the -# run was created, *not* ``github.event.workflow_run.head_sha``, which is the -# commit Vendor Update ran on and so predates the bump it pushed (run -# 38008819343: ``github.sha`` ``b6b95bc8e``, Vendor Update run 38008512299 on -# ``945c3c3d8``). That run was approved three days later and tagged ``main``'s -# tip ``2c0008c3c`` as both ``v1.5.0b9`` and ``conda-1.5.0b9+0``: -# ``action-gh-release`` defaulted ``target_commitish`` to the default branch, -# and ``tag`` ran ``git pull --rebase origin main`` before tagging. ``github``, -# ``tag``, ``pypi`` and ``conda`` also refuse to publish under a tag naming any -# other commit, so a later run for the same version fails loudly instead of -# putting a second tree out under it; finishing a release means re-running the -# run that tagged it. ``github`` creates the ``v*`` tag with the release App's -# token once ``vars.RELEASE_APP_CLIENT_ID`` is set, since GitHub refuses -# ``GITHUB_TOKEN`` a target that differs from ``main`` in a workflow file, and -# ``version_check`` skips the tag-push run that tag starts. -# ``tests/project/test_release_tags_the_run_sha_1634.py`` asserts all of this -# and runs ``tag``'s scripts against a scratch repository. +# ``github.sha``, never a ref that moves while the run waits for approval. On +# ``workflow_run`` that is ``main``'s tip when the run was created, not +# ``github.event.workflow_run.head_sha``, which predates Vendor Update's bump +# (run 38008819343: ``github.sha`` ``b6b95bc8e``, Vendor Update run 38008512299 +# on ``945c3c3d8``). Approved three days later, that run tagged ``main``'s tip +# ``2c0008c3c`` as ``v1.5.0b9`` and ``conda-1.5.0b9+0``. ``github``, ``tag``, +# ``pypi`` and ``conda`` also refuse to publish under a tag naming another +# commit; finishing a release means re-running the run that tagged it. +# tests/project/test_release_tags_the_run_sha_1634.py checks this and runs +# ``tag``'s scripts against a scratch repository. # # #1748: so a version tagged at another commit would fail every later run for -# it: ``v1.5.0b12`` names ``2025da5e5`` with only part of its Anaconda files, -# and each push-raised run spent the test gate and every ``conda`` leg on refusals. -# ``version_check``'s ``owner`` step resolves the ``v*`` tag as the #1634 check -# does, and on a ref that is not a ``v*`` tag, ``PCAPKIT_RELEASED_ELSEWHERE`` -# reading ``'true'`` skips the gate and every publisher -- ``github`` -# mostly skips there already, the tag existing. ``release_status`` then posts -# a notice when that release is complete and a warning naming what it lacks -# when not, and stays green. A ``v*`` push still meets the #1634 check, and -# ``!= 'true'`` sends an unset output there too, as #1052's ``!= 'true'`` does. -# The ``vendor_update`` job below is the issue's other half. +# it (``v1.5.0b12`` names ``2025da5e5`` with only part of its Anaconda files). +# Off a ``v*`` ref, ``PCAPKIT_RELEASED_ELSEWHERE`` reading ``'true'`` skips the +# gate and every publisher, and ``release_status`` reports, green, whether that +# release is complete. A ``v*`` push still meets the #1634 check. The +# ``vendor_update`` job is the issue's other half. jobs: # #1748: a Vendor Update raised by a Unit Tests verdict for another commit - # updates nothing -- its ``verdict`` job finds the verdict does not cover the - # tip it checked out, and ``vendor-update`` skips -- yet the run concludes - # ``success``, and Create Release took it for a real one (Vendor Update - # 38650191283, a second Create Release run for ``0cdd3cdd1``). Measured: that - # run's jobs read ``success``/``skipped``/``skipped``, and 38655782606, whose - # jobs all skipped, concluded ``skipped``; ``verdict`` has to run to - # decide, so Vendor Update could not conclude otherwise without a failed or - # cancelled run. This job reads that run's ``vendor-update`` job instead, with - # "List jobs for a workflow run", whose ``filter`` defaults to ``latest``, - # "jobs from the most recent execution of the workflow run": + # updates nothing -- ``vendor-update`` skips -- yet concludes ``success`` + # (Vendor Update 38650191283), since its ``verdict`` job has to run to decide; + # 38655782606, whose jobs all skipped, concluded ``skipped``. So this reads + # that run's ``vendor-update`` job with "List jobs for a workflow run", whose + # ``filter`` defaults to ``latest``: # https://docs.github.com/en/rest/actions/workflow-jobs#list-jobs-for-a-workflow-run - # It needs the "Actions" repository permission, read: - # https://docs.github.com/en/rest/authentication/permissions-required-for-github-apps # ``updated`` is ``'false'`` only for a ``skipped`` job; anything else it cannot # read leaves it unset, and ``version_check`` runs as before. # - # The concurrency group above cannot see this: it reads only the event, and - # that Vendor Update concluded ``success``, so the run joins - # ``main``'s group and can displace a run pending there. It releases - # nothing in that run's place, so the displaced run's release waits for - # the next run on ``main``: the tip's own Vendor Update if the tip's - # Unit Tests run has not completed yet, else the next push's or - # Saturday's. + # The concurrency group reads only the event, so such a run still joins + # ``main``'s group and can displace a run pending there. It releases nothing + # in that run's place, so the displaced run's release waits for the next run + # on ``main``: the tip's own Vendor Update if the tip's Unit Tests run has not + # completed yet, else the next push's or Saturday's. vendor_update: name: Triggering Vendor Update runs-on: ubuntu-latest @@ -266,25 +174,19 @@ jobs: version_check: name: Check Version runs-on: ubuntu-latest - # The outermost gate, moved here from ``unit-tests`` by #1052: a - # ``workflow_run`` trigger proceeds only when the Vendor Update run that - # triggered it concluded ``success``, and a tag push always proceeds. This - # job is read-only -- it installs the package and reads public - # endpoints -- so running it ahead of the test gate risks nothing. + # Read-only -- it installs the package and reads public endpoints -- so + # running it ahead of the test gate risks nothing (#1052). # - # #1634: a ``v*`` push by the release App's bot skips too. That is the tag - # ``github``'s Create Release makes with the App's token, which, unlike - # ``GITHUB_TOKEN``'s, starts a run; the run that made it releases that - # commit. Without this the second run would rerun the gate, ask for a - # second approval, and race the first run's uploads from another - # concurrency group. Whether that push fires, and carries the bot's - # ``github.actor_id``, is unmeasured; if not, this never matches. + # #1634: a ``v*`` push by the release App's bot skips: that is the tag + # ``github`` makes with the App's token, which, unlike ``GITHUB_TOKEN``'s, + # starts a run, and the run that made it releases that commit. Otherwise it + # would rerun the gate, ask for a second approval, and race the first run's + # uploads from another concurrency group. Whether that push fires with the + # bot's ``github.actor_id`` is unmeasured; if not, this never matches. # - # #1748: and a Vendor Update whose ``vendor-update`` job skipped, as - # ``vendor_update`` above reads it. ``!cancelled()`` because that job skips - # on a push, and "If a job fails or is skipped, all jobs that need it are - # skipped unless the jobs use a conditional expression that causes the job - # to continue": + # #1748: and a Vendor Update whose ``vendor-update`` job skipped. + # ``!cancelled()`` because ``vendor_update`` skips on a push, and a skipped + # need skips its dependents unless the ``if:`` says otherwise: # https://docs.github.com/en/actions/how-tos/write-workflows/choose-what-workflows-do/use-jobs needs: [ vendor_update ] if: >- @@ -301,9 +203,8 @@ jobs: ref: ${{ github.sha }} fetch-depth: 0 - # #1660: .github/python-versions.toml, read with the runner's own python3: - # `latest` for this job and the ``pypi`` and ``conda`` matrices, from this - # run's commit, and the tags the completeness checks below count until + # #1660: `latest` for this job and the ``pypi`` and ``conda`` matrices, + # from this run's commit, and the tags the completeness checks count until # the version is tagged. - name: Read the Python versions id: python-versions @@ -324,33 +225,30 @@ jobs: python -m pip install -e . - echo "PCAPKIT_VERSION=$(python -c 'import pcapkit; print(pcapkit.__version__)')" >> $GITHUB_OUTPUT - echo "PCAPKIT_PRERELEASE=$(python -c 'from packaging.version import Version; import pcapkit; print(str(Version(pcapkit.__version__).is_prerelease).lower())')" >> $GITHUB_OUTPUT - echo "PCAPKIT_CONDA_LABEL=$(python -c 'from packaging.version import Version; import pcapkit; print("dev" if Version(pcapkit.__version__).is_prerelease else "main")')" >> $GITHUB_OUTPUT + { + echo "PCAPKIT_VERSION=$(python -c 'import pcapkit; print(pcapkit.__version__)')" + echo "PCAPKIT_PRERELEASE=$(python -c 'from packaging.version import Version; import pcapkit; print(str(Version(pcapkit.__version__).is_prerelease).lower())')" + echo "PCAPKIT_CONDA_LABEL=$(python -c 'from packaging.version import Version; import pcapkit; print("dev" if Version(pcapkit.__version__).is_prerelease else "main")')" + } >> "$GITHUB_OUTPUT" - uses: mukunku/tag-exists-action@5c39604fe8aef7e65acb6fbcf96ec580f7680313 # v1.7.0 id: check_tag with: tag: "v${{ steps.get_version.outputs.PCAPKIT_VERSION }}" - # #888: ``PCAPKIT_TAG_EXISTS`` answers "was this tagged", which ``github`` - # above legitimately uses since the ``v*`` tag *is* its own artefact. It is - # the wrong question for ``tag``, ``pypi`` and ``conda``, whose artefacts - # are a different tag, PyPI, and Anaconda respectively -- a half-failed - # release leaves the ``v*`` tag behind while those are still - # incomplete, and gating them on it is what made every one of them skip - # silently. Each gets its own evidence step below instead. + # #888: ``tag``'s own evidence. ``PCAPKIT_TAG_EXISTS`` answers only for + # ``github``: a half-failed release leaves the ``v*`` tag behind with the + # other targets incomplete. - uses: mukunku/tag-exists-action@5c39604fe8aef7e65acb6fbcf96ec580f7680313 # v1.7.0 id: check_conda_tag with: tag: "conda-${{ steps.get_version.outputs.PCAPKIT_VERSION }}+0" - # #1660: the checks below count what the version was released with. - # Once its ``v*`` tag exists, that is the tier file at the tag's commit, - # read by that commit's own util/python_versions.py, not this run's file: - # a version added to the supported tier since was never built for it, and - # counting it would read a finished release as incomplete, run the gate and - # the publishers, and fail #1634's tag check on every run until the next + # #1660: once the ``v*`` tag exists, count what the version was released + # with: the tier file at the tag's commit, read by that commit's + # util/python_versions.py. A version added to the tier since was never + # built for it, and counting it would read a finished release as + # incomplete and fail #1634's tag check on every run until the next # release. A tag older than the file counts the tiers the file was added # with, which #1677 copied unchanged from the workflows of the time. - name: Read the released Python versions @@ -392,32 +290,15 @@ jobs: echo "pypi-wheels=${wheels}" >> "$GITHUB_OUTPUT" echo "conda-builds=${builds}" >> "$GITHUB_OUTPUT" - # PyPI evidence: compare the file count for this version against the one - # expected -- a wheel from each supported leg (``pypi``'s ``python-version`` - # axis, whose wheel tags ``PCAPKIT_PYPI_WHEELS`` lists from the tiers read - # above), plus the sdist that the "No Source Distribution" step keeps only - # on the ``latest`` leg. A release-level 200 only proves *some* - # file landed, not all of them, so presence alone -- ``PCAPKIT_TAG_EXISTS``'s - # mistake, repeated -- is not enough. Only the supported tier's wheels are - # counted (#1633): a KTLO leg's wheel neither completes the release nor, - # when that leg fails, leaves it incomplete for every later run to retry. - # - # The per-version endpoint's own ``releases`` mapping was empty on every - # request checked while writing this, including against a version that - # genuinely has every file -- so this reads ``urls``, the field the - # per-version endpoint actually populates for the requested version. - # ``/pypi/pypcapkit/json`` (no version) does populate ``releases``, but at - # ~950KB against this endpoint's ~15KB, for one release's worth of data. - # - # This endpoint is Fastly-fronted with a 900s max-age, so a check right - # after an upload can read a stale response -- but only stale in the - # direction of *undercounting*: a file just uploaded is invisible until the - # cache catches up, never the reverse, since PyPI never removes a file once - # accepted. So the only failure mode staleness can cause here is treating a - # genuinely-complete release as incomplete, which just re-enters the - # ``pypi`` job's matrix -- harmless, because every leg's ``skip-existing: - # true`` absorbs the redundant attempt. It cannot cause the opposite - # mistake of calling an incomplete release complete. + # PyPI evidence: a wheel per supported-tier leg (``PCAPKIT_PYPI_WHEELS``; + # #1633 -- a KTLO wheel neither completes the release nor, failing, leaves + # it incomplete) plus the sdist the ``latest`` leg keeps. A 200 alone + # proves only that *some* file landed. It reads ``urls``: this endpoint's + # ``releases`` was empty on every request checked, and the versionless + # ``/pypi/pypcapkit/json`` is ~950KB to this one's ~15KB. Fastly caches it + # (900s max-age), but staleness can only undercount -- PyPI never removes + # a file -- which re-enters ``pypi``, harmless under ``skip-existing: + # true``; it cannot call an incomplete release complete. - name: Check PyPI publish completeness id: check_pypi env: @@ -429,30 +310,20 @@ jobs: # One wheel per supported leg, plus the sdist. read -r -a wheels <<< "$PCAPKIT_PYPI_WHEELS" expected=$(( ${#wheels[@]} + 1 )) - # `--retry-connrefused` and the `|| status=000` fallback keep a - # transient pypi.org outage from failing this step outright: under - # `set -e` a bare curl failure here would fail `version_check`, - # which every one of `tag`/`pypi`/`conda` requires to have - # succeeded -- coupling the whole release to index availability, - # which it was not coupled to before this check existed. - # "Incomplete" (status != 200) is the safe default either way. + # `--retry-connrefused` and `|| echo 000` keep a pypi.org outage from + # failing `version_check` under `set -e`, which every publisher + # requires; "incomplete" is the safe default. status="$(curl -s --retry 3 --retry-connrefused \ -o /tmp/pypi_release.json -w '%{http_code}' \ "https://pypi.org/pypi/pypcapkit/${PCAPKIT_VERSION}/json" || echo 000)" count=0 if [ "$status" = "200" ]; then - # A `python3 -c` argument with indented source only compiles on - # Python 3.14 -- every earlier interpreter (3.9 through 3.13, - # measured directly) raises `IndentationError: unexpected indent` - # on the leading whitespace this YAML block scalar's own - # indentation would otherwise force onto the source. This job's - # setup-python step installs `latest` first, but the snippet does - # not lean on that; the conda job's check runs before its own. - # Feeding one line per `printf` argument - # keeps each line's YAML indentation off the actual Python source - # `python3 -` reads from stdin, so nothing is indented that - # Python did not itself put there. + # One line per `printf` argument, so the block scalar's indentation + # never reaches the source `python3 -` reads: indented `python3 -c` + # source compiles only on Python 3.14 (3.9 through 3.13 raise + # `IndentationError: unexpected indent`, measured). Not leaning on + # the setup-python above: the conda job's check runs before its own. count="$(printf '%s\n' \ "import json, os" \ "with open('/tmp/pypi_release.json') as f:" \ @@ -472,14 +343,10 @@ jobs: echo "http $status, found $count of $expected expected files for ${PCAPKIT_VERSION} on PyPI" echo "PCAPKIT_PYPI_COMPLETE=$complete" >> "$GITHUB_OUTPUT" - # Anaconda evidence, same shape as PyPI's above but for the ``conda`` matrix - # (each platform its legs publish, each Python read above). Unlike PyPI's, this - # endpoint reported ``cf-cache-status: DYNAMIC`` on every request checked - # while writing this, i.e. not cached -- so there is no staleness direction to - # reason about here. This is the job-level cost saver mirroring ``pypi``'s; - # ``conda``'s own per-leg checks (see that job) are the actual correctness - # fix and still run underneath it, since a release one leg short of - # complete must not have the missing leg mistaken for "nothing to do". + # Anaconda evidence, as PyPI's above, per platform and Python: the + # job-level cost saver, under which ``conda``'s per-leg checks, the + # correctness fix, still run. This endpoint read ``cf-cache-status: + # DYNAMIC`` (uncached) on every request checked, so no staleness applies. - name: Check Anaconda publish completeness id: check_conda env: @@ -500,9 +367,7 @@ jobs: count=0 if [ "$status" = "200" ]; then - # Same Python-3.14-only-dedent hazard as the PyPI check above -- - # one line per `printf` argument, no line indented in the actual - # source `python3 -` reads. + # The same `printf` form as the PyPI check above. count="$(printf '%s\n' \ "import json, os" \ "with open('/tmp/anaconda_release.json') as f:" \ @@ -528,11 +393,9 @@ jobs: echo "http $status, found $count of $expected expected platform/Python combinations for ${PCAPKIT_VERSION} on Anaconda" echo "PCAPKIT_CONDA_COMPLETE=$complete" >> "$GITHUB_OUTPUT" - # #1748: which commit the ``v*`` tag names, resolved as the #1634 check in - # ``github``, ``pypi`` and ``conda`` resolves it, so the gates below skip - # exactly the runs that check would refuse. ``PCAPKIT_RELEASED_ELSEWHERE`` - # is ``true`` only for a tag naming another commit; no tag, or this commit, - # is ``false``. + # #1748: resolved as #1634's check in ``github``, ``pypi`` and ``conda`` + # resolves it, so the gates skip exactly the runs that check would refuse. + # ``PCAPKIT_RELEASED_ELSEWHERE`` is ``true`` only for a tag elsewhere. - name: Check which commit the version's tag names id: owner env: @@ -561,7 +424,6 @@ jobs: PCAPKIT_PYPI_COMPLETE: ${{ steps.check_pypi.outputs.PCAPKIT_PYPI_COMPLETE }} PCAPKIT_CONDA_COMPLETE: ${{ steps.check_conda.outputs.PCAPKIT_CONDA_COMPLETE }} PCAPKIT_CONDA_LABEL: ${{ steps.get_version.outputs.PCAPKIT_CONDA_LABEL }} - # #1748: the commit the ``v*`` tag names, and whether it is another one. PCAPKIT_RELEASED_AT: ${{ steps.owner.outputs.PCAPKIT_RELEASED_AT }} PCAPKIT_RELEASED_ELSEWHERE: ${{ steps.owner.outputs.PCAPKIT_RELEASED_ELSEWHERE }} # #1660: from .github/python-versions.toml, for ``pypi`` and ``conda``. @@ -569,15 +431,9 @@ jobs: PYTHON_PYPI: ${{ steps.python-versions.outputs.pypi }} PYTHON_CONDA: ${{ steps.python-versions.outputs.conda }} - # No ``environment:`` here, and none needed: this is a test gate with no outward - # effect, and a job that calls a reusable workflow may not declare one anyway. - # - # #1052: runs only when a release will actually happen -- a ``v*`` tag push, - # or any target still incomplete by its own evidence. See the - # note above ``jobs:`` for why ``!= 'true'`` and why a skip here can never - # let a publisher through. #1748: nor on a version tagged at another commit, - # unless this is a ``v*`` push, on the same clause as ``tag``, ``pypi`` and - # ``conda``. + # No ``environment:``: no outward effect, and a job calling a reusable + # workflow may not declare one. Runs exactly when a publisher could (#1052, + # #1748); see the note above ``jobs:``. unit-tests: name: Release test gate needs: [ version_check ] @@ -599,22 +455,14 @@ jobs: github: name: GitHub Release runs-on: ubuntu-latest - # Gated: publishes the Release, and creates the ``v*`` tag -- at - # ``github.sha`` (#1634) -- that every job below then verifies before - # publishing under it. This is the release's only approval (#887) -- - # see the note above ``jobs:``. Still gated on ``PCAPKIT_TAG_EXISTS``, - # unlike the publishing jobs below it: the tag that output answers for is this - # job's own artefact, not a proxy for someone else's -- see the #888 note - # above ``jobs:``. ``unit-tests`` is listed because the gate follows - # ``version_check`` (#1052); this ``if:`` calls no status-check function, - # so the implicit ``success()`` requires the gate to have passed. The - # release App's private key is a ``github-release`` environment secret, so - # no job reads it before this approval, and no other job reads it at all. - # #1748: a tag naming another commit is a tag that exists, so - # ``PCAPKIT_TAG_EXISTS`` already skips this job off a ``v*`` ref -- unless - # the tag was pushed between ``check_tag`` and ``owner``, which read it - # separately. The same clause as the other publishers' covers that too, so the - # gate runs whenever any publisher would. + # The release's only approval (#887). Gated on ``PCAPKIT_TAG_EXISTS`` + # because the ``v*`` tag is its own artefact (#888). Its ``if:`` calls no + # status-check function, so the implicit ``success()`` requires the gate + # (#1052). The release App's private key is a ``github-release`` secret, so + # no job reads it before this approval, and no other job at all. #1748: + # ``PCAPKIT_TAG_EXISTS`` already skips a tag at another commit off a ``v*`` + # ref, unless it was pushed between ``check_tag`` and ``owner``; the other + # publishers' clause covers that too. environment: github-release permissions: contents: write @@ -629,27 +477,13 @@ jobs: ref: ${{ github.sha }} fetch-depth: 0 - # ``CHANGELOG.md`` is generated by ``util/changelog_md.py`` from the newest - # entry in the ``docs/source/changelog.rst`` toctree, so it describes the - # version being *prepared* -- which is not always the version being - # *released*. The two agree on a normal release and disagree on every - # automated one, because ``util/bump_version.py`` bumps without ever adding - # an entry: ``1.5.0b3`` becomes ``1.5.0b4``, and a final release becomes - # ``.post1``. A ``.postN`` is the weekly vendor registry refresh, carries no - # library change, and is deliberately recorded as an "Also released" line - # inside its base version's entry rather than as an entry of its own. - # - # Handing the file over unconditionally would therefore republish the base - # version's notes as the body of a release that changed no library code. So - # the body is used only when the generated file's own heading names the - # version being released, and otherwise the release keeps the generated notes - # alone -- which for a vendor refresh is the honest summary, since the commit - # list is the only thing that did change. Both outcomes are announced in the - # log rather than left to be inferred from the published result. - # - # Only the version token is compared, not the rest of the heading, so an - # entry still reading "-- unreleased" is matched and dated-or-not does not - # change the decision. That case gets its own warning below instead. + # CHANGELOG.md (util/changelog_md.py) holds the newest changelog entry: the + # version being prepared, which differs from the one released on every + # automated bump, since util/bump_version.py adds no entry and a ``.postN`` + # vendor refresh is an "Also released" line in its base version's entry. + # So it is the body only when its heading's version token is this version + # ("-- unreleased" still matches, with a warning); otherwise the generated + # notes alone, the honest summary of a vendor refresh. Both are logged. - name: Select release body id: release_body shell: bash @@ -681,14 +515,11 @@ jobs: # #1634: for a target that "adds or modifies any file under # .github/workflows/ relative to the repository's default branch", # https://docs.github.com/en/rest/releases/releases#create-a-release - # requires of "GitHub App installation tokens" the "Workflows" repository - # permission (write), whereas "The GITHUB_TOKEN available to GitHub - # Actions cannot be authorized for this". With no ``owner`` or - # ``repositories`` the token is "scoped to only the current repository", - # and the action's ``post`` step revokes it (v3.3.0's README, which ``v3`` - # names). Skipped while ``vars.RELEASE_APP_CLIENT_ID`` is unset, which - # reads as ``''``, so Create Release spends ``GITHUB_TOKEN`` until the App - # is set up. + # requires the "Workflows" permission (write), and "The GITHUB_TOKEN + # available to GitHub Actions cannot be authorized for this". With no + # ``owner`` or ``repositories`` the token is "scoped to only the current + # repository", and the action's ``post`` step revokes it (v3.3.0's + # README). Skipped while ``vars.RELEASE_APP_CLIENT_ID`` is unset (``''``). - name: Mint the release App's token id: app_token if: ${{ vars.RELEASE_APP_CLIENT_ID != '' }} @@ -703,36 +534,25 @@ jobs: id: create_release uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 with: - # allowUpdates: true - # artifacts: | - # dist/* - # An empty ``body_path`` is skipped by the action's own truthy guard, so - # an unset output here means "no body" rather than "read the file ''". - # ``generate_release_notes`` stays on either way: the action fetches the - # generated notes itself and joins them as ``\n\n``, so - # the hand-written entry leads and the pull-request list follows. + # An empty ``body_path`` is skipped by the action's truthy guard, and + # it joins ``\n\n``, so the entry leads and the + # pull-request list follows. body_path: ${{ steps.release_body.outputs.path }} generate_release_notes: true - # makeLatest: true name: PyPCAPKit v${{ needs.version_check.outputs.PCAPKIT_VERSION }} prerelease: ${{ needs.version_check.outputs.PCAPKIT_PRERELEASE }} tag_name: "v${{ needs.version_check.outputs.PCAPKIT_VERSION }}" - # #1634: on ``workflow_run`` this step creates the tag, and an unset - # target would put it on the default branch -- whatever ``main`` had - # become by approval. On a ``v*`` push the tag already exists, and the - # check below proves it names ``github.sha``, so this is ``''``, which - # the action reads as unset: the REST docs apply the workflow-file rule - # to a given target, else to the default branch's latest commit, so a - # target here would expose a hand-pushed tag to it under - # ``GITHUB_TOKEN``. ``pypi`` and ``conda`` upload with ``gh release - # upload``, which never calls Update a release, so nothing here - # re-checks this target once the release exists. cron-conda.yml's - # upload does, naming the default branch as its target: see - # releasing.rst. + # #1634: on ``workflow_run`` this creates the tag, and an unset target + # would be whatever ``main`` had become by approval. On a ``v*`` push + # the tag exists and the check below proves it names ``github.sha``, so + # this is ``''`` (unset): the REST docs apply the workflow-file rule to + # a given target, so one here would expose a hand-pushed tag to it + # under ``GITHUB_TOKEN``. ``pypi`` and ``conda`` upload with ``gh + # release upload``, which never calls Update a release; cron-conda.yml's + # upload does, naming the default branch: see releasing.rst. target_commitish: ${{ github.event_name == 'workflow_run' && github.sha || '' }} - # The App's token, or ``GITHUB_TOKEN`` when the step above skipped and - # left its output empty. "A non-empty explicit token overrides - # GITHUB_TOKEN" (3.0.2's ``action.yml``), so no ``env:`` names one. + # "A non-empty explicit token overrides GITHUB_TOKEN" (3.0.2's + # ``action.yml``), so no ``env:`` names one. token: ${{ steps.app_token.outputs.token || secrets.GITHUB_TOKEN }} # #1634: the same check ``tag``, ``pypi`` and ``conda`` make before they @@ -756,16 +576,11 @@ jobs: echo "${tag} -> ${run_commit}" done - # #1634: a refusal here is most likely the workflow-file rule quoted above - # the token step, which GitHub's 2023-11-02 changelog words more narrowly - # -- a target that itself modifies a workflow file and has no branch head - # or tag; which reading is enforced is unmeasured. What to fix depends on - # the token Create Release spent: ``GITHUB_TOKEN`` cannot be authorized, - # so the release App needs setting up; the App's token can be, so its - # permissions or installation are wrong. For a final release the - # action creates a draft and publishes it with Update a release in this - # same step; a re-run would reuse a draft left behind, target and all, so - # it is deleted first. + # #1634: a refusal is most likely the workflow-file rule quoted at "Mint + # the release App's token", which GitHub's 2023-11-02 changelog words more + # narrowly; which reading is enforced is unmeasured. For a final release + # the action creates a draft and publishes it in this same step; a re-run + # would reuse a leftover draft, target and all, so it is deleted first. - name: Explain a refused release tag if: ${{ failure() && steps.create_release.outcome == 'failure' }} env: @@ -781,28 +596,16 @@ jobs: echo "::error title=Could not create v${PCAPKIT_VERSION} at ${PCAPKIT_RUN_SHA} (#1634)::If Create Release failed with 404 Not Found or 403 'Resource not accessible by integration', the likely cause is GitHub's workflow-file rule: its REST docs for Create a release require a token authorized to modify workflows for a commit that adds or modifies a file under .github/workflows/ relative to the default branch, which this commit may if main has changed one since. ${fix} Then delete any draft release v${PCAPKIT_VERSION} this left behind, since a re-run would reuse it and its target, and re-run this job. Any other failure is not this." tag: - # "(release)" distinguishes this from the identically named job in - # cron-conda.yml; both workflows run on a push to main, so the two produced - # two check rows reading `Conda Tag` with nothing to tell them apart. + # "(release)": cron-conda.yml has a ``Conda Tag`` too, and both run on a + # push to main, so the two check rows could not be told apart. name: Conda Tag (release) runs-on: ubuntu-latest - # Pushes a commit to ``main`` and the ``conda-*`` tag. ``permissions: {}`` - # does not constrain it -- the write goes through the deploy key and PAT below, - # not the run's token. It has no required reviewer of its own (#887), and - # depends on ``github`` rather than on ``version_check`` alone, precisely so - # the single ``github-release`` approval still gates it -- without that - # ``needs:``, this job would start the moment ``version_check`` finished, - # ahead of any approval. See the note above ``jobs:``. - # - # #888: gated on its own evidence (whether ``conda-+0`` already - # exists) rather than on ``PCAPKIT_TAG_EXISTS``, and opened with - # ``!cancelled()`` so that ``github`` legitimately skipping (its own tag - # already existing) does not cascade-skip this job before its ``if:`` is - # even evaluated -- see the #888 note above ``jobs:``. - # - # #1634: "Create Tag" tags ``github.sha`` before anything here looks at - # ``main``, and "Reset Build Number" is a separate commit on top of - # ``main`` that never moves the tag. Its rules are in its own comment. + # Writes through the deploy key and PAT below, not the run's token, so + # ``permissions: {}`` does not constrain it. Needs ``github`` so the one + # approval gates it (#887), and reads its own evidence under + # ``!cancelled()`` (#888); see the note above ``jobs:``. #1634: "Create + # Tag" tags ``github.sha`` before anything looks at ``main``, and "Reset + # Build Number" is a separate commit that never moves the tag. environment: conda-tag permissions: {} needs: [ github, unit-tests, version_check ] @@ -818,10 +621,8 @@ jobs: with: ssh-private-key: ${{ secrets.PYPCAPKIT_ACTIONS_DEPLOY_KEY }} - # A detached checkout of this run's commit. This job used to check out a - # local ``main`` and run ``git pull --rebase origin main`` before tagging, - # which fast-forwarded it onto ``main``'s tip first -- ``b6b95bc8e`` to - # ``2c0008c3c`` in job 115087308551 (#1634). + # Detached at this run's commit (#1634): pulling ``main`` before tagging + # moved ``b6b95bc8e`` to ``2c0008c3c`` in job 115087308551. - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -838,11 +639,10 @@ jobs: git config --local user.email "github-actions[bot]@users.noreply.github.com" git config --local user.name "github-actions[bot]" - # ``conda-+0`` goes on this run's commit, next to the ``v*`` tag - # ``github`` put there -- and only if that tag is there: a ``v*`` tag on - # another commit means the release belongs to that commit, not this one. - # An existing ``conda-+0`` on this commit is a re-run and is left - # alone; one on another commit is never moved. + # ``conda-+0`` goes on this run's commit, and only if the ``v*`` + # tag is there too: one elsewhere means the release is that commit's. An + # existing ``conda-+0`` here is a re-run, left alone; one + # elsewhere is never moved. - name: Create Tag env: PCAPKIT_VERSION: ${{ needs.version_check.outputs.PCAPKIT_VERSION }} @@ -872,26 +672,18 @@ jobs: exit 1 fi - # The reset is a commit of its own on top of ``main``, never folded into - # the tag above and never a rewrite: ``main`` is only ever fast-forwarded, - # by a plain ``git push`` of one new commit whose parent is the tip just - # fetched. It sets the tip's ``conda/build`` to 0, so the next Conda - # Update bump of this version is ``+1`` -- unless: + # A commit of its own on ``main``, only ever fast-forwarded by a plain push + # of one commit onto the tip just fetched. It sets ``conda/build`` to 0, so + # the next Conda Update bump of this version is ``+1`` -- unless: # - # * this run's commit is not on ``main`` (a ``v*`` tag on a side branch) - # -- ``main`` is left alone, with a warning; - # * the tip already says build 0 -- the usual case, since - # ``util/bump_version.py`` writes 0 with every bump; - # * the tip carries another ``__version__`` -- its counter belongs to - # that version, left alone; - # * origin already has a ``conda-+N`` tag other than ``+0`` -- - # rewinding would let a later bump reuse that number, so ``main`` is - # left alone, with a warning. + # * this run's commit is not on ``main`` -- left alone, with a warning; + # * the tip already says 0 -- usual, as util/bump_version.py writes 0; + # * the tip carries another ``__version__`` -- its counter, left alone; + # * origin has a ``conda-+N`` tag other than ``+0`` -- rewinding + # would let a later bump reuse that number; left alone, with a warning. # - # #1634's own shape -- this commit at build 0, then three Conda Update - # bumps that #1632 left untagged -- gets the reset: the tip's 3 becomes 0. - # A rejected push means ``main`` moved in between; the whole decision is - # made again from a fresh fetch, three times at most. + # A rejected push means ``main`` moved: the decision is made again from a + # fresh fetch, three times at most. - name: Reset Build Number env: PCAPKIT_VERSION: ${{ needs.version_check.outputs.PCAPKIT_VERSION }} @@ -950,22 +742,10 @@ jobs: pypi: name: PyPI distribution for Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }} runs-on: ubuntu-latest - # Named ``pypi`` rather than the upstream snippet's ``release``: see the note - # above ``jobs:`` for why each target gets its own. #641 was that this line - # stayed commented while the rest of the same snippet did not -- the - # ``id-token: write`` that *enables* trusted publishing was re-added below, so - # the block read as disabled as a unit when only its gate still was. #887 - # removed this environment's own required reviewer; it is gated only - # transitively, through ``needs: [ github, unit-tests, version_check ]`` - # below, and the name still matters because PyPI's trusted-publishing claim - # match is keyed on it. - # - # #888: gated on its own evidence (PyPI's file count for this version) rather - # than on ``PCAPKIT_TAG_EXISTS`` -- this job is already idempotent - # (``skip-existing: true`` below), so the evidence check here is a cost - # saver, not the correctness fix. Opened with ``!cancelled()`` for the same - # reason as ``tag`` above: ``github`` legitimately skipping must not - # cascade-skip this job. See the #888 note above ``jobs:``. + # The environment's name matters: PyPI's trusted-publishing claim is matched + # on it (#641, #887; see the note above ``jobs:``). #888: its evidence is a + # cost saver, not the correctness fix, as ``skip-existing: true`` already + # makes it idempotent. environment: pypi permissions: contents: write @@ -988,26 +768,20 @@ jobs: ## one leg failing must not cancel its siblings: each leg still has its own ## wheel to attach to the GitHub Release, and a cancelled leg never does fail-fast: false - # #1660: CPython and PyPy from .github/python-versions.toml, through - # ``version_check``. #1633: the supported tier is the ``python-version`` - # axis, whose wheels version_check counts. The KTLO tier -- end-of-life, - # still built and published but never blocking -- is added by - # ``include``, without the versions that need an older runner than this - # job's ``ubuntu-latest``, leaving the 3.10 language level. Each step - # that can fail on one carries `continue-on-error: ${{ matrix.ktlo == - # true }}`, the step-level form, so ``needs.pypi.result`` cannot see it - # -- see the same choice in unit-tests.yml's ``test`` job. #1672: each - # step after setup-python waits on the ``outcome`` of the one it needs, - # so a KTLO leg stops at its first failure, where a supported leg stops, - # rather than building with the image's ``python`` or uploading nothing; - # and ``KTLO summary`` reports each failed step as a warning. A KTLO leg - # also needs its own ``ktlo_*`` output above, which - # tests/project/test_python_matrix_tiers_1633.py checks. + # #1660: from .github/python-versions.toml through ``version_check``. + # #1633: the supported tier is the ``python-version`` axis, whose wheels + # ``version_check`` counts; ``include`` adds the KTLO tier (end-of-life, + # published, never blocking) without the versions needing an older runner. + # Steps that can fail carry step-level ``continue-on-error``, so + # ``needs.pypi.result`` cannot see a KTLO failure, as in unit-tests.yml's + # ``test`` job. #1672: each step after setup-python waits on the + # ``outcome`` of the one it needs, so a KTLO leg stops at its first + # failure, and "KTLO summary" warns per failed step. A KTLO leg also needs + # its own ``ktlo_*`` output above (test_python_matrix_tiers_1633.py). matrix: ${{ fromJSON(needs.version_check.outputs.PYTHON_PYPI) }} steps: - # #1634: this run's commit, not ``ref: v`` -- that tag was - # whatever ``main`` had become by approval time -- and then the check - # below that the tag these files are filed under names it too. + # #1634: this run's commit, not ``ref: v``, which was whatever + # ``main`` had become by approval; the next step checks the tag names it. - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -1057,7 +831,6 @@ jobs: run: | python -m build - # rename wheel's python tag python util/wheel_rename.py dist/*.whl - name: No Source Distribution for Python ${{ matrix.python-version }} @@ -1073,7 +846,6 @@ jobs: continue-on-error: ${{ matrix.ktlo == true }} uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: - # password: ${{ secrets.PYPI_API_TOKEN }} skip-existing: true attestations: false @@ -1086,27 +858,19 @@ jobs: continue-on-error: true uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: - # password: ${{ secrets.TEST_PYPI_API_TOKEN }} repository-url: https://test.pypi.org/legacy/ skip-existing: true attestations: false # #1634: ``gh release upload``, not ``action-gh-release``, which calls - # Update a release before every upload (3.0.2's ``src/github.ts:631-683``). - # GitHub's REST docs re-check a release's existing target -- ``github.sha`` - # on ``workflow_run`` -- against the workflow-file rule there, so a + # Update a release before every upload (3.0.2's ``src/github.ts``), where + # GitHub re-checks the release's target against the workflow-file rule: a # workflow change reaching ``main`` after ``github`` ran would fail this - # step after PyPI had published. Upload a release asset's docs state no - # such rule; ``gh`` reads the release and POSTs each file to its - # ``upload_url``. #1640: a file already on the release is left alone, where - # the action deleted and re-uploaded it, and a rebuild is not - # byte-identical -- but only an asset in state ``uploaded``: after a 502, - # Upload a release asset "may leave an empty asset with a state of - # starter", which ``--clobber`` deletes before uploading the file again. - # Failures the action did not have: no release fails ``gh release - # view`` with "release not found", where the action created one with the - # default branch as its target, and a pattern matching nothing is an - # ``::error``, where it only warned. ``conda`` runs the same script. + # after PyPI had published. #1640: a file already ``uploaded`` is left + # alone, as a rebuild is not byte-identical; after a 502 an asset "may + # leave an empty asset with a state of starter", which ``--clobber`` + # replaces. A missing release fails ("release not found") where the action + # created one on the default branch. ``conda`` runs the same script. - name: Upload package to GitHub Release id: upload if: ${{ steps.publish.outcome == 'success' }} @@ -1146,11 +910,10 @@ jobs: done done <<< "$PCAPKIT_ASSETS" - # #1633: what this KTLO leg's steps did before ``continue-on-error``, - # for ``release_status`` -- ``steps.*.outcome`` covers the steps with an - # ``id`` above, i.e. every one that can fail here except the TestPyPI - # upload, which no leg treats as a failure, and #1634's tag check, which - # fails every leg alike and so blocks every leg. + # #1633: this KTLO leg's outcome despite ``continue-on-error``, for + # ``release_status``. ``steps.*.outcome`` covers every step with an ``id``: + # all that can fail here but the TestPyPI upload, never a failure, and + # #1634's tag check, which fails every leg alike. - name: Record this KTLO leg's outcome id: ktlo if: ${{ !cancelled() && matrix.ktlo == true }} @@ -1189,21 +952,9 @@ jobs: conda: name: Conda deployment (release) on ${{ matrix.os }} with Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} - # Uploads to anaconda.org/jarryshaw with ``ANACONDA_TOKEN``. This job never had - # an ``environment:`` at all, not even a commented one, so it was the half of - # #641 that was not written down as a thing to turn on. #887 later removed - # this environment's own required reviewer; gating is transitive, through - # ``needs: [ tag, github, unit-tests, version_check ]`` below. See the note - # above ``jobs:``. - # - # #888: gated on Anaconda's aggregate evidence for this version (job-level - # cost saver, mirroring ``pypi``) rather than on ``PCAPKIT_TAG_EXISTS``, and - # opened with ``!cancelled()`` for the same reason as ``tag`` and ``pypi`` - # above. Unlike them, an aggregate check is not the correctness fix here -- - # ``anaconda/actions/upload-package`` has no ``skip-existing``, only - # ``force:`` (never set below), so each *leg* also checks whether its own - # platform/Python distribution already exists and skips only its own upload - # step if so. See "Check whether this leg is already on Anaconda" below. + # Uploads with ``ANACONDA_TOKEN``, gated through ``needs:`` (#641, #887; see + # the note above ``jobs:``). #888: the aggregate evidence is a cost saver; + # "Check whether this leg is already on Anaconda" is the correctness fix. environment: anaconda permissions: contents: write @@ -1229,11 +980,10 @@ jobs: - macos-26-intel - ubuntu-latest # - windows-latest # skip for now - # #1633: the supported tier only, from .github/python-versions.toml - # through ``version_check`` (#1660). There is no KTLO 3.10 leg: the list - # util/conda_vendor.py derives is for the supported tier, so it leaves - # out typing-extensions, which pcapkit/utilities/compat.py imports below - # 3.11, and pins the newest release of each dependency, a rising floor. + # #1633, #1660: the supported tier only. No KTLO 3.10 leg: the list + # util/conda_vendor.py derives omits typing-extensions, which + # pcapkit/utilities/compat.py imports below 3.11, and pins each + # dependency's newest release, a rising floor. python-version: ${{ fromJSON(needs.version_check.outputs.PYTHON_CONDA) }} steps: # #1634: this run's commit, not ``ref: conda-+0``, then the check @@ -1262,11 +1012,8 @@ jobs: echo "${tag} -> ${run_commit}" done - # #888: this is the per-leg correctness fix -- see the job comment above. - # Anaconda's read API reported ``cf-cache-status: DYNAMIC`` on every - # request checked while writing this, i.e. not cached, so unlike PyPI's - # evidence check in ``version_check`` there is no staleness to reason - # about here: this reads the live state. + # #888: the per-leg correctness fix. Uncached, per ``version_check``'s + # Anaconda check, so this reads the live state. - name: Check whether this leg is already on Anaconda id: check_leg env: @@ -1300,20 +1047,10 @@ jobs: exists=false if [ "$status" = "200" ]; then - # This step runs before `actions/setup-python@v7` below, so - # `python3` here is the runner image's default -- 3.12 on - # ubuntu-latest, 3.13 on macos-latest, neither of which is 3.14. - # A `python3 -c` argument with indented source only compiles on - # 3.14 (measured directly: 3.9 through 3.13 all raise - # `IndentationError: unexpected indent` on exactly the leading - # whitespace this YAML block scalar's own indentation would - # otherwise force onto the source); every earlier interpreter - # fails this step outright under `set -e` -- and only on the - # branch entered when Anaconda already has *something* for this - # version, i.e. only on the retry this feature exists for. Feeding - # one line per `printf` argument instead keeps each line's YAML - # indentation off the actual Python source `python3 -` reads from - # stdin, so nothing is indented that Python did not put there. + # The `printf` form of version_check's checks, and needed here: this + # runs before setup-python, so `python3` is the image's (3.12 on + # ubuntu-latest, 3.13 on macos-latest), and indented `python3 -c` + # source would fail under `set -e` -- only on the retry this is for. exists="$(printf '%s\n' \ "import json" \ "with open('/tmp/anaconda_release.json') as f:" \ @@ -1522,26 +1259,15 @@ jobs: done done <<< "$PCAPKIT_ASSETS" - # #888: the point of this job is that a run which released nothing must say - # why, in one place, regardless of which upstream job or gate is what made it - # release nothing. ``version_check`` can skip (a ``workflow_run`` whose Vendor - # Update did not conclude ``success``) before any evidence output the jobs - # above rely on even exists, and ``unit-tests`` can fail, or skip (#1052) - # because that evidence says nothing is left to publish, so this job - # cannot simply read those outputs the way - # ``tag``/``pypi``/``conda`` do -- it has to run unconditionally and - # reconstruct what happened from ``needs.*.result`` and the trigger context - # instead. ``if: !cancelled()`` is what makes that possible: without it, the - # default status check this file relies on everywhere else (see the note - # above ``jobs:``) would skip this job the moment anything upstream skipped, - # which is precisely the situation it exists to report on. ``!cancelled()`` - # rather than ``always()``, same as ``tag``/``pypi``/``conda`` above, so that - # cancelling the run still cancels this report rather than having it run - # anyway with a run it cannot fully see. + # #888: a run that released nothing says why, in one place. ``version_check`` + # can skip before any evidence exists, and ``unit-tests`` can fail or skip + # (#1052), so this reconstructs the outcome from ``needs.*.result`` and the + # trigger. ``!cancelled()`` keeps a skipped upstream from skipping it, and is + # chosen over ``always()`` so that cancelling the run cancels this report too + # rather than leaving it to describe a run it cannot fully see. # - # No ``environment:`` -- it has no outward effect, only annotations, and - # ``permissions: {}`` reflects that: it reads only the ``needs:`` context and - # the trigger event, never the repository. + # No ``environment:`` and ``permissions: {}``: it only annotates, from the + # ``needs:`` context and the trigger event. release_status: name: Release status runs-on: ubuntu-latest @@ -1575,27 +1301,21 @@ jobs: run: | set -euo pipefail - # #1633: pypi's KTLO legs (3.10, pypy3.10) never reach - # needs.pypi.result -- their steps carry continue-on-error -- so - # nothing below can warn or fail because of one, and their wheels are - # not in PCAPKIT_PYPI_COMPLETE's count. Each leg records its own - # outcome as a job output instead, reported here as a notice. + # #1633: pypi's KTLO legs reach neither needs.pypi.result + # (continue-on-error) nor PCAPKIT_PYPI_COMPLETE's count, so each is + # only reported here, as a notice, from its own job output. ktlo_310="${PCAPKIT_PYPI_KTLO_310:-}" ktlo_pypy310="${PCAPKIT_PYPI_KTLO_PYPY310:-}" if [ -n "${ktlo_310}${ktlo_pypy310}" ]; then echo "::notice title=KTLO legs (non-blocking)::pypi 3.10=${ktlo_310:-not run} pypy3.10=${ktlo_pypy310:-not run}. End-of-life Python, built and published but never blocking (#1633); neither result changes this run's verdict." fi - # Case 1: version_check itself never ran. On a `workflow_run` - # trigger this means the triggering Vendor Update run did not - # conclude `success` -- most commonly because it found nothing to - # update (`skipped`), which is routine and gets a quiet notice; any - # other conclusion gets a warning, because it means Vendor Update - # itself needs attention, even though it is not #888 -- no tag exists - # here that could be left stranded. On a push it is quiet only for - # the release App's own tag (#1634), which version_check skips by - # design; an unset bot id never matches here. #1748: a Vendor Update - # that concluded `success` without running vendor-update is quiet too. + # Case 1: version_check never ran. On `workflow_run`, Vendor Update did + # not conclude `success`: `skipped`, nothing to update, gets a quiet + # notice; anything else a warning, as Vendor Update needs attention -- + # but no tag exists to strand, so it is not #888. On a push it is quiet + # only for the release App's own tag (#1634); an unset bot id never + # matches. #1748: a `success` that never ran vendor-update is quiet too. if [ "$PCAPKIT_VERSION_CHECK" = "skipped" ]; then if [ "$PCAPKIT_EVENT_NAME" = "push" ] && [ -n "$PCAPKIT_RELEASE_APP_BOT_ID" ] \ && [ "$PCAPKIT_ACTOR_ID" = "$PCAPKIT_RELEASE_APP_BOT_ID" ]; then @@ -1613,28 +1333,24 @@ jobs: exit 0 fi - # Case, still before any evidence exists: version_check ran and did - # not succeed. A workflow failure, not #888 -- but still worth saying - # plainly why nothing released. + # version_check ran and did not succeed: a workflow failure, not #888. if [ "$PCAPKIT_VERSION_CHECK" != "success" ]; then echo "::warning title=Release blocked before it could start::version_check did not succeed (result: '${PCAPKIT_VERSION_CHECK}'), so neither the release test gate nor any release job ran. This is a workflow failure, not #888." exit 0 fi - # The release test gate ran and did not pass, so every release job - # skipped behind it. A test failure, not #888, and already red in - # this run's checks. + # The gate did not pass, so every release job skipped: a test failure, + # not #888, and already red in this run's checks. if [ "$PCAPKIT_UNIT_TESTS" = "failure" ] || [ "$PCAPKIT_UNIT_TESTS" = "cancelled" ]; then echo "::warning title=Release blocked by the test gate::the release test gate concluded '${PCAPKIT_UNIT_TESTS}', so no release job ran for ${PCAPKIT_VERSION}. This is a test failure, not #888." exit 0 fi - # #1748: the version is tagged at another commit, and the gate and - # every publisher skipped for it -- which they do only off a v* ref. - # That release is that commit's (#1634), so this run has nothing to - # do: a notice when it is complete, and a warning naming what it - # lacks when not, both green. A publisher that ran instead -- on a v* - # push -- meets #1634's check and is reported below. + # #1748: tagged at another commit, with the gate and every publisher + # skipped for it, which happens only off a v* ref. That release is that + # commit's (#1634): a notice if complete, else a warning naming what it + # lacks, both green. A publisher that ran on a v* push meets #1634's + # check and is reported below. if [ "${PCAPKIT_RELEASED_ELSEWHERE:-}" = "true" ] && [ "$PCAPKIT_GITHUB_JOB" = "skipped" ] \ && [ "$PCAPKIT_TAG_JOB" = "skipped" ] && [ "$PCAPKIT_PYPI_JOB" = "skipped" ] \ && [ "$PCAPKIT_CONDA_JOB" = "skipped" ]; then @@ -1650,26 +1366,17 @@ jobs: exit 0 fi - # From here, version_check succeeded, so the evidence outputs above - # are meaningful. A *skipped* gate (#1052) falls through on purpose: - # it skips only when every evidence output reads 'true' on a non-v* - # ref, which the reconciliation below reports as "Nothing to - # release". If the gate's predicate ever drifts from the publishers' - # and skips with work outstanding, every publisher skips behind it - # and the same reconciliation reaches the loud #888 branch instead. + # From here the evidence outputs are meaningful. A skipped gate (#1052) + # falls through on purpose: it skips only when every evidence output + # reads 'true' off a v* ref, reported as "Nothing to release" below; if + # its predicate drifts, every publisher skips behind it and this + # reaches the loud #888 branch instead. # - # Reconcile each release job against its OWN evidence, - # not against each other. A self-heal is a MIXED outcome by design - # -- github/tag skip because their artefacts already existed going - # in, pypi/conda run and succeed because theirs did not -- and - # demanding uniformity across all of them (either all skipped, or all - # succeeded, as an earlier revision of this job did) reads that - # mix as the #888 shape itself, firing the error below on the run - # that just finished the release. "Reconciled" means: this target - # either actually succeeded, or skipped *because* its own evidence - # already said it was done -- anything else (skipped while its - # evidence says incomplete, or an outright failure/cancellation) - # is not reconciled. + # Reconcile each job against its OWN evidence, not against each other: + # a self-heal mixes jobs that skipped because their artefact existed + # with jobs that ran, and demanding uniformity would read that mix as + # #888 on the run that finished the release. Reconciled = succeeded, or + # skipped with its own evidence saying done. reconciled_github=false [ "$PCAPKIT_GITHUB_JOB" = "success" ] && reconciled_github=true [ "$PCAPKIT_GITHUB_JOB" = "skipped" ] && [ "$PCAPKIT_TAG_EXISTS" = "true" ] && reconciled_github=true @@ -1701,17 +1408,14 @@ jobs: fi done - # Case 2: every target reconciled by skipping, none of them ran. - # Quiet -- this is the steady state most runs are in. + # Case 2: every target reconciled by skipping -- the usual steady state. if [ "$all_reconciled" = "true" ] && [ "$all_skipped" = "true" ]; then echo "::notice title=Nothing to release::pypcapkit ${PCAPKIT_VERSION} is already fully released -- the v* tag, the conda-*+0 tag and every PyPI and Anaconda file version_check expects are present. This run correctly released nothing." exit 0 fi - # Every target reconciled, but at least one of them actually ran -- - # either a fresh full release (every one `success`) or a self-heal - # (a mix of `success` and legitimately-`skipped`). Both are the - # release working as intended, so both are quiet. + # Every target reconciled and at least one ran: a fresh release or a + # self-heal, both quiet. if [ "$all_reconciled" = "true" ]; then echo "::notice title=Release complete::pypcapkit ${PCAPKIT_VERSION} is now fully released -- github=${PCAPKIT_GITHUB_JOB} tag=${PCAPKIT_TAG_JOB} pypi=${PCAPKIT_PYPI_JOB} conda=${PCAPKIT_CONDA_JOB}. A 'skipped' entry there means that target's own artefact already existed before this run, not that anything was missed." exit 0 @@ -1722,12 +1426,9 @@ jobs: exit 0 fi - # Case 3, the shape #888 is actually about: at least one target - # skipped despite its own evidence saying it is still incomplete, - # and nothing else failed or was cancelled to explain that. With - # the evidence-based `if:` above this should be unreachable -- a - # target's own incomplete evidence makes its job run, not skip -- - # so reaching this branch means that assumption itself broke. Loud, - # and a real failure: this must never report green. + # Case 3, the #888 shape: a target skipped despite its own evidence + # saying incomplete, with nothing failed or cancelled to explain it. + # The evidence-based `if:`s should make this unreachable, so reaching + # it means that assumption broke: loud, and never green. echo "::error title=Stranded partial release (#888)::${PCAPKIT_VERSION} is only partially published (tag exists=${PCAPKIT_TAG_EXISTS}, conda tag exists=${PCAPKIT_CONDA_TAG_EXISTS}, pypi complete=${PCAPKIT_PYPI_COMPLETE}, conda complete=${PCAPKIT_CONDA_COMPLETE}) but github=${PCAPKIT_GITHUB_JOB} tag=${PCAPKIT_TAG_JOB} pypi=${PCAPKIT_PYPI_JOB} conda=${PCAPKIT_CONDA_JOB} -- at least one target skipped despite its own evidence saying it is not done. Re-running this workflow should let the incomplete jobs self-heal; if this recurs on the next run too, the evidence checks in version_check are the first thing to check." exit 1 diff --git a/.github/workflows/cron-conda.yml b/.github/workflows/cron-conda.yml index 7b30a53151..26a273fecc 100644 --- a/.github/workflows/cron-conda.yml +++ b/.github/workflows/cron-conda.yml @@ -4,49 +4,27 @@ permissions: contents: read on: - # #1967: GitHub Pages, Vendor Update and Conda Update run eight hours - # apart on Saturdays, so each runs `head`'s test gate alone, mostly on a - # commit of its own, and `vendor-update` and `conda-update` do not queue - # on `repository-maintenance` together. None reads another's output, and - # Conda Update comes last so its upload finds the release Create Release - # made after Vendor Update (#1634). workflows.rst gives the evidence. + # #1967: eight hours after Vendor Update, so its upload finds the release + # Create Release made after that (#1634); cron-vendor.yml's schedule says more. schedule: - cron: '0 18 * * 6' # Saturdays at 18:00 UTC - # No `push: branches: [main]` here. `unit-tests.yml`'s own push trigger - # already runs the full matrix for this commit; calling it again on every push - # with `gate-only: true` re-ran the single-version "Gate" job a second time - # for the same tree (#715). Chaining off `Unit Tests`' completion means the gate - # never runs twice, and `conda-update` below just reads the verdict that - # already exists. `.event == 'push'` is checked in that job's `if` rather - # than here, because `workflow_run` fires for a pull request's completion - # too and a `branches:` filter here would be the only thing stopping that. + # No `push` trigger, and pull requests' runs refused in `conda-update`'s + # `if`, as in cron-vendor.yml (#715). workflow_run: workflows: ["Unit Tests"] types: [completed] -# This workflow commits and pushes, so a cancelled run can leave the bump -# half-applied. Kept keyed on the ref, unlike the validation workflows: this -# publishes to Anaconda, and letting two runs upload the same version -# concurrently is worse than dropping a superseded duplicate. -# -# A `workflow_run` is keyed on the commit `Unit Tests` verified instead (#1643). -# Each `Unit Tests` completion raises a run here, and one whose verdict is for an -# older commit only runs `head` and skips -- but in a shared group it would -# still displace a pending run whose verdict matches, since "any existing pending -# job or workflow in the same concurrency group will be canceled" (GitHub, -# "Control the concurrency of workflows and jobs"). -# -# Note this group is not what serialises the repository writes -- the -# `conda-update` job below carries its own `repository-maintenance` group, shared -# with cron-vendor, and that is what actually stops two jobs committing at once. +# Never cancelled in progress: a cancelled run can leave the bump half-applied, +# and two runs uploading one version to Anaconda is worse than dropping a +# superseded duplicate. Keyed as cron-vendor.yml's group is, and why (#1643); +# `conda-update`'s `repository-maintenance` group serialises the writes. concurrency: group: conda-update-${{ github.event.workflow_run.head_sha || github.ref }} cancel-in-progress: false jobs: - # #1967: the test gate, on the schedule only, and the `verdict` on a - # `workflow_run`, shared with Vendor Update and GitHub Pages. cron-head.yml - # says when each runs. + # #1967: the test gate on the schedule, `verdict` on a `workflow_run`; + # shared with Vendor Update and GitHub Pages. head: name: Conda gate permissions: @@ -56,24 +34,11 @@ jobs: conda-update: name: Update requirements.txt runs-on: ubuntu-latest - # #1968: the deploy key is a `conda-tag` environment secret, and only - # a job naming the environment can read it. Keep that environment free of - # required reviewers, or every scheduled bump waits for an approval. + # #1968: as for cron-vendor.yml's `vendor-update`. environment: conda-tag needs: [ head ] - # `needs` cannot itself be conditional, and `head`'s jobs skip on every - # path, so this job must not inherit a skip. `!cancelled()` rather than - # `always()` so that cancelling the run still cancels the update; a gate - # that actually failed still blocks. - # - # The `workflow_run` clause is the real gate for a push to main: it - # reads the verdict `Unit Tests` already reached instead of re-running it, - # and `head`'s `verified` says whether that verdict covers the commit - # checked out below (#1643). `.event == 'push'` and `.head_branch == 'main'` - # both matter -- `workflow_run` fires for *every* completion of that - # workflow, including one triggered by a pull request, and without this - # check a green PR would upload a conda distribution built from that PR's - # tree. + # As cron-vendor.yml's `vendor-update` (#1643); without the `push`/`main` + # checks a green PR would upload a conda distribution built from its tree. if: >- !cancelled() && needs.head.result != 'failure' && (github.event_name != 'workflow_run' || ( @@ -95,25 +60,9 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - # Deliberately the default `github.sha` (main's tip), not - # `github.event.workflow_run.head_sha`. Pinning to the latter once - # ran here and CodeQL flagged it as "checkout of untrusted code in a - # privileged context" -- `workflow_run` can also be raised by a pull - # request's `Unit Tests` run, and that field would then be a fork's - # commit. The `if:` above already keeps that path from reaching this - # step, but making the ref itself always resolve to a trusted - # default-branch commit removes the pattern outright rather than - # relying on CodeQL tracing through the guard. The push below rebases - # onto main's actual tip before writing, or does not write at all, so - # pinning to the exact tested sha would have bought nothing here - # either. Whether this commit is one that verdict covers is - # `head`'s question (#1643); whether what lands after it is, Push - # Changes' (#1632). - # - # The push below re-points `origin` at an SSH remote and authenticates - # with the loaded deploy key, not with this checkout's credential -- - # so there is nothing here for a persisted token to do, and one less - # live credential for anything running in between to find. + # `github.sha`, and no persisted token, as in cron-vendor.yml. Whether + # the verdict covers this commit is `head`'s question (#1643); whether + # it covers what lands after it, Push Changes' (#1632). persist-credentials: false fetch-depth: 0 @@ -127,32 +76,24 @@ jobs: with: python-version: ${{ steps.python-versions.outputs.latest }} - # `.[all]`, which is what `util/conda_vendor.py` below vendors (#1814): - # it derives `conda/requirements.txt` from `dependencies` and the `all` - # extra, walks what they require through the installed metadata, and - # pins each package at the version installed here, but for its - # `PLATFORM_PINS` (#1827). So this install is the step that resolves the - # newest releases, and of the script's runs only the next one reads the - # network, through `requests`, which `.[all]` does not bring. + # `.[all]` is what util/conda_vendor.py vendors (#1814), pinning each + # package at the version installed here but for its `PLATFORM_PINS` + # (#1827), so this install resolves the newest releases. `requests` is + # for the next step, the script's only run that reads the network. - name: Install and Setup run: | set -x python -m pip install -U pip setuptools wheel packaging requests - python -m pip install -e .[all] - - # #1851: asks PyPI for each `PLATFORM_PINS` package's releases and moves - # its pin in conda/platform-pins.toml up to the newest release, up to the - # one installed above, with a wheel for the pinned platform, renewing its - # review-by date; a pin that reaches the installed release, or already - # pins it, goes. - # The step below then writes the moved pin into conda/requirements.txt, - # and Commit Changes commits both. The table is not one of - # util/verdict_covers_checkout.py's bot-managed paths, which says why: - # the workflows chained off Unit Tests wait for such a commit's own run - # (#1643). A package PyPI cannot be read for keeps its pin, with a - # warning, and a pin past its review-by date gets one too; either way the - # step passes. + python -m pip install -e '.[all]' + + # #1851: moves each `PLATFORM_PINS` pin in conda/platform-pins.toml up to + # PyPI's newest release with a wheel for the pinned platform, no further + # than the one installed above, and renews its review-by date; a pin that + # reaches the installed release goes. Not one of + # util/verdict_covers_checkout.py's bot-managed paths, which says why + # (#1643). An unreadable package keeps its pin and a pin past its + # review-by date warns; neither fails the step. - name: Refresh the platform pins run: python util/conda_vendor.py --refresh-pins @@ -196,33 +137,21 @@ jobs: Updated conda distribution at $(date). The following files have been changed: $CHANGED_FILES" - # Writes `pushed` and `commit`, which `CONDA_CHANGED` and `CONDA_COMMIT` - # below hand to `conda-tag` and `conda-dist` (#1632). - # - # The verdict this run acted on -- `head`'s `verdict`, or on the schedule - # its test gate's -- covers `$GITHUB_SHA`, the checkout, and nothing after - # it. A commit that lands on `main` during the run would otherwise be - # rebased under the bump, pushed, tagged and uploaded with no verdict of - # its own. - # So the tip is fetched first and the commits a rebase onto it would bring - # in are put to #1643's rule, util/verdict_covers_checkout.py, run from - # this checkout and not from the tip it is judging. If they are not - # covered, nothing is pushed: the tip's own `Unit Tests` run raises this - # workflow again, and that run bumps. Otherwise the rebase is onto that - # same fetched commit, so what is pushed is what was judged. + # #1632: writes `pushed` and `commit` for `conda-tag` and `conda-dist`. + # The verdict covers `$GITHUB_SHA` only, so the commits a rebase onto the + # fetched tip would bring in are put to #1643's rule, run from this + # checkout. Not covered: nothing is pushed, and the tip's own Unit Tests + # run raises this workflow again. Covered: rebase onto that same commit, + # so what is pushed is what was judged. # - # A bump then counts only while `conda/build` still differs from the tip. - # Two runs can compute the same bump from one checkout -- the schedule and - # a `workflow_run`, or a re-run, which uses "the same `GITHUB_SHA`" - # (GitHub, "Re-running workflows and jobs") -- and the second one's rebase - # drops it as "previously applied", while `files_changed` still reads - # 'true'. Conda Update 38325136710 and 38325153882 were such a pair, both - # on 2e941bbb7 and raised 2 s after two different `Unit Tests` runs, for - # 9083163f5 and for 01151b13f, completed; `verdict` refuses both. - # Asking about that file rather than "is a commit left to push" also - # covers a rebase that keeps part of the commit but drops its bump: that - # remainder would land under a build already tagged, so it is not pushed, - # and the next run recomputes it under a fresh build. + # A bump counts only while `conda/build` still differs from the tip. Two + # runs from one checkout -- the schedule and a `workflow_run`, or a + # re-run, which uses "the same `GITHUB_SHA`" (GitHub, "Re-running + # workflows and jobs") -- compute the same bump, and the second's rebase + # drops it while `files_changed` still reads 'true' (Conda Update + # 38325136710 and 38325153882, both on 2e941bbb7). Asking about the file + # also refuses a rebase that keeps part of the commit but drops its bump, + # which would land under a build already tagged. - name: Push Changes id: push-changes if: steps.verify-changed-files.outputs.files_changed == 'true' @@ -262,14 +191,10 @@ jobs: status=0 git push origin "${commit}:${GITHUB_REF}" || status=$? if [ "$status" -ne 0 ]; then - # #1649: a merge landing after the fetch above has the push - # refused as a non-fast-forward. That is main moving past this - # checkout one step later than the rule can see, so it ends as - # the rule's refusal does. Only main moved to a commit without - # `$commit` in its history is that race: at `$tip` the push - # failed for another reason, and over `$commit` it landed though - # git says it did not. Those, and a fetch that fails, fail the - # step. + # #1649: a merge after the fetch makes this a non-fast-forward, + # which ends as the rule's refusal does. Only main moved to a + # commit lacking `$commit` is that race; main still at `$tip`, + # main already holding `$commit`, or a failed fetch fail the step. git fetch origin "${GITHUB_REF#refs/heads/}" || exit "$status" moved="$(git rev-parse FETCH_HEAD)" ancestry=0 @@ -295,9 +220,11 @@ jobs: run: | set -ex - echo "PCAPKIT_VERSION=$(python -c 'import pcapkit; print(pcapkit.__version__)')" >> $GITHUB_OUTPUT - echo "PCAPKIT_CONDA_LABEL=$(python -c 'from packaging.version import Version; import pcapkit; print("dev" if Version(pcapkit.__version__).is_prerelease else "main")')" >> $GITHUB_OUTPUT - echo "PCAPKIT_BUILD=$(cat conda/build)" >> $GITHUB_OUTPUT + { + echo "PCAPKIT_VERSION=$(python -c 'import pcapkit; print(pcapkit.__version__)')" + echo "PCAPKIT_CONDA_LABEL=$(python -c 'from packaging.version import Version; import pcapkit; print("dev" if Version(pcapkit.__version__).is_prerelease else "main")')" + echo "PCAPKIT_BUILD=$(cat conda/build)" + } >> "$GITHUB_OUTPUT" outputs: # Not `files_changed`: see Push Changes above. @@ -315,35 +242,22 @@ jobs: name: Conda Tag (update) runs-on: ubuntu-latest # #1968: the deploy key and the `PYPCAPKIT` PAT are `conda-tag` environment - # secrets, as for create-release.yml's `tag`. Keep that environment free of - # required reviewers, or every scheduled bump waits for an approval. + # secrets, as for create-release.yml's `tag`; keep it free of reviewers. environment: conda-tag needs: [ conda-update ] - # `!cancelled()` is what made this job reachable at all on the - # `workflow_run` path (#1632), when the test gate was a job of this file and - # skipped on that event. It is kept now that the gate is in `head` (#1967): - # a job of `head` skips on every path, and this job must not hinge on how - # Actions reports such a call. Without a status-check function, "a default - # status check of `success()` is applied unless you include one of these - # functions" (GitHub, "Evaluate expressions in workflows and actions", - # Status check functions). The skip it reacts to is not only a - # direct `needs:` entry's: "If a run contains a series of jobs that need - # each other, a failure or skip applies to all jobs in the dependency chain - # from the point of failure or skip onwards" (GitHub, "Workflow syntax for - # GitHub Actions", `jobs..needs`). So `conda-update` escaping it - # with its own `!cancelled()` does not escape it for this job. Run - # 38325136710 shows it: "Update requirements.txt" succeeded, pushing - # f4f94bcad, and both this job and `conda-dist` reported `skipped` with - # their own conditions true. So the bumps #1632 lists were committed - # and never tagged or uploaded. + # #1632: `!cancelled()`, since `head`'s jobs skip on every path (#1967) and + # without a status-check function "a default status check of `success()` + # is applied" (GitHub, "Evaluate expressions in workflows and actions"). + # The skip cascades past `conda-update`'s own `!cancelled()`: "a failure or + # skip applies to all jobs in the dependency chain from the point of + # failure or skip onwards" (GitHub, "Workflow syntax for GitHub Actions", + # `jobs..needs`). Run 38325136710: "Update requirements.txt" + # pushed f4f94bcad, and this job and `conda-dist` reported `skipped` with + # their own conditions true, so the bump was never tagged or uploaded. # - # `needs.conda-update.result == 'success'` is then what the lost implicit - # check was doing, and it has to be said explicitly: `!cancelled()` bypasses - # the cascade for a legitimate skip, so without it a failed or cancelled - # update -- which still publishes the outputs of the steps that did run -- - # would have this job tag from a half-finished run, e.g. one whose push - # landed and whose Get Version then failed, leaving no build. Same shape as - # create-release.yml's `tag` job. + # `needs.conda-update.result == 'success'` restores the lost check, so a + # failed or cancelled update, which still publishes its finished steps' + # outputs, is never tagged. Same shape as create-release.yml's `tag` job. if: >- !cancelled() && needs.conda-update.result == 'success' && needs.conda-update.outputs.CONDA_CHANGED == 'true' @@ -355,12 +269,10 @@ jobs: with: ssh-private-key: ${{ secrets.PYPCAPKIT_ACTIONS_DEPLOY_KEY }} - # The commit Push Changes pushed, not `main`'s tip: a commit landing after - # the push is neither the bump nor covered by this run's verdict, and - # tagging the tip -- a `git pull` and `HEAD`, as this job did -- would have - # `conda-dist` build and upload it (#1632). `fetch-depth: 0` fetches "all - # history for all branches and tags" (actions/checkout), so Create Tag sees - # a tag that already exists. + # The commit Push Changes pushed, not main's tip, which is neither the + # bump nor covered by this run's verdict (#1632). `fetch-depth: 0` fetches + # "all history for all branches and tags" (actions/checkout), so Create + # Tag sees a tag that already exists. - name: Checkout code uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -368,18 +280,11 @@ jobs: token: ${{ secrets.PYPCAPKIT }} fetch-depth: 0 - # `env:` rather than splicing `needs.*.outputs` straight into the script -- - # CodeQL flags the latter as code injection, since Actions substitutes - # `${{ }}` into the script text before the shell ever parses it, so a - # value containing shell metacharacters runs as script rather than data. - # Routing it through `env:` and reading `$PCAPKIT_VERSION`/`$PCAPKIT_BUILD` - # keeps it a shell variable, which the shell can only ever treat as data. - # - # Tags `$CONDA_COMMIT` by name rather than `HEAD`. A tag that already tags - # it is left alone and the job stays green, so a re-run of this job goes - # on to `conda-dist`; a tag on any other commit means this build number - # already names another tree, so the job fails, the tag is not moved, and - # nothing is uploaded under it. + # `env:`, not `${{ }}` in the script, which CodeQL flags as code + # injection. Tags `$CONDA_COMMIT` by name: a tag already on it is left + # and the job stays green, so a re-run goes on to `conda-dist`; a tag on + # another commit means the build number names another tree, so the job + # fails, the tag stays, and nothing is uploaded under it. - name: Create Tag env: CONDA_COMMIT: ${{ needs.conda-update.outputs.CONDA_COMMIT }} @@ -414,20 +319,13 @@ jobs: name: Conda deployment (update) on ${{ matrix.os }} with Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} # #1968: `ANACONDA_TOKEN` is an `anaconda` environment secret, as for - # create-release.yml's `conda`. Keep that environment free of required - # reviewers, or every leg of a scheduled upload waits for an approval. + # create-release.yml's `conda`; keep it free of reviewers. environment: anaconda needs: [ conda-tag, conda-update ] - # #1632, exactly as for `conda-tag` above -- this job was skipped by the - # same implicit `success()` on the same runs. - # - # `needs.conda-tag.result == 'success'` strictly, with no `|| 'skipped'` - # escape hatch of the kind create-release.yml's `conda` job grants its - # `tag`: the checkout below pins `ref: conda-+`, a tag known - # to name the pushed commit only once `conda-tag` has succeeded. There `tag` - # skips when the tag already exists, which is a reason to carry on; here - # `conda-tag` skips only when no bump was pushed, so carrying on would mean - # failing in `actions/checkout` instead of skipping cleanly. + # #1632, as for `conda-tag`. `conda-tag` must succeed, with no `skipped` + # escape as create-release.yml's `conda` job grants its `tag`: the checkout + # pins the tag, which names the pushed commit only then, and `conda-tag` + # skips only when no bump was pushed. if: >- !cancelled() && needs.conda-update.result == 'success' && needs.conda-tag.result == 'success' && @@ -573,17 +471,14 @@ jobs: echo EOF } >> "$GITHUB_OUTPUT" - # #1814: the package declares no run requirement but Python, so the - # packages it vendors in `pcapkit/_extern` are all it has, and package - # discovery on its own ships them as bare modules, without their data - # files. This unpacks the package built for this runner and checks that - # every vendored distribution has every file its RECORD lists and imports - # from there, with no site-packages on the path, before anything is - # uploaded. #1822: it also fails unless the package, and every native - # binary in it, is for this runner's platform, so a relabelled package - # cannot pass on the runner that built it. setup-python's interpreter, not - # conda's: it is the matrix version the package was built for, which - # conda's base is not on 3.15. + # #1814: the package's only run requirement is Python, so it has only what + # it vendors in `pcapkit/_extern`, and package discovery alone ships those + # without their data files. This checks every vendored distribution has + # every file its RECORD lists and imports from there, with no + # site-packages on the path. #1822: it also fails unless the package and + # every native binary in it are for this runner's platform. + # setup-python's interpreter, not conda's: it is the matrix version the + # package was built for, which conda's base is not on 3.15. - name: Smoke-test the vendored packages env: PCAPKIT_CONDA_PACKAGE: ${{ steps.conda_build.outputs.host_package }} @@ -602,14 +497,12 @@ jobs: # #1634: the upload below never makes the release. ncipollo/release-action # v1.21.0 creates a missing one, and its tag, at ``commit`` # (src/Action.ts:72-92), so a bump made while Create Release waits on - # approval would tag the pending version at main's tip, and Create - # Release's tag check would fail. It also retargets a draft at ``commit`` - # (:85-92,107-120), which ``omitDraftDuringUpdate`` keeps a draft, but - # Create Release's ``workflow_run`` run finds the draft and resets its - # target to the run's sha (softprops/action-gh-release v3.0.2, - # src/github.ts:636-646,832-866). Skipping a draft is a precaution - # against retargeting a release Create Release owns. Only the GitHub - # upload is skipped; the build is on anaconda.org regardless. + # approval would tag the pending version at main's tip and fail Create + # Release's tag check. It also retargets a draft at ``commit`` + # (:85-92,107-120), though Create Release's ``workflow_run`` run resets a + # draft's target to its sha (softprops/action-gh-release v3.0.2, + # src/github.ts:636-646,832-866); skipping a draft is a precaution. Only + # the GitHub upload is skipped; the build is on anaconda.org regardless. - name: Check the GitHub release exists id: release if: ${{ matrix.os == 'ubuntu-latest' }} @@ -646,11 +539,9 @@ jobs: # # #1657: that update also sends ``name``, ``body``, ``prerelease`` and # ``draft``, which unset inputs make the tag, "" and false - # (src/Inputs.ts:70-92,128-135,210-240), so each bump wiped the - # release's name and notes. Each ``omit*DuringUpdate`` makes its field - # undefined (:210-252), which Octokit drops from the request - # (dist/index.js:38369-38388), so the release keeps what Create Release - # wrote. + # (src/Inputs.ts:70-92,128-135,210-240), so each bump wiped the release's + # name and notes. Each ``omit*DuringUpdate`` leaves its field undefined + # (:210-252), and Octokit drops it (dist/index.js:38369-38388). - name: Upload package to GitHub Release if: ${{ matrix.os == 'ubuntu-latest' && steps.release.outputs.published == 'true' }} uses: ncipollo/release-action@339a81892b84b4eeb0f6e744e4574d79d0d9b8dd # v1.21.0 diff --git a/.github/workflows/cron-head.yml b/.github/workflows/cron-head.yml index 9f236d2024..aea685a733 100644 --- a/.github/workflows/cron-head.yml +++ b/.github/workflows/cron-head.yml @@ -1,14 +1,11 @@ name: Cron Head -# #1967: the head that Conda Update, Vendor Update and GitHub Pages each ran as -# two jobs of their own, a test gate and a `verdict`, kept in step by a test -# that compared the three copies. Each now calls this file as its `head` job and -# gates its own work on the call's result and its `verified` output. One task: -# saying whether the commit the caller checked out may be acted on. +# #1967: the shared `head` job of Conda Update, Vendor Update and GitHub Pages. +# One task: saying whether the commit the caller checked out may be acted on. +# Each caller gates its work on this call's result and its `verified` output. # -# Reusable only. Every expression here reads the *caller's* `github` context, -# so `github.event_name` is the event that started Conda Update, Vendor Update -# or GitHub Pages, and `github.sha` is the commit it checked out. +# Reusable only, so every expression reads the *caller's* `github` context: +# `github.event_name` is the caller's event, `github.sha` its checkout. on: workflow_call: outputs: @@ -19,17 +16,15 @@ on: it does not, and unset on any other event, where `verdict` skips. value: ${{ jobs.verdict.outputs.verified }} -# A called workflow cannot hold more than its caller grants, and each caller -# grants this one `contents: read`. That is also what the `unit-tests.yml` call -# below is granted, as Create Release's gate is. +# A called workflow cannot hold more than its caller grants; each grants +# `contents: read`, which is also what the `unit-tests.yml` call below gets. permissions: contents: read jobs: - # Only earns its runners when no completed Unit Tests run has verified this - # tree: on the Saturday schedule, since Unit Tests has no schedule trigger of - # its own. A `workflow_run` carries its own verdict, read by `verdict` below, - # and a pull request never acts, so both skip the gate (#715). + # Runs only when no completed Unit Tests run has verified this tree, i.e. on + # the schedule. A `workflow_run` carries its own verdict, read by `verdict` + # below, and a pull request never acts, so both skip the gate (#715). unit-tests: name: Test gate if: ${{ github.event_name != 'pull_request' && github.event_name != 'workflow_run' }} @@ -39,13 +34,11 @@ jobs: with: gate-only: true - # #1643: `github.sha`, which every checkout of the caller takes, is main's tip - # when the `workflow_run` was raised -- not the commit the completed `Unit - # Tests` run verified, and in a merge wave the two differ. This job compares - # them with util/verdict_covers_checkout.py, whose docstring gives the rule, - # and the caller acts on a `workflow_run` only when `verified` is 'true'. Its - # `if:` is the caller's `workflow_run` clause, so for a run raised by a pull - # request's or a failed `Unit Tests` run nothing here runs, as before. + # #1643: `github.sha`, which the caller checks out, is main's tip when the + # `workflow_run` was raised -- not the commit the completed Unit Tests run + # verified, and in a merge wave the two differ. util/verdict_covers_checkout.py, + # whose docstring gives the rule, compares them; the caller acts on a + # `workflow_run` only when `verified` is 'true'. verdict: name: Verdict covers this commit if: >- @@ -61,7 +54,7 @@ jobs: verified: ${{ steps.verdict.outputs.verified }} steps: # The default ref, as the caller's acting job uses, so both see the same - # `github.sha`. Twenty commits leaves room for the bot commits the rule + # `github.sha`. The depth leaves room for the bot commits the rule # accepts; a verdict further back is not found, and so not accepted. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/cron-python-versions.yml b/.github/workflows/cron-python-versions.yml index a106344daa..da6325de5a 100644 --- a/.github/workflows/cron-python-versions.yml +++ b/.github/workflows/cron-python-versions.yml @@ -1,24 +1,20 @@ name: Python Versions # #1660: keeps .github/python-versions.toml current. `detect` recomputes the -# tiers from python.org's release cycle, PyPy's releases, setup-python's builds -# and the hosted runner images (util/update_python_versions.py, whose docstring -# gives the rules), rewrites the file and the prose the audit tests pin to it, -# and runs that audit. When something changed, `pr` pushes the change to -# `bot/python-versions` and opens that branch's pull request, or updates the -# open one, for the owner to merge. Nothing here pushes `main` or edits ruleset -# 23497679: the pull request body lists the required contexts to change by hand. +# tiers (util/update_python_versions.py's docstring gives the rules), rewrites +# the file and the prose the audit tests pin to it, and runs that audit. `pr` +# proposes any change on `bot/python-versions` for the owner to merge. Nothing +# here pushes `main` or edits ruleset 23497679: the pull request body lists the +# required contexts to change by hand. # -# Only these triggers. A push to `bot/python-versions` starts the pull -# request's own checks, which is why `pr` spends the release App's token rather -# than GITHUB_TOKEN, whose pushes start nothing; nothing it starts runs this +# `pr` pushes with the release App's token, not GITHUB_TOKEN, whose pushes +# start nothing, so the pull request's own checks run; none of them runs this # workflow again. on: schedule: - cron: '41 5 * * 1' # Mondays at 05:41 UTC workflow_dispatch: -# Nothing by default; each job asks for what it needs. permissions: {} # One run at a time, so two runs cannot race for the branch. A queued run is @@ -63,16 +59,12 @@ jobs: - name: Install PyYAML, requests and markdown-it-py run: python -m pip install PyYAML requests markdown-it-py - # Fetches the sources, rewrites the files, and writes the proposal to - # $RUNNER_TEMP/proposal. `publish` is false when nothing changed, or when - # `bot/python-versions` has an open pull request and a commit it holds - # ahead of main proposes these tiers, hand edits or merges of main on top - # or not (an anonymous, commits-only fetch), so an unmerged proposal is not - # pushed, nor its approval asked for, every week. When that branch holds - # commits it did not make and proposes other tiers, this fails as - # `publish` would refuse, before any approval. A branch with no open pull - # request was merged or closed, and is proposed afresh. The token reads - # that pull request's state. + # Writes the proposal to $RUNNER_TEMP/proposal. `publish` is false when + # nothing changed, or when `bot/python-versions`' open pull request + # already proposes these tiers, so an unmerged proposal is not re-pushed, + # nor its approval asked for, every week. A branch holding commits it did + # not make that proposes other tiers fails here, before any approval, as + # `publish` would refuse it. The token reads the pull request's state. - name: Recompute the tiers id: update env: @@ -127,13 +119,12 @@ jobs: echo "::error title=The audit fails on main::the tiers need no change, but the audit fails; see above." exit 1 - # The release App's private key is a secret of the `github-release` - # environment, which needs the owner's approval, so he is asked only on a run - # that has something to propose. The token is contents and pull requests - # write and nothing else: no `workflows`, so it cannot push a workflow file, - # and `update` writes none. `publish` force-pushes `bot/python-versions` alone, - # and while its pull request is open refuses it if it holds commits it did not - # make; with none open it recreates the branch from main. + # The release App's key is a secret of the `github-release` environment, + # which needs the owner's approval, so he is asked only when there is + # something to propose. The token has no `workflows` permission, so it cannot + # push a workflow file, and `update` writes none. `publish` force-pushes + # `bot/python-versions` alone, refusing while its pull request is open if it + # holds commits it did not make; with none open it recreates it from main. pr: name: Open or update the pull request needs: detect diff --git a/.github/workflows/cron-vendor.yml b/.github/workflows/cron-vendor.yml index 3323fca633..29e9be38fa 100644 --- a/.github/workflows/cron-vendor.yml +++ b/.github/workflows/cron-vendor.yml @@ -2,57 +2,39 @@ name: "Vendor Update" on: # #1967: GitHub Pages, Vendor Update and Conda Update run eight hours - # apart on Saturdays, so each runs `head`'s test gate alone, mostly on a - # commit of its own, and `vendor-update` and `conda-update` do not queue - # on `repository-maintenance` together. None reads another's output, and - # Conda Update comes last so its upload finds the release Create Release - # made after Vendor Update (#1634). workflows.rst gives the evidence. + # apart on Saturdays, so each runs `head`'s test gate alone and the + # `repository-maintenance` jobs do not queue together. Conda Update comes + # last so its upload finds the release Create Release made after Vendor + # Update (#1634). workflows.rst gives the evidence. schedule: - cron: '0 10 * * 6' # Saturdays at 10:00 UTC - # No `push: branches: [main]` here. `unit-tests.yml`'s own push trigger - # already runs the full matrix for this commit; calling it again on every push - # with `gate-only: true` re-ran the single-version "Gate" job a second time - # for the same tree (#715). Chaining off `Unit Tests`' completion means the gate - # never runs twice, and `vendor-update` below just reads the verdict that - # already exists. `.event == 'push'` is checked in that job's `if` rather - # than here, because `workflow_run` fires for a pull request's completion - # too and a `branches:` filter here would be the only thing stopping that. + # No `push` trigger: Unit Tests' own push run already reaches a verdict, and + # calling it again with `gate-only: true` re-ran the gate on the same tree + # (#715). `workflow_run` also fires for a pull request's run; the + # `vendor-update` job's `if` refuses those, not a `branches:` filter here. workflow_run: workflows: ["Unit Tests"] types: [completed] -# Least privilege by default, since `workflow_run` is in the trigger list -- -# that event can be raised by a pull request's `Unit Tests` run completing, so -# the ambient token this workflow gets should default to nothing rather than -# whatever the repository's default is. Every job below already declares its -# own narrower `permissions:`, so this changes nothing they rely on; it only -# closes the gap for anything added later that forgets to. +# Nothing by default, since a pull request's Unit Tests run can raise +# `workflow_run`; each job declares its own `permissions:`. permissions: {} -# This workflow commits and pushes, so a cancelled run can leave the bump -# half-applied. Kept keyed on the ref, unlike the validation workflows: a dropped -# run here just means a crawl that did not happen twice, and the next schedule -# picks the registries up again, whereas a dropped validation run means a commit -# nobody ever checked. -# -# A `workflow_run` is keyed on the commit `Unit Tests` verified instead (#1643). -# Each `Unit Tests` completion raises a run here, and one whose verdict is for an -# older commit only runs `head` and skips -- but in a shared group it would -# still displace a pending run whose verdict matches, since "any existing pending -# job or workflow in the same concurrency group will be canceled" (GitHub, -# "Control the concurrency of workflows and jobs"). -# -# Note this group is not what serialises the repository writes -- the -# `vendor-update` job below carries its own `repository-maintenance` group, shared -# with cron-conda, and that is what actually stops two jobs committing at once. +# Never cancelled in progress: a cancelled run can leave the bump half-applied, +# while a dropped duplicate only means a crawl not run twice. A `workflow_run` is +# keyed on the commit Unit Tests verified (#1643): in a shared group, a run for +# an older verdict would displace a pending one whose verdict matches, since +# "any existing pending job or workflow in the same concurrency group will be +# canceled" (GitHub, "Control the concurrency of workflows and jobs"). The +# repository writes are serialised by `vendor-update`'s `repository-maintenance` +# group, shared with cron-conda, not by this one. concurrency: group: vendor-update-${{ github.event.workflow_run.head_sha || github.ref }} cancel-in-progress: false jobs: - # #1967: the test gate, on the schedule only, and the `verdict` on a - # `workflow_run`, shared with Conda Update and GitHub Pages. cron-head.yml - # says when each runs. + # #1967: the test gate on the schedule, `verdict` on a `workflow_run`; + # shared with Conda Update and GitHub Pages. head: name: Vendor gate permissions: @@ -61,23 +43,16 @@ jobs: vendor-update: runs-on: macos-latest - # #1968: the deploy key is a `conda-tag` environment secret, and only - # a job naming the environment can read it. Keep that environment free of - # required reviewers, or every scheduled bump waits for an approval. + # #1968: the deploy key is a `conda-tag` environment secret, readable only + # by a job naming it. Keep it free of required reviewers, or every + # scheduled bump waits for an approval. environment: conda-tag needs: [ head ] - # `needs` cannot itself be conditional, and `head`'s jobs skip on every - # path, so this job must not inherit a skip. `!cancelled()` rather than - # `always()` so that cancelling the run still cancels the update; a gate - # that actually failed still blocks. - # - # The `workflow_run` clause is the real gate for a push to main: it - # reads the verdict `Unit Tests` already reached instead of re-running it, - # and `head`'s `verified` says whether that verdict covers the commit - # checked out below (#1643). `.event == 'push'` and `.head_branch == 'main'` - # both matter -- `workflow_run` fires for *every* completion of that - # workflow, including one triggered by a pull request, and without this - # check a green PR would push a vendor-registry commit from that PR's tree. + # `head`'s jobs skip on every path, so this job must not inherit a skip; + # `!cancelled()`, not `always()`, so cancelling still cancels and a failed + # gate still blocks. On `workflow_run`, `verified` says Unit Tests' verdict + # covers the checkout (#1643), and the `push`/`main` checks stop a green + # pull request's run from pushing a vendor commit from that PR's tree. if: >- !cancelled() && needs.head.result != 'failure' && (github.event_name != 'workflow_run' || ( @@ -99,22 +74,12 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - # Deliberately the default `github.sha` (main's tip), not - # `github.event.workflow_run.head_sha`. Pinning to the latter once - # ran here and CodeQL flagged it as "checkout of untrusted code in a - # privileged context" -- `workflow_run` can also be raised by a pull - # request's `Unit Tests` run, and that field would then be a fork's - # commit. The `if:` above already keeps that path from reaching this - # step, but making the ref itself always resolve to a trusted - # default-branch commit removes the pattern outright rather than - # relying on CodeQL tracing through the guard. The push below rebases - # onto main's actual tip before writing regardless, so pinning to the - # exact tested sha would have bought nothing here either. Whether this - # commit is one that verdict covers is `head`'s question (#1643). - # The push below re-points `origin` at an SSH remote and authenticates - # with the loaded deploy key, not with this checkout's credential -- - # so there is nothing here for a persisted token to do, and one less - # live credential for anything running in between to find. + # The default `github.sha` (main's tip), not `workflow_run.head_sha`, + # which CodeQL flagged as "checkout of untrusted code in a privileged + # context": a pull request's run can raise `workflow_run`. The push + # rebases onto main's tip anyway; whether the verdict covers this + # commit is `head`'s question (#1643). No persisted token: the push + # authenticates with the deploy key over SSH. persist-credentials: false # #1660: `latest` from .github/python-versions.toml, read with the @@ -134,16 +99,11 @@ jobs: python -m pip install -U pip setuptools wheel packaging python -m pip install -U isort - # `all` carries `pycrate`, a core addon under #910 (see its - # own comment in pyproject.toml), but not `requests`/`beautifulsoup4`: - # #910 moved those to the `dev` extra instead, which is - # for the toolchain rather than for a vendor crawl. `vendor` is - # installed alongside `all` for those two: `pcapkit-vendor` below - # regenerates every crawler with no target given, and some of them - # import `bs4` at module scope while `pcapkit.vendor.default` imports - # `requests` the same way (#507). See the `vendor` extra's own - # comment in pyproject.toml for the full reasoning. - python -m pip install -e .[all,vendor] + # `vendor` for `requests` and `beautifulsoup4`, which `all` lacks + # since #910: `pcapkit-vendor` with no target runs every crawler, and + # some import them at module scope (#507). See the `vendor` extra's + # comment in pyproject.toml. + python -m pip install -e '.[all,vendor]' - name: Update Vendor run: | @@ -152,40 +112,25 @@ jobs: export PCAPKIT_VENDOR_RETRY=10 export PCAPKIT_CI_MODE=1 - # `pcapkit-vendor` (#872) exits non-zero if any target raised, and - # the owner's ruling on that is: discard a failed sub-vendor's own - # changes, keep every zero-exited one's, and never fail the whole - # job over it -- the targets that DID succeed still need isort, - # diff, commit and push below. `set +e`/`set -e` around the one - # command (not a bare `pcapkit-vendor || true`) so $vendor_exit - # actually carries the exit code instead of it being thrown away. + # #872, the owner's ruling: a failed sub-vendor's changes are + # discarded, every zero-exited one's are kept, and the job does not + # fail over it. `set +e`, not `|| true`, so $vendor_exit keeps the code. set +e pcapkit-vendor 2> "$RUNNER_TEMP/vendor-stderr.log" vendor_exit=$? set -e - # Always replayed into the step log, success or failure: this is - # each failing target's qualified name and exception repr (from - # __main__.py's run()) plus its VendorRuntimeWarning line, i.e. - # exactly what a live, uncaptured `pcapkit-vendor` would have - # written to stderr directly. `|| true` deliberately: this line - # runs after `set -e` is restored above, and if $RUNNER_TEMP were - # ever unset or unwritable the redirect on the previous command - # would already have failed to create the log file at all -- - # `cat` on a missing file returns 1 under `-e` and would abort the - # whole step right here, exactly the "one dead crawler kills every - # good regeneration" failure mode this handling exists to remove. + # Replayed whatever the exit: each failing target's name and + # exception, plus its VendorRuntimeWarning. `|| true` so a log that + # was never created cannot abort the step under `-e` and lose the + # good regenerations. cat "$RUNNER_TEMP/vendor-stderr.log" >&2 || true if [ "$vendor_exit" -ne 0 ]; then - # Anchored at the start of the line: the unanchored `failed: ` - # also matches inside the VendorRuntimeWarning line whenever the - # exception's own text happens to contain "failed: " (e.g. - # ConnectionError('request failed: 503')), since that line embeds - # the same {error!r} __main__.py's run() already put in the - # print() line above it -- double-counting one failure as two - # ::warning:: annotations. Only the print() line starts with the - # bare qualified name; the warning line starts with a file path. + # Anchored: the VendorRuntimeWarning line embeds the same + # {error!r} and so can contain "failed: " too (e.g. + # ConnectionError('request failed: 503')), double-counting one + # failure. Only run()'s print() line starts with the qualified name. failed_targets=$(grep -E '^[A-Za-z_][A-Za-z0-9_.]* failed: ' \ "$RUNNER_TEMP/vendor-stderr.log" || true) @@ -212,35 +157,21 @@ jobs: fi fi - # `find -mindepth 2`, not a `pcapkit/const/*/*.py` shell glob (#765): - # this job runs on `macos-latest`, whose default `bash` on PATH is - # the system's, unlike the Makefile's own `SHELL := $(shell command - # -v bash ...)`, which is confirmed (elsewhere) to pick up a modern - # one -- `shopt -s globstar` is not a safe assumption here, and - # `find` needs neither globstar nor a modern bash to reach any - # depth. `-mindepth 2` keeps the same floor the old glob had: it - # still skips `pcapkit/const/__init__.py` itself. - isort -l100 -ppcapkit $(find pcapkit/const -mindepth 2 -name '*.py') - - # #1853: neither gate in `head` sees the regenerated tree -- the test gate - # runs on the schedule alone, and `verdict` accepts the Unit Tests result for - # the commit from before the crawl -- so 1.5.0b13 was committed with a test - # failing on it. The owner's ruling: test the regenerated tree here, before - # anything is committed, under #872's rule above. A module the tests fail - # with is put back as HEAD has it and the rest go on to be committed, as - # for a sub-vendor that raised. util/vendor_test_gate.py, whose docstring - # gives the rule, decides which modules those are. + # `find`, not a glob (#765): macOS's system bash has no globstar. + # `-mindepth 2` skips `pcapkit/const/__init__.py`. + find pcapkit/const -mindepth 2 -name '*.py' -exec isort -l100 -ppcapkit {} + + + # #1853: both gates in `head` judge the commit from before the crawl, so + # 1.5.0b13 was committed with a test failing on it. The owner's ruling: + # test it here under #872's rule -- a module the tests fail with goes + # back to HEAD's copy, the rest are committed. util/vendor_test_gate.py's + # docstring gives the rule. # - # `continue-on-error`, so the tests never fail the job. The script exits - # zero once it has decided; if this step fails or times out instead, the - # tree may hold modules no run passed with, and the next step discards - # every regenerated module. The test extra is installed here because only - # this step needs it; it carries psutil too, which the script samples a - # run's memory with (#1894). The script's budget bounds its runs together, - # and `timeout-minutes` sits above it as the backstop. Run serially, pytest - # peaked at 0.7 GB resident over tests/const, measured locally, so the - # memory cap leaves headroom on this runner; files are capped at 2 GiB, as - # in experimental.yml. + # `continue-on-error`, so the tests never fail the job; if the step fails + # or times out, the next one discards every regenerated module. `.[test]` + # brings psutil, which samples a run's memory (#1894); `timeout-minutes` + # backs up the script's budget. Serial pytest peaked at 0.7 GB resident + # over tests/const, measured locally; files capped as in experimental.yml. - name: Test the regenerated tree id: vendor-tests continue-on-error: true @@ -253,9 +184,8 @@ jobs: --max-memory-mib 4096 --max-file-size-mib 2048 -- \ python -m pytest -q -p no:cacheprovider tests/const tests/vendor - # What the step above left is untested if it did not finish, so HEAD's - # copy of every module goes back, new modules are removed, and nothing - # from this crawl is committed. Not a failure either, per #872. + # The tree is untested if the step above did not finish, so nothing from + # this crawl is committed. Not a failure either (#872). - name: Discard untested const changes if: steps.vendor-tests.outcome != 'success' run: | @@ -269,17 +199,13 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" # #1968: `git status` in place of tj-actions/verify-changed-files@v17, so - # no third-party code runs while the deploy key is loaded. The outputs are - # the action's with no `files:` input, which it never had here: - # `files_changed`, and `changed_files`, the paths joined by spaces and - # written only when there are some. Every changed tracked file counts, - # untracked files are listed one by one, and ignored files are left out. - # A staged change counts too, which the action's `git diff` missed; - # nothing before this step stages one. git C-quotes a path holding a - # space, a quote, a control character or a non-ASCII byte, so the list is - # one line. Such a path still splits on its space, and - # util/vendor_code_changed.py counts a name it cannot read as code. The - # steps below read the list through `env:`, so a path is never shell text. + # no third-party code runs while the deploy key is loaded. Same outputs as + # the action with no `files:` input: `files_changed`, and `changed_files`, + # space-joined, written only when non-empty. Tracked changes count, staged + # ones too; untracked files are listed one by one; ignored files never. + # git C-quotes an unusual path, so the list is one line; a path with a + # space still splits, and util/vendor_code_changed.py counts a name it + # cannot read as code. Read through `env:`, so a path is never shell text. # cron-conda.yml runs the same script. - name: Verify Changed files id: verify-changed-files @@ -294,12 +220,9 @@ jobs: echo "files_changed=false" >> "$GITHUB_OUTPUT" fi - # #1674: only a code change cuts a beta. util/vendor_code_changed.py, - # whose docstring gives the rule, writes `code_changed=false` when every - # change is to comments or docstrings; those are committed below without - # a bump. Anything else bumps, as before, including this step failing: - # `continue-on-error` leaves `code_changed` unset, and the bump reads - # `!= 'false'`. + # #1674: only a code change cuts a beta. util/vendor_code_changed.py + # writes `code_changed=false` for comment- and docstring-only changes; + # anything else bumps, including this step failing, which leaves it unset. - name: Classify Changed files id: classify if: steps.verify-changed-files.outputs.files_changed == 'true' @@ -317,8 +240,8 @@ jobs: python util/bump_version.py isort -l100 -ppcapkit --skip-glob '**/__init__.py' pcapkit - isort -l100 -ppcapkit $(find pcapkit/const -mindepth 2 -name '*.py') - isort -l100 -ppcapkit $(find pcapkit/vendor -mindepth 2 -name '*.py') + find pcapkit/const -mindepth 2 -name '*.py' -exec isort -l100 -ppcapkit {} + + find pcapkit/vendor -mindepth 2 -name '*.py' -exec isort -l100 -ppcapkit {} + - name: Commit changes id: commit @@ -330,29 +253,22 @@ jobs: run: | git config --local user.email "github-actions[bot]@users.noreply.github.com" git config --local user.name "github-actions[bot]" - # #1685: `-a` stages tracked files only, so a module the crawl added - # for a new registry was bumped for but never committed, and every - # later crawl bumped for it again. Stage pcapkit-vendor's output first, - # new and deleted modules included; `-a` still takes the bump's files. - # Quoted, so git matches the pattern at any depth, not the shell at - # one; `.py` only, so a backup pcapkit-vendor could not remove stays out. + # #1685: `-a` skips untracked files, so a new module was bumped for but + # never committed, and every later crawl bumped again. Quoted, so git + # matches at any depth; `.py` only, so a leftover backup stays out. git add -A -- 'pcapkit/const/*.py' git commit -am"Bumped version to $(python -c 'import pcapkit; print(pcapkit.__version__)') Updated vendor constant enumerations at $(date). The following files have been changed: $CHANGED_FILES" - # #1674: the same commit without the bump. Create Release still runs when - # this workflow completes, as after a crawl that changed nothing, and sees - # the same version, so nothing new is published. The push starts Unit - # Tests, whose completion raises this workflow again on the commit made - # here. If that crawl changes comments once more, it does not commit them - # on top of this commit; the next push or schedule picks them up. So - # documentation commits cannot chain, however the registries behave. The - # Unit Tests run's head commit is in the `workflow_run` payload - # (octokit/webhooks' workflow-run.schema.json, `head_commit`). `-a` is - # enough here: a new or deleted module is code to the classifier, so it - # takes the bump path above and never reaches this step (#1685). + # #1674: the same commit without the bump; Create Release sees the same + # version and publishes nothing. Its push raises this workflow again, and + # a crawl changing comments once more on top of this commit is held for + # the next push or schedule, so documentation commits cannot chain. The + # verified commit is the payload's `head_commit` (octokit/webhooks' + # workflow-run.schema.json). `-a` suffices: a new or deleted module is + # code, so it takes the bump path (#1685). - name: Commit documentation changes id: commit-docs if: >- @@ -390,19 +306,3 @@ jobs: git remote set-url origin git@github.com:JarryShaw/PyPCAPKit.git git pull --rebase origin "${GITHUB_REF#refs/heads/}" git push origin "HEAD:${GITHUB_REF}" - - # - name: Create Tag - # if: steps.verify-changed-files.outputs.files_changed == 'true' - # run: | - # version=$(python -c 'import pcapkit; print(pcapkit.__version__)') - # git tag "v$version" -m "New distribution [$version] \ - - # Update vendor constant enumerations at $(date). The following files have been changed: ${{ steps.verify-changed-files.outputs.changed_files }}" - - # - name: Push changes - # uses: ad-m/github-push-action@master - # if: steps.verify-changed-files.outputs.files_changed == 'true' - # with: - # tags: true - # github_token: ${{ secrets.GITHUB_TOKEN }} - # branch: ${{ github.ref }} diff --git a/.github/workflows/deploy-pages.yml b/.github/workflows/deploy-pages.yml index e6d3d0af2f..9f49b37738 100644 --- a/.github/workflows/deploy-pages.yml +++ b/.github/workflows/deploy-pages.yml @@ -1,57 +1,43 @@ name: GitHub Pages on: - # No `push: branches: [main]` here. `unit-tests.yml`'s own push trigger already - # runs the full matrix for this commit; calling it again with `gate-only: true` - # on every push re-ran the single-version "Gate" job a second time for the same - # tree (#715). Chaining off `Unit Tests`' completion instead means the gate never - # runs twice, and this workflow simply waits for the verdict that already - # exists. `workflow_run` fires for *any* completion of that workflow, PR or - # push, so the jobs below filter on `.event == 'push'` themselves rather than - # trusting a `branches:` filter here to do it -- see the note on `deploy-pages`. + # No `push` trigger: chaining off `Unit Tests`' completion reuses its verdict + # instead of running the gate a second time for the same tree (#715). + # `workflow_run` fires for any completion, PR or push, so `deploy-pages` + # filters on the event itself. workflow_run: workflows: ["Unit Tests"] types: [completed] - # #1967: GitHub Pages, Vendor Update and Conda Update run eight hours - # apart on Saturdays, so each runs `head`'s test gate alone, mostly on a - # commit of its own, and `vendor-update` and `conda-update` do not queue - # on `repository-maintenance` together. None reads another's output, and - # Conda Update comes last so its upload finds the release Create Release - # made after Vendor Update (#1634). workflows.rst gives the evidence. + # #1967: GitHub Pages, Vendor Update and Conda Update run eight hours apart + # on Saturdays, so each runs `head`'s test gate alone and `vendor-update` and + # `conda-update` do not queue on `repository-maintenance` together. Conda + # Update comes last so its upload finds the release Create Release made after + # Vendor Update (#1634). workflows.rst gives the evidence. schedule: - cron: '0 2 * * 6' # Saturdays at 02:00 UTC pull_request: - # Was '**', which matched pull requests against every branch and made this - # the one workflow that fired outside the main-targeting set the others - # use. Every pull request here targets main, so '**' only ever added runs. branches: [main] permissions: contents: read -# Deliberately still keyed on the ref, unlike the validation workflows. This one -# publishes a *latest state* rather than a verdict on a commit, so coalescing is -# correct: when two pushes land close together the pending build is dropped and -# the newer one publishes, and deploying the older commit's docs after the newer -# ones would actively be wrong. GitHub Pages also permits only one live -# deployment, so serialising here is a requirement rather than a saving. +# Keyed on the ref, unlike the validation workflows: this publishes a latest +# state, not a verdict, so a pending build a newer push supersedes is rightly +# dropped, and Pages permits only one live deployment anyway. # -# A `workflow_run` is keyed on the commit `Unit Tests` verified instead (#1643). -# One whose verdict is for an older commit only runs `head` and skips, and in -# the shared group it would still displace a pending run whose verdict matches, -# since "any existing pending job or workflow in the same concurrency group will -# be canceled" (GitHub, "Control the concurrency of workflows and jobs"). The -# coalescing and serialising above then happen in `deploy-pages`' own `ci-` -# group, which only a run whose verdict covers its checkout reaches. +# A `workflow_run` keys on the commit `Unit Tests` verified instead (#1643): +# one whose verdict is for an older commit only runs `head` and skips, and in a +# shared group would cancel a pending run whose verdict matches. Coalescing +# then happens in `deploy-pages`' own `ci-` group, which only a run whose +# verdict covers its checkout reaches. concurrency: group: pages-${{ github.event.workflow_run.head_sha || github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: - # #1967: the test gate, on the schedule only, and the `verdict` on a - # `workflow_run`, shared with Conda Update and Vendor Update. Pull requests - # never published anyway, so on one both skip. cron-head.yml says when each - # runs. + # #1967: the test gate on the schedule and the `verdict` on a `workflow_run`, + # shared with Conda Update and Vendor Update; cron-head.yml says when each + # runs. On a pull request both skip. head: name: Docs gate permissions: @@ -60,18 +46,12 @@ jobs: deploy-pages: needs: [ head ] - # `needs` cannot itself be conditional, and `head`'s jobs skip on every - # path (both on a pull request), so this job must not inherit a skip. - # `!cancelled()` rather than `always()` so that cancelling the run still - # cancels the docs build; a gate that actually failed still blocks. - # - # The `workflow_run` clause is the real gate for a push to main: it - # reads the verdict `Unit Tests` already reached instead of re-running it, - # and `head`'s `verified` says whether that verdict covers the commit - # checked out below (#1643). `.event == 'push'` and `.head_branch == 'main'` - # both matter -- `workflow_run` fires for *every* completion of that - # workflow, including one triggered by a pull request, and without this - # check a green PR would publish docs from that PR's tree. + # A job of `head` skips on every path, so this must not inherit a skip; + # `!cancelled()` rather than `always()` so cancelling the run still cancels + # the build. The `workflow_run` clause is the real gate for a push to main: + # `head`'s `verified` says whether the verdict covers the commit checked out + # below (#1643), and without `.event == 'push'` and `.head_branch == 'main'` + # a green PR would publish docs from that PR's tree. if: >- !cancelled() && needs.head.result != 'failure' && (github.event_name != 'workflow_run' || ( @@ -86,19 +66,13 @@ jobs: - name: Checkout 🛎ī¸ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - # Deliberately the default `github.sha` (main's tip), not - # `github.event.workflow_run.head_sha`. Pinning to the latter once - # ran here and CodeQL flagged it as "checkout of untrusted code in a - # privileged context" -- `workflow_run` can also be raised by a pull - # request's `Unit Tests` run, and that field would then be a fork's - # commit. The `if:` above already keeps that path from reaching this - # step, but making the ref itself always resolve to a trusted - # default-branch commit removes the pattern outright rather than - # relying on CodeQL tracing through the guard. `main`'s tip is also - # the *right* commit for docs, which should reflect current main, - # not a slightly older one -- so this is not a compromise here. - # Whether this commit is one that verdict covers is `head`'s - # question (#1643). + # The default `github.sha` (main's tip), not + # `github.event.workflow_run.head_sha`, which CodeQL flagged as + # "checkout of untrusted code in a privileged context": on a pull + # request's `Unit Tests` run it is a fork's commit. A trusted ref + # removes the pattern rather than relying on the `if:` guard, and + # main's tip is the right commit for docs anyway. Whether the verdict + # covers it is `head`'s question (#1643). persist-credentials: false # #1660: `latest` from .github/python-versions.toml, read with the @@ -111,17 +85,12 @@ jobs: with: python-version: ${{ steps.python-versions.outputs.latest }} - # Builds the Sphinx documentation into `docs/build/html`, which the deploy - # step below publishes. `PCAPKIT_SPHINX` is what makes the - # `if SPHINX_TYPE_CHECKING:` TypedDicts visible to autodoc; `PCAPKIT_DEVMODE` - # puts the library logger at DEBUG on stderr and leaves tracebacks untrimmed, - # so an autodoc import failure is legible in the run log. Both are read at - # import time, so they are exported before anything imports `pcapkit` -- - # `docs/source/conf.py` sets `PCAPKIT_SPHINX` itself for the same reason. - # `.[all,dev]`, not `.[all]` alone: autodoc imports every module, including - # the engine wrappers, so it needs the same imports resolvable that - # `lint.yml` does and for the same reason (#910's `dev` extra, added when - # that issue narrowed `all` to core addons only). + # `PCAPKIT_SPHINX` makes the `if SPHINX_TYPE_CHECKING:` TypedDicts visible + # to autodoc; `PCAPKIT_DEVMODE` logs at DEBUG with untrimmed tracebacks, + # so an autodoc import failure is legible. Both are read at import time, + # so they are exported before anything imports `pcapkit`. `.[all,dev]` + # because autodoc imports the engine wrappers too, which need #910's + # `dev` extra, as in Lint. - name: Install and Build 🔧 run: | set -x @@ -131,7 +100,7 @@ jobs: python -m pip install -U pip setuptools wheel python -m pip install -r docs/requirements.txt - python -m pip install -e .[all,dev] + python -m pip install -e '.[all,dev]' rm -rf docs/build make -C docs html @@ -150,15 +119,11 @@ jobs: include-hidden-files: true retention-days: 1 - # #1968: the push to `gh-pages`, split out of `deploy-pages` so that the job - # running the tree's own code, a pull request's included, holds only the - # top-level `contents: read`. This job runs no project code, and never on a - # pull request. It needs `contents: write` to push `gh-pages`, and `actions: - # read` to download the site `deploy-pages` uploaded in this run. That job's - # result is checked explicitly, under `!cancelled()`: a job of `head` skips on - # every path, and a skip anywhere in the chain skips a job left with the - # implicit `success()` (#1632). Its own group keeps one push to `gh-pages` at - # a time, and its check below keeps an older build from replacing a newer one. + # #1968: the push to `gh-pages`, split out so that the job running the tree's + # own code, a pull request's included, holds only `contents: read`. This one + # runs no project code, and never on a pull request. `deploy-pages`' result is + # checked explicitly under `!cancelled()`: a skip anywhere in the chain skips + # a job left with the implicit `success()` (#1632). publish-pages: name: Publish to gh-pages needs: [ deploy-pages ] @@ -187,16 +152,12 @@ jobs: gh run download "$GITHUB_RUN_ID" --repo "$GITHUB_REPOSITORY" --name pages-html --dir docs/build/html - # Builds finish in any order, and one group per job cannot order them: a - # job joining a group cancels the one pending there, so sharing the build's - # group would cancel the next build. So each deploy names its commit, and - # this step reads the commit `gh-pages` last published. Where that one is - # newer than this run's -- this run's commit is behind it on `main` -- the - # deploy is skipped. Anything else publishes, as before: no `gh-pages`, a - # head this workflow did not write, an API error, the same commit, or one - # that is not an ancestor. A head check alone would also skip when the - # newer commit's own run never publishes, a red one, leaving the docs - # older than this build. + # Builds finish in any order, and a concurrency group cannot order them (a + # job joining one cancels the one pending there). So each deploy names its + # commit, and this skips only when `gh-pages` holds a build this run's + # commit is behind on main; anything else, an API error included, + # publishes. Comparing heads alone would also skip when the newer commit's + # run never publishes, a red one, leaving the docs older than this build. - name: Check gh-pages holds no newer build id: fresh env: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 45c19d661a..98e3ee5773 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,103 +1,56 @@ name: "Lint" -# The linters this repository already maintains -- vermin, pylint, mypy and -# bandit -- had Makefile targets and no CI presence at all, so nothing noticed -# when their output changed. (`isort` does appear in cron-vendor.yml, but as a -# formatter that rewrites the generated constants, not as a check.) This workflow -# is where they run, with actionlint over the workflows themselves (#1967). +# vermin, pylint, mypy and bandit, which had Makefile targets but no CI +# presence, plus actionlint over the workflows (#1967). `isort` is not among +# them: it appears only in cron-vendor.yml, as a formatter that rewrites the +# generated constants, not as a check. # -# Every one of them is advisory -- `continue-on-error: true` on each step -- and -# that is a measurement result rather than a preference. This header records no -# finding counts on purpose: #753 was filed when most of the figures it once -# recorded (on 15189abff) had already drifted -- mypy and pylint by the issue's -# own numbers, and vermin too, which had moved before the issue's own text said -# it "was not re-checked". Only bandit held. A written-down count invites -# exactly this, with no way to tell staleness from disagreement, so each step -# writes its result to the run summary instead; read the numbers there. +# Every step is advisory (`continue-on-error: true`): none is clean, and a job +# that is red the day it lands teaches everyone to scroll past red. Promote a +# step by deleting its `continue-on-error` once its count reaches zero. Each +# step writes its result to the run summary; read the numbers there. This +# header records no counts on purpose: most of those #753 recorded had drifted +# by the time it was filed. # -# pylint's R class is the one that moves without the code moving. Repeated runs -# on an identical tree were measured to disagree on it, entirely through -# `R0401` cyclic-import, whose count depends on module-processing order rather -# than on the code -- `duplicate-code` and every other R check held steady -# across all of them, as did E, W and C. A change in R between two runs is -# therefore no evidence on its own that anything changed. +# pylint's R class moves without the code moving: `R0401` cyclic-import +# depends on module-processing order, so runs on one tree disagree on it. A +# change in R alone is no evidence that anything changed. # -# #767 cleaned up `PYLINT_FLAGS`. Its `--disable=` list named checks pylint has -# genuinely removed -- `old-division`, `no-absolute-import` and -# `input-builtin` (-> `R0022 useless-option-value`, and pylint's own message -# says so: "was removed from pylint") -- plus `eq-without-hash`, which is not -# gone, only unloaded: `pylint.extensions.eq_without_hash` still ships `W1641`, -# but nothing in `--load-plugins=` ever named it, so `--disable=eq-without-hash` -# was as unrecognised as if the check had been removed (-> `W0012 -# unknown-option-value`). Same shape of breakage as `emptystring` below, a -# different cause. All of them are dropped either way -- loading that plugin -# to enable `eq-without-hash` properly found nothing when measured (no `W1641` -# over `pcapkit/`), so there is nothing lost in dropping it instead. -# -# `--load-plugins=` also named `pylint.extensions.emptystring`, which no -# longer exists in the installed pylint (-> `E0013 bad-plugin-value` every -# run). Unlike the entries above, though, the check it used to provide did not -# stop running when the plugin did: `compare-to-empty-string` was folded into -# the core `refactoring` checker at pylint 3.0 as -# `use-implicit-booleaness-not-comparison-to-string` (C1804), reachable under -# either name with no plugin load at all. #767's own premise -- that the -# broken plugin reference made the check "inert" and it "has never run" -- is -# wrong: `--list-msgs-enabled` run under the old flags (broken plugin -# reference included) and the new ones side by side gives the same -# enabled-message list either way, and C1804 fires under the old flags on a -# minimal repro despite the `E0013`. So this half of `PYLINT_FLAGS`' change is -# flag hygiene, not enablement: it drops a dead plugin reference and keeps the -# check it used to gate under that check's current name, and the only thing -# that actually changes is that `E0013` (and the Command-line noise of the -# entries above) stops firing. With those gone, every message the job's own -# grep-based count (the `pylint` step below) tallies is about the code. -# -# None of them is clean, so none can block: a job that is red the day -# it lands teaches everyone to scroll past red, which costs more than the checks -# are worth. Advisory steps keep the numbers visible (each writes its result to -# the run summary) without that damage, and promoting one to blocking is a matter -# of deleting its `continue-on-error` line once its count reaches zero. +# #767's `PYLINT_FLAGS` cleanup was flag hygiene, not enablement. It dropped +# `--disable=` entries pylint no longer recognises (`R0022`, `W0012`; loading +# the unloaded `eq-without-hash` plugin instead found no `W1641`) and the dead +# `pylint.extensions.emptystring` plugin (`E0013`), whose check still runs as +# core C1804 `use-implicit-booleaness-not-comparison-to-string` -- measured +# with `--list-msgs-enabled`, contrary to #767's premise. Every message the +# `pylint` step tallies is now about the code. # # Promotion order, cheapest first: -# * bandit -- most of its findings are B104 "binding to all interfaces" on -# `address: ... = '0.0.0.0'` *default arguments* in MH/MIP option builders, -# which bind no socket. This package already suppresses findings with -# `# nosec`, so triaging what is left is the whole job. -# * mypy -- part of its output is `unused-ignore`, i.e. `# type: ignore` -# comments that are no longer needed. Deleting those is risk-free. -# * vermin -- blocked on a decision, not on cleanup: see the step below. -# * pylint -- not realistically promotable at this rule set. Most of its -# warnings are `unused-argument`, `redefined-builtin` and -# `super-init-not-called`, each of which is inherent to the -# schema DSL: protocol fields are legitimately named `next`/`type`/`id`, and -# the `if TYPE_CHECKING: def __init__(...)` stubs exist precisely so as not -# to call `super().__init__`. Narrowing to `--enable=E,F` does not rescue it -# either -- what remains is mostly `unsubscriptable-object` and +# * bandit -- most findings are B104 on `address: ... = '0.0.0.0'` default +# arguments in MH/MIP option builders, which bind no socket; triage the +# rest with `# nosec`. +# * mypy -- part of its output is `unused-ignore`, risk-free to delete. +# * vermin -- blocked on a decision, not on cleanup: see its step. +# * pylint -- not realistically promotable: `unused-argument`, +# `redefined-builtin` and `super-init-not-called` are inherent to the +# schema DSL, and `--enable=E,F` leaves `unsubscriptable-object` and # `no-member` false positives off the same metaprogramming. on: - # Deliberately *not* `push: [main]`, which is the trigger the other validation - # workflows here use. Those gate something: a failing unit-test or CodeQL run - # on main is a commit someone must look at. These steps gate nothing, so a - # second advisory run per merge -- on a commit that already got one on its pull - # request -- would buy no information while competing for runners that are - # visibly oversubscribed (pull requests on this repository have been sitting at - # 21 queued checks for ten minutes at a time). The weekly schedule covers - # main's own drift, which is the only thing the push trigger would have added. + # No `push: [main]`: these steps gate nothing, so a second advisory run per + # merge, on a commit its pull request already linted, buys no information + # while competing for oversubscribed runners. The schedule covers main's drift. pull_request: branches: [main] schedule: - # Saturdays, an hour clear of the other weekly jobs: CodeQL runs at 02:00, - # Python Compatibility at 04:00 and Vendor Update at 10:00. + # Saturdays, an hour clear of the other Saturday jobs. - cron: '0 6 * * 6' workflow_dispatch: permissions: contents: read -# Only pull-request runs may be superseded -- see the note in unit-tests.yml. A -# ref-keyed group drops a *pending* run when a newer one joins it, so scheduled -# and manual runs are keyed on the run id to give each one its own group. +# Only pull-request runs may be superseded (see unit-tests.yml): a shared group +# drops a *pending* run when a newer one joins, so other events key on the run id. concurrency: group: lint-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -111,19 +64,18 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # #1967: actionlint checks every workflow under .github/workflows, and runs - # the image's shellcheck over each `run:` block. First, as it needs no - # Python and is the quickest verdict. Advisory like the steps below, since - # shellcheck still reports style and info findings across the workflows. + # actionlint checks every workflow, running the image's shellcheck over + # each `run:` block. First, as it needs no Python. It stays advisory while + # shellcheck findings remain in unit-tests.yml and experimental.yml (#1967). # # The release binary is pinned by version and sha256, from the release's # checksums file. Dependabot does not see this pin; bump both together. - # These files use YAML anchors, which actionlint reads from 1.7.10 on. + # The workflows use YAML anchors, which actionlint reads from 1.7.10 on. # - # The `-ignore` hides a false positive. `concurrency.queue` is a documented + # The `-ignore` hides a false positive: `concurrency.queue` is a documented # GitHub key (project-status.yml uses it) that no actionlint release knows - # yet: rhysd/actionlint#657, with the fix in rhysd/actionlint#654. Drop the - # pattern once a release with that fix is pinned here. + # yet (rhysd/actionlint#657, fixed by rhysd/actionlint#654). Drop the + # pattern once a release with that fix is pinned. - name: actionlint continue-on-error: true timeout-minutes: 5 @@ -158,21 +110,13 @@ jobs: exit "$status" - # One interpreter, not a matrix. Unit Tests and Python Compatibility sweep - # the supported tier, plus the non-blocking KTLO legs (#1633), because - # they check that the package *runs* everywhere; these tools read the - # source, and running them several times over would produce several - # copies of the same findings for several times the runner cost. - # - # `latest` from .github/python-versions.toml, like every single-interpreter - # job (#1661). Moving off 3.14 changes no count. Measured at 0bc157660 - # on setup-python's 3.14.8 and 3.15.0 builds: pylint 4.1.3 (astroid - # 4.3.4) reports the same C, E and W messages on both, message for message; - # only R0401 cyclic-import differs, as it does between two runs on one - # interpreter (see the header). bandit, mypy and vermin print identical - # reports. To test a 3.15 without C `_decimal`, use setup-python's build, - # which has none: faking it with `sys.modules['_decimal'] = None` makes - # astroid crash on pcapkit/toolkit/dpkt.py on 3.14.8 too (#1677). + # One interpreter, not a matrix: these tools read the source, so more + # interpreters would only repeat the same findings. `latest` from + # .github/python-versions.toml, like every single-interpreter job (#1661). + # Measured at 0bc157660, setup-python's 3.14.8 and 3.15.0 give identical + # reports bar R0401 (see the header). To test a 3.15 without C `_decimal`, + # use setup-python's build, which has none: faking it with + # `sys.modules['_decimal'] = None` crashes astroid on 3.14.8 too (#1677). - name: Read the Python versions id: python-versions run: python3 util/python_versions.py --github-output @@ -181,36 +125,24 @@ jobs: with: python-version: ${{ steps.python-versions.outputs.latest }} - # `.[all,dev]` rather than a bare install: pylint and mypy resolve imports, - # so findings depend on what is importable. #910 narrowed `all` to core - # addons only (`cli`, `crypto`, `NGAP`), so the `dev` extra it added is - # what puts `dpkt`/`scapy`/`pyshark`/`pypcapfile` back for this job -- - # without it, narrowing `all` would have grown the count below rather - # than left it alone. Note `pypcap` and `pypcapfile` are marked - # `python_version < '3.12'` in pyproject.toml, and `pcap-ct` is in no - # extra installed here, so each is absent by design -- that is the source - # of pylint's `import-error` messages, and they will persist until those - # engines are installed here. + # `.[all,dev]`: pylint and mypy resolve imports, so findings depend on what + # is importable, and `dev` carries the engines #910 took out of `all`. + # `pypcap` and `pypcapfile` are `python_version < '3.12'` and `pcap-ct` is + # in no extra installed here, so pylint's `import-error` messages for them + # are by design. - name: Install package and lint tools run: | set -x python -m pip install -U pip setuptools wheel - python -m pip install -e .[all,dev] + python -m pip install -e '.[all,dev]' python -m pip install -U vermin pylint mypy bandit - # Every step below runs the *Makefile* target rather than spelling the - # flags out again, with `RUN=` emptying the `pipenv run` prefix the targets - # use locally. That is what keeps this workflow and a developer's `make - # pylint` running the same check -- restating the flag sets here would let - # the two drift the first time either changed, which is how "passes on my - # machine" starts. - # - # `continue-on-error: true` reports the step's real verdict as a non- - # blocking annotation and lets the following steps run, so one tool's - # findings never hide another's. The `status=$?` dance is needed because - # the default shell is `bash -e`: without it the summary lines after a - # non-zero tool would never execute. + # Each step runs its Makefile target, `RUN=` emptying the local `pipenv + # run` prefix, so this and a developer's `make pylint` cannot drift apart. + # `continue-on-error: true` lets every tool run whatever another found; + # `set +e` and `status=` keep the summary lines running under the default + # `bash -e`. - name: bandit continue-on-error: true @@ -227,7 +159,7 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" - exit $status + exit "$status" - name: mypy continue-on-error: true @@ -244,20 +176,15 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" - exit $status + exit "$status" - # `vermin-ci`, not `vermin`: the `vermin` target redirects the report into - # temp/, so a run that found violations stops at that line with the report - # in a file rather than in this log. Same flags, via $(VERMIN_FLAGS). + # `vermin-ci`, not `vermin`: that target redirects the report into temp/, + # out of this log. Same flags, via $(VERMIN_FLAGS). # - # This one needs a decision rather than a cleanup before it can block, and - # the decision is not CI's to make. vermin.ini sets `targets = 3.6` while - # the code's real floor is 3.11 -- `enum.StrEnum` and - # `enum.show_flag_values` are 3.11 members, and modules across the package - # import `typing_extensions`. pyproject.toml's `requires-python` claims - # `>=3.6, <4`, so the declared floor and the actual one are five releases - # apart. Whichever way that is resolved -- raise `requires-python`, or - # raise `targets` to match it -- belongs in its own change. + # Blocking needs a decision, not a cleanup, and not CI's: vermin.ini's + # `targets = 3.6` and `requires-python`'s `>=3.6, <4` both sit below the + # code's real floor, 3.11 (`enum.StrEnum`, `enum.show_flag_values`). + # Raising either belongs in its own change. - name: vermin continue-on-error: true run: | @@ -274,10 +201,10 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" - exit $status + exit "$status" - # Last because it is by far the slowest -- minutes over ~90k lines -- so - # the quick verdicts reach the summary before it starts. + # Last because it is by far the slowest, so the quick verdicts reach the + # summary first. - name: pylint continue-on-error: true run: | @@ -294,4 +221,4 @@ jobs: echo '```' } >> "$GITHUB_STEP_SUMMARY" - exit $status + exit "$status" diff --git a/.github/workflows/project-status.yml b/.github/workflows/project-status.yml index f0bd9da271..456dee5625 100644 --- a/.github/workflows/project-status.yml +++ b/.github/workflows/project-status.yml @@ -22,10 +22,8 @@ name: Project Status # `env:` rather than interpolated into the script, and re-reads everything else # (labels, state) from the API. # -# tests/project/test_project_status.py asserts each of them over this file -- no -# checkout or `uses:`, no `${{ }}` inside a `run:` script, and no event-payload -# reference outside `env:` other than the item number -- so a later edit that -# breaks one fails a test rather than a security review. +# tests/project/test_project_status.py asserts each of them over this file, so a +# later edit that breaks one fails a test rather than a security review. # # `workflow_dispatch` is the other way in, and it is not a fork boundary: it runs # the workflow file *of the branch selected* with PROJECT_TOKEN in scope. Only @@ -33,12 +31,10 @@ name: Project Status # workflow, so this trusts collaborators rather than forks -- the same trust the # rest of the repository's secrets rely on. # -# Event runs and reconciles are in different concurrency groups, so they can -# overlap. A reconcile reads an item, an event run then sets a fresher Status, -# and the reconcile writes its older answer over it. The window is the length of -# one reconcile, and the damage is self-repairing: the item's next event, or the -# next nightly run, re-reads the labels and sets it right. Serializing the two -# would mean one global group, which lets a reconcile hold every event run. +# Event runs and reconciles are in different concurrency groups, so a reconcile +# can write an older Status over a fresher one. That self-repairs on the item's +# next event or the next nightly run; one global group instead would let a +# reconcile hold every event run. # # PROJECT_TOKEN is a classic PAT with `project` and `repo` scopes, because # GITHUB_TOKEN cannot write a user-owned project. Without it every run skips with a @@ -50,17 +46,14 @@ on: pull_request_target: types: [opened, reopened, labeled, unlabeled, closed, ready_for_review] schedule: - # Nightly backstop for events this workflow missed -- a run dropped because - # its item's concurrency queue below was full, a GitHub outage, or a label - # changed while the secret was absent. 00:37 UTC sits well clear of the - # Saturday jobs (CodeQL and GitHub Pages at 02:00, Python Compatibility at - # 04:00, Lint at 06:00, Vendor Update at 10:00, Conda Update at 18:00 since - # #1967), and off the hour mark that GitHub's scheduler queues most heavily. + # Nightly backstop for events this workflow missed -- a run dropped from a + # full concurrency queue, a GitHub outage, or a label changed while the + # secret was absent. 00:37 UTC is clear of the Saturday jobs (#1967) and off + # the hour, which GitHub's scheduler queues most heavily. - cron: '37 0 * * *' workflow_dispatch: -# The job authenticates with PROJECT_TOKEN throughout and never uses GITHUB_TOKEN, -# so it is granted nothing. +# The job uses PROJECT_TOKEN only, never GITHUB_TOKEN. permissions: {} # Event runs are keyed on the item number, so a label swap -- which fires an @@ -76,17 +69,11 @@ permissions: {} # GitHub documents both, and that the two keys together are a validation error: # https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency # -# The board still ends right, whatever order the queue runs in -- GitHub states -# that order is not guaranteed. Runs on one item never overlap, and each re-reads -# the labels rather than trusting its payload. So the last run to execute starts -# after every other run on the item has finished, including the one started by -# the final label event. It therefore reads the final labels, and its write is -# the last. The cost is that a burst of N events makes N runs instead of -# collapsing into one, each a handful of API calls. -# -# Scheduled and manual reconciles share their own group and queue the same way. -# A second reconcile waiting behind the first repeats its work, which is cheap at -# nightly cadence. +# Queue order is not guaranteed, but the board still ends right: runs on one +# item never overlap and each re-reads the labels, so the last to execute reads +# the final labels and writes last. The cost is one run per event, not one per +# burst. Scheduled and manual reconciles share their own group, queued the same +# way. concurrency: group: project-status-${{ (github.event.issue.number || github.event.pull_request.number) && format('item-{0}', github.event.issue.number || github.event.pull_request.number) || 'reconcile' }} queue: max diff --git a/.github/workflows/python-compatibility.yml b/.github/workflows/python-compatibility.yml index 3b65e6cb37..0ba8762381 100644 --- a/.github/workflows/python-compatibility.yml +++ b/.github/workflows/python-compatibility.yml @@ -11,13 +11,10 @@ on: permissions: contents: read -# Only pull-request runs may be superseded. Keying every event on the ref (as -# this did) coalesces all of main's pushes into one group, and GitHub cancels a -# *pending* run whenever a newer one joins the group -- `cancel-in-progress: -# false` protects a run that has already started, never one still queued. So a -# commit merged while its predecessor was still running lost its validation -# entirely. Keying non-PR events on the run id gives each one its own group, -# which is what "validate every commit" actually requires. +# Only pull-request runs may be superseded. GitHub cancels a *pending* run when +# a newer one joins its group, whatever `cancel-in-progress` says, so a shared +# group for main's pushes would leave commits unvalidated; other events key on +# the run id. concurrency: group: python-compatibility-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} @@ -45,18 +42,16 @@ jobs: run: python3 util/python_versions.py --github-output compatibility: - # Not named "Python ${{ matrix.python-version }}": that name collided - # exactly with the Unit Tests matrix job of the same name, and `gh pr checks` - # showed two rows reading `Python 3.10`, one a 13-second import smoke test - # and one a three-minute test run, with nothing to tell them apart. + # `Compat`, so `gh pr checks` cannot show two `Python 3.N` rows, this and + # Unit Tests', with nothing to tell them apart. # - # #1633's tiers. The supported tier, 3.11-3.15, keeps the plain name, - # which is what ruleset 23497679 requires one context per version of. The - # KTLO tier, end-of-life 3.7-3.10, is suffixed ` (KTLO)`, so no required - # context can match one. Its steps carry `continue-on-error`, as in - # unit-tests.yml's `test` job: job-level, a failed leg's check run concluded - # `failure`, so every commit showed red (#1672). 3.7 fails "Verify package" - # by design. No 3.6: setup-python builds it only for retired Ubuntu images. + # #1633's tiers. A supported leg keeps the plain name, one of ruleset + # 23497679's required contexts. A KTLO leg is suffixed ` (KTLO)`, so no + # required context can match it, and its steps carry `continue-on-error`, + # as in unit-tests.yml's `test` job: job-level, a failed leg's check run + # concluded `failure`, so every commit showed red (#1672). 3.7 fails "Verify + # package" by design. No 3.6: setup-python builds it only for retired + # Ubuntu images. name: Compat Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }} needs: [python-versions] runs-on: ${{ matrix.os || 'ubuntu-latest' }} @@ -64,13 +59,10 @@ jobs: ANNOTATION_LEVEL: ${{ matrix.ktlo && 'warning' || 'error' }} strategy: fail-fast: false - # Both tiers, as for the `test` job in unit-tests.yml (#1660): a KTLO leg - # carries `ktlo: true`, and `os` where `ubuntu-latest` cannot provide it. - # setup-python publishes 3.8 and 3.9 builds up to Ubuntu 24.04 only, and - # `ubuntu-latest` moves to 26.04 from 2026-10-19, hence their pin. 3.7's - # newest is for the deprecated 22.04; see the `test` job in unit-tests.yml. - # Each supported leg is one of ruleset 23497679's required contexts, so a - # tier change edits the ruleset too. + # Both tiers, as for unit-tests.yml's `test` job (#1660), which explains + # the `os` pins: a KTLO leg carries `ktlo: true`, and `os` where + # `ubuntu-latest` has no setup-python build for it. A tier change edits + # ruleset 23497679 too. matrix: ${{ fromJSON(needs.python-versions.outputs.compatibility) }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -95,11 +87,10 @@ jobs: python -m pip install -e . # One annotation saying why, so a failed leg -- such as 3.7's, where the - # editable install succeeds and the sources do not compile -- is legible - # from the run summary without opening the log. ANNOTATION_LEVEL makes it - # a warning on a KTLO leg (#1672). Each title escapes `,` as `%2C`: the - # runner splits a command's properties on `,`, so the KTLO leg's title - # would otherwise stop at "(KTLO" (actions/runner + # install succeeds and the sources do not compile -- is legible from the + # run summary; a warning on a KTLO leg (#1672). Each title escapes `,` as + # `%2C`: the runner splits a command's properties on `,`, so the KTLO + # title would stop at "(KTLO" (actions/runner # src/Runner.Common/ActionCommand.cs:100, at 5f0b3ddcc). - name: Verify package id: verify diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index b26503ecc6..dc219b9953 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -49,13 +49,13 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s ``version`` field to match is **not** a safe shortcut: it blocks the release. ``tests/project/test_bump_version.py``'s ``RepositoryCitationTests.test_the_citation_file_names_the_packaged_version`` -(``:494-514``) asserts the two agree, and ``create-release.yml``'s -``unit-tests`` job (``:581-597``) calls ``unit-tests.yml`` with +asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job calls +``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1798-1873``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every -publishing job requires it to have succeeded (``:621,808,974,1210``), so a +publishing job's ``needs:`` requires it to have succeeded, so a citation left behind fails the gate well before any approval is requested. Either run the script, or move both fields by hand in the same commit. @@ -64,27 +64,26 @@ the Release, and every upload are the workflow's job from here. Other checks run unconditionally on the same path, though none is a step in *this* process: the ``changelog`` job (``unit-tests.yml:1641-1667``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under -:file:`docs/source/changelog/`, and the release-body step warns, without -failing, if that entry's heading still reads "unreleased" -(``create-release.yml:677-678``). Each enforces what the changelog convention +:file:`docs/source/changelog/`, and ``github``'s "Select release body" step +warns, without failing, if that entry's heading still reads "unreleased". Each +enforces what the changelog convention already asks for. Ways to Start a Release ----------------------- -``create-release.yml``'s triggers -(``.github/workflows/create-release.yml:2-9``) are all first-class -- none is -a workaround for another: +``create-release.yml``'s ``on:`` triggers are all first-class -- none is a +workaround for another: ``push: tags: ['v*']`` Pushing a tag matching ``v*`` yourself. This chooses **which commit** to - release, not which version: ``version_check``'s checkout - (``:298-302``) pins ``ref: ${{ github.sha }}``, which on this trigger is - the commit the pushed tag points at, and reads :data:`pcapkit.__version__` out - of *that* tree (``:327``). Every downstream tag and release name is built + release, not which version: ``version_check``'s "Checkout code" step pins + ``ref: ${{ github.sha }}``, which on this trigger is the commit the pushed + tag points at, and its "Get Version" step reads :data:`pcapkit.__version__` + out of *that* tree. Every downstream tag and release name is built from that output -- ``v${{ needs.version_check.outputs.PCAPKIT_VERSION }}`` - is ``github``'s ``tag_name`` at ``:719`` and the release ``pypi`` and - ``conda`` upload to at ``:1117`` and ``:1493``, always from + is the ``tag_name`` of ``github``'s "Create Release" step and the release + ``pypi`` and ``conda`` upload to, always from ``PCAPKIT_VERSION``, never from ``github.ref_name`` -- so the version actually released is whatever ``__version__`` says on the tagged commit, not the name of the tag that was pushed. The two agree only if the tag @@ -121,7 +120,7 @@ common Vendor Update run that bumped nothing costs a minute, not a full-suite run. A run that may release waits for the one ahead of it on the same ref and never -cancels it (``create-release.yml:30-32``), since two Vendor Update runs +cancels it (the ``concurrency`` block), since two Vendor Update runs completing close together raise two runs on the same tip. A group holds one pending run, which a newer one cancels, so a run whose Vendor Update did not conclude ``success`` is keyed on its own ``github.run_id`` instead @@ -131,7 +130,7 @@ there; only a newer run that may release, reading ``main``'s newer tip, can. A Vendor Update that concluded ``success`` with ``vendor-update`` skipped -- its Unit Tests verdict was for another commit -- releases nothing either since `#1748 `__: the -``vendor_update`` job reads that run's jobs (``:230-264``), and +``vendor_update`` job reads that run's jobs, and ``version_check`` skips. The group cannot see that, so such a run still joins the ref's group; if it displaces a pending run, that release waits for the next Vendor Update on ``main``. @@ -148,27 +147,28 @@ days later, and tagged ``main``'s tip ``2c0008c3c`` as both ``v1.5.0b9`` and the default branch, and ``tag`` pulled ``main`` before tagging. * On ``workflow_run``, where ``github`` creates the ``v*`` tag, it passes - ``target_commitish`` ``github.sha`` (``:732``); on a ``v*`` push the tag + ``target_commitish`` ``github.sha``; on a ``v*`` push the tag already exists and it passes none. Either way it then checks that the tag names that commit. -* ``tag`` cuts ``conda-+0`` at that commit (``:846-873``) before it - reads ``main`` at all; the :file:`conda/build` reset is a separate commit on - top of ``main`` (``:895-948``), described under `Precautions`_. +* ``tag``'s "Create Tag" step cuts ``conda-+0`` at that commit before + it reads ``main`` at all; the :file:`conda/build` reset, "Reset Build + Number", is a separate commit on top of ``main``, described under + `Precautions`_. * ``pypi`` and ``conda`` check out that commit and, before building, check that every tag they publish under names it, as ``tag`` does for the ``v*`` tag. A run created for a *later* commit of the same version therefore never publishes a second tree under that version. ``version_check`` resolves the - ``v*`` tag the same way first (``:536-554``), and off a ``v*`` ref a tag - naming another commit skips the gate and every publisher + ``v*`` tag the same way first, in "Check which commit the version's tag + names", and off a ``v*`` ref a tag naming another commit skips the gate and every publisher (`#1748 `__): ``Release status`` posts a ``::notice`` if that release is complete, or a - ``::warning`` naming both commits and what is missing (``:1638-1648``), and - the run stays green. A ``v*`` push of such a version still fails with the + ``::warning`` naming both commits and what is missing, and the run stays + green. A ``v*`` push of such a version still fails with the check's ``::error``. A partial release is finished by re-running the run that tagged it. -* ``pypi`` and ``conda`` attach their files with ``gh release upload`` - (``:1110-1147``, ``:1488-1523``), not ``action-gh-release``, which calls Update - a release before every upload (3.0.2's ``src/github.ts:631-683``). A file +* ``pypi`` and ``conda`` attach their files with ``gh release upload``, in + their "Upload package to GitHub Release" steps, not ``action-gh-release``, + which calls Update a release before every upload (3.0.2's ``src/github.ts:631-683``). A file the release already has is left alone (`#1640 `__): the action deleted and re-uploaded it, and a rebuild against live PyPI is not @@ -262,13 +262,12 @@ The owner's choice on `#1634 `__: ``github`` creates the release with a GitHub App installation token, which Create a release accepts for a target that changes a workflow file when it has the "Workflows" -repository permission (write). "Mint the release App's token" -(``create-release.yml:692-700``) runs +repository permission (write). "Mint the release App's token" runs ``actions/create-github-app-token`` v3.3.0, pinned to its commit, with ``permission-contents: write`` and ``permission-workflows: write`` and no ``owner`` or ``repositories``, so the token is "scoped to only the current repository" and revoked in the action's ``post`` step (v3.3.0's README). Only -Create Release spends it (``:736``). +Create Release spends it. While ``vars.RELEASE_APP_CLIENT_ID`` is unset, which reads as "an empty string", the step skips and Create Release spends ``GITHUB_TOKEN``, so the workflow keeps working, refusals included, until the App exists. A PAT would @@ -295,16 +294,16 @@ One-time setup: Unlike ``GITHUB_TOKEN``'s, the App's tag starts a tag-push run. It would rerun the release test gate, ask for a second approval and race the first run's -uploads from another concurrency group, so ``version_check`` skips a ``push`` -whose ``github.actor_id`` is ``vars.RELEASE_APP_BOT_ID`` (``:290-294``), and -``release_status`` reports it with ``::notice title=Nothing to release`` -(``:1600-1602``). A tag pushed by a person runs as before. Whether a tag made +uploads from another concurrency group, so ``version_check``'s ``if:`` skips a +``push`` whose ``github.actor_id`` is ``vars.RELEASE_APP_BOT_ID``, and +``release_status`` reports it with ``::notice title=Nothing to release``. A +tag pushed by a person runs as before. Whether a tag made through the Releases API fires ``push``, and with the bot as actor, is unmeasured; if not, the guard never matches and the cost is one approval to reject. A refused Create Release (404, or 403 "Resource not accessible by -integration") is explained by "Explain a refused release tag" (``:769-781``) +integration") is explained by "Explain a refused release tag" according to the token spent: with ``GITHUB_TOKEN`` it lists the setup above; with the App's, it points at the App's permissions and installation. For a final release, first delete any draft ``v`` the refused run left: @@ -455,9 +454,8 @@ Only ``github-release`` carries a required reviewer; ``conda-tag``, ``pypi`` and ``anaconda`` kept their environments but had their reviewers removed on `#887 `__. What still makes one approval mean *the whole release* is the ``needs:`` -graph: ``tag`` and ``pypi`` both depend on ``github`` -(``.github/workflows/create-release.yml:808,974``), and ``conda`` depends on -``tag`` and ``github`` (``:1210``) -- so nothing downstream of ``github`` can +graph: ``tag`` and ``pypi`` both depend on ``github``, and ``conda`` depends on +``tag`` and ``github`` -- so nothing downstream of ``github`` can start before it is approved, and rejecting it leaves nothing tagged and nothing published. Before that removal ``tag`` depended on ``version_check`` alone, so removing its reviewer without moving this dependency would have let @@ -503,8 +501,8 @@ back, and neither is needed, since a plain re-run self-heals; see `Recovery`_ below. ``environment: pypi`` **stays even though its reviewer is gone.** ``pypi`` -publishes through PyPI's OIDC trusted publishing (``environment: pypi`` at -``:969``, ``id-token: write`` at ``:973``), and PyPI's trusted-publisher +publishes through PyPI's OIDC trusted publishing (the job's +``environment: pypi`` and ``id-token: write``), and PyPI's trusted-publisher configuration can be scoped to a GitHub Actions environment name. If this project's publisher on PyPI is scoped that way, removing the ``pypi`` name -- not just its reviewer -- would break the upload with a claim mismatch; @@ -515,7 +513,7 @@ the name regardless, since there is no upside to removing it. ``conda-tag`` **writes to** ``main``\ **, so a release is not read-only on the branch.** After cutting ``conda-+0`` at the run's commit, the ``tag`` job pushes a commit setting :file:`conda/build` to ``0`` on top of -``main`` (``.github/workflows/create-release.yml:895-948``) -- a plain +``main`` (its "Reset Build Number" step) -- a plain fast-forward push, never a rewrite -- so the next Conda Update bump of that version is ``+1``. There is none when the run's commit is not on ``main``, nor when ``main``'s tip already says ``0`` (the usual case, since diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index ceb97a6e39..8a7a6ae37d 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -263,10 +263,10 @@ re-derives the list: **Conda Update** fires on completion of **Unit Tests**. * ``.github/workflows/deploy-pages.yml``, its ``workflow_run`` trigger -- **GitHub Pages** fires on completion of **Unit Tests**. -* ``.github/workflows/create-release.yml:6-9`` -- **Create Release** fires on - completion of **Vendor Update**. -* ``.github/workflows/coverage-comment.yml:21-23`` -- **Coverage Comment** fires - on completion of **Unit Tests**. +* ``.github/workflows/create-release.yml``, its ``workflow_run`` trigger -- + **Create Release** fires on completion of **Vendor Update**. +* ``.github/workflows/coverage-comment.yml``, its ``workflow_run`` trigger -- + **Coverage Comment** fires on completion of **Unit Tests**. ``workflow_run`` fires for *every* completion of the named workflow -- success, failure, or skipped -- and regardless of what triggered it, @@ -315,7 +315,7 @@ the caller's own workflow run, as one of the caller's own jobs, rather than starting an independent run after the caller finishes. The calls target ``unit-tests.yml``, always with ``gate-only: true``, and ``cron-head.yml``: -* ``.github/workflows/create-release.yml:595`` -- job ``unit-tests`` calls +* ``.github/workflows/create-release.yml``, job ``unit-tests`` -- calls ``./.github/workflows/unit-tests.yml`` with ``gate-only: true``. * ``.github/workflows/cron-head.yml``, job ``unit-tests`` -- calls ``./.github/workflows/unit-tests.yml`` with ``gate-only: true``. @@ -401,7 +401,7 @@ above -- carries: !(github.event_name == 'push' && github.actor_id == vars.RELEASE_APP_BOT_ID) && needs.vendor_update.outputs.updated != 'false' }} -(``create-release.yml:290-294``). Its third clause skips the tag-push run that +Its third clause skips the tag-push run that the release App's own tag starts (:doc:`releasing`, "The Release App"). On the ``workflow_run`` path, this evaluates **false**, and the job is skipped outright, whenever the upstream Vendor Update run's own conclusion was @@ -415,13 +415,12 @@ its ``verdict`` job found the Unit Tests verdict was for another commit -- which the ``vendor_update`` job reads from that run's jobs first (observed on run 38650191283). ``!cancelled()`` keeps a skipped ``vendor_update``, on a push, from skipping it. ``unit-tests`` then depends on that job with -``needs: [ version_check ]`` (``create-release.yml:583``) and an ``if:`` that +``needs: [ version_check ]`` and an ``if:`` that calls no status-check function, so a skipped dependency skips it too -- and every job past it is gated the same way: ``github`` needs ``[ unit-tests, -version_check ]`` (``:621``), ``tag`` needs ``[ github, unit-tests, -version_check ]`` (``:808``), ``pypi`` needs ``[ github, unit-tests, -version_check ]`` (``:974``), and ``conda`` needs ``[ tag, github, -unit-tests, version_check ]`` (``:1210``). +version_check ]``, ``tag`` and ``pypi`` need ``[ github, unit-tests, +version_check ]``, and ``conda`` needs ``[ tag, github, unit-tests, +version_check ]``. .. mermaid:: @@ -446,8 +445,8 @@ unit-tests, version_check ]`` (``:1210``). class UTX,VC,GH,TAGJ,PY,CD skipped This is a transitive reduction, as in :doc:`releasing`: ``version_check`` and -``unit-tests`` are also direct ``needs:`` of ``tag`` and ``pypi`` -(``create-release.yml:808,974``) and of ``conda`` (``:1210``), but ``github`` +``unit-tests`` are also direct ``needs:`` of ``tag``, ``pypi`` and ``conda``, +but ``github`` already implies those edges, since it depends on both itself. Drawing them would add lines without changing which job can start before which, or which job a skip in ``version_check`` reaches. @@ -470,7 +469,7 @@ one can skip the very first attempt, before any tag is ever created, purely because the upstream Vendor Update run did not itself conclude ``success``. Such a run is also kept out of the release's concurrency group -(``create-release.yml:30-32``, +(its ``concurrency`` block, `#1705 `__). It is keyed on its own ``github.run_id`` rather than on the ref, so it cannot cancel a run waiting there with release work. Keyed on the ref, run 38499295798 joined at @@ -692,19 +691,19 @@ grepping every workflow file for its name: - ``unit-tests.yml:1964`` * - ``Compat Python 3.11`` - job ``compatibility``, matrix leg ``3.11`` - - ``python-compatibility.yml:60``, ``.github/python-versions.toml:20`` + - ``python-compatibility.yml``, ``.github/python-versions.toml:20`` * - ``Compat Python 3.12`` - job ``compatibility``, matrix leg ``3.12`` - - ``python-compatibility.yml:60``, ``.github/python-versions.toml:21`` + - ``python-compatibility.yml``, ``.github/python-versions.toml:21`` * - ``Compat Python 3.13`` - job ``compatibility``, matrix leg ``3.13`` - - ``python-compatibility.yml:60``, ``.github/python-versions.toml:22`` + - ``python-compatibility.yml``, ``.github/python-versions.toml:22`` * - ``Compat Python 3.14`` - job ``compatibility``, matrix leg ``3.14`` - - ``python-compatibility.yml:60``, ``.github/python-versions.toml:23`` + - ``python-compatibility.yml``, ``.github/python-versions.toml:23`` * - ``Compat Python 3.15`` - job ``compatibility``, matrix leg ``3.15`` - - ``python-compatibility.yml:60``, ``.github/python-versions.toml:24`` + - ``python-compatibility.yml``, ``.github/python-versions.toml:24`` That is the list after `#1633 `__, which added ``Compat Python 3.15`` and removed ``Compat Python 3.10``. Moving a version @@ -712,8 +711,8 @@ between the tiers below means editing the ruleset in the same change. ``Required checks passed`` is defined by exactly one job (``unit-tests.yml:1964``). ``Compat Python`` is defined by one, the -``compatibility`` job in ``python-compatibility.yml`` (``:60``), whose matrix -(``:74``) comes from :file:`.github/python-versions.toml`: the required legs +``name:`` of the ``compatibility`` job in ``python-compatibility.yml``, whose +``matrix`` comes from :file:`.github/python-versions.toml`: the required legs are its supported tier (``python-versions.toml:20-24``), and its KTLO legs (``python-versions.toml:16-19``) have names ending in ``(KTLO)``, so no required context matches one. A comment at diff --git a/tests/project/test_conda_platform_pin_refresh_1851.py b/tests/project/test_conda_platform_pin_refresh_1851.py index 3dd4778ca3..e49ed41949 100644 --- a/tests/project/test_conda_platform_pin_refresh_1851.py +++ b/tests/project/test_conda_platform_pin_refresh_1851.py @@ -663,7 +663,7 @@ def runs(step: 'dict[str, Any]') -> 'list[str]': def test_the_refresh_runs_between_the_install_and_the_writer(self) -> None: refresh = self.index(lambda step: self.runs(step) == ['python util/conda_vendor.py --refresh-pins']) - install = self.index(lambda step: 'python -m pip install -e .[all]' in self.runs(step)) + install = self.index(lambda step: "python -m pip install -e '.[all]'" in self.runs(step)) writer = self.index(lambda step: 'python util/conda_vendor.py' in self.runs(step)) changed = self.index(lambda step: step.get('id') == 'verify-changed-files') self.assertLess(install, refresh) diff --git a/tests/project/test_unit_tests_queue.py b/tests/project/test_unit_tests_queue.py index 77a8bd64f2..a3af58fd21 100644 --- a/tests/project/test_unit_tests_queue.py +++ b/tests/project/test_unit_tests_queue.py @@ -626,69 +626,17 @@ def test_the_scan_is_not_blind(self) -> None: } -#: Every ``create-release.yml`` citation in these documents, as (the job its lines -#: sit in, ``None`` above ``jobs:``; text the first line must contain; text the -#: last line must contain). Lines repeat across jobs -- ``tag`` and ``pypi`` share -#: their ``needs:``, ``pypi`` and ``conda`` their upload step -- so the job is -#: checked too. A list such as ``:619,784`` is one citation per number. -RELEASE_CITATIONS: 'dict[str, list[tuple[Optional[str], str, Optional[str]]]]' = { - 'docs/source/contributing/releasing.rst': [ - (None, 'push:', '- completed'), - (None, 'concurrency:', 'cancel-in-progress: false'), - ('vendor_update', ' vendor_update:', 'esac'), - ('version_check', 'if: >-', "needs.vendor_update.outputs.updated != 'false' }}"), - ('version_check', '- name: Checkout code', 'fetch-depth: 0'), - ('version_check', "- name: Check which commit the version's tag names", - 'PCAPKIT_RELEASED_ELSEWHERE=${elsewhere}'), - ('version_check', 'print(pcapkit.__version__)', None), - ('unit-tests', ' unit-tests:', 'gate-only: true'), - ('github', 'needs: [ unit-tests, version_check ]', None), - ('github', "grep -qi 'unreleased'", 'title=Changelog entry is undated'), - ('github', "- name: Mint the release App's token", 'permission-workflows: write'), - ('github', 'tag_name: "v${{ needs.version_check.outputs.PCAPKIT_VERSION }}"', None), - ('github', "target_commitish: ${{ github.event_name == 'workflow_run' && github.sha", None), - ('github', 'token: ${{ steps.app_token.outputs.token || secrets.GITHUB_TOKEN }}', None), - ('github', '- name: Explain a refused release tag', 'title=Could not create v'), - ('tag', 'needs: [ github, unit-tests, version_check ]', None), - ('tag', '- name: Create Tag', 'fi'), - ('tag', '- name: Reset Build Number', 'exit 1'), - ('pypi', 'environment: pypi', None), - ('pypi', 'id-token: write', None), - ('pypi', 'needs: [ github, unit-tests, version_check ]', None), - ('pypi', 'PCAPKIT_RELEASE_TAG: v${{ needs.version_check.outputs.PCAPKIT_VERSION }}', None), - ('pypi', '- name: Upload package to GitHub Release', 'done <<< "$PCAPKIT_ASSETS"'), - ('conda', 'needs: [ tag, github, unit-tests, version_check ]', None), - ('conda', 'PCAPKIT_RELEASE_TAG: v${{ needs.version_check.outputs.PCAPKIT_VERSION }}', None), - ('conda', '- name: Upload package to GitHub Release', 'done <<< "$PCAPKIT_ASSETS"'), - ('release_status', '[ -n "$PCAPKIT_RELEASE_APP_BOT_ID" ]', 'title=Nothing to release'), - ('release_status', 'if [ "${PCAPKIT_RELEASED_ELSEWHERE:-}" = "true" ]', 'is unfinished (#1634)::'), - ], - 'docs/source/contributing/workflows.rst': [ - (None, 'workflow_run:', '- completed'), - (None, 'concurrency:', 'cancel-in-progress: false'), - ('version_check', 'if: >-', "needs.vendor_update.outputs.updated != 'false' }}"), - ('unit-tests', 'needs: [ version_check ]', None), - ('unit-tests', 'uses: ./.github/workflows/unit-tests.yml', None), - ('github', 'needs: [ unit-tests, version_check ]', None), - ('tag', 'needs: [ github, unit-tests, version_check ]', None), - ('pypi', 'needs: [ github, unit-tests, version_check ]', None), - ('conda', 'needs: [ tag, github, unit-tests, version_check ]', None), - ], -} +#: The workflows the prose cites by job and step name, never by line number: +#: trimming a comment in one moves every line below it (#1967). +BY_NAME = ('create-release.yml', 'python-compatibility.yml', 'coverage-comment.yml') -#: A document that names ``create-release.yml`` also cites it bare, as ``:N``, -#: and each bare citation there counts as one of its citations unless listed -#: here with the file it does cite. A bare ``:997`` is what outlived the line it -#: named when #1672 first moved ``conda`` down 29 lines. +#: A document that names one of :data:`BY_NAME` may still cite another file bare, +#: as ``:N``; each such citation is listed here with the file it does cite, and +#: any other bare citation there reads as a line of the named workflow. BARE_ELSEWHERE: 'dict[str, dict[str, str]]' = { 'docs/source/contributing/releasing.rst': { - '494-514': 'tests/project/test_bump_version.py', '631-647': "softprops/action-gh-release 3.0.2's src/github.ts", }, - 'docs/source/contributing/workflows.rst': { - '60': '.github/workflows/python-compatibility.yml', - '74': '.github/workflows/python-compatibility.yml', - }, } #: A line-number citation: ``N``, ``N-M``, or several joined by commas. @@ -701,55 +649,35 @@ def documents() -> 'list[str]': | {str(p.relative_to(ROOT)) for p in ROOT.glob('*.md')}) -def jobs_by_line(lines: 'list[str]') -> 'list[Optional[str]]': - """The job each workflow line sits in, ``None`` above ``jobs:``.""" - owner: 'list[Optional[str]]' = [] - current: 'Optional[str]' = None - seen = False - for line in lines: - seen = seen or line.startswith('jobs:') - match = re.match(r' ([\w-]+):\s*$', line) if seen else None - current = match.group(1) if match else current - owner.append(current) - return owner - +class TestWorkflowsCitedByName(unittest.TestCase): + """The prose cites :data:`BY_NAME`'s workflows by job and step, not by line.""" -class TestDocCitationsResolve(unittest.TestCase): - """The prose's ``unit-tests.yml`` and ``create-release.yml`` line numbers land where it says.""" + def test_no_document_cites_them_by_line_number(self) -> None: + """Neither ``.yml:N`` nor, beside one's name, a bare ``:N`` of it. - def test_every_create_release_citation_lands_on_its_anchor(self) -> None: - """Each citation, bare ones included, lands in its anchor's job and on its lines.""" - release = ROOT / '.github' / 'workflows' / 'create-release.yml' - lines = release.read_text(encoding='utf-8').splitlines() - owner = jobs_by_line(lines) + The owner's decision A on #1967: cite steps by job and step name, not + ``file.yml:N``. A bare citation of another file goes in + :data:`BARE_ELSEWHERE`, and an entry no bare citation matches fails too. + """ for document in documents(): text = (ROOT / document).read_text(encoding='utf-8') - groups = re.findall(rf'create-release\.yml:{SPANS}', text) - if 'create-release.yml' in text: - bare = re.findall(rf'``:{SPANS}``', text) - elsewhere = BARE_ELSEWHERE.get(document, {}) - with self.subTest(document=document, table='BARE_ELSEWHERE'): - self.assertFalse(set(elsewhere) - set(bare), - f'{document} no longer cites these bare') - groups += [group for group in bare if group not in elsewhere] - anchors = RELEASE_CITATIONS.get(document, []) - unmatched = list(anchors) - for first, last in ((int(a), int(b or a)) for group in groups - for a, b in re.findall(r'(\d+)(?:-(\d+))?', group)): - with self.subTest(document=document, citation=f'{first}-{last}'): - self.assertLessEqual(last, len(lines)) - hit = next((a for a in anchors if owner[first - 1] == owner[last - 1] == a[0] - and a[1] in lines[first - 1] - and (a[2] is None or a[2] in lines[last - 1])), None) - self.assertIsNotNone( - hit, f'{document} cites create-release.yml:{first}-{last}, which is ' - f'{lines[first - 1].strip()!r} .. {lines[last - 1].strip()!r} in ' - f'{owner[first - 1]!r}; a bare citation of another file goes in ' - f'BARE_ELSEWHERE') - if hit in unmatched: - unmatched.remove(hit) - with self.subTest(document=document): - self.assertFalse(unmatched, f'{document}: no citation lands on {unmatched}') + for name in BY_NAME: + with self.subTest(document=document, workflow=name): + self.assertEqual(re.findall(rf'{re.escape(name)}:{SPANS}', text), [], + f'{document} cites {name} by line; name the job and step') + if not any(name in text for name in BY_NAME): + continue + bare = set(re.findall(rf'``:{SPANS}``', text)) + elsewhere = set(BARE_ELSEWHERE.get(document, {})) + with self.subTest(document=document, table='BARE_ELSEWHERE'): + self.assertEqual(bare - elsewhere, set(), + f'{document} cites these bare; name the job and step, or list ' + f'the file they cite in BARE_ELSEWHERE') + self.assertEqual(elsewhere - bare, set(), f'{document} no longer cites these bare') + + +class TestDocCitationsResolve(unittest.TestCase): + """The prose's ``unit-tests.yml`` line numbers land where it says.""" def test_every_citation_lands_on_its_anchor(self) -> None: """Each ``unit-tests.yml`` citation's range starts and ends on the line its anchor names."""