From 094d57e20e6111c3a2962049590b97b0624c72fd Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 14 Oct 2026 18:58:00 -0400 Subject: [PATCH] ci(python): move the KTLO legs into jobs nothing gating waits on (#1738) - Owner's option A: the KTLO legs of `test`, `integration` and `pypcap-parity` are now `test-ktlo`, `integration-ktlo` and `pypcap-parity-ktlo` in unit-tests.yml. Neither `required-checks` nor `unittest-ordering` needs them, so a lost runner or the job cap on a KTLO leg no longer fails `Required checks passed`. - Each KTLO job runs its twin's steps by YAML alias, so the step-level `continue-on-error`, warning-level annotations and `KTLO summary` are unchanged; job-level `continue-on-error: true` is the backstop. - Check names are unchanged (`Python 3.x (KTLO)` and so on). - util/python_versions.py writes the three KTLO matrices; the gating matrices hold the supported tier only. - util/ci_timing.py resolves aliased steps and picks the most specific job name template; util/update_python_versions.py reports a leg moving between twins as a rename and counts the new jobs' legs. - workflows.rst and releasing.rst describe it; citations renumbered. Tests: test_ktlo_jobs_gate_nothing_1738.py pins the structure; the 1633, 1660, 1672, 1732, apt-cache and queue tests follow the move. Closes #1738 --- .github/workflows/unit-tests.yml | 156 ++++++++---- docs/source/contributing/releasing.rst | 4 +- docs/source/contributing/workflows.rst | 63 +++-- tests/_dependency_gates.py | 6 +- .../test_ktlo_jobs_gate_nothing_1738.py | 233 ++++++++++++++++++ .../project/test_ktlo_legs_stay_green_1672.py | 24 +- .../project/test_python_matrix_tiers_1633.py | 22 +- tests/project/test_python_versions_1660.py | 25 +- .../test_python_versions_update_1660.py | 2 +- .../test_sweeps_on_latest_only_1732.py | 11 +- tests/project/test_workflow_apt_cache.py | 14 +- util/ci_timing.py | 29 ++- util/python_versions.py | 69 ++++-- util/update_python_versions.py | 29 ++- 14 files changed, 563 insertions(+), 124 deletions(-) create mode 100644 tests/project/test_ktlo_jobs_gate_nothing_1738.py diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index ad68ff519c..6012445805 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -64,8 +64,11 @@ jobs: outputs: latest: ${{ steps.read.outputs.latest }} test: ${{ steps.read.outputs.test }} + test-ktlo: ${{ steps.read.outputs.test-ktlo }} integration: ${{ steps.read.outputs.integration }} + integration-ktlo: ${{ steps.read.outputs.integration-ktlo }} pypcap-parity: ${{ steps.read.outputs.pypcap-parity }} + pypcap-parity-ktlo: ${{ steps.read.outputs.pypcap-parity-ktlo }} engine-supported: ${{ steps.read.outputs.engine-supported }} engine-ktlo: ${{ steps.read.outputs.engine-ktlo }} @@ -89,21 +92,22 @@ jobs: # this workflow used to) therefore ran the whole matrix in every caller # and skipped the gate everywhere. An input is the only reliable way for a # reusable workflow to know how it was invoked. - name: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }} + name: Python ${{ matrix.python-version }} if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }} needs: [changes, python-versions] runs-on: ${{ matrix.os || 'ubuntu-latest' }} timeout-minutes: 45 # #1672: the level of every annotation a failing step writes -- a warning on - # a KTLO leg, which must not show red, an error on a supported one. + # a KTLO leg, which must not show red, an error on a supported one. The + # steps are shared with `test-ktlo`, whose legs carry `ktlo: true`. env: ANNOTATION_LEVEL: ${{ matrix.ktlo && 'warning' || 'error' }} strategy: fail-fast: false - # #1660: .github/python-versions.toml's CPython tiers, through the - # `python-versions` job. The supported tier is the `python-version` axis - # and blocks through `required-checks` (#1633); `include` marks one leg - # `coverage: true` and adds each KTLO leg with `ktlo: true`. + # #1660: .github/python-versions.toml's supported CPython tier, through + # the `python-versions` job, as the `python-version` axis; it blocks + # through `required-checks` (#1633). `include` marks one leg + # `coverage: true`. The KTLO legs are `test-ktlo`'s (#1738). # # Exactly one leg runs under coverage, so measuring it adds no leg to # the queue. It is `latest`, the newest supported version, like every @@ -112,32 +116,12 @@ jobs: # coverage cost 1.8x on 3.15 locally (2.0x on 3.14): a step of about # 15-19 minutes under the job's 45. # - # The KTLO tier (#1633): end-of-life versions that still build and - # test but never block. `required-checks` reads this job's one - # `needs.test.result`, so a KTLO leg cannot fail the job: each step - # that can fail on one carries `continue-on-error: ${{ matrix.ktlo - # == true }}` instead -- the step-level form, whose effect on the job - # is documented, as for engine-tests' KTLO cell. The leg concludes - # success, and the job's last step, `KTLO summary`, reports each - # failed step as a warning (#1672). A version `ubuntu-latest` cannot - # provide carries its `runner` from the file as `os`: 3.8 and 3.9 need - # ubuntu-24.04, as setup-python has no Ubuntu 26.04 build of either - # and `ubuntu-latest` moves to 26.04 from 2026-10-19. - # - # 3.7 needs ubuntu-22.04, the newest image setup-python builds it for - # (3.7.17). It cannot install today -- the `test` extra's `pytest>=8` - # needs 3.8, and the sources use syntax 3.7 cannot parse -- so the leg - # stops at its install step, with one annotation naming pip's reason - # (see that step). ubuntu-22.04 is deprecated: brownouts from - # 2027-03-23, retirement on 2027-04-17 (actions/runner-images#14254). - # A brownout fails the leg before any step runs, which no step-level - # `continue-on-error` can contain, so this leg must go before then. - # # The experimental tier (#1660) runs in experimental.yml, a workflow of # its own, so that nothing here waits on it (#1732). matrix: ${{ fromJSON(needs.python-versions.outputs.test) }} - steps: + # `&test-steps`: `test-ktlo` runs these same steps, by YAML alias. + steps: &test-steps - uses: actions/checkout@v7 id: checkout continue-on-error: ${{ matrix.ktlo == true }} @@ -419,8 +403,47 @@ jobs: echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" done <<< "$failed" + # The KTLO tier (#1633): end-of-life versions that still build and test but + # never block. #1738, the owner's option A: their legs are this job's, not + # `test`'s, and neither `required-checks` nor `unittest-ordering` `needs:` + # it, so a failure no step can contain -- a lost runner, the job cap -- + # fails nothing that gates. The steps are `test`'s, by YAML alias + # (docs.github.com, Reusing workflow configurations, "YAML anchors and + # aliases"), and each that can fail on a KTLO leg carries + # `continue-on-error: ${{ matrix.ktlo == true }}`, so the leg concludes + # success, its annotations are warnings, and `KTLO summary`, the last step, + # reports each failed step as a warning (#1672). The job-level + # `continue-on-error` is the backstop for the rest: it "Prevents a workflow + # run from failing when a job fails" (docs.github.com, Workflow syntax, + # `jobs..continue-on-error`), as on experimental.yml's job, so the + # Unit Tests run its `workflow_run` consumers read stays green. + # + # A version `ubuntu-latest` cannot provide carries its `runner` from the + # file as `os`: 3.8 and 3.9 need ubuntu-24.04, as setup-python has no Ubuntu + # 26.04 build of either and `ubuntu-latest` moves to 26.04 from 2026-10-19. + # 3.7 needs ubuntu-22.04, the newest image setup-python builds it for + # (3.7.17). It cannot install today -- the `test` extra's `pytest>=8` needs + # 3.8, and the sources use syntax 3.7 cannot parse -- so the leg stops at + # its install step, with one annotation naming pip's reason (see that step). + # ubuntu-22.04 is deprecated: brownouts from 2027-03-23, retirement on + # 2027-04-17 (actions/runner-images#14254); a brownout fails the leg before + # any step runs, which only the job-level guard contains. + test-ktlo: + name: Python ${{ matrix.python-version }} (KTLO) + if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }} + needs: [changes, python-versions] + runs-on: ${{ matrix.os || 'ubuntu-latest' }} + timeout-minutes: 45 + continue-on-error: true + env: + ANNOTATION_LEVEL: ${{ matrix.ktlo && 'warning' || 'error' }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.python-versions.outputs.test-ktlo) }} + steps: *test-steps + integration: - name: Integration Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }} + name: Integration Python ${{ matrix.python-version }} if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }} needs: [changes, python-versions] runs-on: ${{ matrix.os || 'ubuntu-latest' }} @@ -430,11 +453,12 @@ jobs: ANNOTATION_LEVEL: ${{ matrix.ktlo && 'warning' || 'error' }} strategy: fail-fast: false - # The two tiers of the `test` job above, for the same reasons, without - # its coverage leg. + # The supported tier of the `test` job above, without its coverage leg; + # the KTLO legs are `integration-ktlo`'s (#1738). matrix: ${{ fromJSON(needs.python-versions.outputs.integration) }} - steps: + # `&integration-steps`: `integration-ktlo` runs these same steps. + steps: &integration-steps - uses: actions/checkout@v7 id: checkout continue-on-error: ${{ matrix.ktlo == true }} @@ -610,6 +634,21 @@ jobs: echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" done <<< "$failed" + # `integration`'s KTLO legs, apart for the reasons on `test-ktlo` (#1738). + integration-ktlo: + name: Integration Python ${{ matrix.python-version }} (KTLO) + if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }} + needs: [changes, python-versions] + runs-on: ${{ matrix.os || 'ubuntu-latest' }} + timeout-minutes: 45 + continue-on-error: true + env: + ANNOTATION_LEVEL: ${{ matrix.ktlo && 'warning' || 'error' }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.python-versions.outputs.integration-ktlo) }} + steps: *integration-steps + # Per-engine coverage for the third-party capture engines (#845): every # (Python version, engine) pair is checked, and a failure -- or a correctly # asserted decline -- names the one engine and interpreter responsible. Each @@ -1102,7 +1141,7 @@ jobs: # step-level `continue-on-error`, for the reason given on the `test` job's # matrix. pypcap-parity: - name: PyPCAP/PyPCAPFile parity Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }} + name: PyPCAP/PyPCAPFile parity Python ${{ matrix.python-version }} if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }} needs: [changes, python-versions] runs-on: ubuntu-latest @@ -1121,7 +1160,8 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.python-versions.outputs.pypcap-parity) }} - steps: + # `&pypcap-parity-steps`: `pypcap-parity-ktlo` runs these same steps. + steps: &pypcap-parity-steps - uses: actions/checkout@v7 id: checkout continue-on-error: ${{ matrix.ktlo == true }} @@ -1333,6 +1373,25 @@ jobs: echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" done <<< "$failed" + # `pypcap-parity`'s KTLO legs, apart for the reasons on `test-ktlo` (#1738). + # Same apt set and cache key; `pypcap-parity`'s supported leg saves it. + pypcap-parity-ktlo: + name: PyPCAP/PyPCAPFile parity Python ${{ matrix.python-version }} (KTLO) + if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }} + needs: [changes, python-versions] + runs-on: ubuntu-latest + timeout-minutes: 45 + continue-on-error: true + env: + APT_PACKAGES: build-essential libpcap-dev tshark + APT_SET: pypcap-parity + APT_CACHE_SAVE: 'false' + ANNOTATION_LEVEL: ${{ matrix.ktlo && 'warning' || 'error' }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.python-versions.outputs.pypcap-parity-ktlo) }} + steps: *pypcap-parity-steps + # GitHub issue #981: two test modules # (tests/protocols/application/test_http_unit.py and # tests/test_base_class_contract.py) each pass alone, and pytest -- the @@ -1550,9 +1609,9 @@ jobs: # to the GitHub Release body: a drifted copy is not merely wrong in the tree, it # is published. # - # Deliberately *not* gated on ``gate-only``, unlike the four jobs above - # (`test`, `integration`, `engine-tests`, `pypcap-parity`), and that is the - # point of putting it here at all. ``create-release.yml`` calls this + # Deliberately *not* gated on ``gate-only``, unlike the matrix jobs above + # (`test`, `integration`, `engine-tests`, `pypcap-parity` and the KTLO ones), + # and that is the point of putting it here at all. ``create-release.yml`` calls this # workflow as its release gate, so an ungated job runs on the release path and # the release body cannot be built from a file that has drifted. The matrix is # skipped per caller because it is expensive and already ran for the commit; @@ -1850,19 +1909,16 @@ jobs: # here: the cell script exits 0 for a cell that declined exactly as # expected, so they contribute to that job's `success` like any other cell. # - # Nor is the KTLO tier (#1633): engine-tests' 3.10 cell, and the 3.7-3.10 - # legs of `test` and `integration` and the 3.10 leg of `pypcap-parity`. - # Each is non-blocking through step-level `continue-on-error`, whose effect - # on the job's result *is* documented (a step's `conclusion` is its result - # after `continue-on-error` is applied), so the job, and this gate, read - # `success` while the failed step stays visible. Not a job-level - # `continue-on-error`: GitHub does not document whether `needs..result` - # sees such a job's failure as `failure` or `success`, so a KTLO-only - # regression could block every merge. The one failure no step can contain is - # a leg that never gets a runner: the 3.7 legs' deprecated ubuntu-22.04 image - # has brownouts from 2027-03-23 -- see the `test` job's matrix. The - # experimental legs (#1660) are in experimental.yml, which nothing here - # needs or waits on (#1732). + # Nor is the KTLO tier (#1633). engine-tests' KTLO cell is non-blocking + # through step-level `continue-on-error`, whose effect on the job's result + # *is* documented (a step's `conclusion` is its result after + # `continue-on-error` is applied), so the job, and this gate, read `success` + # while the failed step stays visible. The KTLO legs of `test`, + # `integration` and `pypcap-parity` are `test-ktlo`, `integration-ktlo` + # and `pypcap-parity-ktlo` (#1738), which this job does not need, so not + # even a failure no step can contain -- a lost runner, the job cap -- can + # fail it. The experimental legs (#1660) are in experimental.yml, which + # nothing here needs or waits on (#1732). # # The docs-only path. On a pull request whose diff touches nothing but # `docs/**` and `*.md` files outside the code directories (the `changes` diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index 8a79912510..362c7369ea 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -52,7 +52,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:467-481``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1717-1787``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1776-1846``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:500,685,850,1085``), so a @@ -62,7 +62,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Two other checks run unconditionally on the same path, though neither is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1565-1584``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1624-1643``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 8c5bef23f6..2916000de3 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -292,15 +292,15 @@ repository target the same reusable workflow, with the same input: * ``.github/workflows/deploy-pages.yml:55`` -- job ``unit-tests`` calls ``./.github/workflows/unit-tests.yml`` with ``gate-only: true``. -``gate-only: true`` selects **three** of ``unit-tests.yml``'s **eleven** jobs, +``gate-only: true`` selects **three** of ``unit-tests.yml``'s **fourteen** jobs, not one: ``gate`` (the full suite, on ``latest``), ``changelog`` (checks ``CHANGELOG.md`` against its source entry) and ``python-versions``, which reads ``gate``'s version (see `Python Versions`_ below). It skips the other -**eight** -- the five matrix jobs, ``test`` (nine Python legs: five -supported, four KTLO), ``integration`` (nine), +**eleven** -- the eight matrix jobs, ``test`` (five Python legs), ``test-ktlo`` +(four), ``integration`` (five), ``integration-ktlo`` (four), ``engine-tests`` (one job for each of five engines, each looping over six -Python versions), ``pypcap-parity`` (two) and ``unittest-ordering`` -(eleven matrix cells), each of which has already run once for this commit +Python versions), ``pypcap-parity`` (one), ``pypcap-parity-ktlo`` (one) and +``unittest-ordering`` (eleven matrix cells), each of which has already run once for this commit from Unit Tests' own ``push`` trigger (and, all but ``unittest-ordering``, which runs on ``main`` pushes only, from its ``pull_request`` trigger), so running any of them again per caller would @@ -309,7 +309,7 @@ and ``project-tests``, and ``required-checks`` (see `Required Status Checks`_ below), gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1553-1559``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1612-1618``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -558,9 +558,9 @@ required context names, on every KTLO leg, a job whose name ends in ``(KTLO)`` last covers the experimental legs that ran in :file:`unit-tests.yml` under ``test``'s 45-minute cap before the split, whose runs stay in the 7-day window for a week after it. The finding keeps its ``H`` id and says it is non-blocking. A -KTLO leg of ``test``, ``integration`` or ``pypcap-parity`` past its job's -``timeout-minutes`` is still cancelled, which ``Required checks passed`` refuses, -so that warning is worth acting on before the cap. +KTLO leg past its job's ``timeout-minutes`` is still cancelled, and its check run +reads red, though since #1738 nothing gating waits on it, so that warning is worth +acting on before the cap. Work Kept Off the Pull-Request Queue (`#1538 `__) ----------------------------------------------------------------------------------------------------- @@ -602,7 +602,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:1881`` + - ``unit-tests.yml:1937`` * - ``Compat Python 3.11`` - job ``compatibility``, matrix leg ``3.11`` - ``python-compatibility.yml:60``, ``.github/python-versions.toml:20`` @@ -624,21 +624,21 @@ which added ``Compat Python 3.15`` and removed ``Compat Python 3.10``. Moving a between the tiers below means editing the ruleset in the same change. ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:1881``). ``Compat Python`` is defined by one, the +(``unit-tests.yml:1937``). ``Compat Python`` is defined by one, the ``compatibility`` job in ``python-compatibility.yml`` (``:60``), whose matrix (``:74``) comes from :file:`.github/python-versions.toml`: the five required legs are its supported tier (``python-versions.toml:20-24``), and its four KTLO legs (``python-versions.toml:16-19``) have names ending in ``(KTLO)``, so no required context matches one. A comment at -``unit-tests.yml:1790`` mentions the string without defining it. +``unit-tests.yml:1849`` mentions the string without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the five ``Compat Python 3.11``-``3.15`` legs, nothing else, as -``unit-tests.yml``'s own comment (``unit-tests.yml:1789-1799``) records. It +``unit-tests.yml``'s own comment (``unit-tests.yml:1848-1858``) records. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- ``test``, ``integration``, ``engine-tests`` and ``pypcap-parity``, plus ``changes`` and ``project-tests`` for the docs-only path -- and checks each result explicitly -(``unit-tests.yml:1880-1934``). Rulesets match check names literally, with no +(``unit-tests.yml:1936-1990``). Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. It accepts the four legs as ``skipped`` only on a docs-only pull request, where @@ -647,7 +647,7 @@ instead. The ``Compat`` legs stay required separately because they come from ``python-compatibility.yml``, and a job cannot ``needs:`` a job in another workflow file. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1883``), +``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1939``), so it is never produced -- and never expected -- on those paths. Python versions come in two tiers (#1633), and an experimental one (#1660, @@ -655,12 +655,19 @@ below), declared in :file:`.github/python-versions.toml` (see `Python Versions`_ The **supported** tier, 3.11-3.15, blocks: a failed leg fails ``Required checks passed`` or its own required ``Compat`` context. The **KTLO** tier -- end-of-life 3.7-3.10, still built and tested -- never blocks, is never a required context, and never shows a commit or a -pull request red (#1672). Each KTLO leg shares its job with supported legs, in -``unit-tests.yml``, ``python-compatibility.yml`` and ``create-release.yml``, and -every such job works the same way: +pull request red (#1672). In ``unit-tests.yml`` the KTLO legs are jobs of their own +since `#1738 `__: ``test-ktlo``, +``integration-ktlo`` and ``pypcap-parity-ktlo``, which neither ``required-checks`` +nor ``unittest-ordering`` ``needs:``, so not even a failure no step can contain, such +as a lost runner or the job cap, reaches a gate. Each runs its gating twin's steps, +by YAML alias (`Reusing workflow configurations +`__), +and keeps its legs' check names, ``Python 3.10 (KTLO)`` and so on. In +``python-compatibility.yml`` and ``create-release.yml`` a KTLO leg shares its job +with supported legs. Every job holding KTLO legs works the same way: * Every step that can fail on a KTLO leg has an ``id`` and - ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:115-125``), + ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:406-423``), unless it is masked on every leg, as the timings uploads are. Its ``outcome`` is then ``failure`` but its ``conclusion`` ``success``, so the job, its check run and ``required-checks`` read ``success``. A step that needs an earlier one's work @@ -671,9 +678,14 @@ every such job works the same way: ``ANNOTATION_LEVEL``: ``warning`` on a KTLO leg, ``error`` on a supported one. ``engine-tests``' KTLO cells, a step of their own with ``continue-on-error: true``, report the same way. -* Never job-level ``continue-on-error``: it passes the *run*, but the leg's check - run still concludes ``failure``, which showed every commit red while - ``Compat Python 3.7 (KTLO)`` carried it. +* Never job-level ``continue-on-error`` alone: it passes the *run*, but the leg's + check run still concludes ``failure``, which showed every commit red while + ``Compat Python 3.7 (KTLO)`` carried it. ``unit-tests.yml``'s three KTLO jobs + carry it on top of the step-level guards, as a backstop for the failures no step + contains, as ``experimental.yml``'s job does; it "prevents a workflow run from + failing when a job fails" (`Workflow syntax + `__), + so the Unit Tests run its ``workflow_run`` consumers read stays green. Two errors remain on a failed KTLO step, neither changing a conclusion: the runner's ``Process completed with exit code N.`` for a ``run:`` step, and an @@ -684,9 +696,10 @@ every leg alike, and masked, the leg would go on to publish. fails one step, with a warning saying why -- ``test`` and ``integration`` at the install step, the ``Compat`` leg at ``Verify package`` -- and concludes ``success``. They run on ``ubuntu-22.04``, which retires on 2027-04-17 after -brownouts from 2027-03-23 (actions/runner-images#14254). A brownout fails a ``test`` or -``integration`` leg before any step runs, where no ``continue-on-error`` reaches, -and so fails ``Required checks passed``: they must go before then (#1660). 3.6 is +brownouts from 2027-03-23 (actions/runner-images#14254). A brownout fails a +``test-ktlo`` or ``integration-ktlo`` leg before any step runs, where no step-level +``continue-on-error`` reaches; since #1738 that fails nothing gating, but the leg's +check run reads red, so they must still go before then (#1660). 3.6 is in no matrix: setup-python builds it only for retired Ubuntu images. Nor are 3.7-3.9 in ``engine-tests``, which installs every interpreter on ``ubuntu-latest``, where setup-python has no Ubuntu 26.04 build of any of them. Nor diff --git a/tests/_dependency_gates.py b/tests/_dependency_gates.py index e0988901b5..da5ad53266 100644 --- a/tests/_dependency_gates.py +++ b/tests/_dependency_gates.py @@ -787,7 +787,7 @@ def _toml_array(text: 'str', name: 'str') -> 'Optional[tuple[Requirement, ...]]' """The ``name = [ ... ]`` array of double-quoted strings in ``text``. A targeted scan rather than :mod:`tomllib`, which is 3.11+ while the oldest - ``test`` leg to run the suite is 3.8 (a KTLO leg, #1633) -- and a guard that skips on + leg to run the suite is 3.8 (``test-ktlo``'s, #1633, #1738) -- and a guard that skips on some legs is the shape of invisibility this whole module exists to remove. Comments are stripped first, quote-aware, because the arrays here are heavily annotated; the brackets inside ``"requests[socks]"`` are what makes @@ -1423,6 +1423,10 @@ def pytest_jobs(workflow: 'Optional[pathlib.Path]' = None) -> 'tuple[Job, ...]': text = (workflow or WORKFLOW).read_text(encoding='utf-8') jobs = [] # type: list[Job] for name, section in job_sections(text).items(): + # A KTLO job (#1738) writes its steps as a YAML alias of its gating + # twin's -- `steps: *test-steps` -- so its slice holds no pytest line and + # is skipped here: it installs and selects what the twin does, so it + # reaches no gate the twin does not. if 'python -m pytest' not in section: continue diff --git a/tests/project/test_ktlo_jobs_gate_nothing_1738.py b/tests/project/test_ktlo_jobs_gate_nothing_1738.py new file mode 100644 index 0000000000..efe9ffa422 --- /dev/null +++ b/tests/project/test_ktlo_jobs_gate_nothing_1738.py @@ -0,0 +1,233 @@ +# -*- coding: utf-8 -*- +"""#1738: unit-tests.yml's KTLO legs are jobs of their own, which nothing gating waits on. + +The owner's option A. Until it, the KTLO legs were legs of ``test``, +``integration`` and ``pypcap-parity``, kept from failing their job by step-level +``continue-on-error`` alone. A failure no step can contain -- a lost runner, the +job cap -- still failed the job, and so ``Required checks passed``, which reads one +``needs..result`` per job, and skipped ``unittest-ordering``, which needs +``test``. So they are ``test-ktlo``, ``integration-ktlo`` and ``pypcap-parity-ktlo`` +now, and this module pins the three things that make that hold: + +* no job that can fail the run -- one without job-level ``continue-on-error`` -- + ``needs:`` a KTLO job, directly or through another job; +* none of those jobs holds a KTLO leg; +* every check name is what it was: rulesets match names literally. + +It also pins that each KTLO job runs its gating twin's steps, keeps the KTLO +mechanics of #1672, and that :file:`util/ci_timing.py` tells the twins apart. + +""" + +from __future__ import annotations + +import importlib.util +import pathlib +import re +import unittest +from typing import TYPE_CHECKING + +from tests.project.test_python_matrix_tiers_1633 import KTLO, KTLO_GUARD, PARITY, REQUIRED, SUPPORTED +from tests.project.test_python_versions_1660 import legs as expand +from tests.project.test_python_versions_1660 import matrix, workflow + +if TYPE_CHECKING: + from typing import Any + +ROOT = pathlib.Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / '.github' / 'workflows' / 'unit-tests.yml' +NAME = 'unit-tests.yml' +#: Each gating job and the job holding its KTLO legs. +TWINS = {'test': 'test-ktlo', 'integration': 'integration-ktlo', 'pypcap-parity': 'pypcap-parity-ktlo'} +#: Each gating job's check-name prefix, as main named its legs before the split. +PREFIX = {'test': 'Python', 'integration': 'Integration Python', 'pypcap-parity': 'PyPCAP/PyPCAPFile parity Python'} +#: The KTLO versions each twin runs; ``pypcap-parity`` runs on ``ubuntu-latest`` and below 3.12 only. +KTLO_OF = {'test': KTLO, 'integration': KTLO, 'pypcap-parity': tuple(v for v in PARITY if v in KTLO)} +SUPPORTED_OF = {'test': SUPPORTED, 'integration': SUPPORTED, + 'pypcap-parity': tuple(v for v in PARITY if v in SUPPORTED)} + + +def jobs() -> 'dict[str, Any]': + return workflow(NAME)['jobs'] + + +def needs(spec: 'dict[str, Any]') -> 'list[str]': + found = spec.get('needs', []) + return [found] if isinstance(found, str) else list(found) + + +def closure(job_id: 'str') -> 'set[str]': + """``job_id`` and every job it waits on, through ``needs:``.""" + found, todo = set(), [job_id] + while todo: + current = todo.pop() + if current not in found: + found.add(current) + todo.extend(needs(jobs()[current])) + return found + + +def matrix_legs(job_id: 'str') -> 'list[dict[str, Any]]': + """The legs of ``job_id``'s matrix, read from the tier file; none for a job without one.""" + if 'matrix' not in jobs()[job_id].get('strategy', {}): + return [] + return expand(matrix(NAME, job_id)) + + +def holds_ktlo(job_id: 'str') -> 'bool': + return any(leg.get('ktlo') is True for leg in matrix_legs(job_id)) + + +def render(template: 'str', leg: 'dict[str, Any]') -> 'str': + """A leg's check name, for the two expression shapes these jobs' names have used.""" + name = template.replace("${{ matrix.ktlo && ' (KTLO)' || '' }}", ' (KTLO)' if leg.get('ktlo') is True else '') + name = name.replace('${{ matrix.python-version }}', leg['python-version']) + assert '${{' not in name, name + return name + + +def gating() -> 'list[str]': + """Every job that can fail the run: each one without job-level ``continue-on-error``.""" + return [job_id for job_id, spec in jobs().items() if spec.get('continue-on-error') is not True] + + +class TestNothingGatingWaitsOnKTLO(unittest.TestCase): + """A KTLO job's failure, of any kind, reaches nothing that gates.""" + + def test_no_gating_job_needs_a_ktlo_job_transitively(self) -> None: + ktlo = {job_id for job_id in jobs() if holds_ktlo(job_id)} + self.assertTrue(ktlo, 'no job holds a KTLO leg') + for job_id in gating(): + with self.subTest(job=job_id): + self.assertEqual(sorted((closure(job_id) - {job_id}) & ktlo), []) + + def test_the_named_gates_wait_on_no_ktlo_job(self) -> None: + """``Required checks passed`` and ``unittest-ordering``, by name, so neither can drop out of :func:`gating`.""" + for job_id in ('required-checks', 'unittest-ordering'): + with self.subTest(job=job_id): + self.assertIn(job_id, gating()) + self.assertEqual(sorted(job for job in closure(job_id) if holds_ktlo(job)), []) + self.assertFalse(set(TWINS.values()) & closure(job_id)) + + def test_no_gating_job_holds_a_ktlo_leg(self) -> None: + for job_id in gating(): + with self.subTest(job=job_id): + self.assertEqual([leg['python-version'] for leg in matrix_legs(job_id) if leg.get('ktlo') is True], []) + + def test_the_gating_jobs_keep_the_supported_legs(self) -> None: + for job_id, versions in SUPPORTED_OF.items(): + with self.subTest(job=job_id): + self.assertIn(job_id, gating()) + self.assertIn(job_id, needs(jobs()['required-checks'])) + self.assertEqual(sorted(leg['python-version'] for leg in matrix_legs(job_id)), sorted(versions)) + + +class TestTheCheckNamesAreUnchanged(unittest.TestCase): + """Each leg is named as it was when both tiers shared a job.""" + + def test_each_leg_keeps_its_name(self) -> None: + for job_id, twin in TWINS.items(): + with self.subTest(job=job_id): + found = {job: sorted(render(jobs()[job]['name'], leg) for leg in matrix_legs(job)) + for job in (job_id, twin)} + self.assertEqual(found, { + job_id: sorted(f'{PREFIX[job_id]} {v}' for v in SUPPORTED_OF[job_id]), + twin: sorted(f'{PREFIX[job_id]} {v} (KTLO)' for v in KTLO_OF[job_id]), + }) + + def test_no_ktlo_name_is_a_required_context(self) -> None: + for twin in TWINS.values(): + with self.subTest(job=twin): + names = {render(jobs()[twin]['name'], leg) for leg in matrix_legs(twin)} + self.assertTrue(names) + self.assertFalse(names & REQUIRED) + + +class TestTheKTLOJobsKeepTheirMechanics(unittest.TestCase): + """#1633 and #1672 hold on each KTLO job as they did on the shared one.""" + + def test_each_runs_its_twins_steps_on_ktlo_legs_only(self) -> None: + for job_id, twin in TWINS.items(): + spec, gate = jobs()[twin], jobs()[job_id] + with self.subTest(job=twin): + self.assertEqual(spec['steps'], gate['steps']) + self.assertEqual(needs(spec), needs(gate)) + self.assertEqual(spec['if'], gate['if']) + self.assertEqual((spec['runs-on'], spec['timeout-minutes']), (gate['runs-on'], gate['timeout-minutes'])) + self.assertEqual(spec['env']['ANNOTATION_LEVEL'], gate['env']['ANNOTATION_LEVEL']) + self.assertTrue(all(leg.get('ktlo') is True for leg in matrix_legs(twin))) + self.assertEqual(spec['steps'][-1]['name'], 'KTLO summary') + self.assertEqual(spec['steps'][-1]['continue-on-error'], KTLO_GUARD) + + def test_each_has_the_job_level_backstop(self) -> None: + for twin in TWINS.values(): + with self.subTest(job=twin): + self.assertIs(jobs()[twin].get('continue-on-error'), True) + + def test_each_reads_its_own_output(self) -> None: + outputs = jobs()['python-versions']['outputs'] + for twin in TWINS.values(): + with self.subTest(job=twin): + self.assertEqual(jobs()[twin]['strategy']['matrix'], + f'${{{{ fromJSON(needs.python-versions.outputs.{twin}) }}}}') + self.assertEqual(outputs[twin], f'${{{{ steps.read.outputs.{twin} }}}}') + + def test_the_steps_are_one_yaml_node(self) -> None: + """An alias, not a copy, so the twins cannot drift: ``steps: &-steps`` and ``steps: *-steps``.""" + text = WORKFLOW.read_text(encoding='utf-8') + for job_id, twin in TWINS.items(): + with self.subTest(job=twin): + self.assertEqual(len(re.findall(rf'(?m)^ steps: &{re.escape(job_id)}-steps$', text)), 1) + self.assertEqual(len(re.findall(rf'(?m)^ steps: \*{re.escape(job_id)}-steps$', text)), 1) + + +def ci_timing() -> 'Any': + spec = importlib.util.spec_from_file_location('ci_timing_1738', ROOT / 'util' / 'ci_timing.py') + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # type: ignore[union-attr] + return module + + +class TestCITimingTellsTheTwinsApart(unittest.TestCase): + """util/ci_timing.py reads the workflow's text, aliases included, and names each leg's job.""" + + @classmethod + def setUpClass(cls) -> None: + cls.module = ci_timing() + cls.specs = cls.module.parse_workflow(WORKFLOW.read_text(encoding='utf-8')) + + def test_a_ktlo_leg_is_its_ktlo_jobs(self) -> None: + for job_id, twin in TWINS.items(): + with self.subTest(job=twin): + ktlo = f'{PREFIX[job_id]} {KTLO_OF[job_id][-1]} (KTLO)' + supported = f'{PREFIX[job_id]} {SUPPORTED_OF[job_id][0]}' + self.assertEqual(self.module.match_job(self.specs, ktlo)['id'], twin) + self.assertEqual(self.module.match_job(self.specs, supported)['id'], job_id) + self.assertTrue(self.module.non_blocking(NAME, ktlo)) + + def test_an_aliased_job_has_its_anchors_steps(self) -> None: + by_id = {spec['id']: spec for spec in self.specs} + for job_id, twin in TWINS.items(): + with self.subTest(job=twin): + self.assertTrue(by_id[job_id]['steps']) + self.assertEqual(by_id[twin]['steps'], by_id[job_id]['steps']) + caps = {step['name']: step['timeout'] for step in by_id['pypcap-parity-ktlo']['steps'] if step['timeout']} + self.assertEqual(caps, {'Plan the system packages': 4, + 'Install libpcap headers, a C toolchain, and tshark': 10}) + + def test_the_critical_path_leaves_the_ktlo_jobs_out(self) -> None: + required = next(spec for spec in self.specs if spec['name'] == self.module.REQUIRED_JOB) + self.assertFalse(set(TWINS.values()) & set(required['needs'])) + # The KTLO leg finishes last, so on main, where it was `test`'s, it led. + run = {'event': 'pull_request', 'conclusion': 'success', 'jobs': [ + {'name': name, 'runner_name': 'GitHub Actions 1', 'conclusion': 'success', + 'started_at': '2026-10-14T00:00:00Z', 'completed_at': end} + for name, end in (('Python 3.15', '2026-10-14T00:20:00Z'), + ('Python 3.10 (KTLO)', '2026-10-14T00:40:00Z'))]} + path = self.module._critical_path({'runs': [run]}, self.specs) # pylint: disable=protected-access + self.assertEqual(path['pull_request']['leader'], 'Python 3.15') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_ktlo_legs_stay_green_1672.py b/tests/project/test_ktlo_legs_stay_green_1672.py index 20754e36f9..a833aa7eea 100644 --- a/tests/project/test_ktlo_legs_stay_green_1672.py +++ b/tests/project/test_ktlo_legs_stay_green_1672.py @@ -59,8 +59,14 @@ KTLO_JOBS = tuple((name, job_id) for name, job_id, _ in KTLO_LEGS) #: The jobs whose steps write annotations of their own on a KTLO leg, so set the level. LEVELLED = (('unit-tests.yml', 'test'), ('unit-tests.yml', 'integration'), - ('unit-tests.yml', 'pypcap-parity'), ('python-compatibility.yml', 'compatibility'), - EXPERIMENTAL_JOB) + ('unit-tests.yml', 'pypcap-parity'), ('unit-tests.yml', 'test-ktlo'), + ('unit-tests.yml', 'integration-ktlo'), ('unit-tests.yml', 'pypcap-parity-ktlo'), + ('python-compatibility.yml', 'compatibility'), EXPERIMENTAL_JOB) +#: unit-tests.yml's KTLO-only jobs (#1738), which nothing gating needs. Their job-level +#: ``continue-on-error: true`` is a backstop for the failures no step contains, on top +#: of the step-level guards, which are what keep a leg's check run green. +BACKSTOPPED = (('unit-tests.yml', 'test-ktlo'), ('unit-tests.yml', 'integration-ktlo'), + ('unit-tests.yml', 'pypcap-parity-ktlo')) #: Steps that stop every leg alike, KTLO or not, by job. #1634's: a leg that cannot #: show its tree is the tag's must not go on to publish, as #: ``test_release_tags_the_run_sha_1634.TestPublishersCheckTheirTags`` pins. @@ -78,6 +84,8 @@ MASKED_ON_EVERY_LEG = { ('unit-tests.yml', 'test'): ('Upload test timings',), ('unit-tests.yml', 'integration'): ('Upload test timings',), + ('unit-tests.yml', 'test-ktlo'): ('Upload test timings',), + ('unit-tests.yml', 'integration-ktlo'): ('Upload test timings',), ('create-release.yml', 'pypi'): ('Publish package to TestPyPI',), } #: ``engine-tests``'s KTLO steps: the KTLO cells, and the setup step of their @@ -276,10 +284,18 @@ class TestNoKTLOFailureTurnsAJobRed(unittest.TestCase): """(a) Whatever step fails on a KTLO leg, the job concludes ``success``.""" def test_no_job_with_a_ktlo_leg_masks_it_at_job_level(self) -> None: - """Job-level, the run passes but the leg's check run concludes ``failure``.""" + """Job-level, the run passes but the leg's check run concludes ``failure``. + + So it is never the mechanism: only :data:`BACKSTOPPED` carries it, as a + backstop on top of the step-level guards the tests below pin (#1738). + + """ for name, job_id in KTLO_JOBS + (('unit-tests.yml', 'engine-tests'),): with self.subTest(workflow=name, job=job_id): - self.assertNotIn('continue-on-error', job(name, job_id)) + if (name, job_id) in BACKSTOPPED: + self.assertIs(job(name, job_id).get('continue-on-error'), True) + else: + self.assertNotIn('continue-on-error', job(name, job_id)) def test_no_step_can_fail_a_ktlo_leg(self) -> None: for name, job_id in KTLO_JOBS: diff --git a/tests/project/test_python_matrix_tiers_1633.py b/tests/project/test_python_matrix_tiers_1633.py index b84d719914..32424533ba 100644 --- a/tests/project/test_python_matrix_tiers_1633.py +++ b/tests/project/test_python_matrix_tiers_1633.py @@ -113,18 +113,19 @@ def tier(impl: 'str', name: 'str') -> 'tuple[str, ...]': #: The jobs a single ``needs..result`` aggregates, so KTLO must stay step-level. AGGREGATED = (('unit-tests.yml', 'test'), ('unit-tests.yml', 'integration'), ('unit-tests.yml', 'pypcap-parity'), ('create-release.yml', 'pypi')) -#: Where each KTLO leg must be, as (workflow, job, KTLO versions). +#: Where each KTLO leg must be, as (workflow, job, KTLO versions). unit-tests.yml's +#: are jobs of their own since #1738, which nothing gating waits on. KTLO_LEGS = ( - ('unit-tests.yml', 'test', KTLO), - ('unit-tests.yml', 'integration', KTLO), - ('unit-tests.yml', 'pypcap-parity', tuple(v for v in PARITY if v in KTLO)), + ('unit-tests.yml', 'test-ktlo', KTLO), + ('unit-tests.yml', 'integration-ktlo', KTLO), + ('unit-tests.yml', 'pypcap-parity-ktlo', tuple(v for v in PARITY if v in KTLO)), ('python-compatibility.yml', 'compatibility', KTLO), ('create-release.yml', 'pypi', KTLO_ON_LATEST + tier('pypy', 'ktlo')), ) #: The jobs 3.7 joins (#1633), each as (workflow, job, install-or-verify step). LEGACY_37 = ( - ('unit-tests.yml', 'test', 'Install package and test dependencies'), - ('unit-tests.yml', 'integration', 'Install package, test and generator dependencies'), + ('unit-tests.yml', 'test-ktlo', 'Install package and test dependencies'), + ('unit-tests.yml', 'integration-ktlo', 'Install package, test and generator dependencies'), ('python-compatibility.yml', 'compatibility', 'Verify package'), ) @@ -176,7 +177,7 @@ def tier(impl: 'str', name: 'str') -> 'tuple[str, ...]': ) PROSE_FILES = ('pyproject.toml', 'README.md', 'SECURITY.md', 'CONTRIBUTING.md', 'Makefile') #: Spelled-out counts, as the workflows page writes them. -WORDS = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, +WORDS = {'zero': 0, 'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, 'six': 6, 'seven': 7, 'eight': 8, 'nine': 9, 'ten': 10, 'eleven': 11, 'twelve': 12, 'thirteen': 13, 'fourteen': 14, 'fifteen': 15} @@ -599,8 +600,8 @@ def test_37_is_a_ktlo_leg_of_exactly_the_three_test_matrices(self) -> None: with self.subTest(workflow=name, job=job_id): found = [leg for leg in legs(name, job_id) if leg['python-version'] == '3.7'] self.assertEqual(found, [{'python-version': '3.7', 'ktlo': True, 'os': 'ubuntu-22.04'}]) - self.assertTrue(spec['name'].endswith("${{ matrix.ktlo && ' (KTLO)' || '' }}")) - joined = {(name, job_id) for name, job_id, _ in MATRICES + self.assertTrue(spec['name'].endswith((' (KTLO)', "${{ matrix.ktlo && ' (KTLO)' || '' }}"))) + joined = {(name, job_id) for name, job_id, _ in MATRICES + KTLO_LEGS if '3.7' in [leg['python-version'] for leg in legs(name, job_id)]} engines = [versions for _, versions in engine_inputs()] if '3.7' in ' '.join(engines).split(): @@ -774,8 +775,11 @@ def test_the_workflows_page_counts_each_matrix_jobs_legs(self) -> None: engines = {version for _, versions in engine_inputs() for version in versions.split()} claims = { r'``test`` \((\w+) Python legs': len(legs('unit-tests.yml', 'test')), + r'``test-ktlo`` \((\w+)\)': len(legs('unit-tests.yml', 'test-ktlo')), r'``integration`` \((\w+)\)': len(legs('unit-tests.yml', 'integration')), + r'``integration-ktlo`` \((\w+)\)': len(legs('unit-tests.yml', 'integration-ktlo')), r'``pypcap-parity`` \((\w+)\)': len(legs('unit-tests.yml', 'pypcap-parity')), + r'``pypcap-parity-ktlo`` \((\w+)\)': len(legs('unit-tests.yml', 'pypcap-parity-ktlo')), r'each looping over (\w+) Python versions': len(engines), } for pattern, count in claims.items(): diff --git a/tests/project/test_python_versions_1660.py b/tests/project/test_python_versions_1660.py index c91ee19aac..a4d848f5f2 100644 --- a/tests/project/test_python_versions_1660.py +++ b/tests/project/test_python_versions_1660.py @@ -268,6 +268,10 @@ def setups(spec: 'dict[str, Any]') -> 'list[dict[str, Any]]': ('cron-conda.yml', 'conda-dist'): 'Conda deployment (update) on ${{ matrix.os }} with Python ${{ matrix.python-version }}', } +#: Each unit-tests.yml job MAIN ran KTLO legs in, and the job those legs run in +#: since #1738, which nothing gating waits on. +KTLO_TWINS = {('unit-tests.yml', 'test'): 'test-ktlo', ('unit-tests.yml', 'integration'): 'integration-ktlo', + ('unit-tests.yml', 'pypcap-parity'): 'pypcap-parity-ktlo'} #: Ruleset 23497679's required contexts at MAIN. REQUIRED_AT_MAIN = {'Required checks passed'} | {f'Compat Python {v}' for v in SUPPORTED_AT_MAIN} #: The tiers MAIN's matrices encode, as this change first wrote them into the file. @@ -565,6 +569,13 @@ def test_every_matrix_expands_to_mains_legs(self) -> None: leg.pop('coverage', None) if leg['python-version'] == TIERS_AT_MAIN['latest']: leg['coverage'] = True + if (name, job_id) in KTLO_TWINS: + # #1738: the KTLO legs, in MAIN's order, are the twin job's. + twin = KTLO_TWINS[(name, job_id)] + with self.subTest(workflow=name, job=twin): + self.assertEqual(legs(matrix(name, twin, self.values)), + [leg for leg in expected if leg.get('ktlo') is True]) + expected = [leg for leg in expected if leg.get('ktlo') is not True] with self.subTest(workflow=name, job=job_id): self.assertEqual(legs(matrix(name, job_id, self.values)), expected) @@ -586,12 +597,22 @@ def test_no_job_name_changes_but_the_gates(self) -> None: """Rulesets match check names literally, so each template and leg name stays. ``test``'s template is :data:`MAIN`'s again since its experimental legs - left it for a job of their own (#1732). + left it for a job of their own (#1732). Since #1738 each job of + :data:`KTLO_TWINS` names its legs with a template of its own, the KTLO + job's ending `` (KTLO)``, so what stays is each leg's rendered name. """ for (name, job_id), template in TEMPLATES_AT_MAIN.items(): with self.subTest(workflow=name, job=job_id): - self.assertEqual(workflow(name)['jobs'][job_id]['name'], template) + if (name, job_id) not in KTLO_TWINS: + self.assertEqual(workflow(name)['jobs'][job_id]['name'], template) + continue + spec = MAIN_MATRICES[(name, job_id)] + before = sorted(render(template, leg) for leg in legs(spec)) + after = sorted(render(workflow(name)['jobs'][job]['name'], leg) + for job in (job_id, KTLO_TWINS[(name, job_id)]) + for leg in legs(matrix(name, job, self.values))) + self.assertEqual(after, before) compat = TEMPLATES_AT_MAIN[('python-compatibility.yml', 'compatibility')] compat_legs = legs(matrix('python-compatibility.yml', 'compatibility', self.values)) blocking = {render(compat, leg) for leg in compat_legs diff --git a/tests/project/test_python_versions_update_1660.py b/tests/project/test_python_versions_update_1660.py index c24b4486c6..c714aadf1f 100644 --- a/tests/project/test_python_versions_update_1660.py +++ b/tests/project/test_python_versions_update_1660.py @@ -619,7 +619,7 @@ def test_a_run_that_changes_nothing_writes_nothing(self) -> None: plan, texts = updater().compute_plan(root, as_of_the_file(snapshot())) self.assertEqual((plan['changes'], texts, plan['notes']), ([], {}, [])) page = ' '.join((root / 'docs/source/contributing/workflows.rst').read_text(encoding='utf-8').split()) - self.assertIn('``test`` (nine Python legs: five supported, four KTLO)', page) + self.assertIn('``test`` (five Python legs), ``test-ktlo`` (four)', page) self.assertIn('``experimental`` (two)', page) def test_a_tier_move_keeps_them_byte_for_byte_after_the_computed_versions(self) -> None: diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py index f76c42c6c9..c77933120f 100644 --- a/tests/project/test_sweeps_on_latest_only_1732.py +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -333,15 +333,22 @@ def test_the_caps(self) -> None: self.assertLess(sum(steps.values()), cap) def test_a_failure_never_turns_the_run_red(self) -> None: - """Job-level ``continue-on-error`` there alone: a ``test`` leg's failure must still fail its gate.""" + """Job-level ``continue-on-error`` there and on the KTLO jobs alone: a ``test`` leg's failure must still fail its gate. + + unit-tests.yml's KTLO jobs carry it too since #1738, which nothing gating + ``needs:`` (``tests/project/test_ktlo_jobs_gate_nothing_1738.py``). + + """ spec = self.own['jobs']['experimental'] self.assertIs(spec.get('continue-on-error'), True) self.assertEqual(spec['env'], {'ANNOTATION_LEVEL': 'warning'}) self.assertEqual(spec['steps'][-1]['name'], 'Experimental summary') + expected = {'experimental.yml': ['experimental'], + 'unit-tests.yml': ['test-ktlo', 'integration-ktlo', 'pypcap-parity-ktlo']} for name in ('unit-tests.yml', 'experimental.yml'): with self.subTest(workflow=name): self.assertEqual([job_id for job_id, entry in workflow(name)['jobs'].items() - if 'continue-on-error' in entry], ['experimental'] if name == 'experimental.yml' else []) + if 'continue-on-error' in entry], expected[name]) def test_it_installs_what_test_installs(self) -> None: """So tests/_dependency_gates.py, which reads unit-tests.yml alone, loses no gate by its leaving.""" diff --git a/tests/project/test_workflow_apt_cache.py b/tests/project/test_workflow_apt_cache.py index 8f8bd5889b..44c052424a 100644 --- a/tests/project/test_workflow_apt_cache.py +++ b/tests/project/test_workflow_apt_cache.py @@ -314,14 +314,22 @@ class TestOneSaverPerKey(unittest.TestCase): """Two jobs saving one key race, and the loser logs ``Unable to reserve cache``.""" def test_only_one_parity_cell_saves(self) -> None: - """APT_CACHE_SAVE selects exactly one of the parity matrix's Python versions.""" + """APT_CACHE_SAVE selects exactly one of the parity matrices' Python versions. + + Since #1738 the KTLO legs are ``pypcap-parity-ktlo``'s, under the same + ``APT_SET`` and so the same key, and that job never saves. + + """ value = job_env('pypcap-parity', 'APT_CACHE_SAVE') match = re.fullmatch(r"\$\{\{ matrix\.python-version == '([\d.]+)' \}\}", value) self.assertIsNotNone(match, value) - # The parity matrix's legs, from .github/python-versions.toml (#1660). + # The parity matrices' legs, from .github/python-versions.toml (#1660). versions = [leg['python-version'] for leg in legs(matrix('unit-tests.yml', 'pypcap-parity'))] self.assertIn(match.group(1), versions) # type: ignore[union-attr] - self.assertGreater(len(versions), 1) + self.assertEqual(job_env('pypcap-parity-ktlo', 'APT_CACHE_SAVE'), "'false'") + self.assertEqual(job_env('pypcap-parity-ktlo', 'APT_SET'), job_env('pypcap-parity', 'APT_SET')) + ktlo = [leg['python-version'] for leg in legs(matrix('unit-tests.yml', 'pypcap-parity-ktlo'))] + self.assertGreater(len(versions + ktlo), 1) def test_each_engine_job_saves_its_own_key(self) -> None: """engine-tests has one job per APT_SET, so each saves.""" diff --git a/util/ci_timing.py b/util/ci_timing.py index ad7a13a142..5d22866f75 100644 --- a/util/ci_timing.py +++ b/util/ci_timing.py @@ -301,10 +301,13 @@ def parse_workflow(text: 'str') -> 'list[dict[str, Any]]': Read off the text by indentation, the way the tests under :file:`tests/project` read these files, because PyYAML is not in the standard library. Every workflow - here keeps job keys at four spaces, which is all this relies on. + here keeps job keys at four spaces, which is all this relies on. A job whose + ``steps:`` is a YAML alias, as unit-tests.yml's KTLO jobs are (#1738), gets the + steps of the job that anchors them, which comes first, as YAML needs. """ jobs = [] # type: list[dict[str, Any]] + anchors = {} # type: dict[str, list[dict[str, Any]]] body = re.split(r'(?m)^jobs:\s*$', text, maxsplit=1) if len(body) != 2: return jobs @@ -316,6 +319,12 @@ def parse_workflow(text: 'str') -> 'list[dict[str, Any]]': needed = [] if needs is None else [ item.strip() for item in _scalar(needs.group(1)).strip('[]').split(',') if item.strip()] template = _scalar(name.group(1)) if name else job_id + steps = _parse_steps(section) + node = re.search(r'(?m)^ steps:[ \t]*([&*])([\w-]+)[ \t]*$', section) + if node and node.group(1) == '&': + anchors[node.group(2)] = steps + elif node: + steps = anchors.get(node.group(2), []) jobs.append({ 'id': job_id, 'name': template, @@ -325,7 +334,7 @@ def parse_workflow(text: 'str') -> 'list[dict[str, Any]]': 'unnamed': name is None, 'timeout': int(timeout.group(1)) if timeout else DEFAULT_TIMEOUT, 'needs': needed, - 'steps': _parse_steps(section), + 'steps': steps, }) return jobs @@ -341,7 +350,7 @@ def _parse_steps(section: 'str') -> 'list[dict[str, Any]]': found = [] # type: list[dict[str, Any]] lines = section.splitlines() try: - start = next(i for i, line in enumerate(lines) if re.match(r'^ steps:\s*$', line)) + start = next(i for i, line in enumerate(lines) if re.match(r'^ steps:(?:[ \t]*&[\w-]+)?\s*$', line)) except StopIteration: return found items = [i for i in range(start + 1, len(lines)) if re.match(r'^ {4,}- ', lines[i])] @@ -367,13 +376,21 @@ def _parse_steps(section: 'str') -> 'list[dict[str, Any]]': def match_job(specs: 'list[dict[str, Any]]', name: 'str') -> 'Optional[dict[str, Any]]': - """The spec a job name in the API came from; a literal name wins over a template.""" + """The spec a job name in the API came from; a literal name wins over a template. + + Of the templates that match, the one with the most literal text wins: + ``Python ${{ matrix.python-version }}`` matches ``Python 3.10 (KTLO)`` too, + which is ``Python ${{ matrix.python-version }} (KTLO)``'s, ``test-ktlo``'s + and not ``test``'s (#1738). + + """ for spec in specs: if spec['literal'] and spec['name'] == name: return spec + matched = [spec for spec in specs if spec['pattern'].fullmatch(name)] + if matched: + return max(matched, key=lambda spec: len(re.sub(r'\$\{\{.*?\}\}', '', spec['name']))) for spec in specs: - if spec['pattern'].fullmatch(name): - return spec if spec['unnamed'] and re.fullmatch(rf'{re.escape(spec["id"])} \(.*\)', name): return spec return None diff --git a/util/python_versions.py b/util/python_versions.py index fe2b5f237f..98d66370d9 100644 --- a/util/python_versions.py +++ b/util/python_versions.py @@ -13,17 +13,22 @@ ``latest`` The newest supported CPython, for the single-interpreter jobs. -``test``, ``integration``, ``compatibility`` +``compatibility`` Both tiers. A KTLO leg carries ``ktlo: true``, and ``os`` where - ``ubuntu-latest`` cannot provide it; ``test`` marks the ``latest`` leg - ``coverage: true``. + ``ubuntu-latest`` cannot provide it. +``test``, ``integration`` + The supported tier alone; ``test`` marks the ``latest`` leg ``coverage: true``. +``test-ktlo``, ``integration-ktlo``, ``pypcap-parity-ktlo`` + Each KTLO version as a leg carrying ``ktlo: true``, and ``os`` as above, for + unit-tests.yml's jobs of the same names, which nothing gating waits on (#1738). + No other unit-tests.yml output holds a KTLO version. ``experimental`` Each experimental version as a leg carrying ``experimental: true``, for experimental.yml's ``experimental`` job, a workflow of its own so that nothing waits on it (#1732). No other output holds an experimental version. ``pypcap-parity`` - The versions below 3.12 that run on ``ubuntu-latest``. + The supported versions below 3.12. ``pypi`` CPython, then PyPy, both tiers, on ``ubuntu-latest``. ``conda`` @@ -187,15 +192,41 @@ def on_latest(data: 'dict[str, Any]', impl: 'str', name: 'str', job: 'str') -> ' return found -def tiered(data: 'dict[str, Any]', impl: 'str' = 'cpython') -> 'dict[str, Any]': - """The supported tier as the axis, each KTLO version added by ``include``.""" +def ktlo_legs(data: 'dict[str, Any]', impl: 'str' = 'cpython') -> 'list[dict[str, Any]]': + """Each KTLO version of ``impl`` as a leg, with the image it needs as ``os``.""" include = [] for version in tier(data, impl, 'ktlo'): leg = {'python-version': version, 'ktlo': True} # type: dict[str, Any] if runner(data, impl, version) is not None: leg['os'] = runner(data, impl, version) include.append(leg) - return {'python-version': tier(data, impl, 'supported'), 'include': include} + return include + + +def tiered(data: 'dict[str, Any]', impl: 'str' = 'cpython') -> 'dict[str, Any]': + """python-compatibility.yml's ``compatibility``: the supported tier as the axis, each KTLO version added by ``include``.""" + return {'python-version': tier(data, impl, 'supported'), 'include': ktlo_legs(data, impl)} + + +def supported_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': + """unit-tests.yml's ``integration``: the supported tier alone, its KTLO legs ``integration-ktlo``'s (#1738).""" + return {'python-version': tier(data, 'cpython', 'supported'), 'include': []} + + +def legs_matrix(found: 'list[dict[str, Any]]') -> 'dict[str, Any]': + """``found`` in the shape of every other matrix here: a ``python-version`` axis that ``include`` marks.""" + return {'python-version': [leg['python-version'] for leg in found], 'include': found} + + +def ktlo_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': + """unit-tests.yml's ``test-ktlo`` and ``integration-ktlo``: the KTLO legs alone, out of the gating jobs (#1738). + + ``Required checks passed`` reads one ``needs..result`` per job, and + ``unittest-ordering`` needs ``test``, so a KTLO leg that fails as a job -- a + lost runner, or the job cap -- failed both from inside ``test``. + + """ + return legs_matrix(ktlo_legs(data)) def experimental_legs(data: 'dict[str, Any]') -> 'list[dict[str, Any]]': @@ -211,8 +242,8 @@ def experimental_legs(data: 'dict[str, Any]') -> 'list[dict[str, Any]]': def test_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': - """unit-tests.yml's ``test``: both tiers, and the one coverage leg on ``latest``.""" - matrix = tiered(data) + """unit-tests.yml's ``test``: the supported tier, and the one coverage leg on ``latest``.""" + matrix = supported_matrix(data) matrix['include'].insert(0, {'python-version': data['latest'], 'coverage': True}) return matrix @@ -227,20 +258,23 @@ def experimental_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': ``include`` marks, which keys each leg by its version. """ - found = experimental_legs(data) - return {'python-version': [leg['python-version'] for leg in found], 'include': found} + return legs_matrix(experimental_legs(data)) def parity_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': - """unit-tests.yml's ``pypcap-parity``: every version below :data:`PARITY_BELOW` it can run. + """unit-tests.yml's ``pypcap-parity``: every supported version below :data:`PARITY_BELOW`. - The axis lists both tiers and ``include`` marks the KTLO ones, as the job did. + Its KTLO ones are ``pypcap-parity-ktlo``'s (#1738). """ supported = [v for v in on_latest(data, 'cpython', 'supported', 'pypcap-parity') if key(v) < PARITY_BELOW] + return {'python-version': supported, 'include': []} + + +def parity_ktlo_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': + """unit-tests.yml's ``pypcap-parity-ktlo``: each KTLO version below :data:`PARITY_BELOW` it can run (#1738).""" ktlo = [v for v in on_latest(data, 'cpython', 'ktlo', 'pypcap-parity') if key(v) < PARITY_BELOW] - return {'python-version': sorted(ktlo + supported, key=key), - 'include': [{'python-version': version, 'ktlo': True} for version in ktlo]} + return legs_matrix([{'python-version': version, 'ktlo': True} for version in ktlo]) def pypi_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': @@ -261,10 +295,13 @@ def wheel_tag(version: 'str') -> 'str': OUTPUTS = { 'latest': lambda data: data['latest'], 'test': test_matrix, + 'test-ktlo': ktlo_matrix, 'experimental': experimental_matrix, - 'integration': tiered, + 'integration': supported_matrix, + 'integration-ktlo': ktlo_matrix, 'compatibility': tiered, 'pypcap-parity': parity_matrix, + 'pypcap-parity-ktlo': parity_ktlo_matrix, 'pypi': pypi_matrix, 'conda': lambda data: tier(data, 'cpython', 'supported'), 'engine-supported': lambda data: '\n'.join(on_latest(data, 'cpython', 'supported', 'engine-tests')), diff --git a/util/update_python_versions.py b/util/update_python_versions.py index c8325bc111..30b71800fd 100644 --- a/util/update_python_versions.py +++ b/util/update_python_versions.py @@ -436,6 +436,25 @@ def leg_names(workflows: 'dict[str, Any]', values: 'dict[str, str]') -> 'dict[tu return found +#: The suffix of a job holding only the KTLO legs of the job named without it (#1738). +KTLO_JOB_SUFFIX = '-ktlo' + + +def with_ktlo_twins(names: 'dict[tuple[str, str], dict[tuple, str]]') -> 'dict[tuple[str, str], dict[tuple, str]]': + """:func:`leg_names` with each KTLO job's legs under its gating twin's, as one job. + + A version moving to KTLO moves from ``test`` to ``test-ktlo`` (#1738); keyed + together, its check run reads as renamed, as when both tiers shared a job. + + """ + found = {} # type: dict[tuple[str, str], dict[tuple, str]] + for (name, job_id), legs in names.items(): + if job_id.endswith(KTLO_JOB_SUFFIX): + job_id = job_id[:-len(KTLO_JOB_SUFFIX)] + found.setdefault((name, job_id), {}).update(legs) + return found + + def required_contexts(workflows: 'dict[str, Any]', values: 'dict[str, str]') -> 'list[str]': """Ruleset 23497679's required contexts for ``values``: the aggregate, and :data:`REQUIRED_JOB`'s blocking legs. @@ -901,12 +920,15 @@ def counted(count: 'int', ends: 'Optional[tuple[Any, Any]]') -> 'Optional[tuple[ lambda f: counted(len(f['ktlo']), lines_of(f, 'ktlo'))), Claim(r'and the (\w+) ``Compat Python (3\.\d+)``-``(3\.\d+)`` legs', lambda f: counted(len(f['required']) - 1, span(f['supported']))), - Claim(r'``test`` \((\w+) Python legs: (\w+) supported, (\w+) KTLO\)', - lambda f: (Word(f['legs']['test']), Word(len(f['supported'])), Word(len(f['ktlo'])))), + # #1738: each gating job's legs, then its KTLO twin's. + Claim(r'``test`` \((\w+) Python legs\)', lambda f: (Word(f['legs']['test']),)), + Claim(r'``test-ktlo`` \((\w+)\)', lambda f: (Word(f['legs']['test-ktlo']),)), Claim(r'``experimental`` \((\w+)\)', lambda f: (Word(f['legs']['experimental']),)), Claim(r'``integration`` \((\w+)\)', lambda f: (Word(f['legs']['integration']),)), + Claim(r'``integration-ktlo`` \((\w+)\)', lambda f: (Word(f['legs']['integration-ktlo']),)), Claim(r'each looping over (\w+) Python versions', lambda f: (Word(f['engines']),)), Claim(r'``pypcap-parity`` \((\w+)\)', lambda f: (Word(f['legs']['pypcap-parity']),)), + Claim(r'``pypcap-parity-ktlo`` \((\w+)\)', lambda f: (Word(f['legs']['pypcap-parity-ktlo']),)), Claim(r'the ``Python (3\.\d+)`` leg of ``test`` \(matrix key ``coverage: true``\)', lambda f: (f['latest'],)), ), @@ -1105,7 +1127,8 @@ def compute_plan(root: 'pathlib.Path', sources: 'dict[str, Any]') -> 'tuple[dict texts[path] = text changed = [path for path in TARGETS if texts[path] != (root / path).read_bytes().decode('utf-8')] - names_before, names_after = leg_names(workflows, before_values), leg_names(workflows, after_values) + names_before = with_ktlo_twins(leg_names(workflows, before_values)) + names_after = with_ktlo_twins(leg_names(workflows, after_values)) renamed, added, removed = [], [], [] for job in sorted(set(names_before) | set(names_after)): old, new = names_before.get(job, {}), names_after.get(job, {})