From 296ad6b714b93305239219185a82b97cd3a8bc76 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 14 Oct 2026 09:50:54 -0400 Subject: [PATCH] ci(python): run GraalPy and free-threaded CPython as experimental, non-blocking test legs (#1660) - python-versions.toml: an `experimental` tier, listed last in its table: free-threaded "3.14t" and a GraalPy table with "graalpy-25.4", spelt as setup-python's python-version reads them. - python_versions.py: load() takes the tier for those two builds only, and refuses them any other tier until they have a wheel, a conda build and a Compat leg; `test` gains one leg each with `experimental: true`, and no other output (integration, Compat, parity, engines, wheels, conda) moves. - unit-tests.yml: the legs are named `Python (experimental)` and never block, by the KTLO mechanics: the step guard holds on both tiers, annotations are warnings, and an `Experimental summary` step reports each failed step. No Compat leg is experimental, so none is a required context. Their setup-python, install and test steps are capped at 5, 10 and 25 minutes, so a slow leg fails a step instead of being cancelled at the job's 45; every other leg's caps are 360, so its job cap binds as before. - ci_timing.py: a step cap written as an expression reads as none. - update_python_versions.py: experimental entries are hand-maintained; every recompute keeps their lines as written, after the computed versions, and the proposal fingerprint leaves them out. - workflows.rst: the tiers section, the step caps and the `test` leg count; unit-tests.yml line citations there and in releasing.rst follow the lines. Tests: tests/project/test_python_versions_1660, _matrix_tiers_1633, _ktlo_legs_stay_green_1672, _python_versions_update_1660, _unit_tests_queue, _ci_timing and _release_gates pass; the new ones fail on origin/main. Part of #1660 --- .github/python-versions.toml | 21 +- .github/workflows/unit-tests.yml | 65 +++++- docs/source/contributing/releasing.rst | 4 +- docs/source/contributing/workflows.rst | 65 ++++-- tests/project/test_ci_timing.py | 11 + .../project/test_ktlo_legs_stay_green_1672.py | 217 +++++++++++++++++- .../project/test_python_matrix_tiers_1633.py | 63 ++++- tests/project/test_python_versions_1660.py | 146 +++++++++++- .../test_python_versions_update_1660.py | 94 +++++++- tests/project/test_release_gates.py | 20 +- util/ci_timing.py | 12 +- util/python_versions.py | 81 +++++-- util/update_python_versions.py | 45 +++- 13 files changed, 754 insertions(+), 90 deletions(-) diff --git a/.github/python-versions.toml b/.github/python-versions.toml index e87cdff8d..378375b76 100644 --- a/.github/python-versions.toml +++ b/.github/python-versions.toml @@ -4,11 +4,11 @@ # the classifiers and docs to it. cron-python-versions.yml recomputes it weekly # and proposes each change, and the ruleset 23497679 edit it needs, as a PR. # -# Tiers: supported (blocking), ktlo (end-of-life, non-blocking) and pre-release -# (non-blocking; none listed today). `runner` is the image a version needs when -# ubuntu-latest cannot provide it. `latest` is the newest supported CPython, -# which the single-interpreter jobs run on. See -# docs/source/contributing/workflows.rst. +# Tiers: supported (blocking), ktlo (end-of-life, non-blocking), pre-release +# (non-blocking; none listed today) and experimental (non-blocking `test` legs; +# see below). `runner` is the image a version needs when ubuntu-latest cannot +# provide it. `latest` is the newest supported CPython, which the +# single-interpreter jobs run on. See docs/source/contributing/workflows.rst. latest = "3.15" @@ -22,11 +22,22 @@ latest = "3.15" "3.13" = { tier = "supported" } "3.14" = { tier = "supported" } "3.15" = { tier = "supported" } +"3.14t" = { tier = "experimental" } [versions.pypy] "pypy3.10" = { tier = "ktlo" } "pypy3.11" = { tier = "supported" } +# Experimental (#1660): free-threaded CPython (a `t` version, above) and +# GraalPy, by its release line, spelt as setup-python reads them. Each runs as +# one non-blocking leg of unit-tests.yml's `test` job, named `(experimental)`, +# and in no other matrix: no wheel, conda build or required context. The weekly +# job has no source for either, so it keeps these lines as written, after the +# other versions of their table. Promoting one once its leg is green is a hand +# edit, which util/python_versions.py refuses until it has those three. +[versions.graalpy] +"graalpy-25.4" = { tier = "experimental" } + # The pins kept off the tiers, each with its reason in the comment above it. # tests/project/test_python_versions_1660.py fails on a pin not listed here, on # an entry that matches no pin, and on an entry without a comment. diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 510168bee..689dc0e80 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -89,15 +89,16 @@ jobs: # this workflow used to) therefore ran the whole matrix in every caller # and skipped the gate everywhere. An input is the only reliable way for a # reusable workflow to know how it was invoked. - name: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO)' || '' }} + name: Python ${{ matrix.python-version }}${{ matrix.experimental && ' (experimental)' || '' }}${{ matrix.ktlo && ' (KTLO)' || '' }} if: ${{ inputs.gate-only != true && needs.changes.outputs.code != 'false' }} needs: [changes, python-versions] runs-on: ${{ matrix.os || 'ubuntu-latest' }} timeout-minutes: 45 # #1672: the level of every annotation a failing step writes -- a warning on - # a KTLO leg, which must not show red, an error on a supported one. + # a KTLO or experimental leg, which must not show red, an error on a + # supported one. env: - ANNOTATION_LEVEL: ${{ matrix.ktlo && 'warning' || 'error' }} + ANNOTATION_LEVEL: ${{ (matrix.ktlo || matrix.experimental) && 'warning' || 'error' }} strategy: fail-fast: false # #1660: .github/python-versions.toml's CPython tiers, through the @@ -132,16 +133,34 @@ jobs: # 2027-03-23, retirement on 2027-04-17 (actions/runner-images#14254). # A brownout fails the leg before any step runs, which no step-level # `continue-on-error` can contain, so this leg must go before then. + # + # The experimental tier (#1660): free-threaded CPython and GraalPy, one + # leg each with `experimental: true`, here and in no other job. They + # never block, by the KTLO mechanics: the same step-level guard, and + # `Experimental summary` reports each failed step as a warning. Nor can + # they be a required context, as no `Compat` leg is experimental. + # + # Neither build's runtime here is measured, and a leg past this job's + # `timeout-minutes` is cancelled, which no step-level guard contains and + # `required-checks` refuses. So on these legs alone, setup-python, the + # install and the tests carry caps of their own, 5, 10 and 25 minutes: a + # step past its cap fails rather than cancels, and `continue-on-error` + # then applies as to any failure (actions/runner + # src/Runner.Worker/StepsRunner.cs:322-353, at 5f0b3ddcc). That leaves 5 + # minutes for the rest, which took at most 0.32, setup-python included, + # on every uncovered leg of runs 38569321543-38569522442. Every other + # leg's caps are 360, the documented maximum, so the job's 45 binds first. matrix: ${{ fromJSON(needs.python-versions.outputs.test) }} steps: - uses: actions/checkout@v7 id: checkout - continue-on-error: ${{ matrix.ktlo == true }} + continue-on-error: ${{ matrix.ktlo == true || matrix.experimental == true }} - uses: actions/setup-python@v7 id: python - continue-on-error: ${{ matrix.ktlo == true }} + continue-on-error: ${{ matrix.ktlo == true || matrix.experimental == true }} + timeout-minutes: ${{ matrix.experimental && 5 || 360 }} with: python-version: ${{ matrix.python-version }} cache: pip @@ -183,14 +202,15 @@ jobs: # than failing against a missing package. The install needs its own # interpreter: if setup-python failed, the image's `python` would be # tested under this leg's name. On a supported leg either fails the job; - # on a KTLO leg the annotation is a warning (#1672). + # on a KTLO or experimental leg the annotation is a warning (#1672). - name: Install package and test dependencies id: install if: ${{ steps.python.outcome == 'success' }} - continue-on-error: ${{ matrix.ktlo == true }} + continue-on-error: ${{ matrix.ktlo == true || matrix.experimental == true }} + timeout-minutes: ${{ matrix.experimental && 10 || 360 }} shell: bash env: - LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} + LEG: Python ${{ matrix.python-version }}${{ matrix.experimental && ' (experimental, non-blocking)' || '' }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} run: | python -m pip install -U pip setuptools wheel if ! python -m pip install -e '.[test,DPKT,crypto,NGAP]' 2>&1 | tee "$RUNNER_TEMP/pip-install.log"; then @@ -212,7 +232,7 @@ jobs: - name: Report available parallelism id: parallelism if: ${{ steps.install.outcome == 'success' }} - continue-on-error: ${{ matrix.ktlo == true }} + continue-on-error: ${{ matrix.ktlo == true || matrix.experimental == true }} run: | nproc python -c "import os; print('cpu_count', os.cpu_count())" @@ -220,7 +240,8 @@ jobs: - name: Run unit tests id: tests if: ${{ steps.install.outcome == 'success' }} - continue-on-error: ${{ matrix.ktlo == true }} + continue-on-error: ${{ matrix.ktlo == true || matrix.experimental == true }} + timeout-minutes: ${{ matrix.experimental && 25 || 360 }} # `--dist load` is pinned explicitly rather than left to default: a # future change to xdist's own default, or someone "optimising" to # `loadfile`/`loadscope`, cannot silently change how this suite is @@ -373,6 +394,27 @@ jobs: path: ${{ runner.temp }}/junit.xml retention-days: 14 + # #1660: an experimental leg's report, as `KTLO summary` below is a KTLO + # leg's, and for the same reasons. Its own step, so that one stays a + # single script shared by every KTLO job; it is skipped on a KTLO leg. + - name: Experimental summary + if: ${{ !cancelled() && matrix.experimental == true }} + continue-on-error: ${{ matrix.experimental == true }} + shell: bash + env: + LEG: Python ${{ matrix.python-version }} (experimental, non-blocking) + STEPS: ${{ toJSON(steps) }} + run: | + failed=$(jq -r 'to_entries[] | select(.value.outcome == "failure") | .key' <<< "$STEPS") + if [ -z "$failed" ]; then + echo "$LEG: no step failed." + exit 0 + fi + while IFS= read -r id; do + echo "::warning title=${id} failed on ${LEG//,/%2C}::Experimental legs never block (#1660), so this job still succeeds; see the ${id} step's log." + echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" + done <<< "$failed" + # #1672: the KTLO leg's report, and the last step of every KTLO matrix # job, in one script. Each step above that can fail on a KTLO leg has an # `id` and `continue-on-error: ${{ matrix.ktlo == true }}`, so its failure @@ -1822,7 +1864,8 @@ jobs: # sees such a job's failure as `failure` or `success`, so a KTLO-only # regression could block every merge. The one failure no step can contain is # a leg that never gets a runner: the 3.7 legs' deprecated ubuntu-22.04 image - # has brownouts from 2027-03-23 -- see the `test` job's matrix. + # has brownouts from 2027-03-23 -- see the `test` job's matrix. `test`'s + # experimental legs (#1660) are non-blocking the same way. # # The docs-only path. On a pull request whose diff touches nothing but # `docs/**` and `*.md` files outside the code directories (the `changes` diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index 205e8a92d..0c0154c5e 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -52,7 +52,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:467-481``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1679-1749``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1721-1791``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:500,685,850,1085``), so a @@ -62,7 +62,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Two other checks run unconditionally on the same path, though neither is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1527-1546``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1569-1588``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index d193074cd..56584d604 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -285,10 +285,10 @@ repository target the same reusable workflow, with the same input: not one: ``gate`` (the full suite, on ``latest``), ``changelog`` (checks ``CHANGELOG.md`` against its source entry) and ``python-versions``, which reads ``gate``'s version (see `Python Versions`_ below). It skips the other -**eight** -- the five matrix jobs, ``test`` (nine Python legs: five -supported, four KTLO), ``integration`` (nine), ``engine-tests`` (one job -for each of five engines, each looping over six Python versions), -``pypcap-parity`` (two) and ``unittest-ordering`` +**eight** -- the five matrix jobs, ``test`` (eleven Python legs: five +supported, four KTLO, two experimental), ``integration`` (nine), +``engine-tests`` (one job for each of five engines, each looping over six +Python versions), ``pypcap-parity`` (two) and ``unittest-ordering`` (eleven matrix cells), each of which has already run once for this commit from Unit Tests' own ``push`` trigger (and, all but ``unittest-ordering``, which runs on ``main`` pushes only, from its ``pull_request`` trigger), so @@ -298,7 +298,7 @@ and ``project-tests``, and ``required-checks`` (see `Required Status Checks`_ below), gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1515-1521``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1557-1563``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -444,7 +444,9 @@ says how many runs each sample holds and which span they cover. That is about 62 calls (estimate) of ``GITHUB_TOKEN``'s 1,000 an hour, on a read-only token (``actions: read``, ``contents: read``). A job is measured by its own outcome: one cancelled at 95% of its cap or later timed out and counts, even in a run that reads -``cancelled`` for that reason; one cancelled earlier was superseded and does not. +``cancelled`` for that reason; one cancelled earlier was superseded and does not. A +step cap written as an expression, as three of ``test``'s are (#1660), differs by +leg and reads as none. The report goes to the job summary and to the ``ci-timing-report`` artifact (90 days), and the newest earlier report is the next run's baseline. It is not a ``workflow_run`` on Unit Tests, which would add a job per run to a runner pool @@ -566,7 +568,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:1841`` + - ``unit-tests.yml:1884`` * - ``Compat Python 3.11`` - job ``compatibility``, matrix leg ``3.11`` - ``python-compatibility.yml:60``, ``.github/python-versions.toml:20`` @@ -588,21 +590,21 @@ which added ``Compat Python 3.15`` and removed ``Compat Python 3.10``. Moving a between the tiers below means editing the ruleset in the same change. ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:1841``). ``Compat Python`` is defined by one, the +(``unit-tests.yml:1884``). ``Compat Python`` is defined by one, the ``compatibility`` job in ``python-compatibility.yml`` (``:60``), whose matrix (``:74``) comes from :file:`.github/python-versions.toml`: the five required legs are its supported tier (``python-versions.toml:20-24``), and its four KTLO legs (``python-versions.toml:16-19``) have names ending in ``(KTLO)``, so no required context matches one. A comment at -``unit-tests.yml:1752`` mentions the string without defining it. +``unit-tests.yml:1794`` mentions the string without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the five ``Compat Python 3.11``-``3.15`` legs, nothing else, as -``unit-tests.yml``'s own comment (``unit-tests.yml:1751-1761``) records. It +``unit-tests.yml``'s own comment (``unit-tests.yml:1793-1803``) records. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- ``test``, ``integration``, ``engine-tests`` and ``pypcap-parity``, plus ``changes`` and ``project-tests`` for the docs-only path -- and checks each result explicitly -(``unit-tests.yml:1840-1894``). Rulesets match check names literally, with no +(``unit-tests.yml:1883-1937``). Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. It accepts the four legs as ``skipped`` only on a docs-only pull request, where @@ -611,12 +613,12 @@ instead. The ``Compat`` legs stay required separately because they come from ``python-compatibility.yml``, and a job cannot ``needs:`` a job in another workflow file. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1843``), +``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1886``), so it is never produced -- and never expected -- on those paths. -Python versions come in two tiers (#1633), declared in -:file:`.github/python-versions.toml` (see `Python Versions`_). The **supported** -tier, 3.11-3.15, blocks: a failed leg fails ``Required checks passed`` or its own +Python versions come in two tiers (#1633), and an experimental one (#1660, +below), declared in :file:`.github/python-versions.toml` (see `Python Versions`_). +The **supported** tier, 3.11-3.15, blocks: a failed leg fails ``Required checks passed`` or its own required ``Compat`` context. The **KTLO** tier -- end-of-life 3.7-3.10, still built and tested -- never blocks, is never a required context, and never shows a commit or a pull request red (#1672). Each KTLO leg shares its job with supported legs, in @@ -624,7 +626,7 @@ pull request red (#1672). Each KTLO leg shares its job with supported legs, in every such job works the same way: * Every step that can fail on a KTLO leg has an ``id`` and - ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:115-125``), + ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:116-126``), unless it is masked on every leg, as the timings uploads are. Its ``outcome`` is then ``failure`` but its ``conclusion`` ``success``, so the job, its check run and ``required-checks`` read ``success``. A step that needs an earlier one's work @@ -657,14 +659,35 @@ in no matrix: setup-python builds it only for retired Ubuntu images. Nor are are they in ``pypcap-parity``, which also runs on ``ubuntu-latest``; none of them was measured on that job. +The **experimental** tier is free-threaded CPython and GraalPy, spelt as +setup-python reads them (``3.14t``, ``graalpy-25.4``). Each runs one leg of +``test``, named ``Python (experimental)``, and nothing else: no +``integration``, ``Compat``, wheel or conda leg, so never a required context. The +legs never block, by the KTLO mechanics above: ``test``'s guard is +``${{ matrix.ktlo == true || matrix.experimental == true }}``, their annotations +are warnings, and ``Experimental summary``, the step before ``KTLO summary``, +reports each failed step. A leg past the job's 45-minute ``timeout-minutes`` would +be cancelled, which no step-level guard contains and ``Required checks passed`` +refuses, and neither build's runtime here is measured. So on these legs alone, +setup-python, the install and the tests have caps of their own, of 5, 10 and 25 +minutes: a step at its cap fails rather than cancels, and the guard holds +(actions/runner ``src/Runner.Worker/StepsRunner.cs:322-353``). The runner's +``has timed out`` error stays, changing no conclusion, as ``Process completed with +exit code N.`` does. Everything but the install and the tests took under 20 +seconds on every uncovered leg of runs 38569321543-38569522442. Every other leg's +caps are 360, the documented maximum, so its job's 45 minutes bind first, as with +none. Promoting one whose leg stays green is a hand edit, +which :file:`util/python_versions.py` refuses until the build has a wheel, a conda +build and a ``Compat`` leg. + Python Versions --------------- Since `#1660 `__, :file:`.github/python-versions.toml` declares every Python version CI runs: one -line per version, with its tier -- ``supported``, ``ktlo``, or ``pre-release``, -which none is today -- and, where ``ubuntu-latest`` cannot provide it, the -``runner`` it needs. ``latest`` names the newest supported CPython, which every +line per version, with its tier -- ``supported``, ``ktlo``, ``experimental``, or +``pre-release``, which none is today -- and, where ``ubuntu-latest`` cannot +provide it, the ``runner`` it needs. ``latest`` names the newest supported CPython, which every job on one interpreter runs. :file:`util/python_versions.py` reads the file with the runner's own ``python3`` -- its ``tomllib`` needs 3.11, and every image that runs it has 3.12 or newer -- and writes each matrix as JSON: a ``python-versions`` @@ -704,7 +727,9 @@ commits it did not make: when the branch holds some and proposes other tiers, ``detect`` fails before asking for the approval. Once that pull request is merged or closed, the next run with something to propose recreates the branch from ``main``, a closed proposal's tiers included. It reports a pre-release -instead of writing it. +instead of writing it. The experimental entries are hand-maintained: no source it +reads rates them, so every run keeps their lines as written, and a hand edit to +one changes no proposal. Deployment ``environment:`` ---------------------------- diff --git a/tests/project/test_ci_timing.py b/tests/project/test_ci_timing.py index afca5e0e5..14f1a6616 100644 --- a/tests/project/test_ci_timing.py +++ b/tests/project/test_ci_timing.py @@ -241,6 +241,17 @@ def test_the_real_unit_tests_workflow(self) -> 'None': ordering, 'Run tests/protocols and the root-level modules under plain unittest'), 20) caps = {spec['id']: spec['timeout'] for spec in specs} self.assertEqual((caps['test'], caps['integration'], caps['engine-tests']), (45, 45, 30)) + # #1660: its experimental legs' caps are expressions, so no leg's is known. + self.assertIsNone(ci_timing.step_cap(test, ci_timing.TESTS_STEP)) + + def test_a_cap_by_leg_reads_as_none(self) -> 'None': + """An expression is no one cap; a literal beside it still reads.""" + specs = ci_timing.parse_workflow( + 'jobs:\n test:\n timeout-minutes: 45\n steps:\n' + ' - name: Run unit tests\n timeout-minutes: ${{ matrix.experimental && 25 || 360 }}\n' + ' - name: Report coverage\n timeout-minutes: 15\n') + self.assertEqual([(step['name'], step['timeout']) for step in specs[0]['steps']], + [('Run unit tests', None), ('Report coverage', 15)]) def test_every_other_workflow_reads(self) -> 'None': """Including CodeQL's four-space step list, and a literal name over a template.""" diff --git a/tests/project/test_ktlo_legs_stay_green_1672.py b/tests/project/test_ktlo_legs_stay_green_1672.py index f45ac3920..224bccec8 100644 --- a/tests/project/test_ktlo_legs_stay_green_1672.py +++ b/tests/project/test_ktlo_legs_stay_green_1672.py @@ -20,6 +20,14 @@ through :mod:`tests.project.test_release_gates`'s evaluator, and the scripts that write annotations are executed. +#1660 added an experimental tier, free-threaded CPython and GraalPy, as legs of +``test`` only. They are non-blocking by the same mechanics, under a guard that +holds on both tiers, and report through ``Experimental summary``, the step before +``KTLO summary``: :class:`TestAnExperimentalLegIsNonBlockingTheSameWay`. Their +setup-python, install and test steps have caps of their own, so a slow or hung leg +fails a step rather than being cancelled with its job: +:class:`TestASlowExperimentalLegTimesOutAStepNotTheJob`. + """ from __future__ import annotations @@ -35,10 +43,12 @@ import unittest from typing import TYPE_CHECKING, NamedTuple -from tests.project.test_python_matrix_tiers_1633 import (ANNOTATION_LEVEL, KTLO_GUARD, KTLO_LEGS, - LEG_37, TITLE_37, is_ktlo, job, legs, - titles, workflow, workflow_commands) -from tests.project.test_python_versions_1660 import live_outputs, resolve +from tests.project.test_python_matrix_tiers_1633 import (EXPERIMENTAL_GUARD, EXPERIMENTAL_JOB, KTLO_GUARD, + KTLO_LEGS, LEG_37, TITLE_37, annotation_level, + is_experimental, is_ktlo, is_nonblocking, job, + leg_label, legs, titles, workflow, + workflow_commands) +from tests.project.test_python_versions_1660 import ROOT, live_outputs, prose, resolve from tests.project.test_release_gates import NULL, STATUS, evaluate_condition, evaluate_expression if TYPE_CHECKING: @@ -89,6 +99,25 @@ SUMMARY_IF = '${{ !cancelled() && matrix.ktlo == true }}' SUMMARY_ENV = {'LEG': "Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }}", 'STEPS': '${{ toJSON(steps) }}'} +#: ``test``'s report of an experimental leg (#1660), the step before ``KTLO summary``. +EXPERIMENTAL_SUMMARY = 'Experimental summary' +EXPERIMENTAL_SUMMARY_IF = '${{ !cancelled() && matrix.experimental == true }}' +EXPERIMENTAL_SUMMARY_ENV = {'LEG': 'Python ${{ matrix.python-version }} (experimental, non-blocking)', + 'STEPS': '${{ toJSON(steps) }}'} +#: Why each summary's warning says the job still succeeds; the scripts differ in nothing else. +KTLO_REASON = 'KTLO legs never block (#1633), so this job still succeeds (#1672)' +EXPERIMENTAL_REASON = 'Experimental legs never block (#1660), so this job still succeeds' +#: ``test``'s steps with a ``timeout-minutes`` of their own (#1660), by ``id``, and the +#: cap on an experimental leg: setup-python, the install and the tests, whose runtimes +#: on GraalPy and free-threaded CPython are unmeasured. +EXPERIMENTAL_CAPS = {'python': 5, 'install': 10, 'tests': 25} +#: The job's minutes those caps leave for the rest of an experimental leg. Everything +#: but the install and the tests took at most 0.32 minutes, setup-python included, on +#: every uncovered leg of runs 38569321543, 38569429406 and 38569522442. +UNCAPPED_MINUTES = 5 +#: Those steps' cap on every other leg: the largest a step's ``timeout-minutes`` may be +#: (docs.github.com, workflow-syntax, ``jobs..steps[*].timeout-minutes``). +DOCUMENTED_MAXIMUM = 360 class Step(NamedTuple): @@ -111,6 +140,19 @@ def summary_step(spec: 'dict[str, Any]') -> 'dict[str, Any]': return found[0] +def experimental_summary_step(spec: 'dict[str, Any]') -> 'dict[str, Any]': + """``test``'s ``Experimental summary`` step (#1660).""" + found = [step for step in spec['steps'] if step.get('name') == EXPERIMENTAL_SUMMARY] + if len(found) != 1: + raise AssertionError(f'expected one {EXPERIMENTAL_SUMMARY!r} step, found {len(found)}') + return found[0] + + +def experimental_legs() -> 'list[dict[str, Any]]': + """``test``'s experimental legs, from the tier file.""" + return [leg for leg in legs(*EXPERIMENTAL_JOB) if is_experimental(leg)] + + def is_engine_ktlo(step: 'dict[str, Any]') -> 'bool': """Whether ``step`` is one of ``engine-tests``'s, which run only the KTLO interpreters.""" return label(step) in ENGINE_KTLO or step.get('with', {}).get('python-version') == ENGINE_KTLO_OUTPUT @@ -125,6 +167,11 @@ def stringify(value: 'object') -> 'str': return str(value) +def interpolate(template: 'str', context: 'dict[str, Any]') -> 'str': + """``template`` with each ``${{ ... }}`` in it evaluated, as a step's ``env:`` value.""" + return re.sub(r'\$\{\{.*?\}\}', lambda match: stringify(evaluate_expression(match.group(0), context)), template) + + @functools.lru_cache(maxsize=None) def workflow_env(name: 'str') -> 'tuple[tuple[str, Any], ...]': """``name``'s workflow-level ``env:``, parsed once.""" @@ -291,14 +338,15 @@ def setUp(self) -> None: self.tmp = pathlib.Path(tmp.name) def test_annotation_level_is_a_warning_on_a_ktlo_leg_only(self) -> None: + """And on an experimental one (#1660), which is non-blocking the same way.""" for name, job_id in LEVELLED: spec = job(name, job_id) with self.subTest(workflow=name, job=job_id): - self.assertEqual((spec.get('env') or {}).get('ANNOTATION_LEVEL'), ANNOTATION_LEVEL) + self.assertEqual((spec.get('env') or {}).get('ANNOTATION_LEVEL'), annotation_level(name, job_id)) for leg in legs(name, job_id): with self.subTest(workflow=name, job=job_id, python=leg['python-version']): self.assertEqual(leg_env(name, spec, leg)['ANNOTATION_LEVEL'], - 'warning' if is_ktlo(leg) else 'error') + 'warning' if is_nonblocking(leg) else 'error') def test_no_step_a_ktlo_leg_runs_writes_an_error(self) -> None: for name, job_id in KTLO_JOBS: @@ -444,7 +492,7 @@ def test_every_failure_still_fails_a_supported_leg(self) -> None: masked = [label(step) for step in spec['steps'] if step.get('continue-on-error') is True] with self.subTest(workflow=name, job=job_id): self.assertEqual(masked, list(MASKED_ON_EVERY_LEG.get((name, job_id), ()))) - for leg in (leg for leg in legs(name, job_id) if not is_ktlo(leg)): + for leg in (leg for leg in legs(name, job_id) if not is_nonblocking(leg)): clean, record = run_leg(name, job_id, spec, leg) with self.subTest(workflow=name, job=job_id, python=leg['python-version']): self.assertEqual(clean, 'success') @@ -458,6 +506,15 @@ def test_every_failure_still_fails_a_supported_leg(self) -> None: for gated in re.findall(r'steps\.([\w-]+)\.outcome', str(step.get('if', ''))): self.assertFalse(continue_on_error(by_id[gated], context), gated) + def test_neither_summary_runs_on_a_supported_leg(self) -> None: + """``test`` has two, one per non-blocking tier (#1660).""" + spec = job(*EXPERIMENTAL_JOB) + position = spec['steps'].index(experimental_summary_step(spec)) + for leg in (leg for leg in legs(*EXPERIMENTAL_JOB) if not is_nonblocking(leg)): + for _, _, record in scenarios(*EXPERIMENTAL_JOB, spec, leg): + with self.subTest(python=leg['python-version']): + self.assertEqual((record[position].outcome, record[-1].outcome), ('skipped', 'skipped')) + def test_a_supported_engine_cell_still_fails_its_job_as_an_error(self) -> None: spec = job('unit-tests.yml', 'engine-tests') loop = next(step for step in spec['steps'] if step.get('name') == 'Run the supported cells') @@ -472,5 +529,151 @@ def test_a_supported_engine_cell_still_fails_its_job_as_an_error(self) -> None: self.assertEqual(conclusion, 'success' if step.get('continue-on-error') is True else 'failure') +class TestAnExperimentalLegIsNonBlockingTheSameWay(unittest.TestCase): + """#1660: ``test``'s experimental legs conclude ``success`` and warn, as (a) and (b) pin for KTLO.""" + + def setUp(self) -> None: + self.spec = job(*EXPERIMENTAL_JOB) + self.summary = experimental_summary_step(self.spec) + self.position = self.spec['steps'].index(self.summary) + self.legs = experimental_legs() + self.assertTrue(self.legs, 'the tier file gives `test` no experimental leg') + + def test_no_step_can_fail_an_experimental_leg(self) -> None: + for leg in self.legs: + runs = scenarios(*EXPERIMENTAL_JOB, self.spec, leg) + self.assertGreater(len(runs), 3, f'test on {leg} runs almost nothing') + for index, conclusion, _ in runs: + with self.subTest(python=leg['python-version'], step=label(self.spec['steps'][index])): + self.assertEqual(conclusion, 'success') + + def test_every_failure_it_masks_reaches_the_experimental_summary(self) -> None: + """It runs after any failure on an experimental leg, and only there; ``KTLO summary`` never does.""" + self.assertIs(self.spec['steps'][self.position + 1], summary_step(self.spec)) + self.assertEqual(self.summary.get('if'), EXPERIMENTAL_SUMMARY_IF) + self.assertEqual(self.summary.get('continue-on-error'), EXPERIMENTAL_GUARD) + self.assertEqual(self.summary.get('shell'), 'bash') + self.assertEqual(self.summary.get('env'), EXPERIMENTAL_SUMMARY_ENV) + for leg in self.legs: + for index, _, record in scenarios(*EXPERIMENTAL_JOB, self.spec, leg): + step = self.spec['steps'][index] + if step is self.summary: + continue + with self.subTest(python=leg['python-version'], step=label(step)): + self.assertNotEqual(record[self.position].outcome, 'skipped') + self.assertEqual(record[-1].outcome, 'skipped') + if step.get('continue-on-error') is not True: + self.assertIn('id', step) + for leg in filter(is_ktlo, legs(*EXPERIMENTAL_JOB)): + for _, _, record in scenarios(*EXPERIMENTAL_JOB, self.spec, leg): + with self.subTest(python=leg['python-version']): + self.assertEqual(record[self.position].outcome, 'skipped') + # One script with the KTLO summary's, so a fix to either reaches both. + self.assertIn(EXPERIMENTAL_REASON, self.summary['run']) + self.assertEqual(self.summary['run'].replace(EXPERIMENTAL_REASON, KTLO_REASON), + summary_step(self.spec)['run']) + + def test_each_step_names_the_leg_by_its_tier(self) -> None: + """``LEG`` in an annotation's title: ``(experimental, non-blocking)``, ``(KTLO, non-blocking)``, or bare.""" + named = [step for step in self.spec['steps'] if 'LEG' in (step.get('env') or {})] + self.assertEqual([label(step) for step in named], + ['Install package and test dependencies', EXPERIMENTAL_SUMMARY, SUMMARY]) + for leg in legs(*EXPERIMENTAL_JOB): + never_runs = {EXPERIMENTAL_SUMMARY: not is_experimental(leg), SUMMARY: not is_ktlo(leg)} + for step in named: + if never_runs.get(step['name'], False): + continue + with self.subTest(python=leg['python-version'], step=label(step)): + self.assertEqual(interpolate(step['env']['LEG'], {'matrix': leg}), leg_label(leg)) + + def test_no_step_an_experimental_leg_runs_writes_an_error(self) -> None: + reached = set() # type: set[int] + for leg in self.legs: + for _, _, record in scenarios(*EXPERIMENTAL_JOB, self.spec, leg): + reached.update(index for index, step in enumerate(record) if step.outcome != 'skipped') + self.assertIn(self.position, reached) + for index in sorted(reached): + step = self.spec['steps'][index] + with self.subTest(step=label(step)): + self.assertLessEqual(commands(step.get('run', '')), KTLO_COMMANDS) + + def test_the_summary_warns_once_per_failed_step(self) -> None: + if shutil.which('bash') is None or shutil.which('jq') is None: # pragma: no cover + self.skipTest('bash and jq, on every Ubuntu runner image, are needed to run the summary') + with tempfile.TemporaryDirectory(prefix='pcapkit-1660-') as tmp: + for leg in self.legs: + name = leg_label(leg) + title = name.replace(',', '%2C') + page = pathlib.Path(tmp) / 'summary.md' + page.write_text('', encoding='utf-8') + steps = {'checkout': 'success', 'python': 'success', 'install': 'failure', 'tests': 'skipped'} + context = {key: {'outputs': {}, 'outcome': outcome, + 'conclusion': 'success' if outcome == 'failure' else outcome} + for key, outcome in steps.items()} + with self.subTest(python=leg['python-version']): + self.assertEqual(interpolate(self.summary['env']['LEG'], {'matrix': leg}), name) + result = bash(self.summary['run'], {'LEG': name, 'STEPS': json.dumps(context), + 'GITHUB_STEP_SUMMARY': str(page)}, pathlib.Path(tmp)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(workflow_commands(result.stdout), [ + f'::warning title=install failed on {title}::{EXPERIMENTAL_REASON}; ' + "see the install step's log."]) + self.assertEqual(titles(result.stdout), [f'install failed on {name}']) + self.assertEqual(page.read_text(encoding='utf-8'), + f'- {name}: step `install` failed; non-blocking, so this job still succeeds.\n') + + +class TestASlowExperimentalLegTimesOutAStepNotTheJob(unittest.TestCase): + """#1660: an experimental leg's slow or hung step fails, which its guard contains. + + A job past its ``timeout-minutes`` is cancelled, which no step-level + ``continue-on-error`` contains, and ``Required checks passed`` takes only + ``success`` from ``test`` on a code diff. A step past its own is failed + instead, and ``continue-on-error`` then applies to it as to any failure + (actions/runner src/Runner.Worker/StepsRunner.cs:322-353, at 5f0b3ddcc) -- + the path :class:`TestAnExperimentalLegIsNonBlockingTheSameWay` runs. + + """ + + def setUp(self) -> None: + self.spec = job(*EXPERIMENTAL_JOB) + self.job_cap = self.spec['timeout-minutes'] + self.capped = [step for step in self.spec['steps'] if 'timeout-minutes' in step] + + def caps(self, leg: 'dict[str, Any]') -> 'dict[str, Any]': + """Each capped step's ``timeout-minutes`` on ``leg``, by ``id``.""" + return {step.get('id', label(step)): evaluate_expression(str(step['timeout-minutes']), {'matrix': leg}) + for step in self.capped} + + def test_an_experimental_legs_steps_end_inside_its_job(self) -> None: + """Each cap a whole number under the job's, the install's under the tests', all of them with room left.""" + self.assertTrue(experimental_legs()) + for leg in experimental_legs(): + caps = self.caps(leg) + with self.subTest(python=leg['python-version']): + self.assertEqual(caps, EXPERIMENTAL_CAPS) + for cap in caps.values(): + self.assertIsInstance(cap, int) + self.assertTrue(0 < cap < self.job_cap, cap) + self.assertLess(caps['install'], caps['tests']) + self.assertLessEqual(sum(caps.values()) + UNCAPPED_MINUTES, self.job_cap) + for step in self.capped: + self.assertTrue(continue_on_error(step, {'matrix': leg}), label(step)) + + def test_every_other_leg_keeps_the_jobs_cap(self) -> None: + """Each cap reads the documented maximum, so the job's 45 minutes bind first, as with none.""" + others = [leg for leg in legs(*EXPERIMENTAL_JOB) if not is_experimental(leg)] + self.assertTrue(others) + for leg in others: + with self.subTest(python=leg['python-version']): + self.assertEqual(self.caps(leg), dict.fromkeys(EXPERIMENTAL_CAPS, DOCUMENTED_MAXIMUM)) + self.assertGreaterEqual(DOCUMENTED_MAXIMUM, self.job_cap) + + def test_the_workflows_page_states_the_caps(self) -> None: + page = prose(ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst') + self.assertEqual(re.findall(r'caps of their own, of (\d+), (\d+) and (\d+) minutes', page), + [tuple(str(cap) for cap in EXPERIMENTAL_CAPS.values())]) + + if __name__ == '__main__': unittest.main() diff --git a/tests/project/test_python_matrix_tiers_1633.py b/tests/project/test_python_matrix_tiers_1633.py index 25ad1cfc8..f14320c57 100644 --- a/tests/project/test_python_matrix_tiers_1633.py +++ b/tests/project/test_python_matrix_tiers_1633.py @@ -87,6 +87,18 @@ def tier(impl: 'str', name: 'str') -> 'tuple[str, ...]': KTLO_GUARD = '${{ matrix.ktlo == true }}' #: The job-level level of every annotation a failing step writes (#1672). ANNOTATION_LEVEL = "${{ matrix.ktlo && 'warning' || 'error' }}" +#: ``test``'s, which also hold on its experimental legs (#1660), and the guard of its +#: ``Experimental summary``, which runs on those legs only. +NONBLOCKING_GUARD = '${{ matrix.ktlo == true || matrix.experimental == true }}' +EXPERIMENTAL_GUARD = '${{ matrix.experimental == true }}' +NONBLOCKING_LEVEL = "${{ (matrix.ktlo || matrix.experimental) && 'warning' || 'error' }}" +#: A step's ``LEG``, the leg's name in its annotations, in a KTLO job and in ``test``. +LEG_TEMPLATE = "Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }}" +NONBLOCKING_LEG_TEMPLATE = ("Python ${{ matrix.python-version }}" + "${{ matrix.experimental && ' (experimental, non-blocking)' || '' }}" + "${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }}") +#: The job holding experimental legs, the only one (#1660). +EXPERIMENTAL_JOB = ('unit-tests.yml', 'test') #: Ruleset 23497679's required contexts, which a tier change edits with the file. REQUIRED = frozenset({'Required checks passed'} | {f'Compat Python {v}' for v in SUPPORTED}) @@ -167,7 +179,8 @@ def tier(impl: 'str', name: 'str') -> 'tuple[str, ...]': PROSE_FILES = ('pyproject.toml', 'README.md', 'SECURITY.md', 'CONTRIBUTING.md', 'Makefile') #: Spelled-out counts, as the workflows page writes them. WORDS = {'one': 1, 'two': 2, 'three': 3, 'four': 4, 'five': 5, - 'six': 6, 'seven': 7, 'eight': 8, 'nine': 9, 'ten': 10} + 'six': 6, 'seven': 7, 'eight': 8, 'nine': 9, 'ten': 10, + 'eleven': 11, 'twelve': 12, 'thirteen': 13, 'fourteen': 14, 'fifteen': 15} def job(name: 'str', job_id: 'str') -> 'dict[str, Any]': @@ -190,6 +203,37 @@ def is_ktlo(leg: 'dict[str, Any]') -> 'bool': return leg.get('ktlo') is True +def is_experimental(leg: 'dict[str, Any]') -> 'bool': + """An experimental leg (#1660): non-blocking like a KTLO one, but not KTLO.""" + return leg.get('experimental') is True + + +def is_nonblocking(leg: 'dict[str, Any]') -> 'bool': + return is_ktlo(leg) or is_experimental(leg) + + +def guard(name: 'str', job_id: 'str') -> 'str': + """The step-level guard of job ``job_id``'s failure-prone steps.""" + return NONBLOCKING_GUARD if (name, job_id) == EXPERIMENTAL_JOB else KTLO_GUARD + + +def annotation_level(name: 'str', job_id: 'str') -> 'str': + """Job ``job_id``'s ``ANNOTATION_LEVEL``.""" + return NONBLOCKING_LEVEL if (name, job_id) == EXPERIMENTAL_JOB else ANNOTATION_LEVEL + + +def leg_template(name: 'str', job_id: 'str') -> 'str': + """Job ``job_id``'s ``LEG``, on each step that names the leg in an annotation.""" + return NONBLOCKING_LEG_TEMPLATE if (name, job_id) == EXPERIMENTAL_JOB else LEG_TEMPLATE + + +def leg_label(leg: 'dict[str, Any]') -> 'str': + """What ``LEG`` reads on ``leg``.""" + if is_experimental(leg): + return f"Python {leg['python-version']} (experimental, non-blocking)" + return f"Python {leg['python-version']}{' (KTLO, non-blocking)' if is_ktlo(leg) else ''}" + + def bash(script: 'str', env: 'dict[str, str]') -> 'subprocess.CompletedProcess[str]': """``script`` under ``bash -e``, GitHub's default shell, with ``env`` added.""" return subprocess.run(['bash', '-e', '-c', script], env=dict(os.environ, **env), @@ -204,7 +248,7 @@ def test_every_matrix_runs_the_supported_tier_as_blocking(self) -> None: for name, job_id, blocking in MATRICES: with self.subTest(workflow=name, job=job_id): spec = job(name, job_id) - versions = sorted({leg['python-version'] for leg in legs(name, job_id) if not is_ktlo(leg)}) + versions = sorted({leg['python-version'] for leg in legs(name, job_id) if not is_nonblocking(leg)}) self.assertEqual(versions, sorted(blocking)) # #1672: not even KTLO-only; job-level, a failed leg's check run is red. self.assertNotIn('continue-on-error', spec) @@ -281,8 +325,11 @@ def test_no_ktlo_leg_can_fail_an_aggregated_job(self) -> None: continue # only ever on a supported leg: the coverage one, the cache saver if label in ('Checkout code', "Check the release tags name this run's commit"): continue # #1634: blocks every leg alike, pinned in its own test module + # Each summary's own guard: it runs on its tier's legs only. + expected = {'KTLO summary': KTLO_GUARD, + 'Experimental summary': EXPERIMENTAL_GUARD}.get(label, guard(name, job_id)) with self.subTest(workflow=name, job=job_id, step=label): - self.assertIn(step.get('continue-on-error'), (True, KTLO_GUARD)) + self.assertIn(step.get('continue-on-error'), (True, expected)) def test_engine_tests_isolates_the_ktlo_interpreter_and_cell(self) -> None: """The KTLO versions ``ubuntu-latest`` can run, in steps of their own that cannot fail the job.""" @@ -535,11 +582,9 @@ def run_step(self, name: 'str', job_id: 'str', step_name: 'str', stub: 'str', spec = job(name, job_id) step = next(step for step in spec['steps'] if step.get('name') == step_name) self.assertEqual(step.get('shell'), 'bash', 'pipefail is what keeps `tee` from hiding a failure') - self.assertEqual(step['env']['LEG'], "Python ${{ matrix.python-version }}" - "${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }}") - self.assertEqual(spec['env']['ANNOTATION_LEVEL'], ANNOTATION_LEVEL) - env = dict(env, LEG=f"Python {leg['python-version']}{' (KTLO, non-blocking)' if is_ktlo(leg) else ''}", - ANNOTATION_LEVEL='warning' if is_ktlo(leg) else 'error') + self.assertEqual(step['env']['LEG'], leg_template(name, job_id)) + self.assertEqual(spec['env']['ANNOTATION_LEVEL'], annotation_level(name, job_id)) + env = dict(env, LEG=leg_label(leg), ANNOTATION_LEVEL='warning' if is_nonblocking(leg) else 'error') bin_dir = self.tmp / 'bin' bin_dir.mkdir(exist_ok=True) python = bin_dir / 'python' @@ -586,7 +631,7 @@ def test_a_leg_that_cannot_install_stops_there(self) -> None: condition = str(step.get('if', '')) if 'matrix.coverage' in condition: continue # the `latest` leg only, which a failed install fails anyway - if step.get('name') == 'KTLO summary': + if step.get('name') in ('KTLO summary', 'Experimental summary'): continue with self.subTest(workflow=name, job=job_id, step=step.get('name') or step.get('uses')): self.assertIn("steps.install.outcome == 'success'", condition) diff --git a/tests/project/test_python_versions_1660.py b/tests/project/test_python_versions_1660.py index 805e7cd76..a91429979 100644 --- a/tests/project/test_python_versions_1660.py +++ b/tests/project/test_python_versions_1660.py @@ -284,9 +284,16 @@ def setups(spec: 'dict[str, Any]') -> 'list[dict[str, Any]]': } +#: ``test``'s name since #1660, which names an experimental leg too. +TEST_TEMPLATE = ("Python ${{ matrix.python-version }}${{ matrix.experimental && ' (experimental)' || '' }}" + "${{ matrix.ktlo && ' (KTLO)' || '' }}") + + def render(template: 'str', leg: 'dict[str, Any]') -> 'str': - """A matrix job's name for one leg, for the two expression shapes these names use.""" + """A matrix job's name for one leg, for the three expression shapes these names use.""" name = template.replace("${{ matrix.ktlo && ' (KTLO)' || '' }}", ' (KTLO)' if leg.get('ktlo') is True else '') + name = name.replace("${{ matrix.experimental && ' (experimental)' || '' }}", + ' (experimental)' if leg.get('experimental') is True else '') name = re.sub(r'\$\{\{ matrix\.([\w-]+) \}\}', lambda match: str(leg[match.group(1)]), name) assert '${{' not in name, name return name @@ -577,10 +584,20 @@ def test_engine_tests_installs_and_runs_mains_cells(self) -> None: 'Run the KTLO cells (non-blocking)': ENGINES_AT_MAIN[0]}) def test_no_job_name_changes_but_the_gates(self) -> None: - """Rulesets match check names literally, so each template and leg name stays.""" + """Rulesets match check names literally, so each template and leg name stays. + + ``test``'s template gained the experimental tier's suffix (#1660), which + names every one of :data:`MAIN`'s legs as before. + + """ for (name, job_id), template in TEMPLATES_AT_MAIN.items(): with self.subTest(workflow=name, job=job_id): - self.assertEqual(workflow(name)['jobs'][job_id]['name'], template) + if (name, job_id) == ('unit-tests.yml', 'test'): + self.assertEqual(workflow(name)['jobs'][job_id]['name'], TEST_TEMPLATE) + self.assertEqual([render(TEST_TEMPLATE, leg) for leg in legs(matrix(name, job_id, self.values))], + [render(template, leg) for leg in legs(matrix(name, job_id, self.values))]) + else: + self.assertEqual(workflow(name)['jobs'][job_id]['name'], template) compat = TEMPLATES_AT_MAIN[('python-compatibility.yml', 'compatibility')] compat_legs = legs(matrix('python-compatibility.yml', 'compatibility', self.values)) blocking = {render(compat, leg) for leg in compat_legs @@ -937,5 +954,128 @@ def test_the_release_checks_counts_follow_the_matrices(self) -> None: self.assertEqual(re.findall(pattern, text), expected) +#: The two builds as setup-python spells them, at actions/setup-python 3892123fb: a +#: free-threaded CPython as ``3.14t`` (src/find-python.ts:240, +#: ``/^(\d+\.\d+(\.\d+)?)(t)$/``), and GraalPy as ``graalpy-`` +#: (src/find-graalpy.ts:115-125; README.md:55, ``graalpy-24.0``), the version being +#: a GraalPy release line, matched as a semver range against oracle/graalpython's +#: release tags (src/install-graalpy.ts:213-248). +SETUP_PYTHON = {'cpython': re.compile(r'3\.\d+t'), 'graalpy': re.compile(r'graalpy-\d+\.\d+')} +#: :data:`TIERS_AT_MAIN` with one experimental build of each, listed last as the file lists them. +EXPERIMENTAL_LEGS = [{'python-version': '3.14t', 'experimental': True}, + {'python-version': 'graalpy-25.4', 'experimental': True}] +TIERS_WITH_EXPERIMENTAL = dict(TIERS_AT_MAIN, versions=dict( + TIERS_AT_MAIN['versions'], + cpython=dict(TIERS_AT_MAIN['versions']['cpython'], **{'3.14t': {'tier': 'experimental'}}), + graalpy={'graalpy-25.4': {'tier': 'experimental'}})) + + +def experimental(data: 'dict[str, Any]') -> 'list[str]': + """Every experimental version in ``data``, CPython's then GraalPy's.""" + return [version for impl in ('cpython', 'pypy', 'graalpy') for version in script().tier(data, impl, 'experimental')] + + +class TestTheExperimentalTier(unittest.TestCase): + """#1660, the owner's option 1: GraalPy and free-threaded CPython run non-blocking ``test`` legs, nothing else. + + :mod:`tests.project.test_ktlo_legs_stay_green_1672` runs those legs' steps to + show they cannot fail the job; this reads the file, the matrices and the names. + + """ + + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-1660-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = pathlib.Path(tmp.name) + + def outputs(self, data: 'dict[str, Any]') -> 'dict[str, str]': + return script().outputs(script().load(write_tiers(self.tmp, data))) + + def test_the_file_lists_free_threaded_cpython_and_graalpy(self) -> None: + """Each spelt as setup-python's ``python-version`` reads it, and nothing else experimental.""" + data = script().load() + for impl, spelling in SETUP_PYTHON.items(): + found = script().tier(data, impl, 'experimental') + with self.subTest(implementation=impl): + self.assertTrue(found, f'no experimental {impl}') + for version in found: + self.assertRegex(version, rf'^{spelling.pattern}$') + self.assertEqual(script().tier(data, 'pypy', 'experimental'), []) + + def test_load_holds_the_tier_to_those_builds(self) -> None: + """Either way round: no other build is experimental, and neither build is anything else.""" + self.assertEqual(script().load(write_tiers(self.tmp, TIERS_WITH_EXPERIMENTAL))['versions'], + TIERS_WITH_EXPERIMENTAL['versions']) + versions = TIERS_WITH_EXPERIMENTAL['versions'] + cpython = {key: value for key, value in versions['cpython'].items() if key != '3.14t'} + cases = { + 'a free-threaded build in a blocking tier': dict(versions, cpython=dict(cpython, **{ + '3.14t': {'tier': 'supported'}})), + 'GraalPy in a blocking tier': dict(versions, graalpy={'graalpy-25.4': {'tier': 'ktlo'}}), + 'a plain CPython experimental': dict(versions, cpython=dict(cpython, **{ + '3.16': {'tier': 'experimental'}})), + 'PyPy experimental': dict(versions, pypy=dict(versions['pypy'], **{ + 'pypy3.12': {'tier': 'experimental'}})), + 'experimental before the rest': dict(versions, cpython=dict({'3.14t': {'tier': 'experimental'}}, + **cpython)), + "setup-python's other GraalPy spelling": dict(versions, graalpy={'graalpy25.4': {'tier': 'experimental'}}), + 'GraalPy lists nothing': dict(versions, graalpy={}), + 'an implementation the script cannot spell': dict(versions, jython={'2.7': {'tier': 'experimental'}}), + } + for label, data in cases.items(): + with self.subTest(case=label), self.assertRaises(ValueError): + script().load(write_tiers(self.tmp, dict(TIERS_WITH_EXPERIMENTAL, versions=data))) + + def test_only_test_gains_a_leg_and_no_other_output_moves(self) -> None: + """No ``integration``, ``Compat``, parity, engine, wheel or conda leg, and no release count.""" + base, values = self.outputs(TIERS_AT_MAIN), self.outputs(TIERS_WITH_EXPERIMENTAL) + self.assertEqual({name: value for name, value in values.items() if name != 'test'}, + {name: value for name, value in base.items() if name != 'test'}) + self.assertEqual(legs(json.loads(values['test'])), legs(json.loads(base['test'])) + EXPERIMENTAL_LEGS) + + def test_on_main_no_job_but_test_runs_an_experimental_version(self) -> None: + """Every matrix and setup-python input any workflow reads from the script, as the file stands.""" + found = experimental(script().load()) + self.assertTrue(found) + read = [] + for name, spec in workflows(): + for job_id, job in spec.get('jobs', {}).items(): + raw = job.get('strategy', {}).get('matrix') + texts = [raw] if isinstance(raw, str) else list((raw or {}).values()) if isinstance(raw, dict) else [] + if not any(isinstance(text, str) and READ.fullmatch(text.strip()) for text in texts): + continue + read.append((name, job_id)) + expected = [{'python-version': version, 'experimental': True} for version in found] \ + if (name, job_id) == ('unit-tests.yml', 'test') else [] + with self.subTest(workflow=name, job=job_id): + self.assertEqual([leg for leg in legs(matrix(name, job_id)) if leg['python-version'] in found], + expected) + self.assertGreaterEqual(len(read), 7, 'the scan is blind') + values = live_outputs() + for output in ('conda', 'engine-supported', 'engine-ktlo', 'pypi-wheels', 'conda-builds', 'pypi'): + with self.subTest(output=output): + self.assertFalse(set(re.findall(r'[\w.-]+', values[output])) & set(found)) + + def test_each_leg_is_named_experimental_and_never_required(self) -> None: + found = experimental(script().load()) + template = workflow('unit-tests.yml')['jobs']['test']['name'] + self.assertEqual(template, TEST_TEMPLATE) + names = [render(template, leg) for leg in legs(matrix('unit-tests.yml', 'test'))] + self.assertEqual(len(names), len(set(names)), names) + self.assertEqual([name for name in names if name.endswith(' (experimental)')], + [f'Python {version} (experimental)' for version in found]) + supported = script().tier(script().load(), 'cpython', 'supported') + required = {'Required checks passed'} | {f'Compat Python {version}' for version in supported} + self.assertFalse(required & set(names)) + compat = [leg['python-version'] for leg in legs(matrix('python-compatibility.yml', 'compatibility'))] + self.assertFalse(set(compat) & set(found)) + + def test_the_workflows_page_names_each_experimental_version(self) -> None: + page = prose(ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst') + listed = re.findall(r'spelt as setup-python reads them \(((?:``[^`]+``(?:, )?)+)\)', page) + self.assertEqual(len(listed), 1, listed) + self.assertEqual(re.findall(r'``([^`]+)``', listed[0]), experimental(script().load())) + + if __name__ == '__main__': unittest.main() diff --git a/tests/project/test_python_versions_update_1660.py b/tests/project/test_python_versions_update_1660.py index d644a8e97..ad3656d98 100644 --- a/tests/project/test_python_versions_update_1660.py +++ b/tests/project/test_python_versions_update_1660.py @@ -414,9 +414,14 @@ def test_an_end_of_life_version_the_file_does_not_list_is_not_added(self) -> Non self.assertNotIn('3.5', result.entries) def test_each_implementation_is_one_entry(self) -> None: - """Another implementation is one more source and entry here, and its spelling in python_versions.py.""" + """Another implementation is one more source and entry here, and its spelling in python_versions.py. + + Bar GraalPy, which is experimental only (#1660): no source here rates it. + + """ module = updater() - self.assertEqual(set(module.IMPLEMENTATIONS), set(module.pv.SPELLING)) + self.assertEqual(set(module.IMPLEMENTATIONS), set(module.pv.SPELLING) - {'graalpy'}) + self.assertTrue(module.pv.experimental_only('graalpy', 'graalpy-25.4')) for name, implementation in module.IMPLEMENTATIONS.items(): with self.subTest(implementation=name): self.assertLessEqual(set(implementation.sources), set(module.SOURCES)) @@ -578,6 +583,91 @@ def test_no_file_under_workflows_is_written_or_restates_a_tier(self) -> None: self.assertEqual([match.group(0) for match in restated.finditer(view)], []) +#: :data:`FROZEN` with one experimental build of each kind (#1660), the CPython one +#: carrying an inline comment, so "as written" is checked byte for byte. +FREE_THREADED = '"3.14t" = { tier = "experimental" } # free-threaded, hand-kept\n' +GRAALPY = '\n# GraalPy, hand-kept.\n[versions.graalpy]\n"graalpy-25.4" = { tier = "experimental" }\n' + + +def with_experimental(text: 'str') -> 'str': + """A tier file's text with :data:`FREE_THREADED` after CPython's last line and :data:`GRAALPY` after PyPy's.""" + for anchor, added in (('"3.15" = { tier = "supported" }\n', FREE_THREADED), + ('"pypy3.11" = { tier = "supported" }\n', GRAALPY)): + if text.count(anchor) != 1: + raise AssertionError(f'{anchor!r} is not one line of the tier file') + text = text.replace(anchor, anchor + added) + return text + + +class TestTheExperimentalEntriesAreHandKept(unittest.TestCase): + """#1660: no source rates free-threaded CPython or GraalPy, so every recompute keeps them as written.""" + + def scratch(self) -> 'pathlib.Path': + """:func:`scratch` with :func:`with_experimental` applied to :data:`FROZEN`, the prose regenerated.""" + root = scratch(self, regenerate=False) + tier_file = root / updater().TIER_FILE + tier_file.write_text(with_experimental(FROZEN.read_text(encoding='utf-8')), encoding='utf-8') + plan, texts = updater().compute_plan(root, as_of_the_file(snapshot())) + self.assertEqual(plan['changes'], []) + self.assertNotIn(updater().TIER_FILE, texts) + for path, text in texts.items(): + (root / path).write_bytes(text.encode('utf-8')) + return root + + def test_a_run_that_changes_nothing_writes_nothing(self) -> None: + root = self.scratch() + plan, texts = updater().compute_plan(root, as_of_the_file(snapshot())) + self.assertEqual((plan['changes'], texts, plan['notes']), ([], {}, [])) + self.assertIn('``test`` (eleven Python legs: five supported, four KTLO, two experimental)', + ' '.join((root / 'docs/source/contributing/workflows.rst').read_text(encoding='utf-8').split())) + + def test_a_tier_move_keeps_them_byte_for_byte_after_the_computed_versions(self) -> None: + root = self.scratch() + before = (root / updater().TIER_FILE).read_text(encoding='utf-8') + plan, texts = updater().compute_plan(root, three_eleven_ends_and_three_sixteen_ships(snapshot())) + self.assertEqual(changes(plan), {'3.11': ({'tier': 'supported'}, {'tier': 'ktlo'}), + '3.16': (None, {'tier': 'supported'}), + 'pypy3.12': (None, {'tier': 'supported'})}) + after = texts[updater().TIER_FILE] + self.assertIn('"3.16" = { tier = "supported" }\n' + FREE_THREADED, after) + self.assertIn('"pypy3.12" = { tier = "supported" }\n' + GRAALPY, after) + self.assertEqual(after.replace('"3.16" = { tier = "supported" }\n', '').replace( + '"pypy3.12" = { tier = "supported" }\n', '').replace( + '"3.11" = { tier = "ktlo" }', '"3.11" = { tier = "supported" }').replace( + 'latest = "3.16"', 'latest = "3.15"'), before) + self.assertEqual(updater().validate(after)['versions']['graalpy'], {'graalpy-25.4': {'tier': 'experimental'}}) + self.assertFalse([name for name in plan['added'] + plan['removed'] + sum(plan['renamed'], []) + if 'experimental' in name], plan) + + def test_a_hand_edit_to_one_is_not_a_new_proposal(self) -> None: + """The fingerprint is the computed tiers', so an open proposal stays recognised across one.""" + module = updater() + data = frozen() + edited = module.validate(with_experimental(FROZEN.read_text(encoding='utf-8'))) + self.assertNotEqual(edited['versions'], data['versions']) + self.assertEqual(module.fingerprint(edited['latest'], edited['versions']), + module.fingerprint(data['latest'], data['versions'])) + moved = copy.deepcopy(edited['versions']) + moved['cpython']['3.11'] = {'tier': 'ktlo'} + self.assertNotEqual(module.fingerprint(edited['latest'], moved), + module.fingerprint(edited['latest'], edited['versions'])) + _, plain, _ = plan_for(self, snapshot()) + plan, _ = module.compute_plan(self.scratch(), snapshot()) + self.assertEqual(plan['fingerprint'], plain['fingerprint']) + + def test_no_experimental_leg_is_proposed_as_a_required_context(self) -> None: + """Were one ever given a ``Compat`` leg, the ruleset edit the body lists would still leave it out.""" + module = updater() + workflows = module.load_workflows(ROOT) + values = module.pv.outputs(frozen()) + compatibility = json.loads(values['compatibility']) + compatibility['include'].append({'python-version': '3.14t', 'experimental': True}) + widened = dict(values, compatibility=json.dumps(compatibility)) + self.assertIn('3.14t', [leg['python-version'] for leg in + module.matrix_legs(workflows, widened)[module.REQUIRED_JOB]]) + self.assertEqual(module.required_contexts(workflows, widened), module.required_contexts(workflows, values)) + + def run_prose_audit(root: 'pathlib.Path') -> 'unittest.TestResult': """The prose classes of the audit, reading ``root`` instead of the checkout. diff --git a/tests/project/test_release_gates.py b/tests/project/test_release_gates.py index 8b9ac2625..6cf65160f 100644 --- a/tests/project/test_release_gates.py +++ b/tests/project/test_release_gates.py @@ -137,8 +137,8 @@ :func:`evaluate_condition` implements the subset this workflow uses and nothing more: ``!``, ``&&``, ``||``, parentheses, ``==``/``!=`` against -single-quoted literals, ``startsWith``, ``contains``, ``fromJSON``, the -status-check functions, and ``needs..result`` / +single-quoted literals, integer literals, ``startsWith``, ``contains``, +``fromJSON``, the status-check functions, and ``needs..result`` / ``needs..outputs.`` / ``github.*`` lookups. ``contains``, ``fromJSON`` and ``always()`` appear in no gate on the tree -- they are implemented because the mutation table needs them *evaluated* rather than @@ -640,10 +640,13 @@ def __repr__(self) -> 'str': #: ``!=``, turning ``a != b`` into a negation followed by a stray ``=``. #: Property paths are tokenised whole, dots and hyphens included, since #: ``needs.unit-tests.result`` is one lookup and not an arithmetic expression. +#: A number is a non-negative integer only, which is what a step's ``timeout-minutes`` +#: needs (#1660), so ``1.5`` or ``-1`` still raises. _EXPR_TOKEN = re.compile( r""" (?P\s+) | (?P&&|\|\||==|!=|\(|\)|,|!) | (?P'(?:[^']|'')*') + | (?P\d+) | (?P[A-Za-z_][A-Za-z0-9_.-]*) """, re.VERBOSE, @@ -784,6 +787,10 @@ def primary(self) -> 'tuple': # doubling it, so `'it''s'` is one literal and not two. return ('lit', text[1:-1].replace("''", "'")) + if kind == 'number': + self.index += 1 + return ('lit', int(text)) + if kind == 'name': self.index += 1 lowered = text.casefold() @@ -2899,12 +2906,21 @@ def test_a_status_function_no_scenario_models_raises(self) -> None: def test_a_doubled_quote_inside_a_literal_is_one_literal(self) -> None: self.assertTrue(self.evaluate("${{ 'it''s' == 'IT''S' }}")) + def test_an_integer_literal_is_a_number(self) -> None: + """What ``&& N || M`` returns, as a step's ``timeout-minutes`` reads it (#1660).""" + self.assertEqual(evaluate_expression('${{ true && 25 || 360 }}', {}), 25) + self.assertEqual(evaluate_expression('${{ false && 25 || 360 }}', {}), 360) + self.assertFalse(self.evaluate('${{ 0 }}')) + self.assertTrue(self.evaluate("${{ 25 == '25' }}")) + def test_syntax_outside_the_subset_raises_rather_than_guessing(self) -> None: """Each of these is a real Actions expression this module does not implement, and each has to fail loudly rather than be approximated. """ for condition in ('${{ 1 > 2 }}', # no ordering operators + '${{ 1.5 }}', # an integer is the only number + '${{ -1 }}', "${{ join(needs.*.result, ',') }}", # no `*` and no `join` '${{ toJSON(needs) }}', # not in the subset '${{ needs.version_check.result == }}', diff --git a/util/ci_timing.py b/util/ci_timing.py index 16c0c4588..82b6231ff 100644 --- a/util/ci_timing.py +++ b/util/ci_timing.py @@ -319,7 +319,13 @@ def parse_workflow(text: 'str') -> 'list[dict[str, Any]]': def _parse_steps(section: 'str') -> 'list[dict[str, Any]]': - """The named steps of one job section, with any step-level cap.""" + """The named steps of one job section, with any step-level cap. + + A cap written as an expression differs by leg -- ``test``'s on its experimental + legs (#1660) -- and the job's spec does not say which leg a name ran, so it + reads as none, as an uncapped step's does. + + """ found = [] # type: list[dict[str, Any]] lines = section.splitlines() try: @@ -342,9 +348,9 @@ def _parse_steps(section: 'str') -> 'list[dict[str, Any]]': if 'name' not in keys: continue template = _scalar(keys['name']) - timeout = keys.get('timeout-minutes') + timeout = _scalar(keys.get('timeout-minutes') or '') found.append({'name': template, 'pattern': template_pattern(template), - 'timeout': int(_scalar(timeout)) if timeout else None}) + 'timeout': int(timeout) if timeout and '${{' not in timeout else None}) return found diff --git a/util/python_versions.py b/util/python_versions.py index c316cf162..0f57574c6 100644 --- a/util/python_versions.py +++ b/util/python_versions.py @@ -16,7 +16,8 @@ ``test``, ``integration``, ``compatibility`` Both tiers. A KTLO leg carries ``ktlo: true``, and ``os`` where ``ubuntu-latest`` cannot provide it; ``test`` marks the ``latest`` leg - ``coverage: true``. + ``coverage: true``, and adds each experimental version as a leg carrying + ``experimental: true``. No other output holds an experimental version. ``pypcap-parity`` The versions below 3.12 that run on ``ubuntu-latest``. ``pypi`` @@ -57,9 +58,14 @@ ROOT = pathlib.Path(__file__).resolve().parents[1] FILE = ROOT / '.github' / 'python-versions.toml' -#: Each implementation, and how its versions are spelled. -SPELLING = {'cpython': re.compile(r'3\.(\d+)'), 'pypy': re.compile(r'pypy3\.(\d+)')} -TIERS = ('supported', 'ktlo', 'pre-release') +#: Each implementation, and how its versions are spelled: as setup-python's +#: ``python-version`` reads them, ``3.14t`` a free-threaded CPython and GraalPy by +#: its own release line (#1660). +SPELLING = {'cpython': re.compile(r'3\.(\d+)t?'), 'pypy': re.compile(r'pypy3\.(\d+)'), + 'graalpy': re.compile(r'graalpy-(\d+)\.(\d+)')} +#: The implementations every file lists; a GraalPy table is optional. +MANDATORY = ('cpython', 'pypy') +TIERS = ('supported', 'ktlo', 'pre-release', 'experimental') #: The images a version may name as its ``runner``. RUNNER = re.compile(r'ubuntu-\d{2}\.04') #: Wheel tag prefix per implementation, as util/wheel_rename.py writes it. @@ -70,11 +76,24 @@ def key(version: 'str') -> 'tuple[int, int]': - """``'3.10'`` or ``'pypy3.10'`` as ``(3, 10)``, for ordering.""" - minor = re.search(r'3\.(\d+)$', version) - if minor is None: + """``'3.10'``, ``'3.10t'`` or ``'pypy3.10'`` as ``(3, 10)``, ``'graalpy-25.4'`` as ``(25, 4)``, for ordering.""" + found = re.fullmatch(r'(?:pypy)?3\.(?P\d+)t?|graalpy-(?P\d+)\.(?P\d+)', version) + if found is None: raise ValueError(f'not a Python version: {version!r}') - return 3, int(minor.group(1)) + if found['minor'] is not None: + return 3, int(found['minor']) + return int(found['line']), int(found['release']) + + +def experimental_only(impl: 'str', version: 'str') -> 'bool': + """Whether ``version`` is a build the experimental tier holds, and no other (#1660). + + Free-threaded CPython and GraalPy: no wheel tag, conda build or ``Compat`` leg + handles either, and the weekly job (:file:`util/update_python_versions.py`) + has no source to compute a tier for one, so it keeps their lines as written. + + """ + return impl == 'graalpy' or (impl == 'cpython' and version.endswith('t')) def load(path: 'pathlib.Path' = FILE) -> 'dict[str, Any]': @@ -88,13 +107,20 @@ def load(path: 'pathlib.Path' = FILE) -> 'dict[str, Any]': instead of running it as something else. An exemption's reason is the comment above it, which :mod:`tomllib` does not keep. + The experimental tier holds free-threaded CPython and GraalPy, and they hold + no other (:func:`experimental_only`): promoting one to a blocking tier needs + its wheel, conda build and ``Compat`` leg first, so that fails here too. They + are listed after the other versions of their table, which keeps each tier's + lines one run for the prose that cites them. + """ data = tomllib.loads(pathlib.Path(path).read_text(encoding='utf-8')) if set(data) - {'latest', 'versions', 'exemptions'}: raise ValueError('the top level holds only "latest", "versions" and "exemptions"') versions = data.get('versions') - if not isinstance(versions, dict) or set(versions) != set(SPELLING): - raise ValueError(f'"versions" maps exactly {sorted(SPELLING)} to their versions') + if not isinstance(versions, dict) or not set(MANDATORY) <= set(versions) <= set(SPELLING): + raise ValueError(f'"versions" maps {sorted(MANDATORY)}, and optionally ' + f'{sorted(set(SPELLING) - set(MANDATORY))}, to their versions') for impl, entries in versions.items(): if not isinstance(entries, dict) or not entries: raise ValueError(f'{impl} lists no versions') @@ -109,10 +135,18 @@ def load(path: 'pathlib.Path' = FILE) -> 'dict[str, Any]': if entry['tier'] == 'pre-release': raise ValueError(f'{where}: no workflow runs pre-release legs yet; wire their ' 'non-blocking mechanics before listing one (#1660)') + if entry['tier'] == 'experimental' and not experimental_only(impl, version): + raise ValueError(f'{where}: the experimental tier holds free-threaded CPython and GraalPy ' + 'only; the weekly job computes the tier of every other version (#1660)') + if entry['tier'] != 'experimental' and experimental_only(impl, version): + raise ValueError(f'{where}: free-threaded CPython and GraalPy are experimental only; give them ' + 'a wheel, a conda build and a Compat leg before a blocking tier (#1660)') if 'runner' in entry and not RUNNER.fullmatch(str(entry['runner'])): raise ValueError(f'{where}: runner {entry["runner"]!r} is not an ubuntu-YY.04 image') - if list(entries) != sorted(entries, key=key): - raise ValueError(f'{impl}: list the versions oldest first') + rest = [version for version, entry in entries.items() if entry['tier'] != 'experimental'] + last = [version for version, entry in entries.items() if entry['tier'] == 'experimental'] + if list(entries) != sorted(rest, key=key) + sorted(last, key=key): + raise ValueError(f'{impl}: list the versions oldest first, the experimental ones after the rest') if data.get('latest') not in tier(data, 'cpython', 'supported'): raise ValueError(f'latest {data.get("latest")!r} is not a supported CPython version') for index, exemption in enumerate(data.get('exemptions', [])): @@ -124,8 +158,8 @@ def load(path: 'pathlib.Path' = FILE) -> 'dict[str, Any]': def tier(data: 'dict[str, Any]', impl: 'str', name: 'str') -> 'list[str]': - """The versions of ``impl`` in tier ``name``, oldest first.""" - return [version for version, entry in data['versions'][impl].items() if entry['tier'] == name] + """The versions of ``impl`` in tier ``name``, oldest first; none for an implementation not listed.""" + return [version for version, entry in data['versions'].get(impl, {}).items() if entry['tier'] == name] def runner(data: 'dict[str, Any]', impl: 'str', version: 'str') -> 'Optional[str]': @@ -160,10 +194,23 @@ def tiered(data: 'dict[str, Any]', impl: 'str' = 'cpython') -> 'dict[str, Any]': return {'python-version': tier(data, impl, 'supported'), 'include': include} +def experimental_legs(data: 'dict[str, Any]') -> 'list[dict[str, Any]]': + """Each experimental version, CPython's then GraalPy's, as a leg of its own (#1660).""" + found = [] + for impl in SPELLING: + for version in tier(data, impl, 'experimental'): + leg = {'python-version': version, 'experimental': True} # type: dict[str, Any] + if runner(data, impl, version) is not None: + leg['os'] = runner(data, impl, version) + found.append(leg) + return found + + def test_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': - """unit-tests.yml's ``test``: both tiers, the one coverage leg on ``latest``.""" + """unit-tests.yml's ``test``: both tiers, the one coverage leg on ``latest``, and the experimental legs.""" matrix = tiered(data) matrix['include'].insert(0, {'python-version': data['latest'], 'coverage': True}) + matrix['include'].extend(experimental_legs(data)) return matrix @@ -180,8 +227,8 @@ def parity_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': def pypi_matrix(data: 'dict[str, Any]') -> 'dict[str, Any]': - """create-release.yml's ``pypi``: one wheel per CPython and PyPy version, both tiers.""" - impls = tuple(SPELLING) + """create-release.yml's ``pypi``: one wheel per CPython and PyPy version, both tiers, none experimental.""" + impls = tuple(WHEEL_PREFIX) return {'python-version': [v for impl in impls for v in on_latest(data, impl, 'supported', 'pypi')], 'include': [{'python-version': v, 'ktlo': True} for impl in impls for v in on_latest(data, impl, 'ktlo', 'pypi')]} diff --git a/util/update_python_versions.py b/util/update_python_versions.py index b49693dc8..16d41b1ac 100644 --- a/util/update_python_versions.py +++ b/util/update_python_versions.py @@ -40,6 +40,11 @@ ``feature`` or ``prerelease`` once setup-python has a build. No workflow runs pre-release legs yet, and :func:`python_versions.load` refuses the tier, so such a version is reported (:data:`HELD`) rather than written. +experimental + Free-threaded CPython and GraalPy, which no source here rates (#1660), are + hand-maintained: every recompute keeps their entries as the file has them, + after the computed versions of their table, and :func:`fingerprint` leaves + them out, so a hand edit to one neither changes nor re-proposes the tiers. PyPy has no release cycle of its own: the Python versions of its newest stable release are supported, an older one the file lists stays KTLO while a stable @@ -48,8 +53,8 @@ end-of-life version the file does not list is never added. Each implementation is one entry of :data:`IMPLEMENTATIONS`, with the sources it -reads, so adding one (GraalPy, free-threaded CPython) is one more source and one -more entry, besides its spelling in :file:`util/python_versions.py`. +reads, so computing another is one more source and one more entry, besides its +spelling in :file:`util/python_versions.py`. The tier file is rewritten line by line: a version whose entry is unchanged keeps its line byte for byte, so comments, the exemptions and the layout survive, and a @@ -432,11 +437,16 @@ def leg_names(workflows: 'dict[str, Any]', values: 'dict[str, str]') -> 'dict[tu def required_contexts(workflows: 'dict[str, Any]', values: 'dict[str, str]') -> 'list[str]': - """Ruleset 23497679's required contexts for ``values``: the aggregate, and :data:`REQUIRED_JOB`'s blocking legs.""" + """Ruleset 23497679's required contexts for ``values``: the aggregate, and :data:`REQUIRED_JOB`'s blocking legs. + + A KTLO leg never blocks, nor an experimental one (#1660), so neither is ever proposed as required. + + """ name, job_id = REQUIRED_JOB template = str(workflows[name]['jobs'][job_id]['name']) legs = matrix_legs(workflows, values)[REQUIRED_JOB] - return [REQUIRED_AGGREGATE] + [render_name(template, leg) for leg in legs if leg.get('ktlo') is not True] + return [REQUIRED_AGGREGATE] + [render_name(template, leg) for leg in legs + if leg.get('ktlo') is not True and leg.get('experimental') is not True] def setup_steps(job: 'dict[str, Any]') -> 'list[dict[str, Any]]': @@ -750,9 +760,17 @@ def validate(rendered: 'str') -> 'dict[str, Any]': return pv.load(path) +def computed(versions: 'dict[str, dict[str, Any]]') -> 'dict[str, dict[str, Any]]': + """``versions`` without the experimental entries, which no rule here computes, and the tables left empty.""" + found = {impl: {version: entry for version, entry in entries.items() if entry['tier'] != 'experimental'} + for impl, entries in versions.items()} + return {impl: entries for impl, entries in found.items() if entries} + + def fingerprint(latest: 'str', versions: 'dict[str, Any]') -> 'str': - """A proposal's identity: its tiers, whatever else moved on ``main``.""" - canonical = json.dumps({'latest': latest, 'versions': versions}, sort_keys=True, separators=(',', ':')) + """A proposal's identity: its computed tiers, whatever else moved on ``main``, hand-kept entries included.""" + canonical = json.dumps({'latest': latest, 'versions': computed(versions)}, sort_keys=True, + separators=(',', ':')) return hashlib.sha256(canonical.encode('utf-8')).hexdigest()[:16] @@ -883,8 +901,9 @@ def counted(count: 'int', ends: 'Optional[tuple[Any, Any]]') -> 'Optional[tuple[ lambda f: counted(len(f['ktlo']), lines_of(f, 'ktlo'))), Claim(r'and the (\w+) ``Compat Python (3\.\d+)``-``(3\.\d+)`` legs', lambda f: counted(len(f['required']) - 1, span(f['supported']))), - Claim(r'``test`` \((\w+) Python legs: (\w+) supported, (\w+) KTLO\)', - lambda f: (Word(f['legs']['test']), Word(len(f['supported'])), Word(len(f['ktlo'])))), + Claim(r'``test`` \((\w+) Python legs: (\w+) supported, (\w+) KTLO, (\w+) experimental\)', + lambda f: (Word(f['legs']['test']), Word(len(f['supported'])), Word(len(f['ktlo'])), + Word(len(f['experimental'])))), Claim(r'``integration`` \((\w+)\)', lambda f: (Word(f['legs']['integration']),)), Claim(r'each looping over (\w+) Python versions', lambda f: (Word(f['engines']),)), Claim(r'``pypcap-parity`` \((\w+)\)', lambda f: (Word(f['legs']['pypcap-parity']),)), @@ -985,6 +1004,7 @@ def make_facts(data: 'dict[str, Any]', values: 'dict[str, str]', workflows: 'dic 'latest': data['latest'], 'supported': pv.tier(data, 'cpython', 'supported'), 'ktlo': pv.tier(data, 'cpython', 'ktlo'), + 'experimental': [version for impl in data['versions'] for version in pv.tier(data, impl, 'experimental')], 'wheels': len(values['pypi-wheels'].split()), 'conda': len(json.loads(values['conda'])), 'legs': {job_id: len(found) for (name, job_id), found in legs.items() if name == 'unit-tests.yml'}, @@ -1043,7 +1063,8 @@ def compute_plan(root: 'pathlib.Path', sources: 'dict[str, Any]') -> 'tuple[dict versions, reasons, notes, latest = {}, {}, [], before['latest'] for impl, implementation in IMPLEMENTATIONS.items(): - result = implementation.compute(before['versions'][impl], context) + current = computed(before['versions']).get(impl, {}) + result = implementation.compute(current, context) kept = {} for version in sorted(result.entries, key=pv.key): entry = result.entries[version] @@ -1057,6 +1078,12 @@ def compute_plan(root: 'pathlib.Path', sources: 'dict[str, Any]') -> 'tuple[dict notes.extend(result.notes) if result.latest is not None: latest = result.latest + # The experimental entries, kept as the file has them, after the computed ones + # (#1660); so is a table no rule computes, GraalPy's. + for impl, entries in before['versions'].items(): + versions.setdefault(impl, {}).update( + (version, entry) for version, entry in entries.items() if entry['tier'] == 'experimental') + versions = {impl: versions[impl] for impl in before['versions']} after = dict(before, latest=latest, versions=versions) new_tier_text = render_tier_file(old_tier_text, latest, versions) validate(new_tier_text)