From 6ef223592edd1dfb8e8b906485b08189ab66f9c7 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 20 Oct 2026 00:30:19 -0400 Subject: [PATCH 1/2] ci(unit-tests): run the whole suite on latest CPython (#1958) - test, integration: a step on the latest leg alone installs the whole suite's extras (Scapy, PyShark, vendor, docs, cli on top of the base); every other leg keeps its install line and runs the ALL tier. - project-tests, gate: install the same set. - tests/_dependency_gates.py: read a latest-only install step as that leg's extras; drop the exclusions it closes (HAS_SCAPY, HAS_PYSHARK, HAS_RUNTIME, HAS_VENDOR_DEPS, HAS_SPHINX, HAS_DOCUTILS, HAS_FURO). - tests/project/test_whole_suite_on_latest_1958.py pins the lines, the leg, the gates closed and the docs; tier-guard, #1732 and #1954 pins follow the new reading. - workflows.rst describes it; unit-tests.yml line citations re-anchored. Selection steps and tests/_tiers.py are unchanged. --- .github/workflows/unit-tests.yml | 38 ++- docs/source/contributing/releasing.rst | 4 +- docs/source/contributing/workflows.rst | 33 +- tests/_dependency_gates.py | 306 +++--------------- tests/project/test_project_pins_once_1954.py | 3 +- .../test_sweeps_on_latest_only_1732.py | 4 +- .../test_whole_suite_on_latest_1958.py | 172 ++++++++++ tests/test_tier_guard.py | 62 ++-- 8 files changed, 316 insertions(+), 306 deletions(-) create mode 100644 tests/project/test_whole_suite_on_latest_1958.py diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 3be705e766..238f863909 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -187,6 +187,23 @@ jobs: exit 1 fi + # #1958, the owner's ruling E: latest CPython runs the whole suite, so its + # leg installs everything the suite gates on that pip can install there -- + # the engines (Scapy, PyShark), html5lib through `vendor`, Sphinx and the + # pinned sphinx-autodoc-typehints through `docs`, emoji through `cli`. The + # tests those gate skip on every other leg, which runs the minimal `ALL` + # tier. tests/_dependency_gates.py reads this line as the `latest` leg's + # extras; tests/project/test_whole_suite_on_latest_1958.py pins it to the + # same set as `integration`, `project-tests` and `gate`. Not here: PCAP_CT, + # which needs a system libpcap that `engine-tests` installs for its own + # cell, and PyPCAP and PyPCAPFile, whose markers resolve to nothing on + # `latest`. Never a KTLO leg, so `test-ktlo` always skips it. + - name: Install the whole suite's dependencies on latest + id: install-latest + if: ${{ steps.install.outcome == 'success' && matrix.python-version == needs.python-versions.outputs.latest }} + continue-on-error: ${{ matrix.ktlo == true }} + run: python -m pip install -e '.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]' + # 7.10 is the first release with `[run] patch = ["subprocess"]`, which # .github/coverage.toml relies on to measure the xdist workers. - name: Install coverage @@ -528,6 +545,14 @@ jobs: exit 1 fi + # #1958: the `test` job's step of the same name. On `latest` this job runs + # the whole fixture tier, so it installs the whole suite's set too. + - name: Install the whole suite's dependencies on latest + id: install-latest + if: ${{ steps.install.outcome == 'success' && matrix.python-version == needs.python-versions.outputs.latest }} + continue-on-error: ${{ matrix.ktlo == true }} + run: python -m pip install -e '.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]' + # `shell: bash` rather than the default, because it runs with -o pipefail: # without it the `tee` below would swallow the generator's exit status. - name: Regenerate sample captures @@ -1785,12 +1810,12 @@ jobs: python-version: ${{ steps.python-versions.outputs.latest }} cache: pip - # `test`'s install line, so a gated pin skips or runs here exactly as it - # did on `latest`. + # The whole suite's set, the line `test` adds on `latest` (#1958), so a + # gated pin runs here exactly as the rest of its tier does there. - name: Install package and test dependencies run: | python -m pip install -U pip setuptools wheel - python -m pip install -e '.[test,DPKT,crypto,NGAP]' + python -m pip install -e '.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]' # See the `test` job above for why this step exists. - name: Report available parallelism @@ -1869,10 +1894,15 @@ jobs: # would be a no-op that misleadingly suggests the extra is exercised by # `gate`; #751's `engine-tests` and `pypcap-parity` jobs cover it for # real, on the 3.10-3.15 (and 3.10-3.11) legs, 3.10 being KTLO in both. + # + # #1958: `vendor`, `docs` and `PyShark` join them, so this is the set the + # `latest` legs install (see `test`'s "Install the whole suite's + # dependencies on latest" step), and the html5lib, Sphinx and PyShark + # gates run here as they do there. - name: Install package, test and generator dependencies run: | python -m pip install -U pip setuptools wheel - python -m pip install -e '.[test,Scapy,DPKT,crypto,cli,NGAP]' + python -m pip install -e '.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]' # See the integration job above for why this step is shaped the way it is. - name: Regenerate sample captures diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index a7ba176d86..ead45ff410 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -52,7 +52,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:581-597``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1842-1912``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1867-1942``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:621,808,974,1210``), so a @@ -62,7 +62,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Other checks run unconditionally on the same path, though none is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1646-1672``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1671-1697``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 48fdb20318..120dd1e97c 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -329,7 +329,7 @@ the same commit several times over. It also skips ``changes``, Checks`_ below), which are gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1634-1640``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1659-1665``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -639,7 +639,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:2003`` + - ``unit-tests.yml:2033`` * - ``Compat Python 3.11`` - job ``compatibility``, matrix leg ``3.11`` - ``python-compatibility.yml:60``, ``.github/python-versions.toml:20`` @@ -661,22 +661,22 @@ which added ``Compat Python 3.15`` and removed ``Compat Python 3.10``. Moving a between the tiers below means editing the ruleset in the same change. ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:2003``). ``Compat Python`` is defined by one, the +(``unit-tests.yml:2033``). ``Compat Python`` is defined by one, the ``compatibility`` job in ``python-compatibility.yml`` (``:60``), whose matrix (``:74``) comes from :file:`.github/python-versions.toml`: the required legs are its supported tier (``python-versions.toml:20-24``), and its KTLO legs (``python-versions.toml:16-19``) have names ending in ``(KTLO)``, so no required context matches one. A comment at -``unit-tests.yml:1915`` mentions the string without defining it. +``unit-tests.yml:1945`` mentions the string without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the ``Compat Python 3.11``-``3.15`` legs, nothing else, as -``unit-tests.yml``'s own comment (``unit-tests.yml:1914-1924``) records. It +``unit-tests.yml``'s own comment (``unit-tests.yml:1944-1954``) records. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- ``test``, ``integration``, ``engine-tests`` and ``pypcap-parity``, plus ``project-tests``, the one run of the project pins, and ``changes`` for the docs-only path -- and checks each result explicitly -(``unit-tests.yml:2002-2058``). Rulesets match check names literally, with no +(``unit-tests.yml:2032-2088``). Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. It accepts the gating jobs as ``skipped`` only on a docs-only pull request, where @@ -685,7 +685,7 @@ It accepts the gating jobs as ``skipped`` only on a docs-only pull request, wher ``python-compatibility.yml``, and a job cannot ``needs:`` a job in another workflow file. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:2005``), +``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:2035``), so it is never produced -- and never expected -- on those paths. .. _ktlo-tier: @@ -709,7 +709,7 @@ and keeps its legs' check names, ``Python 3.10 (KTLO)`` and so on. In with supported legs. Every job holding KTLO legs works the same way: * Every step that can fail on a KTLO leg has an ``id`` and - ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:424-441``), + ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:441-458``), unless it is masked on every leg, as the timings uploads are. Its ``outcome`` is then ``failure`` but its ``conclusion`` ``success``, so the job, its check run and ``required-checks`` read ``success``. A step that needs an earlier one's work @@ -976,6 +976,23 @@ The project pins stay in a job of their own rather than joining the ``latest`` l rather than lengthening it under coverage, and it is also the job that runs the docs readers on a docs-only diff. +Latest CPython runs the whole suite, as the sanity check and the gate (the owner's +ruling on #1958), so it also installs what the whole suite gates on. In ``test`` and +``integration`` the step ``Install the whole suite's dependencies on latest`` runs on +the ``latest`` leg alone, after the install every leg runs, and adds the engines pip +can install there (``Scapy``, ``PyShark``), html5lib through ``vendor``, Sphinx and +the pinned sphinx-autodoc-typehints through ``docs``, and emoji through ``cli``. +``project-tests`` and ``gate``, which run on ``latest`` only, install the same set in +their ``Install package and test dependencies`` and ``Install package, test and +generator dependencies`` steps. The other legs keep their install lines and run the +``ALL`` tier with them, so a test gated on one of these extras skips there and runs on +``latest``. The other engines stay out: ``PCAP_CT`` needs a system libpcap, which +``engine-tests`` installs for its own cell, and the ``PyPCAP`` and ``PyPCAPFile`` +markers resolve to nothing on ``latest``. :file:`tests/_dependency_gates.py` reads the +``latest`` step as that leg's extras, and +:file:`tests/project/test_whole_suite_on_latest_1958.py` pins the set and the gates it +closes. + Project Pins ------------ diff --git a/tests/_dependency_gates.py b/tests/_dependency_gates.py index daa003565b..e548cc3552 100644 --- a/tests/_dependency_gates.py +++ b/tests/_dependency_gates.py @@ -103,7 +103,7 @@ from typing import Iterator, Optional __all__ = [ - 'WORKFLOW', 'PYPROJECT', 'CORE', 'MODULE_PROVIDERS', 'NON_DISTRIBUTION_FLAGS', + 'WORKFLOW', 'PYPROJECT', 'LATEST_LEG_CONDITION', 'CORE', 'MODULE_PROVIDERS', 'NON_DISTRIBUTION_FLAGS', 'MUTUALLY_EXCLUSIVE_IMPORTS', 'DEPENDENCY_GATE_EXCLUSIONS', 'Requirement', 'Gate', 'Job', 'Gap', 'Exclusion', 'AmbiguousProvider', 'requirement_key', 'provided_by', 'declared_requirements', 'extras_providing', @@ -133,6 +133,9 @@ WORKFLOW = _tiers.ROOT / '.github' / 'workflows' / 'unit-tests.yml' #: Where the extras are declared. PYPROJECT = _tiers.ROOT / 'pyproject.toml' +#: The ``if:`` of a step that runs on the ``latest`` leg alone (#1958), which is +#: how :func:`pytest_jobs` tells that leg's install line from every leg's. +LATEST_LEG_CONDITION = 'matrix.python-version == needs.python-versions.outputs.latest' #: Stands in for ``[project] dependencies`` where an extra name is expected: #: what ``pip install -e .`` installs with no extra asked for at all. Spelled @@ -206,8 +209,9 @@ 'pylint': ('pylint',), # ``beautifulsoup4`` ships ``bs4``; ``html5lib`` is an *extra of* that # distribution, so ``beautifulsoup4`` alone does not provide it. That - # distinction is the whole reason ``HAS_CRAWLER_DEPS`` is satisfied in CI - # and ``HAS_VENDOR_DEPS`` is not: the ``test`` extra carries plain + # distinction is why ``HAS_CRAWLER_DEPS`` is satisfied by the ``test`` + # extra on every job and ``HAS_VENDOR_DEPS`` only where ``vendor`` joins it, + # the ``latest`` legs (#1958): the ``test`` extra carries plain # ``beautifulsoup4``, and only ``vendor`` and ``dev`` carry # ``beautifulsoup4[html5lib]``. The second alternative below is not a # second *distribution* -- pyproject.toml never declares bare @@ -249,8 +253,8 @@ # extra declares docutils itself this entry has to change: the bare name goes # live alongside Sphinx, :func:`contested_imports` reports docutils, and the # :data:`MUTUALLY_EXCLUSIVE_IMPORTS` liveness test fails until the entry is - # cut to ``('docutils',)``. If the ``test`` and ``gate`` jobs install that - # extra, its gap closes and the ``HAS_DOCUTILS`` exclusion has to go as well. + # cut to ``('docutils',)``. The ``latest`` legs install the ``docs`` extra + # (#1958), which is what closes the ``HAS_DOCUTILS`` gate there. 'docutils': ('Sphinx', 'docutils'), # The theme ``HAS_FURO`` asks for, whose table wrapper the #1861 transform # test runs. Only the ``docs`` extra carries it. @@ -370,7 +374,9 @@ class Exclusion(NamedTuple): '``pcap`` module and cannot coexist -- see ' "pcapkit/foundation/engines/_pcap_backend.py's own docstring. test and gate " "stay dark on purpose, per #751's ruling that per-engine coverage gets its " - 'own job rather than one more install line on either.' + 'own job rather than one more install line on either. #1958 gave the latest ' + 'legs every engine pip can install there, and PCAP_CT alone needs that ' + "system libpcap, which engine-tests' PCAP_CT cell installs and runs on latest." ), ), 'HAS_PYPCAPFILE': Exclusion( @@ -409,192 +415,6 @@ class Exclusion(NamedTuple): 'three.' ), ), - 'HAS_VENDOR_DEPS': Exclusion( - dark={'test': ('html5lib',), 'gate': ('html5lib',)}, - reason=( - "Ruled onto a non-blocking leg by #738's option (b), on the premise that the " - 'crawlers fetch from IANA and Wikipedia and #518 records four Wikipedia 403s ' - 'and a dead IETF URL -- so a blocking leg would go red on upstream ' - 'availability rather than on this code. Note what is actually missing, which ' - "#738's own inventory gets wrong: requests and bs4 have shipped in the test " - 'extra since #507 and are installed on every job, so these classes are one ' - '*requirement extra* short -- html5lib, which only beautifulsoup4[html5lib] ' - 'provides, i.e. the vendor and all extras.\n\n' - 'engine-tests (#751) used to be that leg: it mirrored test\'s ignore-shape ' - 'exactly (same --ignore flags) and carried vendor on its one shared install ' - 'line, so it reached and closed these unit-tier HAS_VENDOR_DEPS gates as a ' - "side effect of installing everything together. #849's rebuild removes that " - 'shared venv: each matrix.engine cell now installs only "test" plus its own ' - 'engine extra and runs only that engine\'s own explicit TEST_PATHS -- ' - 'tests/toolkit/test_scapy_unit.py and four siblings, never tests/vendor/ -- so ' - 'no cell reaches these gates at all any more, let alone installs vendor for ' - 'them. This is not a *new* gap this guard reports, because ' - ':func:`dependency_gate_gaps` only counts a job that reaches a gate without ' - "installing what it needs; a job that no longer reaches the gate at all is " - 'invisible to it either way, which is exactly the "silence is the failure ' - 'mode" problem this module\'s own docstring opens with. It is a real, ' - 'disclosed regression rather than a silent one: #849\'s own PR description ' - "records vendor coverage as dropped, not restored, and worth its own issue. " - 'test and gate stay dark for their own, unrelated reasons below; engine-tests ' - 'is simply no longer part of the answer for this flag.\n\n' - 'test and gate stay dark for different reasons, not the same one. test ' - 'declines html5lib per the non-blocking-leg ruling itself -- that is the ' - 'whole reason it is dark. gate is not part of the blocking matrix at all -- ' - "it only runs on the release path (workflow_call's gate-only: true), never " - 'on a pull request -- and now that engine-tests no longer carries vendor ' - 'either, no per-PR job does, so this flag\'s only per-PR-visible install ' - 'point is test\'s own decline above. gate could add vendor without ' - 'contradicting the "non-blocking" ruling, but doing so on the one job that ' - 'gates an actual release, for a network-and-parser extra whose own crawlers ' - "#518 already documents as flaky, is not worth the footprint on its own -- " - "and it would not restore the per-PR coverage #849 removed regardless, since " - 'gate never runs on a pull request at all.\n\n' - 'Worth recording since it bears on the ruling itself, though this exclusion ' - 'is not the place to relitigate it: none of the 41 methods this closes make ' - 'a live network call, though not for one reason across all four files. ' - 'test_user_agent_unit.py (11 methods) and test_request_prompt_unit.py (16) ' - 'gate on HAS_VENDOR_DEPS only because the pcapkit.vendor package import ' - 'chain needs requests/bs4/html5lib importable -- html5lib itself is never ' - "used by either file's test bodies, both of which replace requests.get with " - "a recorder for the whole of every case (each file's own module docstring " - 'says so in its own words, not a shared one). test_ipx_packet_unit.py (9 ' - 'methods) is a regression suite for a *retired* scrape -- Packet.LINK is ' - 'None, so _request() short-circuits before ever calling requests.get, and ' - 'test_request_makes_no_network_call asserts exactly that by making ' - 'requests.get and requests.Session.request raise if reached at all; ' - 'html5lib is an import precondition there too, nothing in the file uses it. ' - 'test_ftp_return_code_unit.py (5 methods) is not retired -- ' - 'pcapkit.vendor.ftp.return_code.ReturnCode.LINK is a live Wikipedia URL -- ' - 'and is the one file where html5lib is a *functional* dependency rather ' - 'than an import precondition: its tests build the HTML fixture inline and ' - "hand it straight to ReturnCode.request(), which is " - "bs4.BeautifulSoup(text, 'html5lib'); what keeps it off the network is the " - 'fixture being inline, not an absence of html5lib use. ' - "#518's flakiness is real for HAS_CRAWLER_DEPS's live crawlers, which run " - 'unconditionally since #507; it does not describe any of the above. ' - 'pypcap-parity does not appear here even though it also declines html5lib, ' - 'because its fixture-tier selection never reaches ' - 'tests/vendor/test_ipx_packet_unit.py or its three siblings in the first ' - 'place -- same shape as integration above it, which is why integration is ' - 'not listed either.' - ), - ), - 'HAS_SCAPY': Exclusion( - dark={'test': ('scapy',)}, - reason=( - 'Known, not new: #738 set it aside precisely because Scapy is installed on ' - 'the integration and gate legs, so "they are dark only on the test leg". The ' - 'test job declines Scapy on cost grounds -- its own install-step comment ' - 'contrasts DPKT, which "costs nothing per leg", with Scapy. What was never ' - 'written down is the consequence: the 10 methods it darkens ' - '(test_scapy_unit.py 5, test_scapy_engine.py 2, and one each in ' - 'test_core.py, test_misc.py and test_sctp_unit.py) are all unit-tier, so the ' - 'integration job never reaches them either. The gate job does, but it runs ' - "only where a caller passes gate-only: true -- the three Saturday schedules " - '(deploy-pages, cron-vendor, cron-conda) and a v* release tag -- never on a ' - 'pull request or a push to main. So no per-PR leg runs them at all.\n\n' - "#751's engine-tests job used to install Scapy across the full 3.10-3.14 " - 'matrix and run it against the whole unit tier (same --ignore shape as ' - 'test), closing the "no per-PR leg" problem this exclusion used to describe ' - "for all 10 of this guard's 14 unit-tier HAS_SCAPY methods at once.\n\n" - "#849's rebuild only partially keeps that closure. engine-tests's Scapy cell " - 'now runs an explicit TEST_PATHS list -- tests/toolkit/test_scapy_unit.py, ' - 'tests/foundation/engines/test_scapy_engine.py and ' - "tests/protocols/transport/test_sctp_unit.py -- which covers 8 of the 10 (5 " - '+ 2 + 1), but not the other 2, one each in test_core.py and test_misc.py: ' - "those are simply not in that cell's path list, so no per-PR job reaches " - 'them with Scapy installed any more -- the exact "no per-PR leg runs them ' - 'at all" state from before #751, reopened for those two. This is not ' - 'visible as a wider Gap here: :func:`dependency_gate_gaps` only reports a job ' - 'that reaches a gate without installing for it, and a job that no longer ' - "reaches the gate at all is invisible to it either way, by construction -- " - "which is why this paragraph exists to say so in prose rather than leave it " - "to the tool. test_sctp_unit.py's one method was added to the Scapy cell's " - 'TEST_PATHS at zero new-dark-flag cost: unlike test_core.py and ' - 'test_misc.py, it carries no HAS_DPKT-gated tests of its own, so restoring ' - 'it traded nothing away. test_core.py and test_misc.py each still carry ' - "their own HAS_DPKT-gated tests (see the test job's own install-step " - "comment), and adding them here without DPKT would trade one dark flag for " - 'another on a cell that installs neither -- disclosed rather than ' - 'restored, a decision worth its own review rather than folding into a ' - 'workflow-only CR. The other 4 of the 14 live in ' - 'tests/integration/ or match the *_runtime.py ignore-glob, unaffected by any ' - 'of this, already covered by Scapy on the integration and gate jobs. test ' - 'stays dark on purpose: the ruling on #751 was a dedicated job precisely so ' - 'test would not have to reconsider the cost question it already answered.' - ), - ), - 'HAS_PYSHARK': Exclusion( - dark={'test': ('pyshark',), 'integration': ('pyshark',), 'gate': ('pyshark',)}, - reason=( - 'One of #738\'s seven, left out of #740\'s "cheap, uncontroversial subset" ' - 'and never since installed, so its 4 gated methods run nowhere. Three of them ' - 'assert what PyShark.unsupported_reason says on an interpreter where the ' - 'engine cannot work, so they need the distribution importable to have ' - 'anything to ask.\n\n' - "#751's later ruling asked to try a tshark binary the same way it asked to " - 'try building pypcap: "try to build and if the CI is not a good suit, then ' - 'we ripe it… same for HAS_PYSHARK on tshark dependency." ' - 'test_the_reason_tracks_the_running_interpreter used to call ' - 'PyShark.unsupported_reason() unpatched and hard-assert the reason names ' - '"tshark" on every version below the asyncio ceiling -- true only while ' - 'tshark stayed absent, and it would have turned that assertion from a pass ' - 'into a failure the moment tshark was installed. Fixed by probing the same ' - 'way PyShark.unsupported_reason() itself does -- pyshark\'s own ' - "get_process_path(), not shutil.which(), which its own config.ini " - 'precedence can make disagree with -- matching what its sibling ' - 'test_this_host_really_has_no_tshark_so_the_check_is_not_vacuous already ' - 'used, so the file no longer disagrees with itself about whether tshark may ' - 'be present. engine-tests installs both the distribution and tshark (apt-get, ' - 'with a debconf pre-seed so the postinst prompt does not block) on its own ' - 'PyShark cell, and that cell\'s explicit TEST_PATHS (#849) -- ' - 'tests/toolkit/test_pyshark_unit.py (0 of the 4; it carries no HAS_PYSHARK ' - 'gate of its own, only its DPKT/PyPCAPFile/etc. siblings do) and ' - 'tests/foundation/engines/test_pyshark_engine.py (the other 3) -- reaches 3 ' - 'of the 4 HAS_PYSHARK methods. The fourth, ' - 'tests/integration/test_engine_runtime.py:70\'s ' - 'test_pyshark_engine_is_refused_before_asyncio_can_break, is integration-tier ' - "and outside any engine-tests cell's reach -- it stays dark on the integration " - 'job exactly as recorded above, unaffected by any of this. So the rebuild ' - "from one shared ignore-shape venv to one venv per engine leaves this flag's " - "closure unchanged, closing the test-job gap this exclusion used to describe " - "for those 3. integration and gate stay dark on purpose, per #751's ruling " - 'that per-engine coverage gets its own job.' - ), - ), - 'HAS_RUNTIME': Exclusion( - dark={'test': ('pyshark', 'scapy'), 'gate': ('pyshark',)}, - reason=( - "Absent from #738's inventory of seven, and the one gap this guard found " - "that no earlier pass did. Every other module's HAS_RUNTIME asks only for " - 'the four core dependencies, which pip install -e . always provides; ' - 'tests/foundation/engines/test_runtime_engines.py reuses the name for those ' - 'four plus dpkt, scapy and pyshark, so its two classes (5 methods) skip ' - 'wherever any of the three is absent. It is unit-tier despite the name, ' - 'because the ignore-glob is *_runtime.py and the file is ' - 'test_runtime_engines.py, so the test job is what reaches it. The narrow ' - 'fix -- giving that flag a name of its own, so the skip reason says which ' - 'dependency was missing -- is left for a follow-up: it touches a test ' - "module #751 did not otherwise need to change, and this guard does not " - 'care what a flag is named, only whether the job that reaches it installs ' - 'what it asks for.\n\n' - "#751's engine-tests job used to install DPKT, Scapy and PyShark together in " - "one shared venv, closing this gap as a side effect: all 5 methods ran " - "wherever engine-tests did. #849's rebuild removes that shared venv -- each " - 'matrix.engine cell now installs exactly one engine\'s own extra, so no cell ' - 'installs all three together any more, and none of the five explicit ' - 'TEST_PATHS lists names test_runtime_engines.py at all, so engine-tests no ' - 'longer reaches this gate either. Like the HAS_SCAPY and HAS_VENDOR_DEPS ' - 'entries above, this does not surface as a wider Gap -- a job that stops ' - "reaching a gate is invisible to :func:`dependency_gate_gaps` the same way a " - 'job that never reached it would be -- so it is recorded here in prose. ' - "Disclosed, not silent: #849's own PR description names this exact loss " - '("the multi-engine HAS_RUNTIME gate ... needs DPKT+Scapy+PyShark together") ' - 'and defers restoring it to a follow-up issue rather than reopening this ' - 'workflow-only change to add a seventh venv shape. test and gate stay dark on ' - 'purpose, for the same reason HAS_SCAPY and HAS_PYSHARK above do.' - ), - ), 'HAS_MYPY': Exclusion( dark={'test': ('mypy',), 'gate': ('mypy',)}, reason=( @@ -644,36 +464,12 @@ class Exclusion(NamedTuple): ':meth:`~tests.test_tier_guard.DependencyGateCoverageTests\ .test_each_exclusion_still_describes_a_gap_that_is_really_there` exists to catch if this ' 'entry is not kept in step with it. integration and pypcap-parity select by ' - 'fixture tier and never collect that module, which is why they are not listed, ' - 'the same reason they are absent from HAS_VENDOR_DEPS above. If a lint extra ' + 'fixture tier and never collect that module, which is why they are not listed. ' + 'If a lint extra ' 'is ever declared in pyproject.toml, the right change is to delete this entry ' 'and let the gap close on its own rather than to widen it.' ), ), - 'HAS_SPHINX': Exclusion( - dark={'test': ('sphinx', 'sphinx_autodoc_typehints'), - 'project-tests': ('sphinx', 'sphinx_autodoc_typehints'), - 'gate': ('sphinx', 'sphinx_autodoc_typehints')}, - reason=( - 'The classes it gates (#1775, #1856, #1858-#1863, #1917, #1918 and the enum-value ' - 'hook\'s) load docs/source/conf.py and drive its hooks with the nodes and events ' - 'Sphinx itself builds, so they need Sphinx and the pinned sphinx-autodoc-typehints. ' - 'Only deploy-pages.yml installs those, from docs/requirements.txt, and that ' - 'workflow builds the documentation rather than running pytest; no workflow ' - 'installs the docs extra itself. The same argument declines it here as ' - 'declines HAS_MYPY above: the docs build is already the end-to-end check of ' - 'those hooks, and putting Sphinx, furo and their dependencies on every ' - 'unit-test leg would cost each one an install for these classes alone. The ' - 'flag also requires the installed sphinx-autodoc-typehints to be the release ' - 'docs/requirements.txt pins, since conf.py imports that release\'s private ' - 'modules; that check reads no module, so this entry still names only the ' - 'imports. They run wherever docs/requirements.txt or the docs extra is ' - 'installed, a developer checkout included, and skip visibly elsewhere, saying ' - 'which condition failed. The directive-name and pin-agreement tests need ' - 'nothing and run on every leg, or in project-tests where their module is a ' - 'project pin (#1954).' - ), - ), 'HAS_PYLINT': Exclusion( dark={'project-tests': ('pylint',), 'gate': ('pylint',)}, reason=( @@ -693,36 +489,6 @@ class Exclusion(NamedTuple): 'visibly elsewhere.' ), ), - 'HAS_DOCUTILS': Exclusion( - dark={'test': ('docutils',), 'gate': ('docutils',)}, - reason=( - "The method it gates (#1846) parses Vendor.wrap_comment's output with " - 'docutils and asserts it is nothing but a plain paragraph, so it needs ' - 'docutils, which no pytest job installs. No workflow installs the docs extra; ' - 'deploy-pages.yml gets Sphinx, and with it docutils, from ' - 'docs/requirements.txt, and it builds the documentation rather than running ' - 'pytest. Installing docutils on these jobs would not run it anyway: it sits in ' - 'a class gated on ' - 'HAS_VENDOR_DEPS, which these same jobs leave dark by declining html5lib ' - "(#738, above), so closing this gap would also mean the vendor extra on a " - 'blocking leg. The cost argument of HAS_SPHINX applies too. What runs on ' - 'every leg instead is the ungated committed-const class in the same module, ' - 'which pins the re-rendered lines themselves; the string assertions on the ' - 'wrapper run wherever the vendor extra is installed, and the docutils parse ' - 'wherever docutils is importable as well.' - ), - ), - 'HAS_FURO': Exclusion( - dark={'test': ('furo',), 'gate': ('furo',)}, - reason=( - 'The class it gates (#1861) runs the table-wrapper transform of furo, the ' - 'docs theme, ahead of the conf.py transform that undoes its duplicate ids, ' - 'and is gated on HAS_SPHINX as well. It is declined for the reason ' - 'HAS_SPHINX is: only deploy-pages.yml installs furo, from ' - 'docs/requirements.txt, and no workflow installs the docs extra itself; that ' - 'build is the end-to-end check of the same transform.' - ), - ), } @@ -794,6 +560,10 @@ class Job(NamedTuple): #: :func:`~tests._tiers.fixture_tier_paths` for that tier alone, :data:`None` #: for one whose ``latest`` leg asks for every tier, and for every other shape. tier: 'Optional[str]' = None + #: Extras of the job's ``latest``-only install step (#1958), empty when it has + #: none. That leg's selection holds every other leg's, so a gate the job + #: reaches runs there with these installed; see :func:`dependency_gate_gaps`. + latest_extras: 'tuple[str, ...]' = () class Gap(NamedTuple): @@ -877,7 +647,7 @@ def provided_by(requirements: 'tuple[Requirement, ...]', provider: 'str') -> 'bo the base distribution -- while ``'beautifulsoup4[html5lib]'`` is *not* satisfied by plain ``beautifulsoup4``. That asymmetry is the whole point: it is what separates ``HAS_CRAWLER_DEPS``, which the ``test`` extra - satisfies, from ``HAS_VENDOR_DEPS``, which it does not. + satisfies, from ``HAS_VENDOR_DEPS``, which needs ``vendor`` as well. """ wanted = requirement_key(provider) @@ -1554,14 +1324,6 @@ def pytest_jobs(workflow: 'Optional[pathlib.Path]' = None) -> 'tuple[Job, ...]': jobs.extend(variants) continue - installs = re.findall(r"pip install -e '\.\[([^]]*)\]'", section) - if len(installs) != 1: - raise AssertionError( - f'the {name!r} job runs pytest but has {len(installs)} ' - f"\"pip install -e '.[...]'\" lines, and this guard cannot tell which " - f'extras the run would have' - ) - # Scoped to the step that runs pytest, not to the job, and found by what # the step does rather than by what it is called. Both matter: the # `gate` job's *comment* says "no --ignore, no tier selection" and the @@ -1570,6 +1332,27 @@ def pytest_jobs(workflow: 'Optional[pathlib.Path]' = None) -> 'tuple[Job, ...]': # credits it with reaching only the unit tier. Matching on the step name # would work only until a step is renamed. steps = re.findall(r'(?ms)^ - name: [^\n]*\n(.*?)(?=^ - name:|\Z)', section) + + # #1958: a step whose `if:` runs it on `latest` alone holds that leg's + # install line, read apart from the one every leg runs. + latest_steps = [step for step in steps if LATEST_LEG_CONDITION in step.partition('run:')[0]] + latest_installs = [extras for step in latest_steps + for extras in re.findall(r"pip install -e '\.\[([^]]*)\]'", step)] + if len(latest_installs) > 1: + raise AssertionError( + f'the {name!r} job has {len(latest_installs)} latest-only ' + f"\"pip install -e '.[...]'\" lines, and this guard cannot tell which " + f'extras that leg would have' + ) + installs = re.findall(r"pip install -e '\.\[([^]]*)\]'", section) + installs = [extras for extras in installs if extras not in latest_installs] \ + if latest_installs else installs + if len(installs) != 1: + raise AssertionError( + f'the {name!r} job runs pytest but has {len(installs)} ' + f"\"pip install -e '.[...]'\" lines, and this guard cannot tell which " + f'extras the run would have' + ) running = [step for step in steps if 'python -m pytest' in step] if len(running) != 1: raise AssertionError( @@ -1594,7 +1377,9 @@ def pytest_jobs(workflow: 'Optional[pathlib.Path]' = None) -> 'tuple[Job, ...]': selection = 'whole-suite' jobs.append(Job(name, tuple(extra.strip() for extra in installs[0].split(',')), - selection, tier=tier)) + selection, tier=tier, + latest_extras=tuple(extra.strip() for extra in latest_installs[0].split(',')) + if latest_installs else ())) return tuple(jobs) @@ -1682,7 +1467,10 @@ def dependency_gate_gaps(workflow: 'Optional[pathlib.Path]' = None) -> 'tuple[Ga gaps = {} # type: dict[tuple[str, str], tuple[set[str], list[Gate]]] for job in pytest_jobs(workflow): - available = frozenset(job.extras) | {CORE} + # #1958: a job's `latest` leg runs every test its other legs do, so a + # gate it reaches runs there with that leg's extras too. The other legs + # run the `ALL` tier with the job's own line, and skip what it lacks. + available = frozenset(job.extras) | frozenset(job.latest_extras) | {CORE} for gate in gated_scopes(): if gate.flag in NON_DISTRIBUTION_FLAGS: continue diff --git a/tests/project/test_project_pins_once_1954.py b/tests/project/test_project_pins_once_1954.py index 6c5e1a48ad..8a2a13bc20 100644 --- a/tests/project/test_project_pins_once_1954.py +++ b/tests/project/test_project_pins_once_1954.py @@ -54,7 +54,8 @@ def test_it_runs_on_latest_with_what_test_installs(self) -> None: setup = next(entry for entry in job(*JOB)['steps'] if 'setup-python' in entry.get('uses', '')) self.assertEqual(setup['with']['python-version'], '${{ steps.python-versions.outputs.latest }}') install = step(JOB, 'Install package and test dependencies')['run'] - self.assertIn(step(('unit-tests.yml', 'test'), 'Install package and test dependencies')['run'] + # #1958: what `test` installs on `latest`, the leg the pins would run on. + self.assertIn(step(('unit-tests.yml', 'test'), "Install the whole suite's dependencies on latest")['run'] .split("pip install -e '", 1)[1].split("'", 1)[0], install) def test_it_runs_under_the_caps(self) -> None: diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py index 649a0ea086..d9be81d7b5 100644 --- a/tests/project/test_sweeps_on_latest_only_1732.py +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -460,8 +460,10 @@ def test_it_installs_what_test_installs(self) -> None: self.assertEqual(_dependency_gates.WORKFLOW.name, 'unit-tests.yml') installs = {} for where in (JOB, EXPERIMENTAL): + # #1958: less the step `test` runs on `latest` alone, which no experimental leg is. found = [re.findall(r"pip install -e '\.\[([^]]*)\]'", str(entry.get('run', ''))) - for entry in job(*where)['steps']] + for entry in job(*where)['steps'] + if _dependency_gates.LATEST_LEG_CONDITION not in str(entry.get('if', ''))] installs[where] = [extra for entries in found for extra in entries] self.assertEqual(installs[EXPERIMENTAL], installs[JOB]) self.assertEqual(len(installs[JOB]), 1) diff --git a/tests/project/test_whole_suite_on_latest_1958.py b/tests/project/test_whole_suite_on_latest_1958.py new file mode 100644 index 0000000000..9ff50e3d6a --- /dev/null +++ b/tests/project/test_whole_suite_on_latest_1958.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +"""#1958: latest CPython runs the whole suite, with what the whole suite gates on. + +The owner's rulings on #1958, paraphrased: every implementation runs the same +minimal ``ALL`` tier, and latest CPython runs the whole suite as the sanity check +and the gate; tests that run nowhere get a job. So each job's ``latest`` leg +installs the engines pip can install there, html5lib (``vendor``), Sphinx +(``docs``) and emoji (``cli``), and the ``HAS_*`` gates those extras close run +there. This module pins the install lines, the leg they run on, the gates they +close, and the page that describes them. :mod:`tests._dependency_gates` derives +the gaps; :file:`tests/test_tier_guard.py` checks its reading of the workflow. + +""" + +from __future__ import annotations + +import pathlib +import re +import tempfile +import unittest + +from tests import _dependency_gates, _workflows +from tests.project.test_python_matrix_tiers_1633 import legs +from tests.project.test_python_versions_1660 import script + +ROOT = pathlib.Path(__file__).resolve().parents[2] +DOC = ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst' +WORKFLOW = 'unit-tests.yml' + +#: What a ``latest`` leg installs: every extra a gate asks for that installs there. +WHOLE_SUITE = frozenset({'test', 'Scapy', 'DPKT', 'crypto', 'cli', 'NGAP', 'vendor', 'docs', 'PyShark'}) +#: The step ``test`` and ``integration`` add on ``latest``. +LATEST_STEP = "Install the whole suite's dependencies on latest" +LATEST_IF = ("${{ steps.install.outcome == 'success' && " + 'matrix.python-version == needs.python-versions.outputs.latest }}') +#: The install line every leg of each matrix job runs, unchanged by #1958. +BASE = { + 'test': frozenset({'test', 'DPKT', 'crypto', 'NGAP'}), + 'integration': frozenset({'test', 'Scapy', 'DPKT', 'cli'}), +} +#: The jobs that run on ``latest`` alone, each with one install step. +LATEST_ONLY = { + 'project-tests': 'Install package and test dependencies', + 'gate': 'Install package, test and generator dependencies', +} +#: The gates still dark somewhere: lint tools no extra declares, and the engines +#: ``latest`` cannot install (PCAP_CT needs a system libpcap; the PyPCAP and +#: PyPCAPFile markers stop below 3.12). ``engine-tests`` and ``pypcap-parity`` run them. +STILL_DARK = frozenset({'HAS_MYPY', 'HAS_PYLINT', 'HAS_PCAP_CT', 'HAS_PYPCAP', 'HAS_PYPCAPFILE'}) +#: A test the audit found running nowhere, or in ``gate`` alone, per flag. +DARK_TESTS = { + 'HAS_RUNTIME': 'tests/foundation/engines/test_runtime_engines.py', + 'HAS_SCAPY': 'tests/interface/test_core.py', + 'HAS_PYSHARK': 'tests/integration/test_engine_runtime.py', + 'HAS_VENDOR_DEPS': 'tests/vendor/test_user_agent_unit.py', + 'HAS_DOCUTILS': 'tests/vendor/test_vendor_wrap_comment_unit.py', + 'HAS_SPHINX': 'tests/project/test_docs_ambiguous_xrefs_1775_unit.py', + 'HAS_FURO': 'tests/project/test_docs_table_wrapper_ids_1861_unit.py', +} + + +def step(job_id: 'str', name: 'str') -> 'dict[str, object]': + found = [entry for entry in _workflows.steps(WORKFLOW, job_id) if entry.get('name') == name] + if len(found) != 1: + raise AssertionError(f'expected one {name!r} step in {job_id}, found {len(found)}') + return found[0] + + +def extras(run: 'object') -> 'frozenset[str]': + found = re.findall(r"pip install -e '\.\[([^]]*)\]'", str(run)) + if len(found) != 1: + raise AssertionError(f'expected one editable install in {run!r}, found {len(found)}') + return frozenset(extra.strip() for extra in found[0].split(',')) + + +class TestTheLatestLegsInstallTheWholeSuite(unittest.TestCase): + """Each ``latest`` leg installs :data:`WHOLE_SUITE`; every other leg keeps its line.""" + + def test_the_matrix_jobs_add_it_on_latest_alone(self) -> None: + for job_id in BASE: + with self.subTest(job=job_id): + names = [entry.get('name') for entry in _workflows.steps(WORKFLOW, job_id)] + added = step(job_id, LATEST_STEP) + self.assertEqual(added['if'], LATEST_IF) + self.assertEqual(extras(added['run']), WHOLE_SUITE) + # Right after the install every leg runs, whose outcome it waits on. + install = next(entry for entry in _workflows.steps(WORKFLOW, job_id) if entry.get('id') == 'install') + self.assertEqual(names.index(LATEST_STEP), names.index(install['name']) + 1) + self.assertEqual(extras(install['run']), BASE[job_id]) + + def test_the_jobs_on_latest_alone_install_it(self) -> None: + for job_id, name in LATEST_ONLY.items(): + with self.subTest(job=job_id): + self.assertEqual(extras(step(job_id, name)['run']), WHOLE_SUITE) + + def test_no_ktlo_leg_is_latest(self) -> None: + """The KTLO twins run the same steps by alias, so the ``latest`` step must skip on each.""" + latest = script().load()['latest'] + for twin, job_id in (('test', 'test-ktlo'), ('integration', 'integration-ktlo')): + with self.subTest(job=job_id): + shared = _workflows.steps(WORKFLOW, job_id) + self.assertEqual(shared, _workflows.steps(WORKFLOW, twin)) + self.assertIn(LATEST_STEP, [entry.get('name') for entry in shared]) + versions = [leg['python-version'] for leg in legs(WORKFLOW, job_id)] + self.assertTrue(versions) + self.assertNotIn(latest, versions) + self.assertIn(latest, [leg['python-version'] for leg in legs(WORKFLOW, twin)]) + + +class TestTheDarkTestsRunOnLatest(unittest.TestCase): + """The gates those extras close are no gap on any job; the rest are the known ones.""" + + def setUp(self) -> None: + reason = _dependency_gates._tiers.guard_unavailable_reason() + if reason is not None: + self.skipTest(f'git cannot answer here: {reason}') + + def test_the_guard_reads_the_latest_step_as_that_legs_extras(self) -> None: + jobs = {job.name: job for job in _dependency_gates.pytest_jobs() if job.name != 'engine-tests'} + for job_id, base in BASE.items(): + with self.subTest(job=job_id): + self.assertEqual(frozenset(jobs[job_id].extras), base) + self.assertEqual(frozenset(jobs[job_id].latest_extras), WHOLE_SUITE) + for job_id in LATEST_ONLY: + with self.subTest(job=job_id): + self.assertEqual(frozenset(jobs[job_id].extras), WHOLE_SUITE) + self.assertEqual(jobs['pypcap-parity'].latest_extras, ()) + + def test_an_unconditional_second_install_is_still_refused(self) -> None: + """Only a step gated on ``latest`` is read apart; any other second line is ambiguous.""" + text = (_workflows.WORKFLOWS / WORKFLOW).read_text(encoding='utf-8') + doctored = text.replace(f' if: {LATEST_IF}\n', '', 1) + self.assertNotEqual(doctored, text) + with tempfile.TemporaryDirectory() as tmp: + path = pathlib.Path(tmp) / WORKFLOW + path.write_text(doctored, encoding='utf-8') + with self.assertRaises(AssertionError) as caught: + _dependency_gates.pytest_jobs(path) + self.assertIn("'test' job runs pytest but has 2", str(caught.exception)) + + def test_only_the_known_gates_stay_dark(self) -> None: + self.assertEqual({gap.flag for gap in _dependency_gates.dependency_gate_gaps()}, STILL_DARK) + self.assertEqual(set(_dependency_gates.DEPENDENCY_GATE_EXCLUSIONS), STILL_DARK) + + def test_each_dark_test_runs_on_a_latest_leg(self) -> None: + gaps = {(gap.flag, gap.job) for gap in _dependency_gates.dependency_gate_gaps()} + jobs = [job for job in _dependency_gates.pytest_jobs() if job.name in set(BASE) | set(LATEST_ONLY)] + for flag, module in DARK_TESTS.items(): + with self.subTest(flag=flag): + gates = [gate for gate in _dependency_gates.gated_scopes() + if gate.flag == flag and gate.module == module] + self.assertTrue(gates, f'{module} no longer gates on {flag}') + reaching = {job.name for job in jobs for gate in gates if _dependency_gates.job_reaches(job, gate)} + self.assertTrue(reaching, f'no latest leg collects {module}') + for job_id in sorted(reaching): + self.assertNotIn((flag, job_id), gaps) + + +class TestTheWorkflowsPage(unittest.TestCase): + """:file:`docs/source/contributing/workflows.rst` names the step and this module.""" + + def test_it_describes_the_latest_install(self) -> None: + text = ' '.join(DOC.read_text(encoding='utf-8').split()) + self.assertIn(f'the step ``{LATEST_STEP}`` runs on the ``latest`` leg alone', text) + for job_id, name in LATEST_ONLY.items(): + with self.subTest(job=job_id): + self.assertIn(f'``{name}', text) + self.assertIn(':file:`tests/project/test_whole_suite_on_latest_1958.py`', text) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tier_guard.py b/tests/test_tier_guard.py index 2d3f0585ff..3501b08457 100644 --- a/tests/test_tier_guard.py +++ b/tests/test_tier_guard.py @@ -1870,8 +1870,9 @@ def test_the_project_pins_selection_is_exactly_the_pins(self) -> None: """#1954: ``project-tests`` reaches the pins and nothing else, so a gate moves with its module.""" job = next(job for job in _dependency_gates.pytest_jobs() if job.selection == 'project-pins') self.assertEqual(job.name, 'project-tests') + # #1958: the set `test` installs on `latest`, the leg the pins would run on. self.assertEqual(job.extras, next(job for job in _dependency_gates.pytest_jobs() - if job.name == 'test').extras) + if job.name == 'test').latest_extras) cases = { 'tests/project/test_isort_clean.py': True, 'tests/project/test_public_api.py': False, @@ -2021,37 +2022,29 @@ def test_the_crawler_dependencies_are_satisfied_by_the_test_extra(self) -> None: gaps = {gap.flag for gap in _dependency_gates.dependency_gate_gaps()} self.assertNotIn('HAS_CRAWLER_DEPS', gaps) self.assertNotIn('HAS_CRAWLER_DEPS', _dependency_gates.DEPENDENCY_GATE_EXCLUSIONS) - self.assertIn('HAS_VENDOR_DEPS', _dependency_gates.DEPENDENCY_GATE_EXCLUSIONS) + # The ``test`` extra still lacks html5lib: ``vendor`` closes that gate (#1958). + self.assertNotIn('test', _dependency_gates.extras_providing('html5lib')) + self.assertIn('vendor', _dependency_gates.extras_providing('html5lib')) - def test_the_vendor_extra_closes_engine_tests_but_not_test_or_gate(self) -> None: - """#738's remaining scope: an existing non-blocking leg absorbs it. + def test_the_vendor_extra_closes_every_job_reaching_the_vendor_gates(self) -> None: + """#1958: the ``latest`` legs install ``vendor``, so html5lib is no gap anywhere. - ``engine-tests`` (#751) already reaches these gates through the same - ignore-shape selection as ``test`` -- see this guard's own - ``HAS_VENDOR_DEPS`` exclusion -- and it has never been one of ruleset - 23497679's required checks, so it already was the "job that exists - and reports without gating a merge" #738's ruling asked for. Installing - ``vendor`` there, rather than opening another job, is what closes the - gap on it. It is also the only *per-pull-request* job with - that property: ``gate`` reaches these gates too but never runs on a - pull request at all (only via a release's ``gate-only: true`` call), - and ``integration``'s fixture-tier selection never reaches - :file:`tests/vendor/` in the first place -- so ``engine-tests`` was - the forced choice, not merely a convenient one. - - ``test`` and ``gate`` stay dark for different reasons, not the same - one -- see the exclusion's own reason for why ``gate`` is not merely - "also non-blocking". + #738 ruled these gates onto a non-blocking leg, and ``engine-tests`` was + that leg until #849 narrowed it to explicit engine paths, after which + they ran nowhere. The owner's ruling E on #1958 runs the whole suite on + latest CPython, so ``test``'s ``latest`` leg and ``gate`` install + ``vendor``, and the exclusion that recorded the gap is gone. """ + reaching = {job.name for job in _dependency_gates.pytest_jobs() + for gate in _dependency_gates.gated_scopes() + if gate.flag == 'HAS_VENDOR_DEPS' and _dependency_gates.job_reaches(job, gate)} + self.assertEqual(reaching, {'test', 'gate'}) gaps = {(gap.flag, gap.job) for gap in _dependency_gates.dependency_gate_gaps()} - self.assertNotIn(('HAS_VENDOR_DEPS', 'engine-tests'), gaps) - self.assertIn(('HAS_VENDOR_DEPS', 'test'), gaps) - self.assertIn(('HAS_VENDOR_DEPS', 'gate'), gaps) - - exclusion = _dependency_gates.DEPENDENCY_GATE_EXCLUSIONS['HAS_VENDOR_DEPS'] - self.assertNotIn('engine-tests', exclusion.dark) - self.assertEqual(set(exclusion.dark), {'test', 'gate'}) + for job in sorted(reaching): + with self.subTest(job=job): + self.assertNotIn(('HAS_VENDOR_DEPS', job), gaps) + self.assertNotIn('HAS_VENDOR_DEPS', _dependency_gates.DEPENDENCY_GATE_EXCLUSIONS) def test_the_mypy_gate_is_visible_and_dark_on_every_job_that_reaches_it(self) -> None: """#779: a lint-tier tool gated visibly, and declining the install line anyway. @@ -2068,7 +2061,7 @@ def test_the_mypy_gate_is_visible_and_dark_on_every_job_that_reaches_it(self) -> :data:`~tests._dependency_gates.DEPENDENCY_GATE_EXCLUSIONS` filtering. (#779 reports 4 errors of 9 tests for the same mutation; it was written before #774 added - :meth:`test_the_vendor_extra_closes_engine_tests_but_not_test_or_gate`, + :meth:`test_the_vendor_extra_closes_every_job_reaching_the_vendor_gates`, which is the tenth and the fifth error. Re-measured here rather than copied.) The entry is what makes the gate expressible; the exclusion is what @@ -2387,7 +2380,14 @@ def setUp(self) -> None: def test_removing_crypto_from_the_test_job_is_caught(self) -> None: """#745's worked example, verbatim: "deleting ``crypto`` ... would go green".""" + # #1958: from both of `test`'s lines, the one every leg runs and the one + # `latest` adds, since the second alone runs the gate. The first is pinned + # to its extras by tests/project/test_whole_suite_on_latest_1958.py. doctored = doctored_workflow(self, "'.[test,DPKT,crypto,NGAP]'", "'.[test,DPKT,NGAP]'") + text = doctored.read_text(encoding='utf-8') + latest = "'.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]'" + self.assertIn(latest, text) + doctored.write_text(text.replace(latest, latest.replace('crypto,', '')), encoding='utf-8') gaps = {(gap.flag, gap.job): gap for gap in _dependency_gates.dependency_gate_gaps(doctored)} @@ -2972,11 +2972,11 @@ def test_a_flag_no_extra_could_satisfy_is_not_reported_as_a_gap(self) -> None: if reason is not None: self.skipTest(f'git cannot answer here: {reason}') - self.assertIn('HAS_PYSHARK', + self.assertIn('HAS_PYPCAPFILE', {gap.flag for gap in _dependency_gates.dependency_gate_gaps()}) with unittest.mock.patch.dict(_dependency_gates.NON_DISTRIBUTION_FLAGS, - {'HAS_PYSHARK': 'stood in for this test'}): - self.assertNotIn('HAS_PYSHARK', + {'HAS_PYPCAPFILE': 'stood in for this test'}): + self.assertNotIn('HAS_PYPCAPFILE', {gap.flag for gap in _dependency_gates.dependency_gate_gaps()}) def test_an_unmapped_module_names_itself_and_the_table_to_add_it_to(self) -> None: From 623024fc50a6e9e74daf7796e80f0fd99164695a Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 20 Oct 2026 00:53:06 -0400 Subject: [PATCH 2/2] ci(unit-tests): say where the latest leg gets cli, crypto and NGAP (#1958) - test, integration: the install comments said cli, and crypto and NGAP, are not installed; the latest leg's whole-suite step installs them. - workflows.rst, releasing.rst: unit-tests.yml line citations re-anchored. --- .github/workflows/unit-tests.yml | 12 ++++++++---- docs/source/contributing/releasing.rst | 4 ++-- docs/source/contributing/workflows.rst | 16 ++++++++-------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 238f863909..fd5b2173e9 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -142,10 +142,12 @@ jobs: # crypto (cryptography) and NGAP (pycrate) join it for the same reason, # covering #738's remaining HAS_CRYPTO gate (test_esp_unit.py) and its # HAS_PYCRATE gate (test_ngap_unit.py), both plain - # unit-tier. `cli` (emoji) is deliberately NOT added here: its only + # unit-tier. `cli` (emoji) is deliberately not on this line: its only # HAS_EMOJI-gated tests are in tests/integration/test_cli_subprocess.py, # which this job ignores wholesale, so installing it here would unlock - # nothing. cryptography is the one compiled wheel of the two; measured + # nothing. The `latest` leg gets it from the whole-suite step below + # (#1958), whose set every `latest` job shares, and there too it unlocks + # nothing in this job. cryptography is the one compiled wheel of the two; measured # clean installs on 3.10-3.14 (manylinux wheels, no build step). pycrate # is pure Python but its distribution is large (~238MB, see the NGAP # extra's comment in pyproject.toml) -- still cheap in wall time, just not @@ -517,10 +519,12 @@ jobs: # selection (tests/_tiers.py's fixture_tier_paths()) always includes the # whole of tests/integration/ -- which is where every HAS_EMOJI-gated # method lives (test_cli_subprocess.py). `crypto` and `NGAP` - # are deliberately NOT added here: test_esp_unit.py and test_ngap_unit.py + # are deliberately not on this line: test_esp_unit.py and test_ngap_unit.py # are plain unit-tier and never enter this job's selection, so installing # cryptography/pycrate here would unlock nothing -- they are already - # covered, on every Python version, by the `test` job above. + # covered, on every Python version, by the `test` job above. The `latest` + # leg gets both from the whole-suite step below (#1958), whose set every + # `latest` job shares, and there too they unlock nothing in this job. # PyPCAPFile is NOT added here either, even though this job's selection # also reaches test_new_engine_parity_runtime.py's HAS_PYPCAPFILE # methods -- see the `test` job's comment above for the bug that used to diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index ead45ff410..ed127f3e9a 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -52,7 +52,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:581-597``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1867-1942``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1871-1946``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:621,808,974,1210``), so a @@ -62,7 +62,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Other checks run unconditionally on the same path, though none is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1671-1697``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1675-1701``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 120dd1e97c..5323cd3d95 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -329,7 +329,7 @@ the same commit several times over. It also skips ``changes``, Checks`_ below), which are gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1659-1665``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1663-1669``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -639,7 +639,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:2033`` + - ``unit-tests.yml:2037`` * - ``Compat Python 3.11`` - job ``compatibility``, matrix leg ``3.11`` - ``python-compatibility.yml:60``, ``.github/python-versions.toml:20`` @@ -661,22 +661,22 @@ which added ``Compat Python 3.15`` and removed ``Compat Python 3.10``. Moving a between the tiers below means editing the ruleset in the same change. ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:2033``). ``Compat Python`` is defined by one, the +(``unit-tests.yml:2037``). ``Compat Python`` is defined by one, the ``compatibility`` job in ``python-compatibility.yml`` (``:60``), whose matrix (``:74``) comes from :file:`.github/python-versions.toml`: the required legs are its supported tier (``python-versions.toml:20-24``), and its KTLO legs (``python-versions.toml:16-19``) have names ending in ``(KTLO)``, so no required context matches one. A comment at -``unit-tests.yml:1945`` mentions the string without defining it. +``unit-tests.yml:1949`` mentions the string without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the ``Compat Python 3.11``-``3.15`` legs, nothing else, as -``unit-tests.yml``'s own comment (``unit-tests.yml:1944-1954``) records. It +``unit-tests.yml``'s own comment (``unit-tests.yml:1948-1958``) records. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- ``test``, ``integration``, ``engine-tests`` and ``pypcap-parity``, plus ``project-tests``, the one run of the project pins, and ``changes`` for the docs-only path -- and checks each result explicitly -(``unit-tests.yml:2032-2088``). Rulesets match check names literally, with no +(``unit-tests.yml:2036-2092``). Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. It accepts the gating jobs as ``skipped`` only on a docs-only pull request, where @@ -685,7 +685,7 @@ It accepts the gating jobs as ``skipped`` only on a docs-only pull request, wher ``python-compatibility.yml``, and a job cannot ``needs:`` a job in another workflow file. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:2035``), +``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:2039``), so it is never produced -- and never expected -- on those paths. .. _ktlo-tier: @@ -709,7 +709,7 @@ and keeps its legs' check names, ``Python 3.10 (KTLO)`` and so on. In with supported legs. Every job holding KTLO legs works the same way: * Every step that can fail on a KTLO leg has an ``id`` and - ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:441-458``), + ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:443-460``), unless it is masked on every leg, as the timings uploads are. Its ``outcome`` is then ``failure`` but its ``conclusion`` ``success``, so the job, its check run and ``required-checks`` read ``success``. A step that needs an earlier one's work