From 6ef223592edd1dfb8e8b906485b08189ab66f9c7 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 20 Oct 2026 00:30:19 -0400 Subject: [PATCH 1/4] ci(unit-tests): run the whole suite on latest CPython (#1958) - test, integration: a step on the latest leg alone installs the whole suite's extras (Scapy, PyShark, vendor, docs, cli on top of the base); every other leg keeps its install line and runs the ALL tier. - project-tests, gate: install the same set. - tests/_dependency_gates.py: read a latest-only install step as that leg's extras; drop the exclusions it closes (HAS_SCAPY, HAS_PYSHARK, HAS_RUNTIME, HAS_VENDOR_DEPS, HAS_SPHINX, HAS_DOCUTILS, HAS_FURO). - tests/project/test_whole_suite_on_latest_1958.py pins the lines, the leg, the gates closed and the docs; tier-guard, #1732 and #1954 pins follow the new reading. - workflows.rst describes it; unit-tests.yml line citations re-anchored. Selection steps and tests/_tiers.py are unchanged. --- .github/workflows/unit-tests.yml | 38 ++- docs/source/contributing/releasing.rst | 4 +- docs/source/contributing/workflows.rst | 33 +- tests/_dependency_gates.py | 306 +++--------------- tests/project/test_project_pins_once_1954.py | 3 +- .../test_sweeps_on_latest_only_1732.py | 4 +- .../test_whole_suite_on_latest_1958.py | 172 ++++++++++ tests/test_tier_guard.py | 62 ++-- 8 files changed, 316 insertions(+), 306 deletions(-) create mode 100644 tests/project/test_whole_suite_on_latest_1958.py diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 3be705e766..238f863909 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -187,6 +187,23 @@ jobs: exit 1 fi + # #1958, the owner's ruling E: latest CPython runs the whole suite, so its + # leg installs everything the suite gates on that pip can install there -- + # the engines (Scapy, PyShark), html5lib through `vendor`, Sphinx and the + # pinned sphinx-autodoc-typehints through `docs`, emoji through `cli`. The + # tests those gate skip on every other leg, which runs the minimal `ALL` + # tier. tests/_dependency_gates.py reads this line as the `latest` leg's + # extras; tests/project/test_whole_suite_on_latest_1958.py pins it to the + # same set as `integration`, `project-tests` and `gate`. Not here: PCAP_CT, + # which needs a system libpcap that `engine-tests` installs for its own + # cell, and PyPCAP and PyPCAPFile, whose markers resolve to nothing on + # `latest`. Never a KTLO leg, so `test-ktlo` always skips it. + - name: Install the whole suite's dependencies on latest + id: install-latest + if: ${{ steps.install.outcome == 'success' && matrix.python-version == needs.python-versions.outputs.latest }} + continue-on-error: ${{ matrix.ktlo == true }} + run: python -m pip install -e '.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]' + # 7.10 is the first release with `[run] patch = ["subprocess"]`, which # .github/coverage.toml relies on to measure the xdist workers. - name: Install coverage @@ -528,6 +545,14 @@ jobs: exit 1 fi + # #1958: the `test` job's step of the same name. On `latest` this job runs + # the whole fixture tier, so it installs the whole suite's set too. + - name: Install the whole suite's dependencies on latest + id: install-latest + if: ${{ steps.install.outcome == 'success' && matrix.python-version == needs.python-versions.outputs.latest }} + continue-on-error: ${{ matrix.ktlo == true }} + run: python -m pip install -e '.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]' + # `shell: bash` rather than the default, because it runs with -o pipefail: # without it the `tee` below would swallow the generator's exit status. - name: Regenerate sample captures @@ -1785,12 +1810,12 @@ jobs: python-version: ${{ steps.python-versions.outputs.latest }} cache: pip - # `test`'s install line, so a gated pin skips or runs here exactly as it - # did on `latest`. + # The whole suite's set, the line `test` adds on `latest` (#1958), so a + # gated pin runs here exactly as the rest of its tier does there. - name: Install package and test dependencies run: | python -m pip install -U pip setuptools wheel - python -m pip install -e '.[test,DPKT,crypto,NGAP]' + python -m pip install -e '.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]' # See the `test` job above for why this step exists. - name: Report available parallelism @@ -1869,10 +1894,15 @@ jobs: # would be a no-op that misleadingly suggests the extra is exercised by # `gate`; #751's `engine-tests` and `pypcap-parity` jobs cover it for # real, on the 3.10-3.15 (and 3.10-3.11) legs, 3.10 being KTLO in both. + # + # #1958: `vendor`, `docs` and `PyShark` join them, so this is the set the + # `latest` legs install (see `test`'s "Install the whole suite's + # dependencies on latest" step), and the html5lib, Sphinx and PyShark + # gates run here as they do there. - name: Install package, test and generator dependencies run: | python -m pip install -U pip setuptools wheel - python -m pip install -e '.[test,Scapy,DPKT,crypto,cli,NGAP]' + python -m pip install -e '.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]' # See the integration job above for why this step is shaped the way it is. - name: Regenerate sample captures diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index a7ba176d86..ead45ff410 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -52,7 +52,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:581-597``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1842-1912``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1867-1942``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:621,808,974,1210``), so a @@ -62,7 +62,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Other checks run unconditionally on the same path, though none is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1646-1672``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1671-1697``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 48fdb20318..120dd1e97c 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -329,7 +329,7 @@ the same commit several times over. It also skips ``changes``, Checks`_ below), which are gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1634-1640``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1659-1665``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -639,7 +639,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:2003`` + - ``unit-tests.yml:2033`` * - ``Compat Python 3.11`` - job ``compatibility``, matrix leg ``3.11`` - ``python-compatibility.yml:60``, ``.github/python-versions.toml:20`` @@ -661,22 +661,22 @@ which added ``Compat Python 3.15`` and removed ``Compat Python 3.10``. Moving a between the tiers below means editing the ruleset in the same change. ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:2003``). ``Compat Python`` is defined by one, the +(``unit-tests.yml:2033``). ``Compat Python`` is defined by one, the ``compatibility`` job in ``python-compatibility.yml`` (``:60``), whose matrix (``:74``) comes from :file:`.github/python-versions.toml`: the required legs are its supported tier (``python-versions.toml:20-24``), and its KTLO legs (``python-versions.toml:16-19``) have names ending in ``(KTLO)``, so no required context matches one. A comment at -``unit-tests.yml:1915`` mentions the string without defining it. +``unit-tests.yml:1945`` mentions the string without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the ``Compat Python 3.11``-``3.15`` legs, nothing else, as -``unit-tests.yml``'s own comment (``unit-tests.yml:1914-1924``) records. It +``unit-tests.yml``'s own comment (``unit-tests.yml:1944-1954``) records. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- ``test``, ``integration``, ``engine-tests`` and ``pypcap-parity``, plus ``project-tests``, the one run of the project pins, and ``changes`` for the docs-only path -- and checks each result explicitly -(``unit-tests.yml:2002-2058``). Rulesets match check names literally, with no +(``unit-tests.yml:2032-2088``). Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. It accepts the gating jobs as ``skipped`` only on a docs-only pull request, where @@ -685,7 +685,7 @@ It accepts the gating jobs as ``skipped`` only on a docs-only pull request, wher ``python-compatibility.yml``, and a job cannot ``needs:`` a job in another workflow file. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:2005``), +``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:2035``), so it is never produced -- and never expected -- on those paths. .. _ktlo-tier: @@ -709,7 +709,7 @@ and keeps its legs' check names, ``Python 3.10 (KTLO)`` and so on. In with supported legs. Every job holding KTLO legs works the same way: * Every step that can fail on a KTLO leg has an ``id`` and - ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:424-441``), + ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:441-458``), unless it is masked on every leg, as the timings uploads are. Its ``outcome`` is then ``failure`` but its ``conclusion`` ``success``, so the job, its check run and ``required-checks`` read ``success``. A step that needs an earlier one's work @@ -976,6 +976,23 @@ The project pins stay in a job of their own rather than joining the ``latest`` l rather than lengthening it under coverage, and it is also the job that runs the docs readers on a docs-only diff. +Latest CPython runs the whole suite, as the sanity check and the gate (the owner's +ruling on #1958), so it also installs what the whole suite gates on. In ``test`` and +``integration`` the step ``Install the whole suite's dependencies on latest`` runs on +the ``latest`` leg alone, after the install every leg runs, and adds the engines pip +can install there (``Scapy``, ``PyShark``), html5lib through ``vendor``, Sphinx and +the pinned sphinx-autodoc-typehints through ``docs``, and emoji through ``cli``. +``project-tests`` and ``gate``, which run on ``latest`` only, install the same set in +their ``Install package and test dependencies`` and ``Install package, test and +generator dependencies`` steps. The other legs keep their install lines and run the +``ALL`` tier with them, so a test gated on one of these extras skips there and runs on +``latest``. The other engines stay out: ``PCAP_CT`` needs a system libpcap, which +``engine-tests`` installs for its own cell, and the ``PyPCAP`` and ``PyPCAPFile`` +markers resolve to nothing on ``latest``. :file:`tests/_dependency_gates.py` reads the +``latest`` step as that leg's extras, and +:file:`tests/project/test_whole_suite_on_latest_1958.py` pins the set and the gates it +closes. + Project Pins ------------ diff --git a/tests/_dependency_gates.py b/tests/_dependency_gates.py index daa003565b..e548cc3552 100644 --- a/tests/_dependency_gates.py +++ b/tests/_dependency_gates.py @@ -103,7 +103,7 @@ from typing import Iterator, Optional __all__ = [ - 'WORKFLOW', 'PYPROJECT', 'CORE', 'MODULE_PROVIDERS', 'NON_DISTRIBUTION_FLAGS', + 'WORKFLOW', 'PYPROJECT', 'LATEST_LEG_CONDITION', 'CORE', 'MODULE_PROVIDERS', 'NON_DISTRIBUTION_FLAGS', 'MUTUALLY_EXCLUSIVE_IMPORTS', 'DEPENDENCY_GATE_EXCLUSIONS', 'Requirement', 'Gate', 'Job', 'Gap', 'Exclusion', 'AmbiguousProvider', 'requirement_key', 'provided_by', 'declared_requirements', 'extras_providing', @@ -133,6 +133,9 @@ WORKFLOW = _tiers.ROOT / '.github' / 'workflows' / 'unit-tests.yml' #: Where the extras are declared. PYPROJECT = _tiers.ROOT / 'pyproject.toml' +#: The ``if:`` of a step that runs on the ``latest`` leg alone (#1958), which is +#: how :func:`pytest_jobs` tells that leg's install line from every leg's. +LATEST_LEG_CONDITION = 'matrix.python-version == needs.python-versions.outputs.latest' #: Stands in for ``[project] dependencies`` where an extra name is expected: #: what ``pip install -e .`` installs with no extra asked for at all. Spelled @@ -206,8 +209,9 @@ 'pylint': ('pylint',), # ``beautifulsoup4`` ships ``bs4``; ``html5lib`` is an *extra of* that # distribution, so ``beautifulsoup4`` alone does not provide it. That - # distinction is the whole reason ``HAS_CRAWLER_DEPS`` is satisfied in CI - # and ``HAS_VENDOR_DEPS`` is not: the ``test`` extra carries plain + # distinction is why ``HAS_CRAWLER_DEPS`` is satisfied by the ``test`` + # extra on every job and ``HAS_VENDOR_DEPS`` only where ``vendor`` joins it, + # the ``latest`` legs (#1958): the ``test`` extra carries plain # ``beautifulsoup4``, and only ``vendor`` and ``dev`` carry # ``beautifulsoup4[html5lib]``. The second alternative below is not a # second *distribution* -- pyproject.toml never declares bare @@ -249,8 +253,8 @@ # extra declares docutils itself this entry has to change: the bare name goes # live alongside Sphinx, :func:`contested_imports` reports docutils, and the # :data:`MUTUALLY_EXCLUSIVE_IMPORTS` liveness test fails until the entry is - # cut to ``('docutils',)``. If the ``test`` and ``gate`` jobs install that - # extra, its gap closes and the ``HAS_DOCUTILS`` exclusion has to go as well. + # cut to ``('docutils',)``. The ``latest`` legs install the ``docs`` extra + # (#1958), which is what closes the ``HAS_DOCUTILS`` gate there. 'docutils': ('Sphinx', 'docutils'), # The theme ``HAS_FURO`` asks for, whose table wrapper the #1861 transform # test runs. Only the ``docs`` extra carries it. @@ -370,7 +374,9 @@ class Exclusion(NamedTuple): '``pcap`` module and cannot coexist -- see ' "pcapkit/foundation/engines/_pcap_backend.py's own docstring. test and gate " "stay dark on purpose, per #751's ruling that per-engine coverage gets its " - 'own job rather than one more install line on either.' + 'own job rather than one more install line on either. #1958 gave the latest ' + 'legs every engine pip can install there, and PCAP_CT alone needs that ' + "system libpcap, which engine-tests' PCAP_CT cell installs and runs on latest." ), ), 'HAS_PYPCAPFILE': Exclusion( @@ -409,192 +415,6 @@ class Exclusion(NamedTuple): 'three.' ), ), - 'HAS_VENDOR_DEPS': Exclusion( - dark={'test': ('html5lib',), 'gate': ('html5lib',)}, - reason=( - "Ruled onto a non-blocking leg by #738's option (b), on the premise that the " - 'crawlers fetch from IANA and Wikipedia and #518 records four Wikipedia 403s ' - 'and a dead IETF URL -- so a blocking leg would go red on upstream ' - 'availability rather than on this code. Note what is actually missing, which ' - "#738's own inventory gets wrong: requests and bs4 have shipped in the test " - 'extra since #507 and are installed on every job, so these classes are one ' - '*requirement extra* short -- html5lib, which only beautifulsoup4[html5lib] ' - 'provides, i.e. the vendor and all extras.\n\n' - 'engine-tests (#751) used to be that leg: it mirrored test\'s ignore-shape ' - 'exactly (same --ignore flags) and carried vendor on its one shared install ' - 'line, so it reached and closed these unit-tier HAS_VENDOR_DEPS gates as a ' - "side effect of installing everything together. #849's rebuild removes that " - 'shared venv: each matrix.engine cell now installs only "test" plus its own ' - 'engine extra and runs only that engine\'s own explicit TEST_PATHS -- ' - 'tests/toolkit/test_scapy_unit.py and four siblings, never tests/vendor/ -- so ' - 'no cell reaches these gates at all any more, let alone installs vendor for ' - 'them. This is not a *new* gap this guard reports, because ' - ':func:`dependency_gate_gaps` only counts a job that reaches a gate without ' - "installing what it needs; a job that no longer reaches the gate at all is " - 'invisible to it either way, which is exactly the "silence is the failure ' - 'mode" problem this module\'s own docstring opens with. It is a real, ' - 'disclosed regression rather than a silent one: #849\'s own PR description ' - "records vendor coverage as dropped, not restored, and worth its own issue. " - 'test and gate stay dark for their own, unrelated reasons below; engine-tests ' - 'is simply no longer part of the answer for this flag.\n\n' - 'test and gate stay dark for different reasons, not the same one. test ' - 'declines html5lib per the non-blocking-leg ruling itself -- that is the ' - 'whole reason it is dark. gate is not part of the blocking matrix at all -- ' - "it only runs on the release path (workflow_call's gate-only: true), never " - 'on a pull request -- and now that engine-tests no longer carries vendor ' - 'either, no per-PR job does, so this flag\'s only per-PR-visible install ' - 'point is test\'s own decline above. gate could add vendor without ' - 'contradicting the "non-blocking" ruling, but doing so on the one job that ' - 'gates an actual release, for a network-and-parser extra whose own crawlers ' - "#518 already documents as flaky, is not worth the footprint on its own -- " - "and it would not restore the per-PR coverage #849 removed regardless, since " - 'gate never runs on a pull request at all.\n\n' - 'Worth recording since it bears on the ruling itself, though this exclusion ' - 'is not the place to relitigate it: none of the 41 methods this closes make ' - 'a live network call, though not for one reason across all four files. ' - 'test_user_agent_unit.py (11 methods) and test_request_prompt_unit.py (16) ' - 'gate on HAS_VENDOR_DEPS only because the pcapkit.vendor package import ' - 'chain needs requests/bs4/html5lib importable -- html5lib itself is never ' - "used by either file's test bodies, both of which replace requests.get with " - "a recorder for the whole of every case (each file's own module docstring " - 'says so in its own words, not a shared one). test_ipx_packet_unit.py (9 ' - 'methods) is a regression suite for a *retired* scrape -- Packet.LINK is ' - 'None, so _request() short-circuits before ever calling requests.get, and ' - 'test_request_makes_no_network_call asserts exactly that by making ' - 'requests.get and requests.Session.request raise if reached at all; ' - 'html5lib is an import precondition there too, nothing in the file uses it. ' - 'test_ftp_return_code_unit.py (5 methods) is not retired -- ' - 'pcapkit.vendor.ftp.return_code.ReturnCode.LINK is a live Wikipedia URL -- ' - 'and is the one file where html5lib is a *functional* dependency rather ' - 'than an import precondition: its tests build the HTML fixture inline and ' - "hand it straight to ReturnCode.request(), which is " - "bs4.BeautifulSoup(text, 'html5lib'); what keeps it off the network is the " - 'fixture being inline, not an absence of html5lib use. ' - "#518's flakiness is real for HAS_CRAWLER_DEPS's live crawlers, which run " - 'unconditionally since #507; it does not describe any of the above. ' - 'pypcap-parity does not appear here even though it also declines html5lib, ' - 'because its fixture-tier selection never reaches ' - 'tests/vendor/test_ipx_packet_unit.py or its three siblings in the first ' - 'place -- same shape as integration above it, which is why integration is ' - 'not listed either.' - ), - ), - 'HAS_SCAPY': Exclusion( - dark={'test': ('scapy',)}, - reason=( - 'Known, not new: #738 set it aside precisely because Scapy is installed on ' - 'the integration and gate legs, so "they are dark only on the test leg". The ' - 'test job declines Scapy on cost grounds -- its own install-step comment ' - 'contrasts DPKT, which "costs nothing per leg", with Scapy. What was never ' - 'written down is the consequence: the 10 methods it darkens ' - '(test_scapy_unit.py 5, test_scapy_engine.py 2, and one each in ' - 'test_core.py, test_misc.py and test_sctp_unit.py) are all unit-tier, so the ' - 'integration job never reaches them either. The gate job does, but it runs ' - "only where a caller passes gate-only: true -- the three Saturday schedules " - '(deploy-pages, cron-vendor, cron-conda) and a v* release tag -- never on a ' - 'pull request or a push to main. So no per-PR leg runs them at all.\n\n' - "#751's engine-tests job used to install Scapy across the full 3.10-3.14 " - 'matrix and run it against the whole unit tier (same --ignore shape as ' - 'test), closing the "no per-PR leg" problem this exclusion used to describe ' - "for all 10 of this guard's 14 unit-tier HAS_SCAPY methods at once.\n\n" - "#849's rebuild only partially keeps that closure. engine-tests's Scapy cell " - 'now runs an explicit TEST_PATHS list -- tests/toolkit/test_scapy_unit.py, ' - 'tests/foundation/engines/test_scapy_engine.py and ' - "tests/protocols/transport/test_sctp_unit.py -- which covers 8 of the 10 (5 " - '+ 2 + 1), but not the other 2, one each in test_core.py and test_misc.py: ' - "those are simply not in that cell's path list, so no per-PR job reaches " - 'them with Scapy installed any more -- the exact "no per-PR leg runs them ' - 'at all" state from before #751, reopened for those two. This is not ' - 'visible as a wider Gap here: :func:`dependency_gate_gaps` only reports a job ' - 'that reaches a gate without installing for it, and a job that no longer ' - "reaches the gate at all is invisible to it either way, by construction -- " - "which is why this paragraph exists to say so in prose rather than leave it " - "to the tool. test_sctp_unit.py's one method was added to the Scapy cell's " - 'TEST_PATHS at zero new-dark-flag cost: unlike test_core.py and ' - 'test_misc.py, it carries no HAS_DPKT-gated tests of its own, so restoring ' - 'it traded nothing away. test_core.py and test_misc.py each still carry ' - "their own HAS_DPKT-gated tests (see the test job's own install-step " - "comment), and adding them here without DPKT would trade one dark flag for " - 'another on a cell that installs neither -- disclosed rather than ' - 'restored, a decision worth its own review rather than folding into a ' - 'workflow-only CR. The other 4 of the 14 live in ' - 'tests/integration/ or match the *_runtime.py ignore-glob, unaffected by any ' - 'of this, already covered by Scapy on the integration and gate jobs. test ' - 'stays dark on purpose: the ruling on #751 was a dedicated job precisely so ' - 'test would not have to reconsider the cost question it already answered.' - ), - ), - 'HAS_PYSHARK': Exclusion( - dark={'test': ('pyshark',), 'integration': ('pyshark',), 'gate': ('pyshark',)}, - reason=( - 'One of #738\'s seven, left out of #740\'s "cheap, uncontroversial subset" ' - 'and never since installed, so its 4 gated methods run nowhere. Three of them ' - 'assert what PyShark.unsupported_reason says on an interpreter where the ' - 'engine cannot work, so they need the distribution importable to have ' - 'anything to ask.\n\n' - "#751's later ruling asked to try a tshark binary the same way it asked to " - 'try building pypcap: "try to build and if the CI is not a good suit, then ' - 'we ripe it… same for HAS_PYSHARK on tshark dependency." ' - 'test_the_reason_tracks_the_running_interpreter used to call ' - 'PyShark.unsupported_reason() unpatched and hard-assert the reason names ' - '"tshark" on every version below the asyncio ceiling -- true only while ' - 'tshark stayed absent, and it would have turned that assertion from a pass ' - 'into a failure the moment tshark was installed. Fixed by probing the same ' - 'way PyShark.unsupported_reason() itself does -- pyshark\'s own ' - "get_process_path(), not shutil.which(), which its own config.ini " - 'precedence can make disagree with -- matching what its sibling ' - 'test_this_host_really_has_no_tshark_so_the_check_is_not_vacuous already ' - 'used, so the file no longer disagrees with itself about whether tshark may ' - 'be present. engine-tests installs both the distribution and tshark (apt-get, ' - 'with a debconf pre-seed so the postinst prompt does not block) on its own ' - 'PyShark cell, and that cell\'s explicit TEST_PATHS (#849) -- ' - 'tests/toolkit/test_pyshark_unit.py (0 of the 4; it carries no HAS_PYSHARK ' - 'gate of its own, only its DPKT/PyPCAPFile/etc. siblings do) and ' - 'tests/foundation/engines/test_pyshark_engine.py (the other 3) -- reaches 3 ' - 'of the 4 HAS_PYSHARK methods. The fourth, ' - 'tests/integration/test_engine_runtime.py:70\'s ' - 'test_pyshark_engine_is_refused_before_asyncio_can_break, is integration-tier ' - "and outside any engine-tests cell's reach -- it stays dark on the integration " - 'job exactly as recorded above, unaffected by any of this. So the rebuild ' - "from one shared ignore-shape venv to one venv per engine leaves this flag's " - "closure unchanged, closing the test-job gap this exclusion used to describe " - "for those 3. integration and gate stay dark on purpose, per #751's ruling " - 'that per-engine coverage gets its own job.' - ), - ), - 'HAS_RUNTIME': Exclusion( - dark={'test': ('pyshark', 'scapy'), 'gate': ('pyshark',)}, - reason=( - "Absent from #738's inventory of seven, and the one gap this guard found " - "that no earlier pass did. Every other module's HAS_RUNTIME asks only for " - 'the four core dependencies, which pip install -e . always provides; ' - 'tests/foundation/engines/test_runtime_engines.py reuses the name for those ' - 'four plus dpkt, scapy and pyshark, so its two classes (5 methods) skip ' - 'wherever any of the three is absent. It is unit-tier despite the name, ' - 'because the ignore-glob is *_runtime.py and the file is ' - 'test_runtime_engines.py, so the test job is what reaches it. The narrow ' - 'fix -- giving that flag a name of its own, so the skip reason says which ' - 'dependency was missing -- is left for a follow-up: it touches a test ' - "module #751 did not otherwise need to change, and this guard does not " - 'care what a flag is named, only whether the job that reaches it installs ' - 'what it asks for.\n\n' - "#751's engine-tests job used to install DPKT, Scapy and PyShark together in " - "one shared venv, closing this gap as a side effect: all 5 methods ran " - "wherever engine-tests did. #849's rebuild removes that shared venv -- each " - 'matrix.engine cell now installs exactly one engine\'s own extra, so no cell ' - 'installs all three together any more, and none of the five explicit ' - 'TEST_PATHS lists names test_runtime_engines.py at all, so engine-tests no ' - 'longer reaches this gate either. Like the HAS_SCAPY and HAS_VENDOR_DEPS ' - 'entries above, this does not surface as a wider Gap -- a job that stops ' - "reaching a gate is invisible to :func:`dependency_gate_gaps` the same way a " - 'job that never reached it would be -- so it is recorded here in prose. ' - "Disclosed, not silent: #849's own PR description names this exact loss " - '("the multi-engine HAS_RUNTIME gate ... needs DPKT+Scapy+PyShark together") ' - 'and defers restoring it to a follow-up issue rather than reopening this ' - 'workflow-only change to add a seventh venv shape. test and gate stay dark on ' - 'purpose, for the same reason HAS_SCAPY and HAS_PYSHARK above do.' - ), - ), 'HAS_MYPY': Exclusion( dark={'test': ('mypy',), 'gate': ('mypy',)}, reason=( @@ -644,36 +464,12 @@ class Exclusion(NamedTuple): ':meth:`~tests.test_tier_guard.DependencyGateCoverageTests\ .test_each_exclusion_still_describes_a_gap_that_is_really_there` exists to catch if this ' 'entry is not kept in step with it. integration and pypcap-parity select by ' - 'fixture tier and never collect that module, which is why they are not listed, ' - 'the same reason they are absent from HAS_VENDOR_DEPS above. If a lint extra ' + 'fixture tier and never collect that module, which is why they are not listed. ' + 'If a lint extra ' 'is ever declared in pyproject.toml, the right change is to delete this entry ' 'and let the gap close on its own rather than to widen it.' ), ), - 'HAS_SPHINX': Exclusion( - dark={'test': ('sphinx', 'sphinx_autodoc_typehints'), - 'project-tests': ('sphinx', 'sphinx_autodoc_typehints'), - 'gate': ('sphinx', 'sphinx_autodoc_typehints')}, - reason=( - 'The classes it gates (#1775, #1856, #1858-#1863, #1917, #1918 and the enum-value ' - 'hook\'s) load docs/source/conf.py and drive its hooks with the nodes and events ' - 'Sphinx itself builds, so they need Sphinx and the pinned sphinx-autodoc-typehints. ' - 'Only deploy-pages.yml installs those, from docs/requirements.txt, and that ' - 'workflow builds the documentation rather than running pytest; no workflow ' - 'installs the docs extra itself. The same argument declines it here as ' - 'declines HAS_MYPY above: the docs build is already the end-to-end check of ' - 'those hooks, and putting Sphinx, furo and their dependencies on every ' - 'unit-test leg would cost each one an install for these classes alone. The ' - 'flag also requires the installed sphinx-autodoc-typehints to be the release ' - 'docs/requirements.txt pins, since conf.py imports that release\'s private ' - 'modules; that check reads no module, so this entry still names only the ' - 'imports. They run wherever docs/requirements.txt or the docs extra is ' - 'installed, a developer checkout included, and skip visibly elsewhere, saying ' - 'which condition failed. The directive-name and pin-agreement tests need ' - 'nothing and run on every leg, or in project-tests where their module is a ' - 'project pin (#1954).' - ), - ), 'HAS_PYLINT': Exclusion( dark={'project-tests': ('pylint',), 'gate': ('pylint',)}, reason=( @@ -693,36 +489,6 @@ class Exclusion(NamedTuple): 'visibly elsewhere.' ), ), - 'HAS_DOCUTILS': Exclusion( - dark={'test': ('docutils',), 'gate': ('docutils',)}, - reason=( - "The method it gates (#1846) parses Vendor.wrap_comment's output with " - 'docutils and asserts it is nothing but a plain paragraph, so it needs ' - 'docutils, which no pytest job installs. No workflow installs the docs extra; ' - 'deploy-pages.yml gets Sphinx, and with it docutils, from ' - 'docs/requirements.txt, and it builds the documentation rather than running ' - 'pytest. Installing docutils on these jobs would not run it anyway: it sits in ' - 'a class gated on ' - 'HAS_VENDOR_DEPS, which these same jobs leave dark by declining html5lib ' - "(#738, above), so closing this gap would also mean the vendor extra on a " - 'blocking leg. The cost argument of HAS_SPHINX applies too. What runs on ' - 'every leg instead is the ungated committed-const class in the same module, ' - 'which pins the re-rendered lines themselves; the string assertions on the ' - 'wrapper run wherever the vendor extra is installed, and the docutils parse ' - 'wherever docutils is importable as well.' - ), - ), - 'HAS_FURO': Exclusion( - dark={'test': ('furo',), 'gate': ('furo',)}, - reason=( - 'The class it gates (#1861) runs the table-wrapper transform of furo, the ' - 'docs theme, ahead of the conf.py transform that undoes its duplicate ids, ' - 'and is gated on HAS_SPHINX as well. It is declined for the reason ' - 'HAS_SPHINX is: only deploy-pages.yml installs furo, from ' - 'docs/requirements.txt, and no workflow installs the docs extra itself; that ' - 'build is the end-to-end check of the same transform.' - ), - ), } @@ -794,6 +560,10 @@ class Job(NamedTuple): #: :func:`~tests._tiers.fixture_tier_paths` for that tier alone, :data:`None` #: for one whose ``latest`` leg asks for every tier, and for every other shape. tier: 'Optional[str]' = None + #: Extras of the job's ``latest``-only install step (#1958), empty when it has + #: none. That leg's selection holds every other leg's, so a gate the job + #: reaches runs there with these installed; see :func:`dependency_gate_gaps`. + latest_extras: 'tuple[str, ...]' = () class Gap(NamedTuple): @@ -877,7 +647,7 @@ def provided_by(requirements: 'tuple[Requirement, ...]', provider: 'str') -> 'bo the base distribution -- while ``'beautifulsoup4[html5lib]'`` is *not* satisfied by plain ``beautifulsoup4``. That asymmetry is the whole point: it is what separates ``HAS_CRAWLER_DEPS``, which the ``test`` extra - satisfies, from ``HAS_VENDOR_DEPS``, which it does not. + satisfies, from ``HAS_VENDOR_DEPS``, which needs ``vendor`` as well. """ wanted = requirement_key(provider) @@ -1554,14 +1324,6 @@ def pytest_jobs(workflow: 'Optional[pathlib.Path]' = None) -> 'tuple[Job, ...]': jobs.extend(variants) continue - installs = re.findall(r"pip install -e '\.\[([^]]*)\]'", section) - if len(installs) != 1: - raise AssertionError( - f'the {name!r} job runs pytest but has {len(installs)} ' - f"\"pip install -e '.[...]'\" lines, and this guard cannot tell which " - f'extras the run would have' - ) - # Scoped to the step that runs pytest, not to the job, and found by what # the step does rather than by what it is called. Both matter: the # `gate` job's *comment* says "no --ignore, no tier selection" and the @@ -1570,6 +1332,27 @@ def pytest_jobs(workflow: 'Optional[pathlib.Path]' = None) -> 'tuple[Job, ...]': # credits it with reaching only the unit tier. Matching on the step name # would work only until a step is renamed. steps = re.findall(r'(?ms)^ - name: [^\n]*\n(.*?)(?=^ - name:|\Z)', section) + + # #1958: a step whose `if:` runs it on `latest` alone holds that leg's + # install line, read apart from the one every leg runs. + latest_steps = [step for step in steps if LATEST_LEG_CONDITION in step.partition('run:')[0]] + latest_installs = [extras for step in latest_steps + for extras in re.findall(r"pip install -e '\.\[([^]]*)\]'", step)] + if len(latest_installs) > 1: + raise AssertionError( + f'the {name!r} job has {len(latest_installs)} latest-only ' + f"\"pip install -e '.[...]'\" lines, and this guard cannot tell which " + f'extras that leg would have' + ) + installs = re.findall(r"pip install -e '\.\[([^]]*)\]'", section) + installs = [extras for extras in installs if extras not in latest_installs] \ + if latest_installs else installs + if len(installs) != 1: + raise AssertionError( + f'the {name!r} job runs pytest but has {len(installs)} ' + f"\"pip install -e '.[...]'\" lines, and this guard cannot tell which " + f'extras the run would have' + ) running = [step for step in steps if 'python -m pytest' in step] if len(running) != 1: raise AssertionError( @@ -1594,7 +1377,9 @@ def pytest_jobs(workflow: 'Optional[pathlib.Path]' = None) -> 'tuple[Job, ...]': selection = 'whole-suite' jobs.append(Job(name, tuple(extra.strip() for extra in installs[0].split(',')), - selection, tier=tier)) + selection, tier=tier, + latest_extras=tuple(extra.strip() for extra in latest_installs[0].split(',')) + if latest_installs else ())) return tuple(jobs) @@ -1682,7 +1467,10 @@ def dependency_gate_gaps(workflow: 'Optional[pathlib.Path]' = None) -> 'tuple[Ga gaps = {} # type: dict[tuple[str, str], tuple[set[str], list[Gate]]] for job in pytest_jobs(workflow): - available = frozenset(job.extras) | {CORE} + # #1958: a job's `latest` leg runs every test its other legs do, so a + # gate it reaches runs there with that leg's extras too. The other legs + # run the `ALL` tier with the job's own line, and skip what it lacks. + available = frozenset(job.extras) | frozenset(job.latest_extras) | {CORE} for gate in gated_scopes(): if gate.flag in NON_DISTRIBUTION_FLAGS: continue diff --git a/tests/project/test_project_pins_once_1954.py b/tests/project/test_project_pins_once_1954.py index 6c5e1a48ad..8a2a13bc20 100644 --- a/tests/project/test_project_pins_once_1954.py +++ b/tests/project/test_project_pins_once_1954.py @@ -54,7 +54,8 @@ def test_it_runs_on_latest_with_what_test_installs(self) -> None: setup = next(entry for entry in job(*JOB)['steps'] if 'setup-python' in entry.get('uses', '')) self.assertEqual(setup['with']['python-version'], '${{ steps.python-versions.outputs.latest }}') install = step(JOB, 'Install package and test dependencies')['run'] - self.assertIn(step(('unit-tests.yml', 'test'), 'Install package and test dependencies')['run'] + # #1958: what `test` installs on `latest`, the leg the pins would run on. + self.assertIn(step(('unit-tests.yml', 'test'), "Install the whole suite's dependencies on latest")['run'] .split("pip install -e '", 1)[1].split("'", 1)[0], install) def test_it_runs_under_the_caps(self) -> None: diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py index 649a0ea086..d9be81d7b5 100644 --- a/tests/project/test_sweeps_on_latest_only_1732.py +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -460,8 +460,10 @@ def test_it_installs_what_test_installs(self) -> None: self.assertEqual(_dependency_gates.WORKFLOW.name, 'unit-tests.yml') installs = {} for where in (JOB, EXPERIMENTAL): + # #1958: less the step `test` runs on `latest` alone, which no experimental leg is. found = [re.findall(r"pip install -e '\.\[([^]]*)\]'", str(entry.get('run', ''))) - for entry in job(*where)['steps']] + for entry in job(*where)['steps'] + if _dependency_gates.LATEST_LEG_CONDITION not in str(entry.get('if', ''))] installs[where] = [extra for entries in found for extra in entries] self.assertEqual(installs[EXPERIMENTAL], installs[JOB]) self.assertEqual(len(installs[JOB]), 1) diff --git a/tests/project/test_whole_suite_on_latest_1958.py b/tests/project/test_whole_suite_on_latest_1958.py new file mode 100644 index 0000000000..9ff50e3d6a --- /dev/null +++ b/tests/project/test_whole_suite_on_latest_1958.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +"""#1958: latest CPython runs the whole suite, with what the whole suite gates on. + +The owner's rulings on #1958, paraphrased: every implementation runs the same +minimal ``ALL`` tier, and latest CPython runs the whole suite as the sanity check +and the gate; tests that run nowhere get a job. So each job's ``latest`` leg +installs the engines pip can install there, html5lib (``vendor``), Sphinx +(``docs``) and emoji (``cli``), and the ``HAS_*`` gates those extras close run +there. This module pins the install lines, the leg they run on, the gates they +close, and the page that describes them. :mod:`tests._dependency_gates` derives +the gaps; :file:`tests/test_tier_guard.py` checks its reading of the workflow. + +""" + +from __future__ import annotations + +import pathlib +import re +import tempfile +import unittest + +from tests import _dependency_gates, _workflows +from tests.project.test_python_matrix_tiers_1633 import legs +from tests.project.test_python_versions_1660 import script + +ROOT = pathlib.Path(__file__).resolve().parents[2] +DOC = ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst' +WORKFLOW = 'unit-tests.yml' + +#: What a ``latest`` leg installs: every extra a gate asks for that installs there. +WHOLE_SUITE = frozenset({'test', 'Scapy', 'DPKT', 'crypto', 'cli', 'NGAP', 'vendor', 'docs', 'PyShark'}) +#: The step ``test`` and ``integration`` add on ``latest``. +LATEST_STEP = "Install the whole suite's dependencies on latest" +LATEST_IF = ("${{ steps.install.outcome == 'success' && " + 'matrix.python-version == needs.python-versions.outputs.latest }}') +#: The install line every leg of each matrix job runs, unchanged by #1958. +BASE = { + 'test': frozenset({'test', 'DPKT', 'crypto', 'NGAP'}), + 'integration': frozenset({'test', 'Scapy', 'DPKT', 'cli'}), +} +#: The jobs that run on ``latest`` alone, each with one install step. +LATEST_ONLY = { + 'project-tests': 'Install package and test dependencies', + 'gate': 'Install package, test and generator dependencies', +} +#: The gates still dark somewhere: lint tools no extra declares, and the engines +#: ``latest`` cannot install (PCAP_CT needs a system libpcap; the PyPCAP and +#: PyPCAPFile markers stop below 3.12). ``engine-tests`` and ``pypcap-parity`` run them. +STILL_DARK = frozenset({'HAS_MYPY', 'HAS_PYLINT', 'HAS_PCAP_CT', 'HAS_PYPCAP', 'HAS_PYPCAPFILE'}) +#: A test the audit found running nowhere, or in ``gate`` alone, per flag. +DARK_TESTS = { + 'HAS_RUNTIME': 'tests/foundation/engines/test_runtime_engines.py', + 'HAS_SCAPY': 'tests/interface/test_core.py', + 'HAS_PYSHARK': 'tests/integration/test_engine_runtime.py', + 'HAS_VENDOR_DEPS': 'tests/vendor/test_user_agent_unit.py', + 'HAS_DOCUTILS': 'tests/vendor/test_vendor_wrap_comment_unit.py', + 'HAS_SPHINX': 'tests/project/test_docs_ambiguous_xrefs_1775_unit.py', + 'HAS_FURO': 'tests/project/test_docs_table_wrapper_ids_1861_unit.py', +} + + +def step(job_id: 'str', name: 'str') -> 'dict[str, object]': + found = [entry for entry in _workflows.steps(WORKFLOW, job_id) if entry.get('name') == name] + if len(found) != 1: + raise AssertionError(f'expected one {name!r} step in {job_id}, found {len(found)}') + return found[0] + + +def extras(run: 'object') -> 'frozenset[str]': + found = re.findall(r"pip install -e '\.\[([^]]*)\]'", str(run)) + if len(found) != 1: + raise AssertionError(f'expected one editable install in {run!r}, found {len(found)}') + return frozenset(extra.strip() for extra in found[0].split(',')) + + +class TestTheLatestLegsInstallTheWholeSuite(unittest.TestCase): + """Each ``latest`` leg installs :data:`WHOLE_SUITE`; every other leg keeps its line.""" + + def test_the_matrix_jobs_add_it_on_latest_alone(self) -> None: + for job_id in BASE: + with self.subTest(job=job_id): + names = [entry.get('name') for entry in _workflows.steps(WORKFLOW, job_id)] + added = step(job_id, LATEST_STEP) + self.assertEqual(added['if'], LATEST_IF) + self.assertEqual(extras(added['run']), WHOLE_SUITE) + # Right after the install every leg runs, whose outcome it waits on. + install = next(entry for entry in _workflows.steps(WORKFLOW, job_id) if entry.get('id') == 'install') + self.assertEqual(names.index(LATEST_STEP), names.index(install['name']) + 1) + self.assertEqual(extras(install['run']), BASE[job_id]) + + def test_the_jobs_on_latest_alone_install_it(self) -> None: + for job_id, name in LATEST_ONLY.items(): + with self.subTest(job=job_id): + self.assertEqual(extras(step(job_id, name)['run']), WHOLE_SUITE) + + def test_no_ktlo_leg_is_latest(self) -> None: + """The KTLO twins run the same steps by alias, so the ``latest`` step must skip on each.""" + latest = script().load()['latest'] + for twin, job_id in (('test', 'test-ktlo'), ('integration', 'integration-ktlo')): + with self.subTest(job=job_id): + shared = _workflows.steps(WORKFLOW, job_id) + self.assertEqual(shared, _workflows.steps(WORKFLOW, twin)) + self.assertIn(LATEST_STEP, [entry.get('name') for entry in shared]) + versions = [leg['python-version'] for leg in legs(WORKFLOW, job_id)] + self.assertTrue(versions) + self.assertNotIn(latest, versions) + self.assertIn(latest, [leg['python-version'] for leg in legs(WORKFLOW, twin)]) + + +class TestTheDarkTestsRunOnLatest(unittest.TestCase): + """The gates those extras close are no gap on any job; the rest are the known ones.""" + + def setUp(self) -> None: + reason = _dependency_gates._tiers.guard_unavailable_reason() + if reason is not None: + self.skipTest(f'git cannot answer here: {reason}') + + def test_the_guard_reads_the_latest_step_as_that_legs_extras(self) -> None: + jobs = {job.name: job for job in _dependency_gates.pytest_jobs() if job.name != 'engine-tests'} + for job_id, base in BASE.items(): + with self.subTest(job=job_id): + self.assertEqual(frozenset(jobs[job_id].extras), base) + self.assertEqual(frozenset(jobs[job_id].latest_extras), WHOLE_SUITE) + for job_id in LATEST_ONLY: + with self.subTest(job=job_id): + self.assertEqual(frozenset(jobs[job_id].extras), WHOLE_SUITE) + self.assertEqual(jobs['pypcap-parity'].latest_extras, ()) + + def test_an_unconditional_second_install_is_still_refused(self) -> None: + """Only a step gated on ``latest`` is read apart; any other second line is ambiguous.""" + text = (_workflows.WORKFLOWS / WORKFLOW).read_text(encoding='utf-8') + doctored = text.replace(f' if: {LATEST_IF}\n', '', 1) + self.assertNotEqual(doctored, text) + with tempfile.TemporaryDirectory() as tmp: + path = pathlib.Path(tmp) / WORKFLOW + path.write_text(doctored, encoding='utf-8') + with self.assertRaises(AssertionError) as caught: + _dependency_gates.pytest_jobs(path) + self.assertIn("'test' job runs pytest but has 2", str(caught.exception)) + + def test_only_the_known_gates_stay_dark(self) -> None: + self.assertEqual({gap.flag for gap in _dependency_gates.dependency_gate_gaps()}, STILL_DARK) + self.assertEqual(set(_dependency_gates.DEPENDENCY_GATE_EXCLUSIONS), STILL_DARK) + + def test_each_dark_test_runs_on_a_latest_leg(self) -> None: + gaps = {(gap.flag, gap.job) for gap in _dependency_gates.dependency_gate_gaps()} + jobs = [job for job in _dependency_gates.pytest_jobs() if job.name in set(BASE) | set(LATEST_ONLY)] + for flag, module in DARK_TESTS.items(): + with self.subTest(flag=flag): + gates = [gate for gate in _dependency_gates.gated_scopes() + if gate.flag == flag and gate.module == module] + self.assertTrue(gates, f'{module} no longer gates on {flag}') + reaching = {job.name for job in jobs for gate in gates if _dependency_gates.job_reaches(job, gate)} + self.assertTrue(reaching, f'no latest leg collects {module}') + for job_id in sorted(reaching): + self.assertNotIn((flag, job_id), gaps) + + +class TestTheWorkflowsPage(unittest.TestCase): + """:file:`docs/source/contributing/workflows.rst` names the step and this module.""" + + def test_it_describes_the_latest_install(self) -> None: + text = ' '.join(DOC.read_text(encoding='utf-8').split()) + self.assertIn(f'the step ``{LATEST_STEP}`` runs on the ``latest`` leg alone', text) + for job_id, name in LATEST_ONLY.items(): + with self.subTest(job=job_id): + self.assertIn(f'``{name}', text) + self.assertIn(':file:`tests/project/test_whole_suite_on_latest_1958.py`', text) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_tier_guard.py b/tests/test_tier_guard.py index 2d3f0585ff..3501b08457 100644 --- a/tests/test_tier_guard.py +++ b/tests/test_tier_guard.py @@ -1870,8 +1870,9 @@ def test_the_project_pins_selection_is_exactly_the_pins(self) -> None: """#1954: ``project-tests`` reaches the pins and nothing else, so a gate moves with its module.""" job = next(job for job in _dependency_gates.pytest_jobs() if job.selection == 'project-pins') self.assertEqual(job.name, 'project-tests') + # #1958: the set `test` installs on `latest`, the leg the pins would run on. self.assertEqual(job.extras, next(job for job in _dependency_gates.pytest_jobs() - if job.name == 'test').extras) + if job.name == 'test').latest_extras) cases = { 'tests/project/test_isort_clean.py': True, 'tests/project/test_public_api.py': False, @@ -2021,37 +2022,29 @@ def test_the_crawler_dependencies_are_satisfied_by_the_test_extra(self) -> None: gaps = {gap.flag for gap in _dependency_gates.dependency_gate_gaps()} self.assertNotIn('HAS_CRAWLER_DEPS', gaps) self.assertNotIn('HAS_CRAWLER_DEPS', _dependency_gates.DEPENDENCY_GATE_EXCLUSIONS) - self.assertIn('HAS_VENDOR_DEPS', _dependency_gates.DEPENDENCY_GATE_EXCLUSIONS) + # The ``test`` extra still lacks html5lib: ``vendor`` closes that gate (#1958). + self.assertNotIn('test', _dependency_gates.extras_providing('html5lib')) + self.assertIn('vendor', _dependency_gates.extras_providing('html5lib')) - def test_the_vendor_extra_closes_engine_tests_but_not_test_or_gate(self) -> None: - """#738's remaining scope: an existing non-blocking leg absorbs it. + def test_the_vendor_extra_closes_every_job_reaching_the_vendor_gates(self) -> None: + """#1958: the ``latest`` legs install ``vendor``, so html5lib is no gap anywhere. - ``engine-tests`` (#751) already reaches these gates through the same - ignore-shape selection as ``test`` -- see this guard's own - ``HAS_VENDOR_DEPS`` exclusion -- and it has never been one of ruleset - 23497679's required checks, so it already was the "job that exists - and reports without gating a merge" #738's ruling asked for. Installing - ``vendor`` there, rather than opening another job, is what closes the - gap on it. It is also the only *per-pull-request* job with - that property: ``gate`` reaches these gates too but never runs on a - pull request at all (only via a release's ``gate-only: true`` call), - and ``integration``'s fixture-tier selection never reaches - :file:`tests/vendor/` in the first place -- so ``engine-tests`` was - the forced choice, not merely a convenient one. - - ``test`` and ``gate`` stay dark for different reasons, not the same - one -- see the exclusion's own reason for why ``gate`` is not merely - "also non-blocking". + #738 ruled these gates onto a non-blocking leg, and ``engine-tests`` was + that leg until #849 narrowed it to explicit engine paths, after which + they ran nowhere. The owner's ruling E on #1958 runs the whole suite on + latest CPython, so ``test``'s ``latest`` leg and ``gate`` install + ``vendor``, and the exclusion that recorded the gap is gone. """ + reaching = {job.name for job in _dependency_gates.pytest_jobs() + for gate in _dependency_gates.gated_scopes() + if gate.flag == 'HAS_VENDOR_DEPS' and _dependency_gates.job_reaches(job, gate)} + self.assertEqual(reaching, {'test', 'gate'}) gaps = {(gap.flag, gap.job) for gap in _dependency_gates.dependency_gate_gaps()} - self.assertNotIn(('HAS_VENDOR_DEPS', 'engine-tests'), gaps) - self.assertIn(('HAS_VENDOR_DEPS', 'test'), gaps) - self.assertIn(('HAS_VENDOR_DEPS', 'gate'), gaps) - - exclusion = _dependency_gates.DEPENDENCY_GATE_EXCLUSIONS['HAS_VENDOR_DEPS'] - self.assertNotIn('engine-tests', exclusion.dark) - self.assertEqual(set(exclusion.dark), {'test', 'gate'}) + for job in sorted(reaching): + with self.subTest(job=job): + self.assertNotIn(('HAS_VENDOR_DEPS', job), gaps) + self.assertNotIn('HAS_VENDOR_DEPS', _dependency_gates.DEPENDENCY_GATE_EXCLUSIONS) def test_the_mypy_gate_is_visible_and_dark_on_every_job_that_reaches_it(self) -> None: """#779: a lint-tier tool gated visibly, and declining the install line anyway. @@ -2068,7 +2061,7 @@ def test_the_mypy_gate_is_visible_and_dark_on_every_job_that_reaches_it(self) -> :data:`~tests._dependency_gates.DEPENDENCY_GATE_EXCLUSIONS` filtering. (#779 reports 4 errors of 9 tests for the same mutation; it was written before #774 added - :meth:`test_the_vendor_extra_closes_engine_tests_but_not_test_or_gate`, + :meth:`test_the_vendor_extra_closes_every_job_reaching_the_vendor_gates`, which is the tenth and the fifth error. Re-measured here rather than copied.) The entry is what makes the gate expressible; the exclusion is what @@ -2387,7 +2380,14 @@ def setUp(self) -> None: def test_removing_crypto_from_the_test_job_is_caught(self) -> None: """#745's worked example, verbatim: "deleting ``crypto`` ... would go green".""" + # #1958: from both of `test`'s lines, the one every leg runs and the one + # `latest` adds, since the second alone runs the gate. The first is pinned + # to its extras by tests/project/test_whole_suite_on_latest_1958.py. doctored = doctored_workflow(self, "'.[test,DPKT,crypto,NGAP]'", "'.[test,DPKT,NGAP]'") + text = doctored.read_text(encoding='utf-8') + latest = "'.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]'" + self.assertIn(latest, text) + doctored.write_text(text.replace(latest, latest.replace('crypto,', '')), encoding='utf-8') gaps = {(gap.flag, gap.job): gap for gap in _dependency_gates.dependency_gate_gaps(doctored)} @@ -2972,11 +2972,11 @@ def test_a_flag_no_extra_could_satisfy_is_not_reported_as_a_gap(self) -> None: if reason is not None: self.skipTest(f'git cannot answer here: {reason}') - self.assertIn('HAS_PYSHARK', + self.assertIn('HAS_PYPCAPFILE', {gap.flag for gap in _dependency_gates.dependency_gate_gaps()}) with unittest.mock.patch.dict(_dependency_gates.NON_DISTRIBUTION_FLAGS, - {'HAS_PYSHARK': 'stood in for this test'}): - self.assertNotIn('HAS_PYSHARK', + {'HAS_PYPCAPFILE': 'stood in for this test'}): + self.assertNotIn('HAS_PYPCAPFILE', {gap.flag for gap in _dependency_gates.dependency_gate_gaps()}) def test_an_unmapped_module_names_itself_and_the_table_to_add_it_to(self) -> None: From 623024fc50a6e9e74daf7796e80f0fd99164695a Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 20 Oct 2026 00:53:06 -0400 Subject: [PATCH 2/4] ci(unit-tests): say where the latest leg gets cli, crypto and NGAP (#1958) - test, integration: the install comments said cli, and crypto and NGAP, are not installed; the latest leg's whole-suite step installs them. - workflows.rst, releasing.rst: unit-tests.yml line citations re-anchored. --- .github/workflows/unit-tests.yml | 12 ++++++++---- docs/source/contributing/releasing.rst | 4 ++-- docs/source/contributing/workflows.rst | 16 ++++++++-------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 238f863909..fd5b2173e9 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -142,10 +142,12 @@ jobs: # crypto (cryptography) and NGAP (pycrate) join it for the same reason, # covering #738's remaining HAS_CRYPTO gate (test_esp_unit.py) and its # HAS_PYCRATE gate (test_ngap_unit.py), both plain - # unit-tier. `cli` (emoji) is deliberately NOT added here: its only + # unit-tier. `cli` (emoji) is deliberately not on this line: its only # HAS_EMOJI-gated tests are in tests/integration/test_cli_subprocess.py, # which this job ignores wholesale, so installing it here would unlock - # nothing. cryptography is the one compiled wheel of the two; measured + # nothing. The `latest` leg gets it from the whole-suite step below + # (#1958), whose set every `latest` job shares, and there too it unlocks + # nothing in this job. cryptography is the one compiled wheel of the two; measured # clean installs on 3.10-3.14 (manylinux wheels, no build step). pycrate # is pure Python but its distribution is large (~238MB, see the NGAP # extra's comment in pyproject.toml) -- still cheap in wall time, just not @@ -517,10 +519,12 @@ jobs: # selection (tests/_tiers.py's fixture_tier_paths()) always includes the # whole of tests/integration/ -- which is where every HAS_EMOJI-gated # method lives (test_cli_subprocess.py). `crypto` and `NGAP` - # are deliberately NOT added here: test_esp_unit.py and test_ngap_unit.py + # are deliberately not on this line: test_esp_unit.py and test_ngap_unit.py # are plain unit-tier and never enter this job's selection, so installing # cryptography/pycrate here would unlock nothing -- they are already - # covered, on every Python version, by the `test` job above. + # covered, on every Python version, by the `test` job above. The `latest` + # leg gets both from the whole-suite step below (#1958), whose set every + # `latest` job shares, and there too they unlock nothing in this job. # PyPCAPFile is NOT added here either, even though this job's selection # also reaches test_new_engine_parity_runtime.py's HAS_PYPCAPFILE # methods -- see the `test` job's comment above for the bug that used to diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index ead45ff410..ed127f3e9a 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -52,7 +52,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:581-597``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1867-1942``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1871-1946``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:621,808,974,1210``), so a @@ -62,7 +62,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Other checks run unconditionally on the same path, though none is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1671-1697``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1675-1701``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 120dd1e97c..5323cd3d95 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -329,7 +329,7 @@ the same commit several times over. It also skips ``changes``, Checks`_ below), which are gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1659-1665``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1663-1669``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -639,7 +639,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:2033`` + - ``unit-tests.yml:2037`` * - ``Compat Python 3.11`` - job ``compatibility``, matrix leg ``3.11`` - ``python-compatibility.yml:60``, ``.github/python-versions.toml:20`` @@ -661,22 +661,22 @@ which added ``Compat Python 3.15`` and removed ``Compat Python 3.10``. Moving a between the tiers below means editing the ruleset in the same change. ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:2033``). ``Compat Python`` is defined by one, the +(``unit-tests.yml:2037``). ``Compat Python`` is defined by one, the ``compatibility`` job in ``python-compatibility.yml`` (``:60``), whose matrix (``:74``) comes from :file:`.github/python-versions.toml`: the required legs are its supported tier (``python-versions.toml:20-24``), and its KTLO legs (``python-versions.toml:16-19``) have names ending in ``(KTLO)``, so no required context matches one. A comment at -``unit-tests.yml:1945`` mentions the string without defining it. +``unit-tests.yml:1949`` mentions the string without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the ``Compat Python 3.11``-``3.15`` legs, nothing else, as -``unit-tests.yml``'s own comment (``unit-tests.yml:1944-1954``) records. It +``unit-tests.yml``'s own comment (``unit-tests.yml:1948-1958``) records. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- ``test``, ``integration``, ``engine-tests`` and ``pypcap-parity``, plus ``project-tests``, the one run of the project pins, and ``changes`` for the docs-only path -- and checks each result explicitly -(``unit-tests.yml:2032-2088``). Rulesets match check names literally, with no +(``unit-tests.yml:2036-2092``). Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. It accepts the gating jobs as ``skipped`` only on a docs-only pull request, where @@ -685,7 +685,7 @@ It accepts the gating jobs as ``skipped`` only on a docs-only pull request, wher ``python-compatibility.yml``, and a job cannot ``needs:`` a job in another workflow file. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:2035``), +``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:2039``), so it is never produced -- and never expected -- on those paths. .. _ktlo-tier: @@ -709,7 +709,7 @@ and keeps its legs' check names, ``Python 3.10 (KTLO)`` and so on. In with supported legs. Every job holding KTLO legs works the same way: * Every step that can fail on a KTLO leg has an ``id`` and - ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:441-458``), + ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:443-460``), unless it is masked on every leg, as the timings uploads are. Its ``outcome`` is then ``failure`` but its ``conclusion`` ``success``, so the job, its check run and ``required-checks`` read ``success``. A step that needs an earlier one's work From b733cd0427ad0088c629f181368356cd62efa413 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 20 Oct 2026 09:56:04 -0400 Subject: [PATCH 3/4] test(tiers): make ALL a curated opt-in tier (#1958) - tests/_tiers.py: module_tier() is the declared tier, else DEFAULT_TIER (LATEST); rule_tier() goes. all_tier_paths() lists ALL, and branches_on_interpreter() flags a module that compares the interpreter itself. The docstring states the curation criterion (#1963). - Curated modules opt in with `__tier__ = 'ALL'` and a reason: version gates, GC/weakref, int/struct limits, pickling and copying, metaclass and enum machinery, IO, threading and hooks, engines whose releases vary by interpreter, the hand-picked round trips, and a smoke slice per protocol family. Six vendor markers with no such reason are gone. - tests/test_tier_guard.py: ALL is opt-in, every ALL module declares it, and a module that branches on the interpreter must state its tier; the mixed-directory split test no longer assumes tests/integration is all ALL. test_sweeps_on_latest_only_1732 compares what the ALL fixture selection drops with the LATEST modules it reaches. - tests/project/test_curated_all_tier_1958.py pins the opt-in rule, the strict subset, the ruling-B round trips and the docs. - workflows.rst "Test Tiers" and the `test` job's `Run unit tests` comment describe it. Tier guard, #1732, #1954 and #1958 pins pass on CPython 3.14. --- .github/workflows/unit-tests.yml | 4 +- docs/source/contributing/workflows.rst | 67 +++++---- tests/_tiers.py | 104 ++++++++++---- tests/cli/test_main.py | 4 + .../test_const_apptype_lazy_1538_unit.py | 3 + tests/const/test_const_enum_builtin_parity.py | 3 + tests/const/test_const_enum_get.py | 4 + tests/const/test_const_enum_lookup.py | 4 + ..._register_loud_in_quiet_block_1916_unit.py | 4 + tests/corekit/test_decorators.py | 4 + .../test_enum_get_quiet_fallback_1845_unit.py | 4 + tests/corekit/test_enum_lookup_base_unit.py | 4 + .../test_enum_lookup_reparent_877_unit.py | 4 + .../test_fields_byteorder_1519_unit.py | 4 + .../corekit/test_fields_copy_dispatch_unit.py | 4 + tests/corekit/test_fields_field.py | 4 + tests/corekit/test_fields_numbers.py | 4 + ...est_fields_numbers_bit_length_1488_unit.py | 4 + ..._fields_numbers_signed_decode_1274_unit.py | 4 + .../test_fields_numbers_width_repair.py | 4 + tests/corekit/test_fields_strings.py | 4 + .../test_info_child_finalisation_1490_unit.py | 4 + tests/corekit/test_infoclass.py | 4 + tests/corekit/test_io.py | 4 + .../test_io_peekable_stream_copy_1524_unit.py | 4 + ...o_seekable_reader_short_reads_1586_unit.py | 4 + .../test_io_stream_proxies_1516_unit.py | 4 + tests/corekit/test_ipv4interface_mask_unit.py | 4 + tests/corekit/test_module.py | 4 + tests/corekit/test_multidict.py | 4 + tests/corekit/test_multiinfo_1484_unit.py | 4 + .../test_ordered_multidict_copy_unit.py | 4 + .../corekit/test_sentinel_py_abc_1960_unit.py | 4 + .../test_sentinel_singleton_1842_unit.py | 4 + tests/corekit/test_sentinels_housing_unit.py | 4 + .../corekit/test_stringfield_lossless_unit.py | 4 + tests/dumpkit/test_common_unit.py | 4 + tests/dumpkit/test_plist_date_unit.py | 4 + tests/foundation/engines/test_engine_base.py | 3 + .../foundation/engines/test_pcap_ct_engine.py | 4 + .../foundation/engines/test_pypcap_engine.py | 4 + .../engines/test_pypcapfile_engine.py | 4 + .../foundation/engines/test_pyshark_engine.py | 4 + tests/foundation/reassembly/test_ip.py | 4 + .../test_ip_max_length_edge_1739_unit.py | 4 + .../reassembly/test_reassembly_base.py | 3 + tests/foundation/reassembly/test_tcp.py | 4 + tests/foundation/registry/test_protocols.py | 4 + tests/foundation/test_extraction.py | 4 + tests/foundation/test_extraction_no_eof.py | 4 + ...xtraction_non_seekable_stream_1586_unit.py | 4 + tests/foundation/test_extraction_ownership.py | 4 + tests/foundation/traceflow/test_tcp.py | 4 + .../traceflow/test_traceflow_base.py | 3 + tests/integration/test_engine_runtime.py | 4 + tests/integration/test_frame_iteration.py | 4 + tests/interface/test_core.py | 4 + tests/project/test_curated_all_tier_1958.py | 129 ++++++++++++++++++ tests/project/test_public_api.py | 4 + .../test_sweeps_on_latest_only_1732.py | 18 ++- tests/protocols/application/test_http_unit.py | 4 + tests/protocols/internet/test_ipv4_unit.py | 4 + .../test_ipv6_ext_chain_limit_1604_unit.py | 4 + .../internet/test_ipv6_jumbogram_unit.py | 4 + tests/protocols/internet/test_ipv6_unit.py | 4 + tests/protocols/link/test_link_unit.py | 4 + .../misc/pcap/test_header_frame_unit.py | 4 + ...t_pcapng_tzone_tsoffset_range_1340_unit.py | 4 + tests/protocols/misc/test_pcapng_unit.py | 4 + .../schema/test_enum_schema_registry_unit.py | 4 + .../test_schema_metaclass_abc_cache_unit.py | 4 + tests/protocols/schema/test_schema_unit.py | 4 + .../test_construction_keyword_check_unit.py | 4 + .../test_frame_layer_limit_1610_unit.py | 4 + ...tion_generator_deadline_scale_1428_unit.py | 3 + tests/protocols/test_protocol_base_unit.py | 4 + .../test_protocol_code_registration_unit.py | 3 + tests/protocols/test_protocol_make_unit.py | 3 + .../protocols/transport/test_tcp_udp_unit.py | 4 + tests/test_base_class_contract.py | 4 + tests/test_final_enforcement.py | 3 + tests/test_periodic_gc_1964_unit.py | 4 + tests/test_support_helpers.py | 3 + tests/test_tier_guard.py | 45 +++++- .../test_dpkt_timestamp_copy_1647_unit.py | 4 + tests/toolkit/test_dpkt_unit.py | 4 + .../toolkit/test_engine_pcap_trace_runtime.py | 4 + tests/toolkit/test_pcap_ct_unit.py | 4 + tests/toolkit/test_pypcap_unit.py | 4 + tests/toolkit/test_pypcapfile_unit.py | 4 + tests/utilities/test_compat.py | 3 + tests/utilities/test_config_unit.py | 4 + tests/utilities/test_exceptions_excepthook.py | 4 + tests/utilities/test_logging.py | 4 + tests/utilities/test_quiet_exceptions.py | 4 + tests/utilities/test_stacklevel.py | 4 + tests/vendor/test_ftp_return_code_unit.py | 4 - .../test_re_sub_positional_flag_unit.py | 3 + tests/vendor/test_request_prompt_unit.py | 4 - tests/vendor/test_user_agent_unit.py | 4 - tests/vendor/test_vendor_dest_path_unit.py | 4 - tests/vendor/test_vendor_exit_code_unit.py | 4 - .../test_vendor_missing_extra_1850_unit.py | 3 + tests/vendor/test_vendor_ngap_unit.py | 4 - 104 files changed, 657 insertions(+), 89 deletions(-) create mode 100644 tests/project/test_curated_all_tier_1958.py diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index fd5b2173e9..b7b8736ecf 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -234,8 +234,8 @@ jobs: # timings, uploaded by the last step of this job. # # #1958, the test tiers: tests/_tiers.py's module_tier() puts every test - # module in `ALL` or `LATEST`, by the `__tier__` constant a module states - # or by the rule where it states none, and `LATEST` runs on `latest` + # module in `ALL`, a curated list each stating `__tier__ = 'ALL'` (#1963), + # or else in `LATEST`, and `LATEST` runs on `latest` # alone, which is also the coverage leg, so coverage keeps its # selection. Every other leg -- `test-ktlo`'s too, as it runs these # steps -- passes each module latest_tier_paths() prints as an diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 5323cd3d95..35e791c2a7 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -915,30 +915,49 @@ and fixture-dependent tiers, which decide what a module may read and so which jo collects it; a test tier decides on which of that job's legs it runs. ``module_tier()`` in :file:`tests/_tiers.py` decides, in one place, for every -``tests/**/test_*.py``: - -* **The marker.** A module may state its tier outright with a module-level - ``__tier__ = 'ALL'`` or ``__tier__ = 'LATEST'``, read by AST, under a ``#:`` - comment that gives the reason. It wins over the rule. It is a module constant - rather than a pytest marker, so ``unittest-ordering``'s plain ``unittest`` runs - read the same module. A value that is not one of the two string literals, or a - marker assigned twice or anywhere but at the module's top level, fails the tier - guard. -* **The rule.** Otherwise a module is ``LATEST`` when its run does not vary by - interpreter, by the same AST test as the project pins below: it reads the - repository's text, a workflow or the docs, or drives a ``util/`` script, and - neither imports ``pcapkit`` nor compares the Python version. Everything that - exercises ``pcapkit`` is ``ALL``. - -The marker says what the AST cannot see. Each real sweep -- a round trip over every -class or registry entry a table enumerates, every offset of a packet or every -capture -- is marked ``LATEST``, as the owner ruled on #1732 and #1958; the -hand-picked round trips, whatever their names, run on ``ALL``, and so do the #1202 -edge-case tables, whose cases are each written out by hand. A module that imports ``pcapkit`` only to read its text or -structure, or that drives a docs hook or a generator other legs already run, is -marked ``LATEST``; one that the rule would send to ``latest`` but that depends on -the interpreter, such as the tier guard itself, which :file:`tests/conftest.py` runs at -collection on every leg, is marked ``ALL``. +``tests/**/test_*.py``. ``ALL`` is a curated, opt-in list (#1958, #1963): short enough +to save time on every leg, the same on every leg so QA stays consistent, and enough +on its own to validate a change on each interpreter, while ``latest`` runs the whole +suite as the gate. + +* **The marker.** A module joins ``ALL`` with a module-level ``__tier__ = 'ALL'``, + read by AST, under a ``#: Test tier (#1958): `` comment. It is a module + constant rather than a pytest marker, so ``unittest-ordering``'s plain + ``unittest`` runs read the same module. A value that is not ``'ALL'`` or + ``'LATEST'``, or a marker assigned twice or anywhere but at the module's top + level, fails the tier guard. +* **The default.** A module that states no tier is ``LATEST``, so a new module runs + on ``latest`` until it says why every leg needs it. ``__tier__ = 'LATEST'`` is + optional, and records why a module someone might expect on every leg is not; a + module that compares ``sys.version_info`` or ``sys.implementation`` itself has + to state its tier either way, so a version gate never lands on ``latest`` alone + by omission. + +A module opts in when it exercises behaviour that can differ between interpreters +or versions: + +* compat shims and version or implementation gates; +* C-accelerated versus pure paths, and code sensitive to garbage collection, weak + references or finalisation; +* ``int``, ``bytes`` and ``struct`` edge cases, and ``sys.maxsize`` or recursion + limits; +* pickling and copying, metaclass and ABC machinery, and enum mechanics; +* stream and buffer semantics, threading and locking, and the warning and + exception hooks; +* a third-party engine whose installable releases vary by interpreter, such as + ``pypcap`` and ``pypcapfile``, which install below 3.12 only; +* the hand-picked round trips, whatever their names, and the #1202 edge-case + tables, whose cases are each written out by hand, as the owner ruled on #1958; + and +* the smoke slice: a representative module per protocol family and per package, + so each leg still drives the main parse and build paths. + +Everything else is ``LATEST``: the per-protocol field-by-field tests, whose +behaviour is interpreter-independent pure Python, each real sweep -- a round trip +over every class or registry entry a table enumerates, every offset of a packet or +every capture, as the owner ruled on #1732 and #1958 -- and the docs, project and +structure tests. The tier guard itself is ``ALL``, since :file:`tests/conftest.py` +runs it at collection on every leg. Which legs run which tier: diff --git a/tests/_tiers.py b/tests/_tiers.py index 5bf864cd3d..465d6cadf7 100644 --- a/tests/_tiers.py +++ b/tests/_tiers.py @@ -47,10 +47,12 @@ for real, and the complete pytest selection the fixture-dependent tier needs in order to still exercise them. * :func:`module_tier` -- which *test tier* does this module run in (#1958): - ``ALL``, on every leg, or ``LATEST``, on ``latest`` alone? A ``__tier__`` - constant in the module (:func:`declared_tier`) answers first, and - :func:`rule_tier` otherwise. :func:`latest_tier_paths` lists the ``LATEST`` - modules, which every leg but ``latest`` leaves out. + ``ALL``, on every leg, or ``LATEST``, on ``latest`` alone? ``ALL`` is opt-in: + a module is in it only when it says so with ``__tier__ = 'ALL'`` + (:func:`declared_tier`), and :data:`DEFAULT_TIER`, ``LATEST``, otherwise. + :func:`all_tier_paths` and :func:`latest_tier_paths` list the two tiers; every + leg but ``latest`` leaves the second out. See "The curated ``ALL`` tier" below + for which modules opt in. * :func:`is_project_pin` and :func:`project_pin_paths` -- which :file:`tests/project` modules give one answer on every interpreter, and so run once per push, in the ``project-tests`` job, rather than on every leg (#1954)? @@ -114,6 +116,40 @@ capture reads go through :func:`~tests._support.sample_path`, and that is the door with the lock on it. +The curated ``ALL`` tier +------------------------ + +``ALL`` is a tailored list rather than whatever a rule guesses (#1958, #1963): it +is what every leg off ``latest`` runs, so it has to be short enough to save time, +the same on every leg so QA stays consistent, and enough on its own to validate a +change on each interpreter. ``latest`` runs the whole suite, and is the gate. + +A module opts in with ``__tier__ = 'ALL'`` when it exercises behaviour that can +differ between interpreters or versions: + +* compat shims and ``sys.version_info`` or ``sys.implementation`` gates; +* C-accelerated versus pure paths, and code sensitive to garbage collection, + weak references or finalisation; +* ``int``, ``bytes`` and :mod:`struct` edge cases, and ``sys.maxsize`` or + recursion limits; +* pickling and copying, metaclass and ABC machinery, and enum mechanics; +* stream and buffer semantics, threading and locking, and the warning and + exception hooks; +* a third-party engine whose installable releases vary by interpreter, which + ``latest`` alone could not cover; +* the hand-picked round trips, the ``*_edge_roundtrip_unit`` tables among them, + which the owner ruled run on ``ALL`` as part of QA; and +* the smoke slice: a representative module per protocol family and per + package, so each leg still drives the main parse and build paths. + +Everything else is ``LATEST`` by default: the per-protocol field-by-field tests, +whose behaviour is interpreter-independent pure Python, the real sweeps, and the +docs, project and structure tests. A module that branches on the interpreter +itself (:func:`branches_on_interpreter`) has to state its tier either way, so a +version gate never lands on ``latest`` alone by omission; a ``LATEST`` marker is +otherwise optional, and records why a module someone might expect on every leg +is not. + Nothing here imports :mod:`tests._support` -- the dependency runs the other way, and adding the reverse edge would make it a cycle. Nothing here imports :mod:`pcapkit`, :mod:`pytest` or any optional engine either, so the guard is @@ -141,7 +177,8 @@ 'ROOT', 'TESTS_ROOT', 'SAMPLE_ROOT', 'REGENERATE_SAMPLES_CMD', 'UNIT_TIER_SELECTION', 'FIXTURE_TIER_SUFFIXES', 'FIXTURE_TIER_DIRS', 'FIXTURE_TIER_SELECTOR', 'TIER_ALL', 'TIER_LATEST', 'TIERS', 'TIER_MARKER', 'LATEST_TIER_SELECTOR', 'FIXTURE_ALL_TIER_SELECTOR', - 'InvalidTierError', 'suite_modules', 'declared_tier', 'rule_tier', 'module_tier', 'latest_tier_paths', + 'InvalidTierError', 'DEFAULT_TIER', 'suite_modules', 'declared_tier', 'branches_on_interpreter', 'module_tier', + 'all_tier_paths', 'latest_tier_paths', 'PROJECT_PINS_DIR', 'PROJECT_PINS_SELECTOR', 'PROJECT_PIN_PACKAGE', 'LOADABLE_DIRS', 'varies_by_interpreter', 'is_project_pin', 'project_pin_paths', 'is_unit_tier', 'committed_captures', 'committed_capture_names', @@ -195,6 +232,9 @@ TIER_LATEST = 'LATEST' #: Every test tier, the values a :data:`TIER_MARKER` may take. TIERS = (TIER_ALL, TIER_LATEST) +#: The tier of a module that states none: ``ALL`` is opt-in (#1958, #1963), so a +#: new module runs on ``latest``, the gate, until it says why every leg needs it. +DEFAULT_TIER = TIER_LATEST #: The module-level constant that states a module's test tier outright, e.g. #: ``__tier__ = 'LATEST'``, read by AST in :func:`declared_tier`. A constant rather #: than a pytest marker, so plain :mod:`unittest` runs see the same module. @@ -282,7 +322,7 @@ class TierGuardWarning(UserWarning): class InvalidTierError(ValueError): """A test module's :data:`TIER_MARKER` cannot be read as one of :data:`TIERS`. - Raised rather than ignored: a marker that silently fell back to the rule would + Raised rather than ignored: a marker that silently fell back to the default would put the module on legs its author ruled it off, or the other way round. """ @@ -896,7 +936,7 @@ def declared_tier(path: 'pathlib.Path | str') -> 'Optional[str]': Read by AST, never by importing the module: one module-level statement, ``__tier__ = 'ALL'`` or ``__tier__ = 'LATEST'``, the value a string literal. - A module that does not parse states nothing here, and :func:`rule_tier` + A module that does not parse states nothing here, and :data:`DEFAULT_TIER` answers for it. Args: @@ -912,19 +952,15 @@ def declared_tier(path: 'pathlib.Path | str') -> 'Optional[str]': return _declared_tier(_test_module(path)) -def rule_tier(path: 'pathlib.Path | str') -> 'str': - """The test tier the rule gives the module at ``path`` (#1958). +def branches_on_interpreter(path: 'pathlib.Path | str') -> 'bool': + """Whether the test module at ``path`` itself compares the running interpreter (#1958). - :data:`TIER_LATEST` when its run does not :func:`vary by interpreter - ` -- it reads the repository's text, a workflow or the - docs, or drives a ``util/`` script, and imports no ``pcapkit`` -- since one run - on ``latest`` is then every leg's answer. :data:`TIER_ALL` for everything - else, which is everything that exercises ``pcapkit``. What the AST cannot - see -- that a module is a sweep over every case of a table or every capture, - that one which imports ``pcapkit`` still answers a question about the - repository, or that one which imports nothing still depends on the - interpreter -- is stated with :data:`TIER_MARKER` instead; see - :func:`module_tier`. + By its own AST alone, unlike :func:`varies_by_interpreter`, which follows what + the module runs: a comparison of ``sys.version_info``, ``sys.implementation``, + ``platform.python_version()`` or the like, anywhere in it. Such a module has to + state its tier with :data:`TIER_MARKER`, so a version gate never runs on + ``latest`` alone just because nobody opted it in to ``ALL``. A module that does + not parse branches on nothing here. Args: path: Path to a test module, absolute or relative to :data:`ROOT`. @@ -933,18 +969,20 @@ def rule_tier(path: 'pathlib.Path | str') -> 'str': ValueError: ``path`` is not a test module under :file:`tests/`. """ - return TIER_ALL if varies_by_interpreter(_test_module(path)) else TIER_LATEST + tree = _parse_module(_test_module(path)) + return tree is not None and any(_compares_interpreter(node) for node in ast.walk(tree)) def module_tier(path: 'pathlib.Path | str') -> 'str': """The test tier of the module at ``path``: :data:`TIER_ALL` or :data:`TIER_LATEST` (#1958). - Its :func:`declared_tier` where it states one, its :func:`rule_tier` - otherwise, so every test module has exactly one. ``ALL`` modules run on every - leg; ``LATEST`` ones on ``latest`` alone, which is also the coverage leg, the - ``gate`` job's interpreter, and every ``unittest-ordering`` cell's. Within a - tier, the unit and fixture-dependent tiers still decide which job collects a - module. + Its :func:`declared_tier` where it states one, :data:`DEFAULT_TIER` otherwise, + so every test module has exactly one and ``ALL`` is opt-in: see "The curated + ``ALL`` tier" in this module's docstring for which modules opt in. ``ALL`` + modules run on every leg; ``LATEST`` ones on ``latest`` alone, which is also + the coverage leg, the ``gate`` job's interpreter, and every + ``unittest-ordering`` cell's. Within a tier, the unit and fixture-dependent + tiers still decide which job collects a module. Args: path: Path to a test module, absolute or relative to :data:`ROOT`. @@ -954,8 +992,18 @@ def module_tier(path: 'pathlib.Path | str') -> 'str': InvalidTierError: Its marker is malformed; see :func:`declared_tier`. """ - candidate = _test_module(path) - return _declared_tier(candidate) or rule_tier(candidate) + return _declared_tier(_test_module(path)) or DEFAULT_TIER + + +@functools.lru_cache(maxsize=1) +def all_tier_paths() -> 'tuple[str, ...]': + """Every :data:`TIER_ALL` test module, relative to :data:`ROOT`, sorted. + + The curated tier every leg runs: the complement of :func:`latest_tier_paths` + within :func:`suite_modules`, and each of them marked ``__tier__ = 'ALL'``. + + """ + return tuple(path for path in suite_modules() if module_tier(path) == TIER_ALL) @functools.lru_cache(maxsize=1) diff --git a/tests/cli/test_main.py b/tests/cli/test_main.py index 166aa76468..526beedd36 100644 --- a/tests/cli/test_main.py +++ b/tests/cli/test_main.py @@ -48,6 +48,10 @@ from tests._support import ISOLATED_PREFIXES, isolate_modules, snapshot_modules +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + ROOT = pathlib.Path(__file__).resolve().parents[2] #: The :data:`sys.modules` region this file stands things in for, and therefore diff --git a/tests/const/test_const_apptype_lazy_1538_unit.py b/tests/const/test_const_apptype_lazy_1538_unit.py index 0615b6509e..2b40365caf 100644 --- a/tests/const/test_const_apptype_lazy_1538_unit.py +++ b/tests/const/test_const_apptype_lazy_1538_unit.py @@ -39,6 +39,9 @@ from types import ModuleType from typing import Any +#: Test tier (#1958): branches on the running interpreter's version or implementation. +__tier__ = 'ALL' + ROOT = pathlib.Path(__file__).resolve().parents[2] #: The names ``dir(pcapkit.const.reg.apptype)`` listed on the eager package diff --git a/tests/const/test_const_enum_builtin_parity.py b/tests/const/test_const_enum_builtin_parity.py index 615d62fe5b..eb2cb244a9 100644 --- a/tests/const/test_const_enum_builtin_parity.py +++ b/tests/const/test_const_enum_builtin_parity.py @@ -66,6 +66,9 @@ if TYPE_CHECKING: from typing import Optional +#: Test tier (#1958): branches on the running interpreter's version or implementation. +__tier__ = 'ALL' + #: An integer no wire field in the library is wide enough to carry. Shared with #: :mod:`tests.const.test_const_enum_get`, which uses the same value for the #: same reason. diff --git a/tests/const/test_const_enum_get.py b/tests/const/test_const_enum_get.py index 8113f5fba1..7d7a3194b7 100644 --- a/tests/const/test_const_enum_get.py +++ b/tests/const/test_const_enum_get.py @@ -73,6 +73,10 @@ from tests._support import ISOLATED_PREFIXES, purge_modules, restore_modules, snapshot_modules +#: Test tier (#1958): exercises metaclass, ABC or enum machinery, which differs by +#: interpreter. +__tier__ = 'ALL' + #: An integer no wire field in the library is wide enough to carry, so every #: registry that bounds its own domain rejects it. Deliberately far outside the #: ``0..0xFF`` range the Mobility Header flag registries check, because a value diff --git a/tests/const/test_const_enum_lookup.py b/tests/const/test_const_enum_lookup.py index 510bc74984..47f868bb94 100644 --- a/tests/const/test_const_enum_lookup.py +++ b/tests/const/test_const_enum_lookup.py @@ -72,6 +72,10 @@ from tests._support import (ISOLATED_PREFIXES, purge_modules, reimport_once_per_class, restore_modules, snapshot_modules) +#: Test tier (#1958): exercises metaclass, ABC or enum machinery, which differs by +#: interpreter. +__tier__ = 'ALL' + #: Fully qualified names of the :class:`~aenum.IntEnum` classes under #: :mod:`pcapkit.const` for which rejecting ``0`` is *correct*, because their #: registries simply do not define ``0`` (registries reserving ``0`` and diff --git a/tests/const/test_const_register_loud_in_quiet_block_1916_unit.py b/tests/const/test_const_register_loud_in_quiet_block_1916_unit.py index c272d97039..6818970993 100644 --- a/tests/const/test_const_register_loud_in_quiet_block_1916_unit.py +++ b/tests/const/test_const_register_loud_in_quiet_block_1916_unit.py @@ -30,6 +30,10 @@ if TYPE_CHECKING: from typing import Callable +#: Test tier (#1958): exercises threading, locking or warning and exception hooks, which +#: differ by interpreter. +__tier__ = 'ALL' + class RegisterLoudInQuietBlock1916Tests(unittest.TestCase): """``register`` on a generated registry logs its rejection inside a quiet block.""" diff --git a/tests/corekit/test_decorators.py b/tests/corekit/test_decorators.py index f4819e19b2..1fb7552649 100644 --- a/tests/corekit/test_decorators.py +++ b/tests/corekit/test_decorators.py @@ -7,6 +7,10 @@ from tests._support import (bootstrap_core_modules, install_fake_payload_protocols, isolate_modules) +#: Test tier (#1958): exercises metaclass, ABC or enum machinery, which differs by +#: interpreter. +__tier__ = 'ALL' + class DecoratorTests(unittest.TestCase): def setUp(self) -> None: diff --git a/tests/corekit/test_enum_get_quiet_fallback_1845_unit.py b/tests/corekit/test_enum_get_quiet_fallback_1845_unit.py index 67e15b789c..be684a6baf 100644 --- a/tests/corekit/test_enum_get_quiet_fallback_1845_unit.py +++ b/tests/corekit/test_enum_get_quiet_fallback_1845_unit.py @@ -47,6 +47,10 @@ from tests.utilities._harness import Recorder +#: Test tier (#1958): exercises threading, locking or warning and exception hooks, which +#: differ by interpreter. +__tier__ = 'ALL' + __all__ = ['QuietFallbackTests', 'QuietRaiseTests', 'LoudPathTests', 'MechanismTests'] #: A value :class:`ToSPrecedence`'s ``_validate_value`` refuses: the field is 3 bits. diff --git a/tests/corekit/test_enum_lookup_base_unit.py b/tests/corekit/test_enum_lookup_base_unit.py index 6738556682..f9693f428c 100644 --- a/tests/corekit/test_enum_lookup_base_unit.py +++ b/tests/corekit/test_enum_lookup_base_unit.py @@ -64,6 +64,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): exercises metaclass, ABC or enum machinery, which differs by +#: interpreter. +__tier__ = 'ALL' + #: The mutating half, which must stay on :class:`~pcapkit.corekit.enum.EnumRegistry`. MUTATING = ('register', 'register_alias', 'register_aliases', '_extend', '_unregistered_member') diff --git a/tests/corekit/test_enum_lookup_reparent_877_unit.py b/tests/corekit/test_enum_lookup_reparent_877_unit.py index 34f2057bf0..86d69dbc6e 100644 --- a/tests/corekit/test_enum_lookup_reparent_877_unit.py +++ b/tests/corekit/test_enum_lookup_reparent_877_unit.py @@ -60,6 +60,10 @@ from pcapkit.corekit.enum import EnumLookup +#: Test tier (#1958): exercises metaclass, ABC or enum machinery, which differs by +#: interpreter. +__tier__ = 'ALL' + __all__ = [ 'ReparentedBasesTests', 'HttpV2FlagsHierarchyTests', 'PureReparentGetTests', 'TransportProtocolGetTests', 'CriticalityGetTests', 'NoMintingTests', diff --git a/tests/corekit/test_fields_byteorder_1519_unit.py b/tests/corekit/test_fields_byteorder_1519_unit.py index 93748be3ef..c163ecf4e5 100644 --- a/tests/corekit/test_fields_byteorder_1519_unit.py +++ b/tests/corekit/test_fields_byteorder_1519_unit.py @@ -27,6 +27,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + SHARED = 'pcapkit.corekit.fields.byteorder' #: Where the helper lived before #1711; removed, not re-exported. diff --git a/tests/corekit/test_fields_copy_dispatch_unit.py b/tests/corekit/test_fields_copy_dispatch_unit.py index 658941c58d..364d791f47 100644 --- a/tests/corekit/test_fields_copy_dispatch_unit.py +++ b/tests/corekit/test_fields_copy_dispatch_unit.py @@ -41,6 +41,10 @@ from pcapkit.corekit.fields.field import FieldBase +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + class FieldCallUsesCopyDunderDirectlyTests(unittest.TestCase): """Every ``__call__`` in :mod:`pcapkit.corekit.fields` calls ``self.__copy__()``. diff --git a/tests/corekit/test_fields_field.py b/tests/corekit/test_fields_field.py index d79722f0a0..36c917527b 100644 --- a/tests/corekit/test_fields_field.py +++ b/tests/corekit/test_fields_field.py @@ -6,6 +6,10 @@ from tests._support import reimport_once_per_class, time_limit +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + class FieldBaseUnpackBoundsTests(unittest.TestCase): """Bounds checking in :meth:`FieldBase.unpack `. diff --git a/tests/corekit/test_fields_numbers.py b/tests/corekit/test_fields_numbers.py index 185af50685..cfbec8a615 100644 --- a/tests/corekit/test_fields_numbers.py +++ b/tests/corekit/test_fields_numbers.py @@ -6,6 +6,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/corekit/test_fields_numbers_bit_length_1488_unit.py b/tests/corekit/test_fields_numbers_bit_length_1488_unit.py index af26cc204b..b64d1d1a6f 100644 --- a/tests/corekit/test_fields_numbers_bit_length_1488_unit.py +++ b/tests/corekit/test_fields_numbers_bit_length_1488_unit.py @@ -20,6 +20,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + #: Octets of a two-octet field, as hex. WIRES = ('0000', '0001', '0fff', 'ffff', 'f000', '8000', '0800', 'f7ff', 'a5a5') diff --git a/tests/corekit/test_fields_numbers_signed_decode_1274_unit.py b/tests/corekit/test_fields_numbers_signed_decode_1274_unit.py index 2add91d246..bb79621f08 100644 --- a/tests/corekit/test_fields_numbers_signed_decode_1274_unit.py +++ b/tests/corekit/test_fields_numbers_signed_decode_1274_unit.py @@ -18,6 +18,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + class TestSignedNumberFieldDecode(unittest.TestCase): """Pin the decoded value and the rebuilt octets of signed fields.""" diff --git a/tests/corekit/test_fields_numbers_width_repair.py b/tests/corekit/test_fields_numbers_width_repair.py index e363a6dd40..0982717132 100644 --- a/tests/corekit/test_fields_numbers_width_repair.py +++ b/tests/corekit/test_fields_numbers_width_repair.py @@ -42,6 +42,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/corekit/test_fields_strings.py b/tests/corekit/test_fields_strings.py index 45d467ca68..c7ad145b4c 100644 --- a/tests/corekit/test_fields_strings.py +++ b/tests/corekit/test_fields_strings.py @@ -4,6 +4,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + class BitFieldTests(unittest.TestCase): """Packing and parsing of :class:`~pcapkit.corekit.fields.strings.BitField`.""" diff --git a/tests/corekit/test_info_child_finalisation_1490_unit.py b/tests/corekit/test_info_child_finalisation_1490_unit.py index a4d07242d4..6ae1574d1b 100644 --- a/tests/corekit/test_info_child_finalisation_1490_unit.py +++ b/tests/corekit/test_info_child_finalisation_1490_unit.py @@ -23,6 +23,10 @@ from tests.corekit._roundtrip import skip_without_runtime +#: Test tier (#1958): depends on garbage collection, weak references or object +#: finalisation. +__tier__ = 'ALL' + ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) #: Parent and child, by ``module:qualname``; ``synthetic`` is declared in the script. diff --git a/tests/corekit/test_infoclass.py b/tests/corekit/test_infoclass.py index ae7807dd40..41e8a41b72 100644 --- a/tests/corekit/test_infoclass.py +++ b/tests/corekit/test_infoclass.py @@ -6,6 +6,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises metaclass, ABC or enum machinery, which differs by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/corekit/test_io.py b/tests/corekit/test_io.py index 992a4c77f3..01af4a150b 100644 --- a/tests/corekit/test_io.py +++ b/tests/corekit/test_io.py @@ -9,6 +9,10 @@ from tests._support import bootstrap_core_modules, load_module, reimport_once_per_class +#: Test tier (#1958): exercises stream and buffer semantics, which differ by +#: interpreter. +__tier__ = 'ALL' + class SeekableReaderTests(unittest.TestCase): def _close_reader(self, reader) -> None: diff --git a/tests/corekit/test_io_peekable_stream_copy_1524_unit.py b/tests/corekit/test_io_peekable_stream_copy_1524_unit.py index f79e3ce41c..5a77d86d1b 100644 --- a/tests/corekit/test_io_peekable_stream_copy_1524_unit.py +++ b/tests/corekit/test_io_peekable_stream_copy_1524_unit.py @@ -23,6 +23,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + #: What :class:`_CopiesItself` returns from its own ``__deepcopy__``. _COPIED = 'the stream copied itself' diff --git a/tests/corekit/test_io_seekable_reader_short_reads_1586_unit.py b/tests/corekit/test_io_seekable_reader_short_reads_1586_unit.py index 5a09e97a1a..4b87bc6e29 100644 --- a/tests/corekit/test_io_seekable_reader_short_reads_1586_unit.py +++ b/tests/corekit/test_io_seekable_reader_short_reads_1586_unit.py @@ -24,6 +24,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises stream and buffer semantics, which differ by +#: interpreter. +__tier__ = 'ALL' + #: Octets the stream holds: more than a few buffers' worth. DATA = bytes(range(256)) * 4 diff --git a/tests/corekit/test_io_stream_proxies_1516_unit.py b/tests/corekit/test_io_stream_proxies_1516_unit.py index 73e622d941..ca8b904f38 100644 --- a/tests/corekit/test_io_stream_proxies_1516_unit.py +++ b/tests/corekit/test_io_stream_proxies_1516_unit.py @@ -22,6 +22,10 @@ from tests._support import reimport_once_per_class, sample_path +#: Test tier (#1958): depends on garbage collection, weak references or object +#: finalisation. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/corekit/test_ipv4interface_mask_unit.py b/tests/corekit/test_ipv4interface_mask_unit.py index 75ff9800dd..d663cd7ef1 100644 --- a/tests/corekit/test_ipv4interface_mask_unit.py +++ b/tests/corekit/test_ipv4interface_mask_unit.py @@ -26,6 +26,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + #: Address ``10.0.0.1`` followed by four mask octets. MASKS = { 'hostmask /24': '0a000001' '000000ff', diff --git a/tests/corekit/test_module.py b/tests/corekit/test_module.py index 6e03b691f3..57cafd499a 100644 --- a/tests/corekit/test_module.py +++ b/tests/corekit/test_module.py @@ -10,6 +10,10 @@ from tests._support import load_module, reimport_once_per_class +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + class ModuleDescriptorTests(unittest.TestCase): def setUp(self) -> None: diff --git a/tests/corekit/test_multidict.py b/tests/corekit/test_multidict.py index 605c32b53e..72065261ff 100644 --- a/tests/corekit/test_multidict.py +++ b/tests/corekit/test_multidict.py @@ -5,6 +5,10 @@ from tests._support import bootstrap_core_modules, reimport_once_per_class +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + class MultiDictTests(unittest.TestCase): def setUp(self) -> None: diff --git a/tests/corekit/test_multiinfo_1484_unit.py b/tests/corekit/test_multiinfo_1484_unit.py index ec8ff0ca01..736f775677 100644 --- a/tests/corekit/test_multiinfo_1484_unit.py +++ b/tests/corekit/test_multiinfo_1484_unit.py @@ -31,6 +31,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + PAIRS = [('a', 1), ('b', 2), ('a', 3)] diff --git a/tests/corekit/test_ordered_multidict_copy_unit.py b/tests/corekit/test_ordered_multidict_copy_unit.py index b411fd1a15..cfbf68970b 100644 --- a/tests/corekit/test_ordered_multidict_copy_unit.py +++ b/tests/corekit/test_ordered_multidict_copy_unit.py @@ -19,6 +19,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + #: Interleaved keys, so grouping by key changes the order. ITEMS = [('a', 1), ('b', 2), ('a', 3)] diff --git a/tests/corekit/test_sentinel_py_abc_1960_unit.py b/tests/corekit/test_sentinel_py_abc_1960_unit.py index 2732e2847f..5a10088226 100644 --- a/tests/corekit/test_sentinel_py_abc_1960_unit.py +++ b/tests/corekit/test_sentinel_py_abc_1960_unit.py @@ -33,6 +33,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises metaclass, ABC or enum machinery, which differs by +#: interpreter. +__tier__ = 'ALL' + #: Repository root, put first on the child interpreter's path so that it #: imports this tree rather than whatever :mod:`pcapkit` is installed. REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) diff --git a/tests/corekit/test_sentinel_singleton_1842_unit.py b/tests/corekit/test_sentinel_singleton_1842_unit.py index 2c8c81ca18..13d5dd5da4 100644 --- a/tests/corekit/test_sentinel_singleton_1842_unit.py +++ b/tests/corekit/test_sentinel_singleton_1842_unit.py @@ -41,6 +41,10 @@ if TYPE_CHECKING: from typing import Any, Callable +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + #: Where every sentinel is defined. CANONICAL_MODULE = 'pcapkit.corekit.sentinels' diff --git a/tests/corekit/test_sentinels_housing_unit.py b/tests/corekit/test_sentinels_housing_unit.py index 1a96d6f7bd..9e605b0ada 100644 --- a/tests/corekit/test_sentinels_housing_unit.py +++ b/tests/corekit/test_sentinels_housing_unit.py @@ -71,6 +71,10 @@ NoDefaultType, NoValueType, NullType) from tests._support import purge_modules +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + #: Every sentinel, as ``(instance name, consumer module, instance, type)``. The #: consumer module is the one that compares against the object, and so still #: imports it from :mod:`pcapkit.corekit.sentinels` -- without re-exporting it. diff --git a/tests/corekit/test_stringfield_lossless_unit.py b/tests/corekit/test_stringfield_lossless_unit.py index 188ea64300..46705b5ca9 100644 --- a/tests/corekit/test_stringfield_lossless_unit.py +++ b/tests/corekit/test_stringfield_lossless_unit.py @@ -31,6 +31,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + #: Charsets whose codecs reject a ``'surrogateescape'`` error handler, so that a #: fallback through one would raise where ``'replace'`` does not. NO_SURROGATE_CHARSETS = ('utf-16', 'utf-16-le', 'utf-16-be', 'utf-32', 'utf-7') diff --git a/tests/dumpkit/test_common_unit.py b/tests/dumpkit/test_common_unit.py index c2d19b6624..aeebb79064 100644 --- a/tests/dumpkit/test_common_unit.py +++ b/tests/dumpkit/test_common_unit.py @@ -16,6 +16,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/dumpkit/test_plist_date_unit.py b/tests/dumpkit/test_plist_date_unit.py index 1cd2233450..7883054e1a 100644 --- a/tests/dumpkit/test_plist_date_unit.py +++ b/tests/dumpkit/test_plist_date_unit.py @@ -24,6 +24,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises stream and buffer semantics, which differ by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/engines/test_engine_base.py b/tests/foundation/engines/test_engine_base.py index 4310e4539c..78e798ce82 100644 --- a/tests/foundation/engines/test_engine_base.py +++ b/tests/foundation/engines/test_engine_base.py @@ -8,6 +8,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): branches on the running interpreter's version or implementation. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/engines/test_pcap_ct_engine.py b/tests/foundation/engines/test_pcap_ct_engine.py index f3effa2f2e..0572306a91 100644 --- a/tests/foundation/engines/test_pcap_ct_engine.py +++ b/tests/foundation/engines/test_pcap_ct_engine.py @@ -28,6 +28,10 @@ from tests._support import close_extractor, reimport_once_per_class, sample_path +#: Test tier (#1958): drives a third-party engine whose installable releases vary by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/engines/test_pypcap_engine.py b/tests/foundation/engines/test_pypcap_engine.py index a1bc946637..f1fbb9b146 100644 --- a/tests/foundation/engines/test_pypcap_engine.py +++ b/tests/foundation/engines/test_pypcap_engine.py @@ -20,6 +20,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): drives a third-party engine whose installable releases vary by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/engines/test_pypcapfile_engine.py b/tests/foundation/engines/test_pypcapfile_engine.py index 1f36588e0d..941e131911 100644 --- a/tests/foundation/engines/test_pypcapfile_engine.py +++ b/tests/foundation/engines/test_pypcapfile_engine.py @@ -28,6 +28,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): drives a third-party engine whose installable releases vary by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/engines/test_pyshark_engine.py b/tests/foundation/engines/test_pyshark_engine.py index f338e235c2..e15a5df368 100644 --- a/tests/foundation/engines/test_pyshark_engine.py +++ b/tests/foundation/engines/test_pyshark_engine.py @@ -48,6 +48,10 @@ import unittest from unittest import mock +#: Test tier (#1958): drives a third-party engine whose installable releases vary by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/reassembly/test_ip.py b/tests/foundation/reassembly/test_ip.py index 7bd1155d84..d4de9dfe6a 100644 --- a/tests/foundation/reassembly/test_ip.py +++ b/tests/foundation/reassembly/test_ip.py @@ -6,6 +6,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/reassembly/test_ip_max_length_edge_1739_unit.py b/tests/foundation/reassembly/test_ip_max_length_edge_1739_unit.py index 6d7fd6b129..fcf0e6bbd5 100644 --- a/tests/foundation/reassembly/test_ip_max_length_edge_1739_unit.py +++ b/tests/foundation/reassembly/test_ip_max_length_edge_1739_unit.py @@ -19,6 +19,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + #: Datagram octets, with no zero octet in them, so a hole shows. DATA = (bytes(range(1, 256)) * 258)[:65536] diff --git a/tests/foundation/reassembly/test_reassembly_base.py b/tests/foundation/reassembly/test_reassembly_base.py index 5f96e9cd92..4d63c2197b 100644 --- a/tests/foundation/reassembly/test_reassembly_base.py +++ b/tests/foundation/reassembly/test_reassembly_base.py @@ -8,6 +8,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): branches on the running interpreter's version or implementation. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/reassembly/test_tcp.py b/tests/foundation/reassembly/test_tcp.py index c93889cbc8..3468253eff 100644 --- a/tests/foundation/reassembly/test_tcp.py +++ b/tests/foundation/reassembly/test_tcp.py @@ -7,6 +7,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/registry/test_protocols.py b/tests/foundation/registry/test_protocols.py index d0d91f6d52..b96a09744b 100644 --- a/tests/foundation/registry/test_protocols.py +++ b/tests/foundation/registry/test_protocols.py @@ -8,6 +8,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises metaclass, ABC or enum machinery, which differs by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/test_extraction.py b/tests/foundation/test_extraction.py index 9e001284de..7da9238b73 100644 --- a/tests/foundation/test_extraction.py +++ b/tests/foundation/test_extraction.py @@ -15,6 +15,10 @@ from tests._support import reimport_once_per_class, sample_path +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) #: Whether the optional DPKT engine can be selected. It is an extra diff --git a/tests/foundation/test_extraction_no_eof.py b/tests/foundation/test_extraction_no_eof.py index 6b66bbd08f..c5444f2cd7 100644 --- a/tests/foundation/test_extraction_no_eof.py +++ b/tests/foundation/test_extraction_no_eof.py @@ -108,6 +108,10 @@ from tests._support import sample_path, scale_timeout, time_limit from tests._tiers import ROOT +#: Test tier (#1958): exercises stream and buffer semantics, which differ by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/test_extraction_non_seekable_stream_1586_unit.py b/tests/foundation/test_extraction_non_seekable_stream_1586_unit.py index 2581e81a80..41176f46c5 100644 --- a/tests/foundation/test_extraction_non_seekable_stream_1586_unit.py +++ b/tests/foundation/test_extraction_non_seekable_stream_1586_unit.py @@ -45,6 +45,10 @@ from tests._support import reimport_once_per_class, scale_timeout from tests._tiers import ROOT +#: Test tier (#1958): exercises stream and buffer semantics, which differ by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) HAS_DPKT = importlib.util.find_spec('dpkt') is not None diff --git a/tests/foundation/test_extraction_ownership.py b/tests/foundation/test_extraction_ownership.py index 9a0d0ca77d..e827559e88 100644 --- a/tests/foundation/test_extraction_ownership.py +++ b/tests/foundation/test_extraction_ownership.py @@ -56,6 +56,10 @@ from tests._support import sample_path +#: Test tier (#1958): depends on garbage collection, weak references or object +#: finalisation. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/traceflow/test_tcp.py b/tests/foundation/traceflow/test_tcp.py index 1f9448222e..6d07b76d82 100644 --- a/tests/foundation/traceflow/test_tcp.py +++ b/tests/foundation/traceflow/test_tcp.py @@ -8,6 +8,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/traceflow/test_traceflow_base.py b/tests/foundation/traceflow/test_traceflow_base.py index cd114f78b0..05c03cea7b 100644 --- a/tests/foundation/traceflow/test_traceflow_base.py +++ b/tests/foundation/traceflow/test_traceflow_base.py @@ -9,6 +9,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): branches on the running interpreter's version or implementation. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/integration/test_engine_runtime.py b/tests/integration/test_engine_runtime.py index e836cddbaa..e0ae06bee1 100644 --- a/tests/integration/test_engine_runtime.py +++ b/tests/integration/test_engine_runtime.py @@ -8,6 +8,10 @@ from tests._support import close_extractor, reimport_once_per_class, sample_path +#: Test tier (#1958): drives a third-party engine whose installable releases vary by +#: interpreter. +__tier__ = 'ALL' + HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) HAS_DPKT = importlib.util.find_spec('dpkt') is not None HAS_SCAPY = importlib.util.find_spec('scapy') is not None diff --git a/tests/integration/test_frame_iteration.py b/tests/integration/test_frame_iteration.py index 325ed764ac..41e06a8876 100644 --- a/tests/integration/test_frame_iteration.py +++ b/tests/integration/test_frame_iteration.py @@ -24,6 +24,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + #: Frames of :file:`http6.cap` that carry an HTTP header block: the request and #: the response of each of the two connections. HTTP_FRAMES = (4, 6, 19, 21) diff --git a/tests/interface/test_core.py b/tests/interface/test_core.py index 2c7f39bece..2c01891541 100644 --- a/tests/interface/test_core.py +++ b/tests/interface/test_core.py @@ -10,6 +10,10 @@ from tests._support import isolate_modules, load_module, reimport_once_per_class, sample_path +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) #: Whether the optional DPKT / Scapy engines can be selected. Both are extras diff --git a/tests/project/test_curated_all_tier_1958.py b/tests/project/test_curated_all_tier_1958.py new file mode 100644 index 0000000000..8fc5ede16b --- /dev/null +++ b/tests/project/test_curated_all_tier_1958.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- +"""#1958, #1963: the ``ALL`` test tier is a curated, opt-in list. + +The owner's rulings, paraphrased: ``ALL`` is a tailored list of tests, to save +time, to keep QA consistent across legs, and to be enough to validate a change on +each of them; every implementation runs the same minimal ``ALL`` tier while +``latest`` runs the whole suite as the gate; and the hand-picked round trips run +on ``ALL`` as part of QA, with only the real sweeps on ``latest``. +:func:`tests._tiers.module_tier` is the one source of truth, and +:file:`tests/test_tier_guard.py` checks that every module lands in exactly one +tier. This module pins the curation itself: a module is in ``ALL`` only when it +says so, ``ALL`` stays a strict subset of the suite, and it keeps the ruling-B +round trips. + +""" + +from __future__ import annotations + +import pathlib +import tempfile +import unittest +import unittest.mock + +from tests import _tiers + +ROOT = pathlib.Path(__file__).resolve().parents[2] +DOC = ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst' +#: The #1202 edge-case tables, each case written out by hand (ruling B). +EDGE_TABLES = tuple(sorted(path.relative_to(ROOT).as_posix() + for path in (ROOT / 'tests' / 'protocols').glob('*/test_*_edge_roundtrip_unit.py'))) +#: The phrase every hand-picked round trip's marker reason carries. +HAND_PICKED = 'hand-picked' + + +def tier_of(source: 'str') -> 'str': + """The tier of a module holding ``source``, under a temporary :file:`tests/`.""" + with tempfile.TemporaryDirectory(prefix='pcapkit-curated-tier-') as temp: + root = pathlib.Path(temp).resolve() + path = root / 'test_probe.py' + path.write_text(source, encoding='utf-8') + with unittest.mock.patch.object(_tiers, 'TESTS_ROOT', root): + return _tiers.module_tier(path) + + +def reason(module: 'str') -> 'str': + """The ``#:`` comment above ``module``'s ``__tier__`` marker, joined into one line.""" + lines = (ROOT / module).read_text(encoding='utf-8').splitlines() + index = next(number for number, line in enumerate(lines) if line.startswith(_tiers.TIER_MARKER)) + comment = [] # type: list[str] + for line in reversed(lines[:index]): + if not line.startswith('#: '): + break + comment.insert(0, line[3:]) + return ' '.join(comment) + + +class TestAllIsOptIn(unittest.TestCase): + """Only ``__tier__ = 'ALL'`` puts a module in ``ALL``.""" + + def test_a_module_without_a_marker_is_latest(self) -> None: + for label, source in { + 'imports pcapkit': 'import pcapkit\n', + 'imports a pcapkit submodule': 'from pcapkit.protocols.link import Ethernet\n', + 'reads text': 'TEXT = "pcapkit"\n', + }.items(): + with self.subTest(case=label): + self.assertEqual(tier_of(source), _tiers.TIER_LATEST) + + def test_the_marker_opts_in(self) -> None: + self.assertEqual(tier_of("import pcapkit\n__tier__ = 'ALL'\n"), _tiers.TIER_ALL) + + def test_every_all_module_states_it(self) -> None: + for module in _tiers.suite_modules(): + if _tiers.module_tier(module) == _tiers.TIER_ALL: + with self.subTest(module=module): + self.assertEqual(_tiers.declared_tier(module), _tiers.TIER_ALL) + + +class TestAllIsAStrictSubset(unittest.TestCase): + """``ALL`` is a curated part of the suite, never the suite, and keeps the ruling-B round trips.""" + + def test_it_is_a_strict_subset_smaller_than_latest(self) -> None: + modules = set(_tiers.suite_modules()) + selected = {module for module in modules if _tiers.module_tier(module) == _tiers.TIER_ALL} + self.assertTrue(selected) + self.assertLess(selected, modules) + self.assertLess(len(selected), len(modules - selected)) + + def test_it_keeps_the_edge_case_tables(self) -> None: + self.assertTrue(EDGE_TABLES) + for module in EDGE_TABLES: + with self.subTest(module=module): + self.assertEqual(_tiers.module_tier(module), _tiers.TIER_ALL) + + def test_it_keeps_every_hand_picked_round_trip(self) -> None: + picked = [module for module in _tiers.suite_modules() + if _tiers.declared_tier(module) is not None and HAND_PICKED in reason(module)] + self.assertTrue(set(EDGE_TABLES) <= set(picked)) + for module in picked: + with self.subTest(module=module): + self.assertEqual(_tiers.module_tier(module), _tiers.TIER_ALL) + + def test_the_sweeps_stay_on_latest(self) -> None: + for module in ('tests/protocols/test_option_roundtrip_unit.py', + 'tests/protocols/test_short_read_roundtrip_unit.py', + 'tests/protocols/test_capture_roundtrip_runtime.py'): + with self.subTest(module=module): + self.assertEqual(_tiers.module_tier(module), _tiers.TIER_LATEST) + + +class TestTheDocsSayIt(unittest.TestCase): + + def test_the_workflows_page_states_the_criterion(self) -> None: + page = ' '.join(DOC.read_text(encoding='utf-8').split()) + section = page[page.index('Test Tiers ----------'):] + for phrase in ("``__tier__ = 'ALL'``", 'opt-in', '#1963', 'smoke slice', 'hand-picked round trips', + 'A module that states no tier is ``LATEST``'): + with self.subTest(phrase=phrase): + self.assertIn(phrase, section) + + def test_the_tier_module_states_the_criterion(self) -> None: + text = ' '.join(_tiers.__doc__.split()) + for phrase in ('The curated ``ALL`` tier', 'opt-in', 'smoke slice', '``*_edge_roundtrip_unit``'): + with self.subTest(phrase=phrase): + self.assertIn(phrase, text) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_public_api.py b/tests/project/test_public_api.py index f9021971b7..ebc388f15f 100644 --- a/tests/project/test_public_api.py +++ b/tests/project/test_public_api.py @@ -144,6 +144,10 @@ class exists at ``pcapkit/const/mh/handover_initiate_status.py`` and is used by import types import unittest +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + #: First path component under ``pcapkit.`` whose subtree the sweep skips. Empty: #: it held ``'vendor'`` until #1772, though ``requests`` and ``beautifulsoup4`` #: were in the ``test`` extra before the exclusion was written, and diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py index d9be81d7b5..515005a102 100644 --- a/tests/project/test_sweeps_on_latest_only_1732.py +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -317,10 +317,20 @@ def flags() -> 'list[str]': f'--junitxml={RUNNER_TEMP}/junit.xml'] def test_the_all_selection_leaves_out_the_latest_modules(self) -> None: - latest = [module for module in _tiers.latest_tier_paths() if not _tiers.is_unit_tier(module)] - self.assertTrue(latest) - modules = [{entry.split('::', 1)[0] for entry in entries} for entries in (self.selection, self.all)] - self.assertEqual(sorted(modules[0] - modules[1]), latest) + """What the ``ALL`` part drops is exactly the ``LATEST`` modules the whole selection reaches.""" + def reached(entries: 'list[str]') -> 'set[str]': + modules = set() # type: set[str] + for entry in entries: + module = entry.split('::', 1)[0] + if (_tiers.ROOT / module).is_dir(): + modules.update(path for path in _tiers.suite_modules() if path.startswith(module.rstrip('/') + '/')) + else: + modules.add(module) + return modules + everything, kept = reached(self.selection), reached(self.all) + dropped = everything - kept + self.assertTrue(dropped) + self.assertEqual(dropped, everything & set(_tiers.latest_tier_paths())) def test_only_latest_runs_them(self) -> None: """``latest`` passes the whole selection; every other leg its ``ALL`` part.""" diff --git a/tests/protocols/application/test_http_unit.py b/tests/protocols/application/test_http_unit.py index 370baf6325..fb39439ba2 100644 --- a/tests/protocols/application/test_http_unit.py +++ b/tests/protocols/application/test_http_unit.py @@ -8,6 +8,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/internet/test_ipv4_unit.py b/tests/protocols/internet/test_ipv4_unit.py index a800ddac38..ea08cfe1a6 100644 --- a/tests/protocols/internet/test_ipv4_unit.py +++ b/tests/protocols/internet/test_ipv4_unit.py @@ -11,6 +11,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/internet/test_ipv6_ext_chain_limit_1604_unit.py b/tests/protocols/internet/test_ipv6_ext_chain_limit_1604_unit.py index 0b900dfd00..7afb473cfa 100644 --- a/tests/protocols/internet/test_ipv6_ext_chain_limit_1604_unit.py +++ b/tests/protocols/internet/test_ipv6_ext_chain_limit_1604_unit.py @@ -27,6 +27,10 @@ from tests._support import reimport_once_per_class from tests.foundation import _roundtrip as wire +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + #: The TCP segment every chain ends in. SEGMENT = wire.tcp(b'payload', seq=1) diff --git a/tests/protocols/internet/test_ipv6_jumbogram_unit.py b/tests/protocols/internet/test_ipv6_jumbogram_unit.py index ec4e3c043a..31d512be8f 100644 --- a/tests/protocols/internet/test_ipv6_jumbogram_unit.py +++ b/tests/protocols/internet/test_ipv6_jumbogram_unit.py @@ -32,6 +32,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + #: The two IPv6 addresses, ``::1`` and ``::2``. ADDRS = bytes(15) + b'\x01' + bytes(15) + b'\x02' #: A TCP header of 20 octets, no options. diff --git a/tests/protocols/internet/test_ipv6_unit.py b/tests/protocols/internet/test_ipv6_unit.py index f227120cbb..eaa96fca89 100644 --- a/tests/protocols/internet/test_ipv6_unit.py +++ b/tests/protocols/internet/test_ipv6_unit.py @@ -7,6 +7,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/link/test_link_unit.py b/tests/protocols/link/test_link_unit.py index 3a7bb32d97..7527a9c147 100644 --- a/tests/protocols/link/test_link_unit.py +++ b/tests/protocols/link/test_link_unit.py @@ -8,6 +8,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/misc/pcap/test_header_frame_unit.py b/tests/protocols/misc/pcap/test_header_frame_unit.py index 5fa536862e..b733134b7f 100644 --- a/tests/protocols/misc/pcap/test_header_frame_unit.py +++ b/tests/protocols/misc/pcap/test_header_frame_unit.py @@ -11,6 +11,10 @@ from tests._support import reimport_once_per_class, sample_path +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/misc/test_pcapng_tzone_tsoffset_range_1340_unit.py b/tests/protocols/misc/test_pcapng_tzone_tsoffset_range_1340_unit.py index 684ffe5ebc..a52e869760 100644 --- a/tests/protocols/misc/test_pcapng_tzone_tsoffset_range_1340_unit.py +++ b/tests/protocols/misc/test_pcapng_tzone_tsoffset_range_1340_unit.py @@ -32,6 +32,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + #: ``datetime`` holds years 1 to 9999, so these are its first and last whole #: seconds since the UNIX epoch. DT_MIN = -62135596800 diff --git a/tests/protocols/misc/test_pcapng_unit.py b/tests/protocols/misc/test_pcapng_unit.py index a5a5243026..9bc08b2280 100644 --- a/tests/protocols/misc/test_pcapng_unit.py +++ b/tests/protocols/misc/test_pcapng_unit.py @@ -23,6 +23,10 @@ from tests._support import ROOT, reimport_once_per_class, sample_path, skip_if_stand_in +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/schema/test_enum_schema_registry_unit.py b/tests/protocols/schema/test_enum_schema_registry_unit.py index d5905229fb..e8e943fe30 100644 --- a/tests/protocols/schema/test_enum_schema_registry_unit.py +++ b/tests/protocols/schema/test_enum_schema_registry_unit.py @@ -23,6 +23,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises metaclass, ABC or enum machinery, which differs by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py b/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py index 9f20c72c2a..fc32f7b016 100644 --- a/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py +++ b/tests/protocols/schema/test_schema_metaclass_abc_cache_unit.py @@ -7,6 +7,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises metaclass, ABC or enum machinery, which differs by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/schema/test_schema_unit.py b/tests/protocols/schema/test_schema_unit.py index 8863c599e9..388338f0e1 100644 --- a/tests/protocols/schema/test_schema_unit.py +++ b/tests/protocols/schema/test_schema_unit.py @@ -10,6 +10,10 @@ from tests._support import reimport_once_per_class, time_limit +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/test_construction_keyword_check_unit.py b/tests/protocols/test_construction_keyword_check_unit.py index b4ef5e8b76..c85367a90c 100644 --- a/tests/protocols/test_construction_keyword_check_unit.py +++ b/tests/protocols/test_construction_keyword_check_unit.py @@ -82,6 +82,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): exercises metaclass, ABC or enum machinery, which differs by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/test_frame_layer_limit_1610_unit.py b/tests/protocols/test_frame_layer_limit_1610_unit.py index a1fea376a8..3263f57307 100644 --- a/tests/protocols/test_frame_layer_limit_1610_unit.py +++ b/tests/protocols/test_frame_layer_limit_1610_unit.py @@ -36,6 +36,10 @@ if TYPE_CHECKING: from typing import Optional +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + #: Little-endian PCAP global header: v2.4, snaplen 65535, Ethernet. GLOBAL = bytes.fromhex('d4c3b2a1' '0200' '0400' '00000000' '00000000' 'ffff0000' '01000000') diff --git a/tests/protocols/test_option_generator_deadline_scale_1428_unit.py b/tests/protocols/test_option_generator_deadline_scale_1428_unit.py index 454a626a15..8cf5e7235d 100644 --- a/tests/protocols/test_option_generator_deadline_scale_1428_unit.py +++ b/tests/protocols/test_option_generator_deadline_scale_1428_unit.py @@ -47,6 +47,9 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): branches on the running interpreter's version or implementation. +__tier__ = 'ALL' + #: The variable :mod:`tests._support` reads, spelled out so a generator reading a #: different name fails here rather than agreeing with itself. SCALE_ENV = 'PCAPKIT_TEST_TIMEOUT_SCALE' diff --git a/tests/protocols/test_protocol_base_unit.py b/tests/protocols/test_protocol_base_unit.py index e6126dee96..5db35cd3c1 100644 --- a/tests/protocols/test_protocol_base_unit.py +++ b/tests/protocols/test_protocol_base_unit.py @@ -10,6 +10,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/test_protocol_code_registration_unit.py b/tests/protocols/test_protocol_code_registration_unit.py index 1b3530a283..23bb89836c 100644 --- a/tests/protocols/test_protocol_code_registration_unit.py +++ b/tests/protocols/test_protocol_code_registration_unit.py @@ -35,6 +35,9 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): branches on the running interpreter's version or implementation. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/test_protocol_make_unit.py b/tests/protocols/test_protocol_make_unit.py index c60fb0fe33..9ac649e4e1 100644 --- a/tests/protocols/test_protocol_make_unit.py +++ b/tests/protocols/test_protocol_make_unit.py @@ -33,6 +33,9 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): branches on the running interpreter's version or implementation. +__tier__ = 'ALL' + def _attr(path: 'str') -> 'Any': module, _, name = path.rpartition('.') diff --git a/tests/protocols/transport/test_tcp_udp_unit.py b/tests/protocols/transport/test_tcp_udp_unit.py index a32ab1a089..b53851ba67 100644 --- a/tests/protocols/transport/test_tcp_udp_unit.py +++ b/tests/protocols/transport/test_tcp_udp_unit.py @@ -14,6 +14,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/test_base_class_contract.py b/tests/test_base_class_contract.py index ef8d5cddb2..934fe03c37 100644 --- a/tests/test_base_class_contract.py +++ b/tests/test_base_class_contract.py @@ -113,6 +113,10 @@ if TYPE_CHECKING: from typing import Any, Iterator +#: Test tier (#1958): exercises metaclass, ABC or enum machinery, which differs by +#: interpreter. +__tier__ = 'ALL' + #: Every ``*Base`` name mapped to the public name it must never be aliased to. #: Keyed on the base rather than the public name because the base is what an #: import statement names; the alias is the thing being outlawed. diff --git a/tests/test_final_enforcement.py b/tests/test_final_enforcement.py index c59db390dc..ff48fb06fb 100644 --- a/tests/test_final_enforcement.py +++ b/tests/test_final_enforcement.py @@ -82,6 +82,9 @@ import unittest import warnings +#: Test tier (#1958): branches on the running interpreter's version or implementation. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/test_periodic_gc_1964_unit.py b/tests/test_periodic_gc_1964_unit.py index 762437d9f7..e7235f2c9c 100644 --- a/tests/test_periodic_gc_1964_unit.py +++ b/tests/test_periodic_gc_1964_unit.py @@ -16,6 +16,10 @@ from tests import conftest +#: Test tier (#1958): depends on garbage collection, weak references or object +#: finalisation. +__tier__ = 'ALL' + class TestPeriodicCollection(unittest.TestCase): diff --git a/tests/test_support_helpers.py b/tests/test_support_helpers.py index cbea1f5f21..a58f9a6cfb 100644 --- a/tests/test_support_helpers.py +++ b/tests/test_support_helpers.py @@ -81,6 +81,9 @@ if TYPE_CHECKING: from typing import Iterator +#: Test tier (#1958): branches on the running interpreter's version or implementation. +__tier__ = 'ALL' + #: Seconds a :class:`TimeLimitTests` body spins waiting for the deadline before it #: gives up and fails by name: ten times the one-second deadline those tests arm, #: so a deadline that fires at all, its repeat included, lands well inside it. diff --git a/tests/test_tier_guard.py b/tests/test_tier_guard.py index 3501b08457..a533c4663d 100644 --- a/tests/test_tier_guard.py +++ b/tests/test_tier_guard.py @@ -520,14 +520,20 @@ def test_every_module_resolves_to_exactly_one_tier(self) -> None: tiers.setdefault(tier, []).append(module) self.assertEqual(set(tiers), set(_tiers.TIERS)) self.assertEqual(list(_tiers.latest_tier_paths()), tiers[_tiers.TIER_LATEST]) + self.assertEqual(list(_tiers.all_tier_paths()), tiers[_tiers.TIER_ALL]) self.assertEqual(sorted(tiers[_tiers.TIER_ALL] + tiers[_tiers.TIER_LATEST]), list(modules)) + for module in tiers[_tiers.TIER_ALL]: + with self.subTest(module=module): + self.assertEqual(_tiers.declared_tier(module), _tiers.TIER_ALL) found = sorted(path.relative_to(_tiers.ROOT).as_posix() for path in _tiers.TESTS_ROOT.rglob('test_*.py')) self.assertEqual(list(modules), found) - def test_a_marker_wins_over_the_rule(self) -> None: + def test_all_is_opt_in(self) -> None: + """#1963: ``ALL`` is a curated list, so only a marker puts a module in it.""" cases = { - 'the rule, exercising pcapkit': ('import pcapkit\n', _tiers.TIER_ALL), - 'the rule, reading text': ('TEXT = "pcapkit"\n', _tiers.TIER_LATEST), + 'no marker, exercising pcapkit': ('import pcapkit\n', _tiers.TIER_LATEST), + 'no marker, reading text': ('TEXT = "pcapkit"\n', _tiers.TIER_LATEST), + 'opted in, exercising pcapkit': ("import pcapkit\n__tier__ = 'ALL'\n", _tiers.TIER_ALL), 'a sweep, marked': ("import pcapkit\n__tier__ = 'LATEST'\n", _tiers.TIER_LATEST), 'an interpreter-bound guard, marked': ("import ast\n__tier__ = 'ALL'\n", _tiers.TIER_ALL), 'annotated': ("import pcapkit\n__tier__: str = 'LATEST'\n", _tiers.TIER_LATEST), @@ -554,6 +560,27 @@ def test_an_invalid_marker_fails(self) -> None: self.assertIn('test_probe.py', str(caught.exception)) self.assertIn(_tiers.TIER_MARKER, str(caught.exception)) + def test_a_module_that_branches_on_the_interpreter_states_its_tier(self) -> None: + """A version gate is never ``LATEST`` by omission: such a module says which tier it is in.""" + cases = { + 'a version gate': ('import sys\nNEW = sys.version_info >= (3, 12)\n', True), + 'an implementation gate': ('import sys\nGRAAL = sys.implementation.name == "graalpy"\n', True), + 'a bare import': ('from sys import version_info\nOLD = version_info < (3, 9)\n', True), + 'pcapkit alone': ('import pcapkit\n', False), + 'a version only read': ('import sys\nprint(sys.version_info)\n', False), + } + for label, (source, expected) in cases.items(): + with self.subTest(case=label), tempfile.TemporaryDirectory(prefix='pcapkit-test-tier-') as temp: + path = write_module(pathlib.Path(temp), 'test_probe.py', source) + with unittest.mock.patch.object(_tiers, 'TESTS_ROOT', pathlib.Path(temp).resolve()): + self.assertIs(_tiers.branches_on_interpreter(path), expected) + for module in _tiers.suite_modules(): + if _tiers.branches_on_interpreter(module): + with self.subTest(module=module): + self.assertIsNotNone(_tiers.declared_tier(module), + f'{module} branches on the interpreter; state its tier with ' + f"{_tiers.TIER_MARKER} = {_tiers.TIER_ALL!r} or {_tiers.TIER_LATEST!r}") + def test_only_a_test_module_has_a_tier(self) -> None: for path in ('tests/_support.py', 'tests/_tiers.py', 'util/ci_timing.py', 'tests/project', 'tests/project/test_no_such_module.py'): @@ -562,7 +589,7 @@ def test_only_a_test_module_has_a_tier(self) -> None: _tiers.module_tier(path) def test_every_marker_states_a_reason(self) -> None: - """A ``__tier__`` the rule would also give says why it is there, as one that overrules it does.""" + """Every ``__tier__`` says why it is there, under a ``#: Test tier (#1958):`` comment.""" for module in _tiers.suite_modules(): if _tiers.declared_tier(module) is None: continue @@ -620,13 +647,17 @@ def test_a_directory_holding_a_latest_module_is_split(self) -> None: """pytest ignores no path it was handed, so the ``ALL`` selection names a mixed directory's modules.""" integration = f'tests/{next(iter(_tiers.FIXTURE_TIER_DIRS))}' members = [path for path in _tiers.suite_modules() if path.startswith(integration + '/')] - self.assertIn(integration, _tiers.fixture_tier_paths(_tiers.TIER_ALL)) - marked = members[0] _tiers.fixture_tier_paths.cache_clear() self.addCleanup(_tiers.fixture_tier_paths.cache_clear) real = _tiers.module_tier with unittest.mock.patch.object(_tiers, 'module_tier', - lambda path: _tiers.TIER_LATEST if path == marked else real(path)): + lambda path: _tiers.TIER_ALL if path in members else real(path)): + self.assertIn(integration, _tiers.fixture_tier_paths(_tiers.TIER_ALL)) + _tiers.fixture_tier_paths.cache_clear() + marked = members[0] + with unittest.mock.patch.object(_tiers, 'module_tier', + lambda path: _tiers.TIER_LATEST if path == marked + else _tiers.TIER_ALL if path in members else real(path)): selected = _tiers.fixture_tier_paths(_tiers.TIER_ALL) self.assertNotIn(integration, selected) self.assertNotIn(marked, selected) diff --git a/tests/toolkit/test_dpkt_timestamp_copy_1647_unit.py b/tests/toolkit/test_dpkt_timestamp_copy_1647_unit.py index 7de7a3f450..6f01fda56a 100644 --- a/tests/toolkit/test_dpkt_timestamp_copy_1647_unit.py +++ b/tests/toolkit/test_dpkt_timestamp_copy_1647_unit.py @@ -36,6 +36,10 @@ from pcapkit.toolkit.dpkt import RecordTimestamp +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) #: Rerun :func:`clones` over :func:`stamps` with :mod:`_decimal` unimportable, as diff --git a/tests/toolkit/test_dpkt_unit.py b/tests/toolkit/test_dpkt_unit.py index a6c1d335c1..63deb45bfd 100644 --- a/tests/toolkit/test_dpkt_unit.py +++ b/tests/toolkit/test_dpkt_unit.py @@ -12,6 +12,10 @@ from tests._support import close_extractor, reimport_once_per_class, sample_path +#: Test tier (#1958): the representative smoke slice for its package's main parse and +#: build paths. +__tier__ = 'ALL' + HAS_DPKT = importlib.util.find_spec('dpkt') is not None RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/toolkit/test_engine_pcap_trace_runtime.py b/tests/toolkit/test_engine_pcap_trace_runtime.py index 2f921f5f2e..a35a1816dd 100644 --- a/tests/toolkit/test_engine_pcap_trace_runtime.py +++ b/tests/toolkit/test_engine_pcap_trace_runtime.py @@ -38,6 +38,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): drives a third-party engine whose installable releases vary by +#: interpreter. +__tier__ = 'ALL' + HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) HAS_DPKT = importlib.util.find_spec('dpkt') is not None diff --git a/tests/toolkit/test_pcap_ct_unit.py b/tests/toolkit/test_pcap_ct_unit.py index cc8226e569..62e429da21 100644 --- a/tests/toolkit/test_pcap_ct_unit.py +++ b/tests/toolkit/test_pcap_ct_unit.py @@ -20,6 +20,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): drives a third-party engine whose installable releases vary by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/toolkit/test_pypcap_unit.py b/tests/toolkit/test_pypcap_unit.py index 67b953e7ef..040d7c24bc 100644 --- a/tests/toolkit/test_pypcap_unit.py +++ b/tests/toolkit/test_pypcap_unit.py @@ -15,6 +15,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): drives a third-party engine whose installable releases vary by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/toolkit/test_pypcapfile_unit.py b/tests/toolkit/test_pypcapfile_unit.py index ec0021c000..9565aaf411 100644 --- a/tests/toolkit/test_pypcapfile_unit.py +++ b/tests/toolkit/test_pypcapfile_unit.py @@ -24,6 +24,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): drives a third-party engine whose installable releases vary by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/utilities/test_compat.py b/tests/utilities/test_compat.py index 55e505b935..c6bae26df8 100644 --- a/tests/utilities/test_compat.py +++ b/tests/utilities/test_compat.py @@ -67,6 +67,9 @@ from tests._support import load_module, purge_modules, reimport_once_per_class from tests._tiers import ROOT +#: Test tier (#1958): branches on the running interpreter's version or implementation. +__tier__ = 'ALL' + # NOTE: :mod:`aenum` is deliberately *not* imported at module scope, however # convenient that would be for the assertions below. A module-level import would # warm the cache issue #687 is about as a side effect of this file being diff --git a/tests/utilities/test_config_unit.py b/tests/utilities/test_config_unit.py index 7f47b7ad83..78d0b833ea 100644 --- a/tests/utilities/test_config_unit.py +++ b/tests/utilities/test_config_unit.py @@ -34,6 +34,10 @@ if TYPE_CHECKING: from typing import Any, Iterator +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + #: Root of the :mod:`pcapkit` package under test. PACKAGE = pathlib.Path(__file__).resolve().parents[2] / 'pcapkit' diff --git a/tests/utilities/test_exceptions_excepthook.py b/tests/utilities/test_exceptions_excepthook.py index 0c1ee6c292..d6eb25b599 100644 --- a/tests/utilities/test_exceptions_excepthook.py +++ b/tests/utilities/test_exceptions_excepthook.py @@ -47,6 +47,10 @@ from tests._support import purge_modules from tests.utilities._harness import bootstrap, capture +#: Test tier (#1958): exercises threading, locking or warning and exception hooks, which +#: differ by interpreter. +__tier__ = 'ALL' + #: Repository root -- three directories up from this file #: (``tests/utilities/test_exceptions_excepthook.py``). Passed to the child #: interpreter as ``PYTHONPATH`` explicitly, rather than relied on via the diff --git a/tests/utilities/test_logging.py b/tests/utilities/test_logging.py index e63be66845..9b2005fedd 100644 --- a/tests/utilities/test_logging.py +++ b/tests/utilities/test_logging.py @@ -9,6 +9,10 @@ from tests._support import load_module, reimport_once_per_class +#: Test tier (#1958): exercises threading, locking or warning and exception hooks, which +#: differ by interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/utilities/test_quiet_exceptions.py b/tests/utilities/test_quiet_exceptions.py index 4ed1a19490..f655aeec26 100644 --- a/tests/utilities/test_quiet_exceptions.py +++ b/tests/utilities/test_quiet_exceptions.py @@ -26,6 +26,10 @@ from tests._support import purge_modules from tests.utilities._harness import bootstrap, capture +#: Test tier (#1958): exercises threading, locking or warning and exception hooks, which +#: differ by interpreter. +__tier__ = 'ALL' + def _unrelated_failure() -> 'int': """Format an unrelated exception and count the traceback's lines. diff --git a/tests/utilities/test_stacklevel.py b/tests/utilities/test_stacklevel.py index d7fca2a91e..8ded5a628b 100644 --- a/tests/utilities/test_stacklevel.py +++ b/tests/utilities/test_stacklevel.py @@ -58,6 +58,10 @@ _T = TypeVar('_T') +#: Test tier (#1958): exercises threading, locking or warning and exception hooks, which +#: differ by interpreter. +__tier__ = 'ALL' + #: The path fragment :func:`~pcapkit.utilities.exceptions.stacklevel` treats as #: "inside :mod:`pcapkit`". MARKER = f'{os.path.sep}pcapkit{os.path.sep}' diff --git a/tests/vendor/test_ftp_return_code_unit.py b/tests/vendor/test_ftp_return_code_unit.py index 6445018c7f..86278e4202 100644 --- a/tests/vendor/test_ftp_return_code_unit.py +++ b/tests/vendor/test_ftp_return_code_unit.py @@ -44,10 +44,6 @@ if TYPE_CHECKING: from typing import Any -#: Test tier (#1958): exercises pcapkit.vendor, which users run on any supported -#: interpreter. -__tier__ = 'ALL' - #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_re_sub_positional_flag_unit.py b/tests/vendor/test_re_sub_positional_flag_unit.py index 380ad26cb3..1866757c1c 100644 --- a/tests/vendor/test_re_sub_positional_flag_unit.py +++ b/tests/vendor/test_re_sub_positional_flag_unit.py @@ -86,6 +86,9 @@ import unittest import warnings +#: Test tier (#1958): branches on the running interpreter's version or implementation. +__tier__ = 'ALL' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_request_prompt_unit.py b/tests/vendor/test_request_prompt_unit.py index 039ba91ec9..ac9e42eae9 100644 --- a/tests/vendor/test_request_prompt_unit.py +++ b/tests/vendor/test_request_prompt_unit.py @@ -88,10 +88,6 @@ if TYPE_CHECKING: from typing import Any, Iterator -#: Test tier (#1958): exercises pcapkit.vendor, which users run on any supported -#: interpreter. -__tier__ = 'ALL' - #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_user_agent_unit.py b/tests/vendor/test_user_agent_unit.py index 9fa7286825..16beba92b7 100644 --- a/tests/vendor/test_user_agent_unit.py +++ b/tests/vendor/test_user_agent_unit.py @@ -50,10 +50,6 @@ if TYPE_CHECKING: from typing import Any, Iterator -#: Test tier (#1958): exercises pcapkit.vendor, which users run on any supported -#: interpreter. -__tier__ = 'ALL' - #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_vendor_dest_path_unit.py b/tests/vendor/test_vendor_dest_path_unit.py index 493e666da3..19f7891e50 100644 --- a/tests/vendor/test_vendor_dest_path_unit.py +++ b/tests/vendor/test_vendor_dest_path_unit.py @@ -123,10 +123,6 @@ if TYPE_CHECKING: from typing import Any -#: Test tier (#1958): exercises pcapkit.vendor, which users run on any supported -#: interpreter. -__tier__ = 'ALL' - #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_vendor_exit_code_unit.py b/tests/vendor/test_vendor_exit_code_unit.py index 50c596253e..2674f30249 100644 --- a/tests/vendor/test_vendor_exit_code_unit.py +++ b/tests/vendor/test_vendor_exit_code_unit.py @@ -63,10 +63,6 @@ import warnings from unittest import mock -#: Test tier (#1958): exercises pcapkit.vendor, which users run on any supported -#: interpreter. -__tier__ = 'ALL' - #: Same gate :file:`tests/vendor/test_crawler_reachability_unit.py` and #: :file:`tests/vendor/test_vendor_dest_path_unit.py` use. Importing #: :mod:`pcapkit.vendor.__main__` imports :mod:`pcapkit.vendor` (for the diff --git a/tests/vendor/test_vendor_missing_extra_1850_unit.py b/tests/vendor/test_vendor_missing_extra_1850_unit.py index 9afffc7ba6..1af475a412 100644 --- a/tests/vendor/test_vendor_missing_extra_1850_unit.py +++ b/tests/vendor/test_vendor_missing_extra_1850_unit.py @@ -53,6 +53,9 @@ if TYPE_CHECKING: from typing import Sequence +#: Test tier (#1958): branches on the running interpreter's version or implementation. +__tier__ = 'ALL' + #: The package under test, found from this file rather than by importing it, so #: that a child interpreter can be pointed at the same tree. PACKAGE = pathlib.Path(__file__).resolve().parents[2] / 'pcapkit' diff --git a/tests/vendor/test_vendor_ngap_unit.py b/tests/vendor/test_vendor_ngap_unit.py index fc01df9840..e6844eab7c 100644 --- a/tests/vendor/test_vendor_ngap_unit.py +++ b/tests/vendor/test_vendor_ngap_unit.py @@ -31,10 +31,6 @@ import unittest from unittest import mock -#: Test tier (#1958): exercises pcapkit.vendor, which users run on any supported -#: interpreter. -__tier__ = 'ALL' - HAS_PYCRATE = importlib.util.find_spec('pycrate_asn1dir') is not None From d963c39bfa7de5369c3473a9f69c080d0275600c Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 20 Oct 2026 10:30:59 -0400 Subject: [PATCH 4/4] test(tiers): opt the remaining interpreter-sensitive modules into ALL (#1958) - Opt in the modules the review found on LATEST despite the criterion: pickling and copying, the warning machinery and exception hooks, struct and int boundary tests, and the modules #1974 gates on GraalPy: fallback cycle, MH timestamp and the two #1541 overrun tests that compare against sys.maxsize. - Opt Scapy's engine and toolkit modules into ALL like the other engines, and PyShark's toolkit and non-packet modules with them. - State each demoted vendor module's LATEST tier and reason. - branches_on_interpreter() also sees sys.flags, sys._is_gil_enabled, sys.maxsize comparisons, sysconfig.get_config_var() and hasattr/getattr probes of sys; its docstring and the guard's message say the check is syntactic and best-effort. - _tiers.py and workflows.rst "Test Tiers" give the Scapy and vendor reasons. Tier guard, #1732, #1954, #1958 pins and every touched module pass on CPython 3.14. --- docs/source/contributing/workflows.rst | 18 +++-- tests/_tiers.py | 69 ++++++++++++++++--- ..._enum_get_exception_provenance_923_unit.py | 4 ++ .../test_enum_lookup_reparent_930_unit.py | 4 ++ tests/corekit/test_fields_misc.py | 4 ++ .../test_fields_numbers_callable_length.py | 4 ++ ..._numbers_port_enum_field_peek_1774_unit.py | 4 ++ ...fields_numbers_port_option_no_mint_unit.py | 4 ++ tests/dumpkit/test_fallback_cycle_unit.py | 4 ++ ...t_pcap_trace_truncated_record_1541_unit.py | 4 ++ .../engines/test_pyshark_non_packet_unit.py | 4 ++ tests/foundation/engines/test_scapy_engine.py | 5 ++ .../test_scapy_pcapng_sections_runtime.py | 5 ++ .../engines/test_scapy_reader_hooks_unit.py | 5 ++ .../test_scapy_reader_interfaces_runtime.py | 5 ++ .../test_scapy_record_length_1717_runtime.py | 5 ++ .../test_scapy_record_length_1717_unit.py | 5 ++ .../test_httpv2_negative_length_unit.py | 4 ++ .../test_ipv6_calipso_make_validation_unit.py | 4 ++ .../test_ipv6_option_helpers_1519_unit.py | 4 ++ .../test_mh_timestamp_info_1886_unit.py | 4 ++ .../test_frame_incl_len_overrun_1541_unit.py | 4 ++ .../test_to_dict_repeated_exthdr_1453_unit.py | 4 ++ .../test_trial_parse_warnings_1580_unit.py | 4 ++ .../test_tcp_mptcp_length_arithmetic_unit.py | 4 ++ tests/test_tier_guard.py | 19 ++++- tests/toolkit/test_pyshark_unit.py | 4 ++ tests/toolkit/test_scapy_linktype_unit.py | 5 ++ tests/toolkit/test_scapy_packet2dict_unit.py | 5 ++ tests/toolkit/test_scapy_unit.py | 5 ++ tests/utilities/test_exceptions_warnings.py | 4 ++ tests/utilities/test_warning_emission.py | 4 ++ tests/utilities/test_warning_filters.py | 4 ++ tests/vendor/test_ftp_return_code_unit.py | 7 ++ tests/vendor/test_request_prompt_unit.py | 7 ++ tests/vendor/test_user_agent_unit.py | 7 ++ tests/vendor/test_vendor_dest_path_unit.py | 7 ++ tests/vendor/test_vendor_exit_code_unit.py | 7 ++ tests/vendor/test_vendor_ngap_unit.py | 7 ++ 39 files changed, 260 insertions(+), 17 deletions(-) diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 35e791c2a7..937da2705d 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -929,9 +929,13 @@ suite as the gate. * **The default.** A module that states no tier is ``LATEST``, so a new module runs on ``latest`` until it says why every leg needs it. ``__tier__ = 'LATEST'`` is optional, and records why a module someone might expect on every leg is not; a - module that compares ``sys.version_info`` or ``sys.implementation`` itself has - to state its tier either way, so a version gate never lands on ``latest`` alone - by omission. + module that compares ``sys.version_info``, ``sys.implementation`` or + ``sys.maxsize``, reads + ``sys.flags``, calls ``sysconfig.get_config_var()`` or probes :mod:`sys` with + ``hasattr``/``getattr`` has to state its tier either way, so a version gate never + lands on ``latest`` alone by omission. The check is syntactic and best-effort: an + alias, a flag imported from ``pcapkit.utilities.compat`` or a gate in a helper is + not seen, and such a module states its tier by hand. A module opts in when it exercises behaviour that can differ between interpreters or versions: @@ -945,7 +949,9 @@ or versions: * stream and buffer semantics, threading and locking, and the warning and exception hooks; * a third-party engine whose installable releases vary by interpreter, such as - ``pypcap`` and ``pypcapfile``, which install below 3.12 only; + ``pypcap`` and ``pypcapfile``, which install below 3.12 only, and Scapy, which the + ``integration`` and ``pypcap-parity`` legs install on every interpreter and a + ``test`` leg off ``latest`` skips; * the hand-picked round trips, whatever their names, and the #1202 edge-case tables, whose cases are each written out by hand, as the owner ruled on #1958; and @@ -956,7 +962,9 @@ Everything else is ``LATEST``: the per-protocol field-by-field tests, whose behaviour is interpreter-independent pure Python, each real sweep -- a round trip over every class or registry entry a table enumerates, every offset of a packet or every capture, as the owner ruled on #1732 and #1958 -- and the docs, project and -structure tests. The tier guard itself is ``ALL``, since :file:`tests/conftest.py` +structure tests. The vendor extra is installed on ``latest`` alone and the crawlers +are pure Python, so the vendor modules are ``LATEST``, each saying so, but for the +file-restore and environment-variable tests, which stay on every leg. The tier guard itself is ``ALL``, since :file:`tests/conftest.py` runs it at collection on every leg. Which legs run which tier: diff --git a/tests/_tiers.py b/tests/_tiers.py index 465d6cadf7..4dca218957 100644 --- a/tests/_tiers.py +++ b/tests/_tiers.py @@ -145,10 +145,17 @@ Everything else is ``LATEST`` by default: the per-protocol field-by-field tests, whose behaviour is interpreter-independent pure Python, the real sweeps, and the docs, project and structure tests. A module that branches on the interpreter -itself (:func:`branches_on_interpreter`) has to state its tier either way, so a -version gate never lands on ``latest`` alone by omission; a ``LATEST`` marker is -otherwise optional, and records why a module someone might expect on every leg -is not. +itself (:func:`branches_on_interpreter`, a syntactic, best-effort check) has to +state its tier either way, so a version gate never lands on ``latest`` alone by +omission; a ``LATEST`` marker is otherwise optional, and records why a module +someone might expect on every leg is not. + +Scapy is opted in like the other third-party engines: the ``integration`` and +``pypcap-parity`` legs install it on every interpreter, and a ``test`` leg off +``latest``, which does not, skips it. The vendor extra is installed on ``latest`` +alone, and the crawlers are pure Python over ``requests`` and ``bs4``, so the vendor +modules are ``LATEST``, each saying so, but for the file-restore and +environment-variable tests, which stay on every leg. Nothing here imports :mod:`tests._support` -- the dependency runs the other way, and adding the reverse edge would make it a cycle. Nothing here imports @@ -952,15 +959,56 @@ def declared_tier(path: 'pathlib.Path | str') -> 'Optional[str]': return _declared_tier(_test_module(path)) +#: Attributes of :mod:`sys` whose mere use asks which build is running, compared or not: +#: ``sys.flags.context_aware_warnings``, ``sys._is_gil_enabled()``. +_BUILD_FACTS = frozenset({'flags', '_is_gil_enabled'}) +#: Builtins that probe :mod:`sys` for an attribute, e.g. ``hasattr(sys, 'tracebacklimit')``. +_PROBES = frozenset({'hasattr', 'getattr'}) + + +def _probes_interpreter(node: 'ast.AST') -> 'bool': + """Whether ``node`` asks the running interpreter something outside a comparison. + + ``sys.flags`` or ``sys._is_gil_enabled``, a comparison with ``sys.maxsize``, a + ``sysconfig.get_config_var()`` call such as ``Py_GIL_DISABLED``'s, or a + ``hasattr``/``getattr`` probe of :mod:`sys`. + + """ + if isinstance(node, ast.Attribute): + return node.attr in _BUILD_FACTS and isinstance(node.value, ast.Name) and node.value.id == 'sys' + if isinstance(node, ast.Compare): + return any(isinstance(part, ast.Attribute) and part.attr == 'maxsize' and isinstance(part.value, ast.Name) + and part.value.id == 'sys' for operand in [node.left] + list(node.comparators) + for part in ast.walk(operand)) + if not isinstance(node, ast.Call): + return False + func = node.func + if isinstance(func, ast.Attribute) and func.attr == 'get_config_var': + return isinstance(func.value, ast.Name) and func.value.id == 'sysconfig' + if isinstance(func, ast.Name) and func.id == 'get_config_var': + return True + return (isinstance(func, ast.Name) and func.id in _PROBES and bool(node.args) + and isinstance(node.args[0], ast.Name) and node.args[0].id == 'sys') + + def branches_on_interpreter(path: 'pathlib.Path | str') -> 'bool': - """Whether the test module at ``path`` itself compares the running interpreter (#1958). + """Whether the test module at ``path`` itself asks which interpreter is running (#1958). By its own AST alone, unlike :func:`varies_by_interpreter`, which follows what the module runs: a comparison of ``sys.version_info``, ``sys.implementation``, - ``platform.python_version()`` or the like, anywhere in it. Such a module has to - state its tier with :data:`TIER_MARKER`, so a version gate never runs on - ``latest`` alone just because nobody opted it in to ``ALL``. A module that does - not parse branches on nothing here. + ``platform.python_version()`` or the like; a use of ``sys.flags`` or + ``sys._is_gil_enabled``; a comparison with ``sys.maxsize``; a + ``sysconfig.get_config_var()`` call, such as + ``Py_GIL_DISABLED``'s; or a ``hasattr``/``getattr`` probe of :mod:`sys`, anywhere + in it. Such a module has to state its tier with :data:`TIER_MARKER`, so a + version gate never runs on ``latest`` alone just because nobody opted it in to + ``ALL``. A module that does not parse branches on nothing here. + + The detection is syntactic and best-effort, not proof that a module is + interpreter-independent: it does not see :mod:`sys` under an alias, a probe of + any other module, a flag imported from :mod:`pcapkit.utilities.compat` or + another helper, or a gate in code the module imports. A module that relies on + any of those states its tier by hand. Args: path: Path to a test module, absolute or relative to :data:`ROOT`. @@ -970,7 +1018,8 @@ def branches_on_interpreter(path: 'pathlib.Path | str') -> 'bool': """ tree = _parse_module(_test_module(path)) - return tree is not None and any(_compares_interpreter(node) for node in ast.walk(tree)) + return tree is not None and any(_compares_interpreter(node) or _probes_interpreter(node) + for node in ast.walk(tree)) def module_tier(path: 'pathlib.Path | str') -> 'str': diff --git a/tests/corekit/test_enum_get_exception_provenance_923_unit.py b/tests/corekit/test_enum_get_exception_provenance_923_unit.py index 4e0d80db6d..06875ce259 100644 --- a/tests/corekit/test_enum_get_exception_provenance_923_unit.py +++ b/tests/corekit/test_enum_get_exception_provenance_923_unit.py @@ -73,6 +73,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): exercises threading, locking or warning and exception hooks, which +#: differ by interpreter. +__tier__ = 'ALL' + __all__ = [ 'EnumKeyErrorClassTests', 'BaseGetProvenanceTests', 'BaseGetQuietnessTests', 'MethodStillMintsTests', 'TransportProtocolCaseFoldingTests', diff --git a/tests/corekit/test_enum_lookup_reparent_930_unit.py b/tests/corekit/test_enum_lookup_reparent_930_unit.py index 4c54d535da..6879729fdc 100644 --- a/tests/corekit/test_enum_lookup_reparent_930_unit.py +++ b/tests/corekit/test_enum_lookup_reparent_930_unit.py @@ -148,6 +148,10 @@ from pcapkit.utilities.logging import logger from tests.utilities._harness import capture +#: Test tier (#1958): exercises threading, locking or warning and exception hooks, which +#: differ by interpreter. +__tier__ = 'ALL' + __all__ = [ 'ReparentedBasesTests', 'GetByNameAndValueTests', 'FailedLookupTests', 'InheritedQuietnessTests', 'NoMintingTests', 'AllSevenInheritTheBareClassmethodTests', diff --git a/tests/corekit/test_fields_misc.py b/tests/corekit/test_fields_misc.py index 973e9e6f93..2cbd94fa86 100644 --- a/tests/corekit/test_fields_misc.py +++ b/tests/corekit/test_fields_misc.py @@ -5,6 +5,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + #: Every address-typed branch of every address-typed #: :class:`~pcapkit.corekit.fields.misc.SwitchField` in the schema tree, as #: ``(module, class, attribute, label, packet)``. The ``packet`` is a selector diff --git a/tests/corekit/test_fields_numbers_callable_length.py b/tests/corekit/test_fields_numbers_callable_length.py index f974622f82..11358a16d4 100644 --- a/tests/corekit/test_fields_numbers_callable_length.py +++ b/tests/corekit/test_fields_numbers_callable_length.py @@ -24,6 +24,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/corekit/test_fields_numbers_port_enum_field_peek_1774_unit.py b/tests/corekit/test_fields_numbers_port_enum_field_peek_1774_unit.py index 61459a253a..1d70ff5bdf 100644 --- a/tests/corekit/test_fields_numbers_port_enum_field_peek_1774_unit.py +++ b/tests/corekit/test_fields_numbers_port_enum_field_peek_1774_unit.py @@ -20,6 +20,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + #: An unassigned port on all four transports, in none of ``_missing_``'s spans. PORT = 50000 diff --git a/tests/corekit/test_fields_numbers_port_option_no_mint_unit.py b/tests/corekit/test_fields_numbers_port_option_no_mint_unit.py index dbdff0dd78..1ed8cf164f 100644 --- a/tests/corekit/test_fields_numbers_port_option_no_mint_unit.py +++ b/tests/corekit/test_fields_numbers_port_option_no_mint_unit.py @@ -52,6 +52,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + class IntCode(aenum.IntEnum): """An :class:`int`-backed registry stand-in. diff --git a/tests/dumpkit/test_fallback_cycle_unit.py b/tests/dumpkit/test_fallback_cycle_unit.py index 13d4cea895..86b572a014 100644 --- a/tests/dumpkit/test_fallback_cycle_unit.py +++ b/tests/dumpkit/test_fallback_cycle_unit.py @@ -30,6 +30,10 @@ except ImportError: # pragma: no cover - not POSIX resource = None # type: ignore[assignment] +#: Test tier (#1958): gates on GraalPy's sys.implementation, so it has to run on that +#: leg. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/dumpkit/test_pcap_trace_truncated_record_1541_unit.py b/tests/dumpkit/test_pcap_trace_truncated_record_1541_unit.py index 7ee7930e38..b8027f6f8e 100644 --- a/tests/dumpkit/test_pcap_trace_truncated_record_1541_unit.py +++ b/tests/dumpkit/test_pcap_trace_truncated_record_1541_unit.py @@ -28,6 +28,10 @@ from tests._support import close_extractor, reimport_once_per_class from tests.foundation._roundtrip import ethernet, ipv4, pcap, pcapng, tcp +#: Test tier (#1958): a record whose claimed length runs towards the interpreter's size +#: limit, which differs by implementation (sys.maxsize, #1963). +__tier__ = 'ALL' + HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) diff --git a/tests/foundation/engines/test_pyshark_non_packet_unit.py b/tests/foundation/engines/test_pyshark_non_packet_unit.py index ebdd21f0b4..70d006a650 100644 --- a/tests/foundation/engines/test_pyshark_non_packet_unit.py +++ b/tests/foundation/engines/test_pyshark_non_packet_unit.py @@ -28,6 +28,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): drives a third-party engine whose installable releases vary by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/foundation/engines/test_scapy_engine.py b/tests/foundation/engines/test_scapy_engine.py index 38353fd812..4d06ba61a3 100644 --- a/tests/foundation/engines/test_scapy_engine.py +++ b/tests/foundation/engines/test_scapy_engine.py @@ -46,6 +46,11 @@ from tests._support import sample_path +#: Test tier (#1958): drives Scapy, whose installable releases vary by interpreter; the +#: integration and pypcap-parity legs install it on every interpreter, and a test leg +#: off latest skips it. +__tier__ = 'ALL' + #: Repository root, which the child interpreter needs on :data:`sys.path` to #: import the :mod:`pcapkit` under test rather than an installed copy. ROOT = pathlib.Path(__file__).resolve().parents[3] diff --git a/tests/foundation/engines/test_scapy_pcapng_sections_runtime.py b/tests/foundation/engines/test_scapy_pcapng_sections_runtime.py index 40ddc31c0b..b2ef2bd83c 100644 --- a/tests/foundation/engines/test_scapy_pcapng_sections_runtime.py +++ b/tests/foundation/engines/test_scapy_pcapng_sections_runtime.py @@ -30,6 +30,11 @@ from tests._support import close_extractor, reimport_once_per_class, sample_path +#: Test tier (#1958): drives Scapy, whose installable releases vary by interpreter; the +#: integration and pypcap-parity legs install it on every interpreter, and a test leg +#: off latest skips it. +__tier__ = 'ALL' + HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) HAS_SCAPY = importlib.util.find_spec('scapy') is not None diff --git a/tests/foundation/engines/test_scapy_reader_hooks_unit.py b/tests/foundation/engines/test_scapy_reader_hooks_unit.py index f0eb2b46f5..440fc8091c 100644 --- a/tests/foundation/engines/test_scapy_reader_hooks_unit.py +++ b/tests/foundation/engines/test_scapy_reader_hooks_unit.py @@ -26,6 +26,11 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): drives Scapy, whose installable releases vary by interpreter; the +#: integration and pypcap-parity legs install it on every interpreter, and a test leg +#: off latest skips it. +__tier__ = 'ALL' + HAS_SCAPY = importlib.util.find_spec('scapy') is not None HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) diff --git a/tests/foundation/engines/test_scapy_reader_interfaces_runtime.py b/tests/foundation/engines/test_scapy_reader_interfaces_runtime.py index ded33cc6f0..02dbad5b9b 100644 --- a/tests/foundation/engines/test_scapy_reader_interfaces_runtime.py +++ b/tests/foundation/engines/test_scapy_reader_interfaces_runtime.py @@ -31,6 +31,11 @@ from tests._support import close_extractor, reimport_once_per_class, sample_path +#: Test tier (#1958): drives Scapy, whose installable releases vary by interpreter; the +#: integration and pypcap-parity legs install it on every interpreter, and a test leg +#: off latest skips it. +__tier__ = 'ALL' + HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) HAS_SCAPY = importlib.util.find_spec('scapy') is not None diff --git a/tests/foundation/engines/test_scapy_record_length_1717_runtime.py b/tests/foundation/engines/test_scapy_record_length_1717_runtime.py index 5a5cd4e3dc..7a4a608883 100644 --- a/tests/foundation/engines/test_scapy_record_length_1717_runtime.py +++ b/tests/foundation/engines/test_scapy_record_length_1717_runtime.py @@ -19,6 +19,11 @@ from tests._support import close_extractor, reimport_once_per_class, sample_path +#: Test tier (#1958): drives Scapy, whose installable releases vary by interpreter; the +#: integration and pypcap-parity legs install it on every interpreter, and a test leg +#: off latest skips it. +__tier__ = 'ALL' + HAS_SCAPY = importlib.util.find_spec('scapy') is not None HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) diff --git a/tests/foundation/engines/test_scapy_record_length_1717_unit.py b/tests/foundation/engines/test_scapy_record_length_1717_unit.py index 6df7769c95..4e9fcf3892 100644 --- a/tests/foundation/engines/test_scapy_record_length_1717_unit.py +++ b/tests/foundation/engines/test_scapy_record_length_1717_unit.py @@ -22,6 +22,11 @@ from tests._support import close_extractor, reimport_once_per_class +#: Test tier (#1958): drives Scapy, whose installable releases vary by interpreter; the +#: integration and pypcap-parity legs install it on every interpreter, and a test leg +#: off latest skips it. +__tier__ = 'ALL' + HAS_SCAPY = importlib.util.find_spec('scapy') is not None HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) diff --git a/tests/protocols/application/test_httpv2_negative_length_unit.py b/tests/protocols/application/test_httpv2_negative_length_unit.py index 55b1017af8..56fe4a6543 100644 --- a/tests/protocols/application/test_httpv2_negative_length_unit.py +++ b/tests/protocols/application/test_httpv2_negative_length_unit.py @@ -33,6 +33,10 @@ import unittest import warnings +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/internet/test_ipv6_calipso_make_validation_unit.py b/tests/protocols/internet/test_ipv6_calipso_make_validation_unit.py index f7b7ecfbb1..162dbde506 100644 --- a/tests/protocols/internet/test_ipv6_calipso_make_validation_unit.py +++ b/tests/protocols/internet/test_ipv6_calipso_make_validation_unit.py @@ -22,6 +22,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + #: The protocol classes sharing the option code, by module and class name. PROTOCOLS = ( ('pcapkit.protocols.internet.hopopt', 'HOPOPT'), diff --git a/tests/protocols/internet/test_ipv6_option_helpers_1519_unit.py b/tests/protocols/internet/test_ipv6_option_helpers_1519_unit.py index 674b2b001a..d6f0ac10cf 100644 --- a/tests/protocols/internet/test_ipv6_option_helpers_1519_unit.py +++ b/tests/protocols/internet/test_ipv6_option_helpers_1519_unit.py @@ -23,6 +23,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + SHARED = 'pcapkit.protocols.schema.internet.ipv6_option' #: Each header's schema module, and the prefix its errors carry. diff --git a/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py b/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py index b000b5541b..93906fa8a4 100644 --- a/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py +++ b/tests/protocols/internet/test_mh_timestamp_info_1886_unit.py @@ -30,6 +30,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/protocols/misc/pcap/test_frame_incl_len_overrun_1541_unit.py b/tests/protocols/misc/pcap/test_frame_incl_len_overrun_1541_unit.py index 4248096bf0..670d6d23f2 100644 --- a/tests/protocols/misc/pcap/test_frame_incl_len_overrun_1541_unit.py +++ b/tests/protocols/misc/pcap/test_frame_incl_len_overrun_1541_unit.py @@ -20,6 +20,10 @@ from tests._support import close_extractor, reimport_once_per_class +#: Test tier (#1958): a record whose claimed length runs towards the interpreter's size +#: limit, which differs by implementation (sys.maxsize, #1963). +__tier__ = 'ALL' + #: Little-endian global header: v2.4, Ethernet. GLOBAL = bytes.fromhex('d4c3b2a1' '0200' '0400' '00000000' '00000000' 'ffff0000' '01000000') #: Sixty octets of packet data, none of them zero. diff --git a/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py b/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py index d78b69b9c7..6a1b89d082 100644 --- a/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py +++ b/tests/protocols/test_to_dict_repeated_exthdr_1453_unit.py @@ -28,6 +28,10 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): exercises pickling and copying, whose protocols differ by +#: interpreter. +__tier__ = 'ALL' + HOP, DST, ROUTE, UDP = 0, 60, 43, 17 MAC = bytes.fromhex('0123456789ab' 'fedcba987654') SRC6 = bytes.fromhex('20010db8' + '00' * 11 + '01') diff --git a/tests/protocols/test_trial_parse_warnings_1580_unit.py b/tests/protocols/test_trial_parse_warnings_1580_unit.py index 61efc8e093..4b95a67e7c 100644 --- a/tests/protocols/test_trial_parse_warnings_1580_unit.py +++ b/tests/protocols/test_trial_parse_warnings_1580_unit.py @@ -40,6 +40,10 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): exercises the warning machinery itself, which free-threaded builds +#: make context-aware. +__tier__ = 'ALL' + #: Snaplen of the capture's interface, and so the octets each packet keeps. SNAPLEN = 10 #: Original length of each packet, longer than the Ethernet header. diff --git a/tests/protocols/transport/test_tcp_mptcp_length_arithmetic_unit.py b/tests/protocols/transport/test_tcp_mptcp_length_arithmetic_unit.py index 75e6bf5530..d5c396367d 100644 --- a/tests/protocols/transport/test_tcp_mptcp_length_arithmetic_unit.py +++ b/tests/protocols/transport/test_tcp_mptcp_length_arithmetic_unit.py @@ -146,6 +146,10 @@ import unittest +#: Test tier (#1958): exercises integer, bytes and struct edge cases, whose limits +#: differ by interpreter. +__tier__ = 'ALL' + #: Header fields shared by the constructed TCP segments here, matching #: :data:`examples.generators.options.TCP_BASE` and #: :data:`tests.protocols.transport.test_tcp_mptcp_subtype_unit.TCP_BASE`. diff --git a/tests/test_tier_guard.py b/tests/test_tier_guard.py index a533c4663d..52511d1999 100644 --- a/tests/test_tier_guard.py +++ b/tests/test_tier_guard.py @@ -561,13 +561,26 @@ def test_an_invalid_marker_fails(self) -> None: self.assertIn(_tiers.TIER_MARKER, str(caught.exception)) def test_a_module_that_branches_on_the_interpreter_states_its_tier(self) -> None: - """A version gate is never ``LATEST`` by omission: such a module says which tier it is in.""" + """A version gate is never ``LATEST`` by omission: such a module says which tier it is in. + + Detection is syntactic and best-effort: :mod:`sys` under an alias, a flag imported + from :mod:`pcapkit.utilities.compat`, or a gate in an imported helper is not seen. + + """ cases = { 'a version gate': ('import sys\nNEW = sys.version_info >= (3, 12)\n', True), 'an implementation gate': ('import sys\nGRAAL = sys.implementation.name == "graalpy"\n', True), 'a bare import': ('from sys import version_info\nOLD = version_info < (3, 9)\n', True), 'pcapkit alone': ('import pcapkit\n', False), 'a version only read': ('import sys\nprint(sys.version_info)\n', False), + 'a build flag': ('import sys\nAWARE = sys.flags.context_aware_warnings\n', True), + 'the GIL probe': ('import sys\nGIL = sys._is_gil_enabled()\n', True), + 'a config var': ('import sysconfig\nFT = sysconfig.get_config_var("Py_GIL_DISABLED")\n', True), + 'hasattr on sys': ('import sys\nSET = hasattr(sys, "tracebacklimit")\n', True), + 'getattr on sys': ('import sys\nLIMIT = getattr(sys, "tracebacklimit", None)\n', True), + 'getattr on another object': ('import os\nSEP = getattr(os, "sep")\n', False), + 'a size limit': ('import sys\nBIG = 1 << 40 > sys.maxsize\n', True), + 'a size limit only read': ('import sys\nprint(sys.maxsize)\n', False), } for label, (source, expected) in cases.items(): with self.subTest(case=label), tempfile.TemporaryDirectory(prefix='pcapkit-test-tier-') as temp: @@ -579,7 +592,9 @@ def test_a_module_that_branches_on_the_interpreter_states_its_tier(self) -> None with self.subTest(module=module): self.assertIsNotNone(_tiers.declared_tier(module), f'{module} branches on the interpreter; state its tier with ' - f"{_tiers.TIER_MARKER} = {_tiers.TIER_ALL!r} or {_tiers.TIER_LATEST!r}") + f"{_tiers.TIER_MARKER} = {_tiers.TIER_ALL!r} or {_tiers.TIER_LATEST!r} " + '(the check is syntactic and best-effort: aliases, probes of other ' + 'modules and flags imported from compat are not seen)') def test_only_a_test_module_has_a_tier(self) -> None: for path in ('tests/_support.py', 'tests/_tiers.py', 'util/ci_timing.py', 'tests/project', diff --git a/tests/toolkit/test_pyshark_unit.py b/tests/toolkit/test_pyshark_unit.py index dc28ee05cb..28d29fb822 100644 --- a/tests/toolkit/test_pyshark_unit.py +++ b/tests/toolkit/test_pyshark_unit.py @@ -15,6 +15,10 @@ if TYPE_CHECKING: from typing import Any, Callable +#: Test tier (#1958): drives a third-party engine whose installable releases vary by +#: interpreter. +__tier__ = 'ALL' + RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/toolkit/test_scapy_linktype_unit.py b/tests/toolkit/test_scapy_linktype_unit.py index b2ce69bba0..3b9602a3d1 100644 --- a/tests/toolkit/test_scapy_linktype_unit.py +++ b/tests/toolkit/test_scapy_linktype_unit.py @@ -17,6 +17,11 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): drives Scapy, whose installable releases vary by interpreter; the +#: integration and pypcap-parity legs install it on every interpreter, and a test leg +#: off latest skips it. +__tier__ = 'ALL' + HAS_SCAPY = importlib.util.find_spec('scapy') is not None HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in ('tbtrim', 'aenum', 'chardet', 'dictdumper')) diff --git a/tests/toolkit/test_scapy_packet2dict_unit.py b/tests/toolkit/test_scapy_packet2dict_unit.py index e116f636f5..de02d8deb4 100644 --- a/tests/toolkit/test_scapy_packet2dict_unit.py +++ b/tests/toolkit/test_scapy_packet2dict_unit.py @@ -7,6 +7,11 @@ from tests._support import reimport_once_per_class +#: Test tier (#1958): drives Scapy, whose installable releases vary by interpreter; the +#: integration and pypcap-parity legs install it on every interpreter, and a test leg +#: off latest skips it. +__tier__ = 'ALL' + HAS_SCAPY = importlib.util.find_spec('scapy') is not None RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/toolkit/test_scapy_unit.py b/tests/toolkit/test_scapy_unit.py index 27b949e453..a81cd7410b 100644 --- a/tests/toolkit/test_scapy_unit.py +++ b/tests/toolkit/test_scapy_unit.py @@ -13,6 +13,11 @@ from tests._support import close_extractor, purge_modules, reimport_once_per_class +#: Test tier (#1958): drives Scapy, whose installable releases vary by interpreter; the +#: integration and pypcap-parity legs install it on every interpreter, and a test leg +#: off latest skips it. +__tier__ = 'ALL' + HAS_SCAPY = importlib.util.find_spec('scapy') is not None RUNTIME_DEPS = ('tbtrim', 'aenum', 'chardet', 'dictdumper') HAS_RUNTIME = all(importlib.util.find_spec(name) is not None for name in RUNTIME_DEPS) diff --git a/tests/utilities/test_exceptions_warnings.py b/tests/utilities/test_exceptions_warnings.py index 9bcc5af685..c87da7ba73 100644 --- a/tests/utilities/test_exceptions_warnings.py +++ b/tests/utilities/test_exceptions_warnings.py @@ -7,6 +7,10 @@ from tests._support import bootstrap_core_modules, reimport_once_per_class +#: Test tier (#1958): exercises threading, locking or warning and exception hooks, which +#: differ by interpreter. +__tier__ = 'ALL' + class ExceptionsWarningsTests(unittest.TestCase): def setUp(self) -> None: diff --git a/tests/utilities/test_warning_emission.py b/tests/utilities/test_warning_emission.py index 1dc38c8b97..e6b7969c0d 100644 --- a/tests/utilities/test_warning_emission.py +++ b/tests/utilities/test_warning_emission.py @@ -27,6 +27,10 @@ from tests._support import purge_modules from tests.utilities._harness import bootstrap, capture +#: Test tier (#1958): exercises the warning machinery itself, which free-threaded builds +#: make context-aware. +__tier__ = 'ALL' + class WarningEmissionTests(unittest.TestCase): def setUp(self) -> None: diff --git a/tests/utilities/test_warning_filters.py b/tests/utilities/test_warning_filters.py index f697e4c3e8..049d30d69a 100644 --- a/tests/utilities/test_warning_filters.py +++ b/tests/utilities/test_warning_filters.py @@ -29,6 +29,10 @@ from tests.utilities import _unrelated_warning from tests.utilities._harness import bootstrap, capture +#: Test tier (#1958): exercises the warning machinery itself, which free-threaded builds +#: make context-aware. +__tier__ = 'ALL' + #: Every warning category the module exports, by name. CATEGORY_NAMES = [ 'BaseWarning', diff --git a/tests/vendor/test_ftp_return_code_unit.py b/tests/vendor/test_ftp_return_code_unit.py index 86278e4202..1669b95686 100644 --- a/tests/vendor/test_ftp_return_code_unit.py +++ b/tests/vendor/test_ftp_return_code_unit.py @@ -44,6 +44,13 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): the vendor extra is installed on latest alone, where the whole +#: suite runs, and the crawlers are pure Python over requests and bs4; the vendor +#: package's interpreter-sensitive parts, file restore and environment-variable parsing, +#: run on every leg in tests/vendor/test_vendor_snapshot_restore_unit.py and +#: tests/vendor/test_vendor_variables_1726_unit.py. +__tier__ = 'LATEST' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_request_prompt_unit.py b/tests/vendor/test_request_prompt_unit.py index ac9e42eae9..98f4a54185 100644 --- a/tests/vendor/test_request_prompt_unit.py +++ b/tests/vendor/test_request_prompt_unit.py @@ -88,6 +88,13 @@ if TYPE_CHECKING: from typing import Any, Iterator +#: Test tier (#1958): the vendor extra is installed on latest alone, where the whole +#: suite runs, and the crawlers are pure Python over requests and bs4; the vendor +#: package's interpreter-sensitive parts, file restore and environment-variable parsing, +#: run on every leg in tests/vendor/test_vendor_snapshot_restore_unit.py and +#: tests/vendor/test_vendor_variables_1726_unit.py. +__tier__ = 'LATEST' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_user_agent_unit.py b/tests/vendor/test_user_agent_unit.py index 16beba92b7..2cf613b881 100644 --- a/tests/vendor/test_user_agent_unit.py +++ b/tests/vendor/test_user_agent_unit.py @@ -50,6 +50,13 @@ if TYPE_CHECKING: from typing import Any, Iterator +#: Test tier (#1958): the vendor extra is installed on latest alone, where the whole +#: suite runs, and the crawlers are pure Python over requests and bs4; the vendor +#: package's interpreter-sensitive parts, file restore and environment-variable parsing, +#: run on every leg in tests/vendor/test_vendor_snapshot_restore_unit.py and +#: tests/vendor/test_vendor_variables_1726_unit.py. +__tier__ = 'LATEST' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_vendor_dest_path_unit.py b/tests/vendor/test_vendor_dest_path_unit.py index 19f7891e50..4e8cb0b7f5 100644 --- a/tests/vendor/test_vendor_dest_path_unit.py +++ b/tests/vendor/test_vendor_dest_path_unit.py @@ -123,6 +123,13 @@ if TYPE_CHECKING: from typing import Any +#: Test tier (#1958): the vendor extra is installed on latest alone, where the whole +#: suite runs, and the crawlers are pure Python over requests and bs4; the vendor +#: package's interpreter-sensitive parts, file restore and environment-variable parsing, +#: run on every leg in tests/vendor/test_vendor_snapshot_restore_unit.py and +#: tests/vendor/test_vendor_variables_1726_unit.py. +__tier__ = 'LATEST' + #: Repository root, i.e. the grandparent of the directory holding this file. ROOT = pathlib.Path(__file__).resolve().parents[2] diff --git a/tests/vendor/test_vendor_exit_code_unit.py b/tests/vendor/test_vendor_exit_code_unit.py index 2674f30249..ef0a15f245 100644 --- a/tests/vendor/test_vendor_exit_code_unit.py +++ b/tests/vendor/test_vendor_exit_code_unit.py @@ -63,6 +63,13 @@ import warnings from unittest import mock +#: Test tier (#1958): the vendor extra is installed on latest alone, where the whole +#: suite runs, and the crawlers are pure Python over requests and bs4; the vendor +#: package's interpreter-sensitive parts, file restore and environment-variable parsing, +#: run on every leg in tests/vendor/test_vendor_snapshot_restore_unit.py and +#: tests/vendor/test_vendor_variables_1726_unit.py. +__tier__ = 'LATEST' + #: Same gate :file:`tests/vendor/test_crawler_reachability_unit.py` and #: :file:`tests/vendor/test_vendor_dest_path_unit.py` use. Importing #: :mod:`pcapkit.vendor.__main__` imports :mod:`pcapkit.vendor` (for the diff --git a/tests/vendor/test_vendor_ngap_unit.py b/tests/vendor/test_vendor_ngap_unit.py index e6844eab7c..4e9b6ef14d 100644 --- a/tests/vendor/test_vendor_ngap_unit.py +++ b/tests/vendor/test_vendor_ngap_unit.py @@ -31,6 +31,13 @@ import unittest from unittest import mock +#: Test tier (#1958): the vendor extra is installed on latest alone, where the whole +#: suite runs, and the crawlers are pure Python over requests and bs4; the vendor +#: package's interpreter-sensitive parts, file restore and environment-variable parsing, +#: run on every leg in tests/vendor/test_vendor_snapshot_restore_unit.py and +#: tests/vendor/test_vendor_variables_1726_unit.py. +__tier__ = 'LATEST' + HAS_PYCRATE = importlib.util.find_spec('pycrate_asn1dir') is not None