From 412ea8e84d58c186521c23b695ebdb4989d6a6e0 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Mon, 5 Oct 2026 22:25:18 -0400 Subject: [PATCH] ci(coverage): report test coverage from a workflow (#1063) - unit-tests.yml: the `test` job's Python 3.14 leg (matrix key `coverage: true`) runs its unchanged pytest selection under `coverage run --rcfile=.github/coverage.toml`, writes the total and a per-package table to the job summary, and uploads the HTML report and the bare numbers as artifacts. The first line flags failed tests. - New coverage-comment.yml (workflow_run on Unit Tests, runs no tests) turns those numbers into one PR comment, edited on later pushes. It stays out of unit-tests.yml because that file is reusable and its callers grant only `contents: read`. The PR number comes from the API, and the artifact is validated, never executed. - .github/coverage.toml repeats pyproject.toml's coverage settings and adds `parallel` and `patch = ["subprocess"]`, so xdist workers are measured, and `core = "ctrace"`: sysmon's per-generation state under the suite's pcapkit re-imports OOM-killed the runner; tests/project/test_coverage_rcfile.py guards the repetition. - workflows.rst / releasing.rst: document the above, re-point line refs. tests/project and tests/test_tier_guard.py pass locally (427 passed). --- .github/coverage.toml | 32 +++++++ .github/workflows/coverage-comment.yml | 111 +++++++++++++++++++++++++ .github/workflows/unit-tests.yml | 102 +++++++++++++++++++++-- docs/source/contributing/releasing.rst | 4 +- docs/source/contributing/workflows.rst | 69 +++++++++++---- tests/project/test_coverage_rcfile.py | 58 +++++++++++++ 6 files changed, 354 insertions(+), 22 deletions(-) create mode 100644 .github/coverage.toml create mode 100644 .github/workflows/coverage-comment.yml create mode 100644 tests/project/test_coverage_rcfile.py diff --git a/.github/coverage.toml b/.github/coverage.toml new file mode 100644 index 0000000000..f2dee3e48e --- /dev/null +++ b/.github/coverage.toml @@ -0,0 +1,32 @@ +# Coverage configuration for the one coverage-measured leg of unit-tests.yml +# (`test`, Python 3.14), passed with `--rcfile`. Coverage reads a single config +# file, so this repeats pyproject.toml's [tool.coverage.*] settings and adds the +# two the CI run needs on top. tests/project/test_coverage_rcfile.py asserts +# every pyproject setting is repeated here unchanged. +# +# parallel -- each process writes its own .coverage.* file, merged afterwards +# by `coverage combine`. +# patch -- "subprocess" (coverage 7.10+) starts measurement in every +# child Python process. pytest-xdist's workers are such children +# (execnet popen), so without it only the controller is measured +# and the -n auto run reports next to nothing. +# core -- "ctrace", not the sys.monitoring default on 3.12+. Thousands of +# tests purge and re-import pcapkit in setUp, and sysmon keeps +# state for every generation's code objects: measured on +# test_http_unit.py alone, peak RSS 0.40 GiB without coverage, +# 4.17 GiB under sysmon and 0.41 GiB under ctrace. Under +# -n auto that growth killed the runner (#1064). + +[tool.coverage.run] +source = [ + "pcapkit", +] +branch = true +parallel = true +patch = [ + "subprocess", +] +core = "ctrace" + +[tool.coverage.report] +show_missing = true diff --git a/.github/workflows/coverage-comment.yml b/.github/workflows/coverage-comment.yml new file mode 100644 index 0000000000..07f8a679fc --- /dev/null +++ b/.github/workflows/coverage-comment.yml @@ -0,0 +1,111 @@ +name: Coverage Comment + +# #1063: posts the coverage the `test` job's Python 3.14 leg measured in +# unit-tests.yml as a pull-request comment. It runs no tests. The comment cannot +# be posted from unit-tests.yml itself: that file is also a reusable workflow, +# and a called workflow can only narrow the token its caller passes, never widen +# it. All four callers grant `contents: read`, so asking for +# `pull-requests: write` there would break every one of them, the release gate +# included. +# +# `workflow_run` runs this file as it is on the default branch, with a token of +# its own, so fork pull requests get a comment too. The price is that the +# triggering run's artifact is untrusted input. So this workflow: +# +# * never checks out or executes anything from the pull request; +# * takes the PR number from the API (the open PR whose head is this run's +# head sha), not from the artifact, and accepts it only if it is all digits; +# * renders its own Markdown from the artifact's numbers, accepting only the +# shapes the `test` job writes, rather than posting text from the run. +on: + workflow_run: + workflows: ["Unit Tests"] + types: [completed] + +permissions: + actions: read + pull-requests: write + +# One comment per PR, so serialise per head branch. A run that finishes after a +# newer push skips itself below, since its sha is no longer the PR's head. +concurrency: + group: coverage-comment-${{ github.event.workflow_run.head_repository.full_name }}-${{ github.event.workflow_run.head_branch }} + cancel-in-progress: false + +jobs: + comment: + name: Coverage comment + if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion != 'cancelled' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + + steps: + - name: Post or update the coverage comment + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + RUN_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + HEAD_OWNER: ${{ github.event.workflow_run.head_repository.owner.login }} + HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + run: | + set -eu + case "$HEAD_SHA" in + *[!0-9a-f]*|'') echo "::error::unexpected head sha"; exit 1 ;; + esac + + # `workflow_run.pull_requests` is empty for fork PRs, so ask the API. + pr=$(gh api -X GET "repos/$REPO/pulls" -f state=open -f head="$HEAD_OWNER:$HEAD_BRANCH" \ + --jq ".[] | select(.head.sha == \"$HEAD_SHA\") | .number" | head -n 1) + case "$pr" in + *[!0-9]*|'') echo "::notice::No open PR has $HEAD_SHA as its head; nothing to comment on."; exit 0 ;; + esac + + dir="$RUNNER_TEMP/coverage" + if ! gh run download "$RUN_ID" --repo "$REPO" --name coverage-summary --dir "$dir"; then + echo "::notice::Run $RUN_ID uploaded no coverage-summary artifact (the coverage leg did not get that far)." + exit 0 + fi + + python3 - "$dir/coverage-summary.json" > "$RUNNER_TEMP/body.md" <<'PY' + import json, os, re, sys + data = json.load(open(sys.argv[1])) + def number(value, ints=False): + ok = isinstance(value, int) if ints else isinstance(value, (int, float)) + if isinstance(value, bool) or not ok or not 0 <= value < 10**7: + sys.exit(f'invalid number in artifact: {value!r}') + return value + tests = data['tests'] + if tests not in ('success', 'failure', 'cancelled', 'skipped'): + sys.exit(f'invalid test outcome in artifact: {tests!r}') + total = number(data['total']) + status = '' if tests == 'success' else f' -- **tests {tests}**, so this is partial' + print(f"**Coverage: {total:.2f}%**{status} (unit tier, Python 3.14, " + f"`{os.environ['HEAD_SHA'][:9]}`, Unit Tests run {os.environ['RUN_CONCLUSION']})") + print() + print('| Package | Statements | Missed | Branches | Partial | Cover |') + print('|---|--:|--:|--:|--:|--:|') + for name, *counts, cover in data['packages']: + if not isinstance(name, str) or not re.fullmatch(r'pcapkit(/\w+| \(top level\))?', name): + sys.exit(f'invalid package name in artifact: {name!r}') + counts = [number(c, ints=True) for c in counts] + if len(counts) != 4: + sys.exit('invalid row in artifact') + print('| `{}` | {} | {} | {} | {} | {:.2f}% |'.format(name, *counts, number(cover))) + print() + print(f"Per-file detail: the `coverage-html` artifact of [this run]({os.environ['RUN_URL']}).") + print() + print('') + PY + + # Edit the one comment this workflow posted, found by its marker and + # author, rather than adding one per push. + id=$(gh api --paginate "repos/$REPO/issues/$pr/comments" --jq \ + '.[] | select(.user.login == "github-actions[bot]" and (.body | contains(""))) | .id' | head -n 1) + if [ -n "$id" ]; then + gh api -X PATCH "repos/$REPO/issues/comments/$id" -F body=@"$RUNNER_TEMP/body.md" > /dev/null + else + gh api -X POST "repos/$REPO/issues/$pr/comments" -F body=@"$RUNNER_TEMP/body.md" > /dev/null + fi diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 264e382636..8e265261bb 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -64,6 +64,15 @@ jobs: - "3.12" - "3.13" - "3.14" + # #1063: exactly one leg runs under coverage, so measuring it adds no + # leg to the queue #1052 is cutting. 3.14 because it had the most + # headroom under the 45-minute cap on main (15.6, 15.8 and 16.8 minutes + # in runs 37393532176, 37392964804 and 37391296686, against 12.8-22.0 + # for the other four), and because coverage's sys.monitoring core, + # which measures branches only from 3.14 on, is its cheapest tracer. + include: + - python-version: "3.14" + coverage: true steps: - uses: actions/checkout@v7 @@ -107,6 +116,12 @@ jobs: python -m pip install -U pip setuptools wheel python -m pip install -e '.[test,DPKT,crypto,NGAP]' + # 7.10 is the first release with `[run] patch = ["subprocess"]`, which + # .github/coverage.toml relies on to measure the xdist workers. + - name: Install coverage + if: ${{ matrix.coverage }} + run: python -m pip install 'coverage>=7.10' + # `-n auto` resolves to whatever this runner reports, and nothing else in # the log says what that was -- print it so the number `-n auto` picked # is a measured fact instead of an assumption about runner size. @@ -116,15 +131,90 @@ jobs: python -c "import os; print('cpu_count', os.cpu_count())" - name: Run unit tests + id: tests # `--dist load` is pinned explicitly rather than left to default: a # future change to xdist's own default, or someone "optimising" to # `loadfile`/`loadscope`, cannot silently change how this suite is - # distributed. - run: >- - python -m pytest -q -n auto --dist load - --ignore=tests/integration - --ignore-glob='*_runtime.py' - --ignore-glob='*_regression.py' + # distributed. The coverage leg runs the identical selection. + env: + COVERAGE: ${{ matrix.coverage }} + run: | + set -- -q -n auto --dist load \ + --ignore=tests/integration \ + --ignore-glob='*_runtime.py' \ + --ignore-glob='*_regression.py' + if [ "$COVERAGE" = "true" ]; then + python -m coverage run --rcfile=.github/coverage.toml -m pytest "$@" + else + python -m pytest "$@" + fi + + # The table goes to the job summary. The numbers also go to an artifact + # that coverage-comment.yml renders as a PR comment once this run + # completes; that comment cannot be posted from here, because this file is + # also a reusable workflow and its callers grant only `contents: read`. + # `!cancelled()` so a failing test still reports what it measured, and + # says the tests failed in the first line. + - name: Report coverage + if: ${{ !cancelled() && matrix.coverage }} + shell: bash + env: + SHA: ${{ github.sha }} + TESTS_OUTCOME: ${{ steps.tests.outcome }} + run: | + set -euo pipefail + python -m coverage combine --rcfile=.github/coverage.toml + python -m coverage report --rcfile=.github/coverage.toml + python -m coverage html --rcfile=.github/coverage.toml -d coverage-html + python -m coverage json --rcfile=.github/coverage.toml -o coverage.json + mkdir -p coverage-comment + # Writes the numbers as JSON for coverage-comment.yml, which renders + # its own Markdown from them rather than posting text from this run. + python - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import collections, json, os + data = json.load(open('coverage.json')) + keys = ('num_statements', 'missing_lines', 'num_branches', 'num_partial_branches', + 'covered_lines', 'covered_branches') + packages = collections.defaultdict(collections.Counter) + for path, info in data['files'].items(): + parts = path.split('/') + name = '/'.join(parts[:2]) if len(parts) > 2 else 'pcapkit (top level)' + packages[name].update({k: info['summary'][k] for k in keys}) + def pct(c): + total = c['num_statements'] + c['num_branches'] + return 100.0 * (c['covered_lines'] + c['covered_branches']) / total if total else 100.0 + rows = [[name, c['num_statements'], c['missing_lines'], c['num_branches'], + c['num_partial_branches'], round(pct(c), 2)] for name, c in sorted(packages.items())] + total = round(data['totals']['percent_covered'], 2) + tests = os.environ['TESTS_OUTCOME'] + json.dump({'total': total, 'tests': tests, 'packages': rows}, + open('coverage-comment/coverage-summary.json', 'w')) + status = '' if tests == 'success' else f' -- **tests {tests}**, so this is partial' + print(f"**Coverage: {total:.2f}%**{status} (unit tier, Python 3.14, `{os.environ['SHA'][:9]}`)") + print() + print('| Package | Statements | Missed | Branches | Partial | Cover |') + print('|---|--:|--:|--:|--:|--:|') + for row in rows: + print('| `{}` | {} | {} | {} | {} | {:.2f}% |'.format(*row)) + print() + print('Per-file detail: the `coverage-html` artifact of this run.') + PY + + - name: Upload coverage HTML report + if: ${{ !cancelled() && matrix.coverage }} + uses: actions/upload-artifact@v7 + with: + name: coverage-html + path: coverage-html/ + retention-days: 30 + + - name: Upload coverage summary + if: ${{ !cancelled() && matrix.coverage }} + uses: actions/upload-artifact@v7 + with: + name: coverage-summary + path: coverage-comment/coverage-summary.json + retention-days: 7 integration: name: Integration Python ${{ matrix.python-version }} diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index b805da8297..16946c82c1 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -46,7 +46,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:362-376``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:989-1058``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1079-1148``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:392,493,573,674``), so a @@ -56,7 +56,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Two other checks run unconditionally on the same path, though neither is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:962-983``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1052-1073``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index a76bbb76a4..f80c191ad3 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -3,7 +3,7 @@ GitHub Actions Workflows .. important:: - The nine workflows under :file:`.github/workflows/` trigger each other, and + The ten workflows under :file:`.github/workflows/` trigger each other, and the chain that results is not visible from any single file -- reading one ``on:`` block never shows what a *different* workflow's completion goes on to start. This page is the repository-wide graph. The release *pipeline* @@ -86,10 +86,18 @@ At a Glance - daily 00:37 - -- - ``issues``, ``pull_request_target``, ``workflow_dispatch`` + * - Coverage Comment + - :file:`coverage-comment.yml` + - -- + - -- + - -- + - **Unit Tests** + - -- .. note:: - No workflow here fires *exclusively* on ``workflow_run``. Each of the four + Only Coverage Comment fires *exclusively* on ``workflow_run``: it has + nothing to do without a completed Unit Tests run. Each of the other four that carry one pairs it with a direct trigger of its own -- a Saturday ``schedule`` for Vendor Update, Conda Update and GitHub Pages, a ``v*`` tag push for Create Release -- for the path where no upstream run exists yet to @@ -125,6 +133,7 @@ than starting a new one. GP["GitHub Pages
deploy-pages.yml"] CR["Create Release
create-release.yml"] PS["Project Status
project-status.yml"] + CC["Coverage Comment
coverage-comment.yml"] PUSH --> CQ PR --> CQ @@ -158,6 +167,7 @@ than starting a new one. UT ==>|workflow_run: completed| CU UT ==>|workflow_run: completed| GP VU ==>|workflow_run: completed| CR + UT ==>|workflow_run: completed| CC VU -.->|uses: gate-only| UT CU -.->|uses: gate-only| UT @@ -167,7 +177,8 @@ than starting a new one. classDef trig fill:none,stroke-dasharray:2 2 class PUSH,PR,TAG,SCHED,DISPATCH,ISSUES,PRT trig -The double relationship between **Unit Tests** and its four dependants is the +The double relationship between **Unit Tests** and the four dependants that +also call it is the part that reading a single file cannot show, and it is deliberate rather than redundant: each of the four calls ``unit-tests.yml`` with ``gate-only: true`` itself (dotted, above) **only when its own trigger is not** ``workflow_run`` @@ -180,7 +191,7 @@ re-running the gate a second time. ``workflow_run`` Edges ---------------------- -Four in total, found by grepping every ``on:`` block rather than trusting a +Five in total, found by grepping every ``on:`` block rather than trusting a hand-maintained list: * ``.github/workflows/cron-vendor.yml:14-16`` -- **Vendor Update** fires on @@ -191,12 +202,14 @@ hand-maintained list: completion of **Unit Tests**. * ``.github/workflows/create-release.yml:6-9`` -- **Create Release** fires on completion of **Vendor Update**. +* ``.github/workflows/coverage-comment.yml:21-23`` -- **Coverage Comment** fires + on completion of **Unit Tests**. ``workflow_run`` fires for *every* completion of the named workflow -- success, failure, or skipped -- and regardless of what triggered it, -including a pull request's own run of Unit Tests. Every one of the four +including a pull request's own run of Unit Tests. Every one of the five downstream workflows therefore re-checks ``github.event.workflow_run.event`` -and ``.head_branch`` (or, for Create Release, ``.conclusion``) itself before +and ``.head_branch`` (or, for Create Release and Coverage Comment, ``.conclusion``) itself before doing anything with an outward effect; none of the filtering happens in the ``on:`` block. @@ -229,7 +242,7 @@ would test the same commit several times over -- and ``required-checks`` (see `Required Status Checks`_ below), gated out by its own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:950-956``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1040-1046``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -324,6 +337,34 @@ branch's tip under that event -- and takes only the item number from the payload secret it skips with a ``::notice::``. The Status mapping it applies is documented in :doc:`conventions/process`. +Coverage (`#1063 `__) +------------------------------------------------------------------------- + +Coverage is measured inside an existing leg rather than by a workflow of its +own: the ``Python 3.14`` leg of ``test`` (matrix key ``coverage: true``) runs +its unchanged pytest selection under ``coverage run +--rcfile=.github/coverage.toml``. That rcfile repeats +:file:`pyproject.toml`'s ``[tool.coverage.*]`` settings +(``tests/project/test_coverage_rcfile.py`` asserts it) and adds ``parallel`` +and ``patch = ["subprocess"]``, without which only the xdist controller is +measured, not its workers, plus ``core = "ctrace"``: under the default +``sys.monitoring`` core, the suite's purge-and-reimport of pcapkit grew one +module's peak RSS from 0.40 to 4.17 GiB and killed the runner. The leg then writes the total and a per-package +table to the job summary and uploads the HTML report as the ``coverage-html`` +artifact (30 days), with the bare numbers as the ``coverage-summary`` artifact. + +The PR comment comes from **Coverage Comment** (:file:`coverage-comment.yml`), +not from ``unit-tests.yml``. That file is also a reusable workflow, a called +workflow can only narrow the token its caller passes, and all four callers grant +``contents: read``, so a ``pull-requests: write`` job there would break them. +Coverage Comment runs no tests. On ``workflow_run`` it uses the default +branch's file and its own token, so fork pull requests get a comment too. It +treats the artifact as untrusted: it never checks out pull-request code. It +takes the PR number from the API, as the open PR whose head is the run's head +sha, and renders its own Markdown from validated numbers. It edits one comment, +found by a hidden marker, on each push. The first line says when the tests +failed. Neither workflow's coverage step is a required check. + Required Status Checks ---------------------- @@ -340,7 +381,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:1200`` + - ``unit-tests.yml:1290`` * - ``Compat Python 3.10`` - job ``compatibility``, matrix leg ``3.10`` - ``python-compatibility.yml:31,41`` @@ -358,16 +399,16 @@ grepping every workflow file for its name: - ``python-compatibility.yml:31,45`` ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:1200``). ``Compat Python`` is defined by two in +(``unit-tests.yml:1290``). ``Compat Python`` is defined by two in ``python-compatibility.yml``: the required ``compatibility`` job (``:31``, ``Compat Python ${{ matrix.python-version }}``, expanding to the five required legs at ``:41-45``) and the non-required ``compatibility-nightly`` job (``:69``, ``Compat Python 3.15 (scheduled)``, see below). A comment in -``unit-tests.yml`` (``:1061``) mentions the string without defining it. +``unit-tests.yml`` (``:1151``) mentions the string without defining it. ``Required checks passed`` is itself an aggregate, not a single check run -- but it stands in for **17** of the ruleset's originally-named 22 contexts, -not all 22. ``unit-tests.yml``'s own comment (``:1060-1081``) accounts for the +not all 22. ``unit-tests.yml``'s own comment (``:1150-1171``) accounts for the 22: five each for ``test`` and ``integration``, five for the live ``Compat Python 3.10``-``3.14`` contexts (emitted by ``python-compatibility.yml``, a *different* workflow file -- nothing in ``unit-tests.yml`` could ever stand @@ -378,17 +419,17 @@ stopped producing once it became a Python x engine matrix, and two for ``test``/``integration``/dead-``Engines``/``pypcap-parity`` slots -- 5 + 5 + 5 + 2 = 17 -- via its own ``needs: [test, integration, engine-tests, pypcap-parity]`` plus an explicit per-dependency check -(``unit-tests.yml:1199-1225``). The five ``Compat`` contexts stay required +(``unit-tests.yml:1289-1315``). The five ``Compat`` contexts stay required exactly as they are and are listed separately in the table above. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``:1202``), so it is never +``if: ${{ always() && inputs.gate-only != true }}`` (``:1292``), so it is never produced -- and never expected -- on those paths. ``Compat Python 3.10``-``3.14`` are five ordinary matrix legs, not an aggregate: ``python-compatibility.yml``'s own comment (``:37-40``) and ``unit-tests.yml``'s matching one for its own, differently-matrixed -``test``/``integration`` jobs (``:58-61``, ``:138``) both say why 3.15 is +``test``/``integration`` jobs (``:58-61``, ``:228``) both say why 3.15 is excluded from every *required* matrix in this repository -- the ruleset's required-checks list stops at 3.14, so a 3.15 leg cannot gate a merge and is kept advisory (``continue-on-error: true`` in ``python-compatibility.yml``'s diff --git a/tests/project/test_coverage_rcfile.py b/tests/project/test_coverage_rcfile.py new file mode 100644 index 0000000000..0203aa974b --- /dev/null +++ b/tests/project/test_coverage_rcfile.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +"""Tests that :file:`.github/coverage.toml` repeats :file:`pyproject.toml`'s coverage settings. + +#1063: the coverage leg of :file:`.github/workflows/unit-tests.yml` runs with +``--rcfile=.github/coverage.toml``, and coverage reads exactly one config file. +So the ``[tool.coverage.*]`` tables in :file:`pyproject.toml` are *not* read on +that run; the rcfile has to repeat them, plus the two keys the xdist run needs +(``parallel`` and ``patch``). A setting changed in :file:`pyproject.toml` alone +would silently not apply in CI, which is the drift asserted against here. + +""" + +from __future__ import annotations + +import pathlib +import unittest + +try: + import tomllib +except ImportError: # Python 3.10 + tomllib = None # type: ignore[assignment] + +ROOT = pathlib.Path(__file__).resolve().parents[2] +PYPROJECT = ROOT / 'pyproject.toml' +RCFILE = ROOT / '.github' / 'coverage.toml' + + +@unittest.skipIf(tomllib is None, 'tomllib needs Python 3.11+') +class CoverageRcfileTests(unittest.TestCase): + """:file:`.github/coverage.toml` against :file:`pyproject.toml`.""" + + def setUp(self) -> None: + for path in (PYPROJECT, RCFILE): + if not path.is_file(): + self.skipTest(f'{path} is not present, e.g. in a source distribution') + self.project = tomllib.loads(PYPROJECT.read_text(encoding='utf-8'))['tool']['coverage'] + self.rcfile = tomllib.loads(RCFILE.read_text(encoding='utf-8'))['tool']['coverage'] + + def test_every_pyproject_setting_is_repeated_unchanged(self) -> None: + """Each ``[tool.coverage.
]`` key has the same value in the rcfile.""" + for section, options in self.project.items(): + for key, value in options.items(): + with self.subTest(setting=f'{section}.{key}'): + self.assertEqual(self.rcfile.get(section, {}).get(key), value) + + def test_the_xdist_workers_are_measured(self) -> None: + """Without these two keys an ``-n auto`` run measures only the controller.""" + run = self.rcfile['run'] + self.assertIs(run.get('parallel'), True) + self.assertIn('subprocess', run.get('patch', ())) + + def test_the_core_is_ctrace(self) -> None: + """sysmon grew one module's peak RSS from 0.40 to 4.17 GiB and killed the runner (#1064).""" + self.assertEqual(self.rcfile['run'].get('core'), 'ctrace') + + +if __name__ == '__main__': + unittest.main()