From 4c3ce4b86cdb64da93b741df7177c9cb2a492046 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Tue, 20 Oct 2026 14:17:34 -0400 Subject: [PATCH] ci: move the repeated workflow shell into util scripts (#1967) - util/leg_summary.py: every `KTLO summary` (unit-tests.yml's test, integration and pypcap-parity, python-compatibility.yml's compatibility) and experimental.yml's `Experimental summary` run it in place of the jq loop, on the image's /usr/bin/python3; the tier words the warning. - util/classify_changes.py: `Classify the diff` in both `changes` jobs runs it in place of the inline heredoc; their checkout runs on every event, as the script is in the tree. - util/coverage_report.py: `Report coverage`'s data check (#1387), combine/report/html/json and summary table, with a stage argument. - Pins follow the steps to the scripts; create-release.yml's inline KTLO summary is run against the script and must match it. - workflows.rst: a Workflow Scripts section, by job and step name; shifted unit-tests.yml line citations renumbered. tests/project: 1336 passed, 71 skipped. --- .github/workflows/experimental.yml | 60 +---- .github/workflows/python-compatibility.yml | 11 +- .github/workflows/unit-tests.yml | 181 ++------------ docs/source/contributing/releasing.rst | 4 +- docs/source/contributing/workflows.rst | 59 ++++- tests/project/test_coverage_data_guard.py | 36 +-- .../project/test_ktlo_legs_stay_green_1672.py | 75 ++++-- tests/project/test_unit_tests_queue.py | 44 ++-- tests/project/test_workflow_scripts_1967.py | 233 ++++++++++++++++++ util/classify_changes.py | 76 ++++++ util/coverage_report.py | 158 ++++++++++++ util/leg_summary.py | 74 ++++++ 12 files changed, 729 insertions(+), 282 deletions(-) create mode 100644 tests/project/test_workflow_scripts_1967.py create mode 100644 util/classify_changes.py create mode 100644 util/coverage_report.py create mode 100644 util/leg_summary.py diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index 1b2067a013..0912917665 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -121,10 +121,11 @@ jobs: id: read run: python3 util/python_versions.py --github-output - # unit-tests.yml's `changes`, whose comment says how a file is classified; the - # script is the same, byte for byte (tests/project/test_sweeps_on_latest_only_1732.py - # pins it). This workflow is never called, so its own pull_request run is every - # pull_request run. + # unit-tests.yml's `changes`: both run util/classify_changes.py, whose docstring + # says how a file is classified (#1967), with the same step + # (tests/project/test_sweeps_on_latest_only_1732.py pins it). This workflow is + # never called, so its own pull_request run is every pull_request run. Every + # event checks out, as the script is in the tree. changes: name: Classify changed files needs: current @@ -136,7 +137,6 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - if: ${{ github.event_name == 'pull_request' }} with: fetch-depth: 2 @@ -144,41 +144,7 @@ jobs: id: classify env: OWN_PR: ${{ github.event_name == 'pull_request' }} - run: | - python3 - <<'PY' - import os - import subprocess - - CODE_DIRS = ('.github/', 'pcapkit/', 'tests/', 'util/', 'examples/') - - def is_docs(path): - if path.startswith(CODE_DIRS): - return False - return path.startswith('docs/') or path.endswith('.md') - - code = 'true' - if os.environ['OWN_PR'] != 'true': - print("Not this workflow's own pull_request run: everything runs.") - else: - # -z: names are NUL-terminated, so a name holding whitespace - # (or one git would otherwise quote) stays one path. - diff = subprocess.run(['git', 'diff', '--name-only', '-z', '--no-renames', 'HEAD^1', 'HEAD'], - capture_output=True, text=True) - files = [path for path in diff.stdout.split('\0') if path] - if diff.returncode != 0 or not files: - print(f'::warning title=Could not list the changed files::git diff exited {diff.returncode} ' - f'with {len(files)} files: {diff.stderr.strip()}. Treating the diff as code.') - else: - code_files = [path for path in files if not is_docs(path)] - print(f'{len(files)} changed files, {len(code_files)} of them code:') - for path in files: - print(f" {'code' if path in code_files else 'docs'} {path}") - if not code_files: - code = 'false' - print(f'code={code}') - with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as file: - print(f'code={code}', file=file) - PY + run: python3 util/classify_changes.py # Its legs never block, by the KTLO mechanics: every step that can fail has # `continue-on-error: true`, so the job concludes success, and `Experimental @@ -624,7 +590,8 @@ jobs: echo "$LEG: tests exited ${code} after ${elapsed} s, scope ${scope}, ${oom} OOM kill(s), ${errors} MemoryError line(s), ${crashed:-0} worker(s) down; ${last}." # #1660: an experimental leg's report, as unit-tests.yml `test`'s `KTLO - # summary` is a KTLO leg's, and for the same reasons. + # summary` is a KTLO leg's, and for the same reasons: the same script, + # util/leg_summary.py, words its warning for this tier (#1967). - name: Experimental summary if: ${{ !cancelled() }} continue-on-error: true @@ -632,13 +599,4 @@ jobs: env: LEG: Python ${{ matrix.python-version }} (experimental, non-blocking) STEPS: ${{ toJSON(steps) }} - run: | - failed=$(jq -r 'to_entries[] | select(.value.outcome == "failure") | .key' <<< "$STEPS") - if [ -z "$failed" ]; then - echo "$LEG: no step failed." - exit 0 - fi - while IFS= read -r id; do - echo "::warning title=${id} failed on ${LEG//,/%2C}::Experimental legs never block (#1660), so this job still succeeds; see the ${id} step's log." - echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" - done <<< "$failed" + run: /usr/bin/python3 util/leg_summary.py experimental diff --git a/.github/workflows/python-compatibility.yml b/.github/workflows/python-compatibility.yml index 2b3e1d3595..3b65e6cb37 100644 --- a/.github/workflows/python-compatibility.yml +++ b/.github/workflows/python-compatibility.yml @@ -131,13 +131,4 @@ jobs: env: LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} STEPS: ${{ toJSON(steps) }} - run: | - failed=$(jq -r 'to_entries[] | select(.value.outcome == "failure") | .key' <<< "$STEPS") - if [ -z "$failed" ]; then - echo "$LEG: no step failed." - exit 0 - fi - while IFS= read -r id; do - echo "::warning title=${id} failed on ${LEG//,/%2C}::KTLO legs never block (#1633), so this job still succeeds (#1672); see the ${id} step's log." - echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" - done <<< "$failed" + run: /usr/bin/python3 util/leg_summary.py ktlo diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6242c7de73..fee10978b5 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -279,7 +279,9 @@ jobs: python util/cap_file_size.py python -m pytest "$@" fi - # The table goes to the job summary. The numbers also go to an artifact + # util/coverage_report.py (#1967): it fails, posting no number, unless every + # measured process saved its data (#1387), then combines it. The table goes + # to the job summary. The numbers also go to an artifact # that coverage-comment.yml renders as a PR comment once this run # completes; that comment cannot be posted from here, because this file is # also a reusable workflow and its callers grant only `contents: read`. @@ -292,78 +294,7 @@ jobs: SHA: ${{ github.sha }} PYTHON: ${{ matrix.python-version }} TESTS_OUTCOME: ${{ steps.tests.outcome }} - run: | - set -euo pipefail - # #1387: a worker killed before its atexit save leaves no .coverage.* - # file, and combine then reports the rest as the whole. So fail, and - # post no number, unless the controller and every xdist worker (its - # direct children in the log above) left one. - python - <<'PY' - import glob, os, re, sys - parent, cmd, atexit_ = {}, {}, {} - log = 'coverage-processes.log' - for line in open(log, errors='replace').readlines() if os.path.exists(log) else []: - if m := re.match(r' *(\d+)\.\w+: New process(: cmd| parent pid): (.*)', line): - (parent if m[2] == ' parent pid' else cmd)[m[1]] = m[3] - elif m := re.match(r' *(\d+)\.\w+: (atexit|sigterm): pid: \d+, (.*)', line): - atexit_[m[1]] = f'{m[2]}: {m[3]}' - root = [p for p, c in cmd.items() if "'run', '--rcfile=.github/coverage.toml'" in c] - workers = sorted(p for p, q in parent.items() if root and q == root[0]) - saved = set(re.findall(r'\.pid(\d+)\.', ' '.join(glob.glob('.coverage.*')))) - missing = [p for p in root + workers if p not in saved] - print(f'controller {root}, {len(workers)} xdist workers, missing data: {missing}') - # #1423: which process it was, and whether coverage's atexit handler ran - # in it at all -- a pid with no `atexit:` line died on a signal before - # saving, one with such a line reached the save and lost it afterwards. - # Those are different bugs, and the log that distinguishes them is kept - # as an artifact by the step below rather than only read here. - for p in missing: - print(f' pid {p}: cmd {cmd.get(p, "")}') - print(f' pid {p}: {atexit_.get(p, "no atexit/sigterm entry -- died before saving")}') - if len(root) != 1 or not workers or missing: - why = f'no data file from pid {missing}' if missing else 'the process log is missing or unreadable' - with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as summary: - print(f'**Coverage not reported:** {why} (#1387); a total would under-count.', - file=summary) - print(f'::error::Coverage data is incomplete: {why}.') - sys.exit(1) - PY - python -m coverage combine --rcfile=.github/coverage.toml - python -m coverage report --rcfile=.github/coverage.toml - python -m coverage html --rcfile=.github/coverage.toml -d coverage-html - python -m coverage json --rcfile=.github/coverage.toml -o coverage.json - mkdir -p coverage-comment - # Writes the numbers as JSON for coverage-comment.yml, which renders - # its own Markdown from them rather than posting text from this run. - python - <<'PY' >> "$GITHUB_STEP_SUMMARY" - import collections, json, os - data = json.load(open('coverage.json')) - keys = ('num_statements', 'missing_lines', 'num_branches', 'num_partial_branches', - 'covered_lines', 'covered_branches') - packages = collections.defaultdict(collections.Counter) - for path, info in data['files'].items(): - parts = path.split('/') - name = '/'.join(parts[:2]) if len(parts) > 2 else 'pcapkit (top level)' - packages[name].update({k: info['summary'][k] for k in keys}) - def pct(c): - total = c['num_statements'] + c['num_branches'] - return 100.0 * (c['covered_lines'] + c['covered_branches']) / total if total else 100.0 - rows = [[name, c['num_statements'], c['missing_lines'], c['num_branches'], - c['num_partial_branches'], round(pct(c), 2)] for name, c in sorted(packages.items())] - total = round(data['totals']['percent_covered'], 2) - tests = os.environ['TESTS_OUTCOME'] - json.dump({'total': total, 'tests': tests, 'packages': rows, 'python': os.environ['PYTHON']}, - open('coverage-comment/coverage-summary.json', 'w')) - status = '' if tests == 'success' else f' -- **tests {tests}**, so this is partial' - print(f"**Coverage: {total:.2f}%**{status} (unit tier, Python {os.environ['PYTHON']}, `{os.environ['SHA'][:9]}`)") - print() - print('| Package | Statements | Missed | Branches | Partial | Cover |') - print('|---|--:|--:|--:|--:|--:|') - for row in rows: - print('| `{}` | {} | {} | {} | {} | {:.2f}% |'.format(*row)) - print() - print('Per-file detail: the `coverage-html` artifact of this run.') - PY + run: python util/coverage_report.py # #1423: the guard above fires on runs whose tests all pass, and the only # record of what became of the lost process is this log -- it carries each @@ -410,14 +341,16 @@ jobs: retention-days: 14 # #1672: the KTLO leg's report, and the last step of every KTLO matrix - # job, in one script. Each step above that can fail on a KTLO leg has an - # `id` and `continue-on-error: ${{ matrix.ktlo == true }}`, so its failure - # leaves the job, and `required-checks`, at success. This step makes the - # failure visible again: one warning and one job-summary line per failed - # step. `toJSON(steps)` holds each step above with an `id`, and its - # `outcome` is the result before `continue-on-error` applied. jq is on - # every Ubuntu runner image. A step masked on every leg, as the timings - # upload is, is not KTLO's to report. Not removable here: the error the + # job, in one script, util/leg_summary.py (#1967). Each step above that + # can fail on a KTLO leg has an `id` and `continue-on-error: ${{ + # matrix.ktlo == true }}`, so its failure leaves the job, and + # `required-checks`, at success. This step makes the failure visible + # again: one warning and one job-summary line per failed step. + # `toJSON(steps)` holds each step above with an `id`, and its `outcome` is + # the result before `continue-on-error` applied. It runs on the image's + # /usr/bin/python3, not the leg's, which may have failed to set up. A + # step masked on every leg, as the timings upload is, is not KTLO's to + # report. Not removable here: the error the # runner itself records for a failed `run:` step, `Process completed with # exit code N.`, whatever its `continue-on-error` (actions/runner # src/Runner.Worker/Handlers/ScriptHandler.cs:354, at 5f0b3ddcc), and an @@ -429,16 +362,7 @@ jobs: env: LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} STEPS: ${{ toJSON(steps) }} - run: | - failed=$(jq -r 'to_entries[] | select(.value.outcome == "failure") | .key' <<< "$STEPS") - if [ -z "$failed" ]; then - echo "$LEG: no step failed." - exit 0 - fi - while IFS= read -r id; do - echo "::warning title=${id} failed on ${LEG//,/%2C}::KTLO legs never block (#1633), so this job still succeeds (#1672); see the ${id} step's log." - echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" - done <<< "$failed" + run: /usr/bin/python3 util/leg_summary.py ktlo # The KTLO tier (#1633): end-of-life versions that still build and test but # never block. #1738, the owner's option A: their legs are this job's, not @@ -665,16 +589,7 @@ jobs: env: LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} STEPS: ${{ toJSON(steps) }} - run: | - failed=$(jq -r 'to_entries[] | select(.value.outcome == "failure") | .key' <<< "$STEPS") - if [ -z "$failed" ]; then - echo "$LEG: no step failed." - exit 0 - fi - while IFS= read -r id; do - echo "::warning title=${id} failed on ${LEG//,/%2C}::KTLO legs never block (#1633), so this job still succeeds (#1672); see the ${id} step's log." - echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" - done <<< "$failed" + run: /usr/bin/python3 util/leg_summary.py ktlo # `integration`'s KTLO legs, apart for the reasons on `test-ktlo` (#1738). integration-ktlo: @@ -1407,16 +1322,7 @@ jobs: env: LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} STEPS: ${{ toJSON(steps) }} - run: | - failed=$(jq -r 'to_entries[] | select(.value.outcome == "failure") | .key' <<< "$STEPS") - if [ -z "$failed" ]; then - echo "$LEG: no step failed." - exit 0 - fi - while IFS= read -r id; do - echo "::warning title=${id} failed on ${LEG//,/%2C}::KTLO legs never block (#1633), so this job still succeeds (#1672); see the ${id} step's log." - echo "- ${LEG}: step \`${id}\` failed; non-blocking, so this job still succeeds." >> "$GITHUB_STEP_SUMMARY" - done <<< "$failed" + run: /usr/bin/python3 util/leg_summary.py ktlo # `pypcap-parity`'s KTLO legs, apart for the reasons on `test-ktlo` (#1738). # Same apt set and cache key; `pypcap-parity`'s supported leg saves it. @@ -1702,22 +1608,18 @@ jobs: # Which of a pull request's changed files are code. This decides # whether the expensive legs run at all (see `required-checks` for how their - # skips are then judged), so it fails safe: anything it cannot classify is - # code. - # - # A file is docs iff it is under `docs/` or ends in `.md`, and is not under - # `.github/`, `pcapkit/`, `tests/`, `util/` or `examples/` -- so a Markdown - # file in one of those (a PR template, a test fixture) is code, and so is - # every other file (`pyproject.toml`, `setup.py`, ...). The diff is the PR - # merge commit against its first parent, which is exactly what merging would - # add; `--no-renames` lists both sides of a rename, so moving a code file - # under `docs/` is still a code change. + # skips are then judged). util/classify_changes.py, which experimental.yml's + # `changes` runs too (#1967), says how a file is classified, and fails safe: + # anything it cannot classify is code. # # Only this workflow's own `pull_request` run is classified. Every other # event reports `code: true`, including a `workflow_call` from a caller # triggered by a pull request: inside a called workflow `github.event_name` # is the caller's, but `github.workflow` is the caller's name too (see the # concurrency comment at the top), and no caller is named "Unit Tests". + # Every event checks out, as the script is in the tree; only the pull + # request's merge commit needs its first parent, but `fetch-depth: 2` costs + # the others one commit. changes: name: Classify changed files if: ${{ inputs.gate-only != true }} @@ -1728,7 +1630,6 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - if: ${{ github.event_name == 'pull_request' && github.workflow == 'Unit Tests' }} with: fetch-depth: 2 @@ -1736,41 +1637,7 @@ jobs: id: classify env: OWN_PR: ${{ github.event_name == 'pull_request' && github.workflow == 'Unit Tests' }} - run: | - python3 - <<'PY' - import os - import subprocess - - CODE_DIRS = ('.github/', 'pcapkit/', 'tests/', 'util/', 'examples/') - - def is_docs(path): - if path.startswith(CODE_DIRS): - return False - return path.startswith('docs/') or path.endswith('.md') - - code = 'true' - if os.environ['OWN_PR'] != 'true': - print("Not this workflow's own pull_request run: everything runs.") - else: - # -z: names are NUL-terminated, so a name holding whitespace - # (or one git would otherwise quote) stays one path. - diff = subprocess.run(['git', 'diff', '--name-only', '-z', '--no-renames', 'HEAD^1', 'HEAD'], - capture_output=True, text=True) - files = [path for path in diff.stdout.split('\0') if path] - if diff.returncode != 0 or not files: - print(f'::warning title=Could not list the changed files::git diff exited {diff.returncode} ' - f'with {len(files)} files: {diff.stderr.strip()}. Treating the diff as code.') - else: - code_files = [path for path in files if not is_docs(path)] - print(f'{len(files)} changed files, {len(code_files)} of them code:') - for path in files: - print(f" {'code' if path in code_files else 'docs'} {path}") - if not code_files: - code = 'false' - print(f'code={code}') - with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as file: - print(f'code={code}', file=file) - PY + run: python3 util/classify_changes.py # #1954, the owner's option 3: the project pins, once per push. They are the # tests/project modules whose run does not vary by interpreter -- each reads diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index b657ea9179..c4dad486d3 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -52,7 +52,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s (``:494-514``) asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job (``:581-597``) calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1871-1946``). +subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1738-1813``). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job requires it to have succeeded (``:621,808,974,1210``), so a @@ -62,7 +62,7 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Other checks run unconditionally on the same path, though none is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1675-1701``) fails outright if +process: the ``changelog`` job (``unit-tests.yml:1581-1607``) fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and the release-body step warns, without failing, if that entry's heading still reads "unreleased" diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 520510f201..4bfa86e4aa 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -329,7 +329,7 @@ the same commit several times over. It also skips ``changes``, Checks`_ below), which are gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no ``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1663-1669``): ``create-release.yml`` feeds +comment (``unit-tests.yml:1569-1575``): ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -463,8 +463,11 @@ and re-importing pcapkit, one module's peak RSS was 4.17 GiB under the default killed the runner. ``sigterm = true`` and a ``pytest_sessionfinish`` hook in :file:`tests/conftest.py` that saves each xdist worker's data early keep a worker that dies during shutdown from losing its file -(`#1423 `__). The leg then -writes the total and a per-package table to the job summary and uploads the HTML +(`#1423 `__). The leg's +``Report coverage`` step then runs :file:`util/coverage_report.py`, which fails, +posting no total, unless the controller and every xdist worker saved a data file +(`#1387 `__), and otherwise +writes the total and a per-package table to the job summary. The leg uploads the HTML report as the ``coverage-html`` artifact (30 days), with the bare numbers as the ``coverage-summary`` artifact and the per-process debug log as ``coverage-processes-log``. @@ -639,7 +642,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:2037`` + - ``unit-tests.yml:1904`` * - ``Compat Python 3.11`` - job ``compatibility``, matrix leg ``3.11`` - ``python-compatibility.yml:60``, ``.github/python-versions.toml:20`` @@ -661,22 +664,22 @@ which added ``Compat Python 3.15`` and removed ``Compat Python 3.10``. Moving a between the tiers below means editing the ruleset in the same change. ``Required checks passed`` is defined by exactly one job -(``unit-tests.yml:2037``). ``Compat Python`` is defined by one, the +(``unit-tests.yml:1904``). ``Compat Python`` is defined by one, the ``compatibility`` job in ``python-compatibility.yml`` (``:60``), whose matrix (``:74``) comes from :file:`.github/python-versions.toml`: the required legs are its supported tier (``python-versions.toml:20-24``), and its KTLO legs (``python-versions.toml:16-19``) have names ending in ``(KTLO)``, so no required context matches one. A comment at -``unit-tests.yml:1949`` mentions the string without defining it. +``unit-tests.yml:1816`` mentions the string without defining it. ``Required checks passed`` is an aggregate, not a single check run. The ruleset requires it and the ``Compat Python 3.11``-``3.15`` legs, nothing else, as -``unit-tests.yml``'s own comment (``unit-tests.yml:1948-1958``) records. It +``unit-tests.yml``'s own comment (``unit-tests.yml:1815-1825``) records. It ``needs:`` the gating jobs of ``unit-tests.yml`` -- ``test``, ``integration``, ``engine-tests`` and ``pypcap-parity``, plus ``project-tests``, the one run of the project pins, and ``changes`` for the docs-only path -- and checks each result explicitly -(``unit-tests.yml:2036-2092``). Rulesets match check names literally, with no +(``unit-tests.yml:1903-1959``). Rulesets match check names literally, with no wildcard, so requiring the matrix cells one by one would mean editing the ruleset whenever a matrix changes; the aggregate keeps the required list fixed. It accepts the gating jobs as ``skipped`` only on a docs-only pull request, where @@ -685,7 +688,7 @@ It accepts the gating jobs as ``skipped`` only on a docs-only pull request, wher ``python-compatibility.yml``, and a job cannot ``needs:`` a job in another workflow file. This job runs on Unit Tests' own ``push``/``pull_request`` triggers; the ``gate-only: true`` reusable calls documented above skip it via -``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:2039``), +``if: ${{ always() && inputs.gate-only != true }}`` (``unit-tests.yml:1906``), so it is never produced -- and never expected -- on those paths. .. _ktlo-tier: @@ -709,14 +712,15 @@ and keeps its legs' check names, ``Python 3.10 (KTLO)`` and so on. In with supported legs. Every job holding KTLO legs works the same way: * Every step that can fail on a KTLO leg has an ``id`` and - ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:443-460``), + ``continue-on-error: ${{ matrix.ktlo == true }}`` (``unit-tests.yml:367-384``), unless it is masked on every leg, as the timings uploads are. Its ``outcome`` is then ``failure`` but its ``conclusion`` ``success``, so the job, its check run and ``required-checks`` read ``success``. A step that needs an earlier one's work waits on that step's ``outcome``: a leg that cannot install stops there, and a ``pypi`` leg stops at its first failure. * The last step, ``KTLO summary``, runs on KTLO legs only and reports each failed - ``outcome`` as a warning and a job-summary line. A step's own annotations take + ``outcome`` as a warning and a job-summary line, through + :file:`util/leg_summary.py` (see `Workflow Scripts (#1967)`_). A step's own annotations take ``ANNOTATION_LEVEL``: ``warning`` on a KTLO leg, ``error`` on a supported one. ``engine-tests``' KTLO cells, a step of their own with ``continue-on-error: true``, report the same way. @@ -1080,6 +1084,39 @@ above). ``unit-tests.yml`` runs pytest, unittest or :file:`util/run_unittest_leg.py` without the script, and runs the script against a small limit. +Workflow Scripts (`#1967 `__) +--------------------------------------------------------------------------------- + +Shell that several steps repeat lives in a :file:`util/` script, which the steps +run, rather than in a composite action: a composite action's inner steps cannot +take ``timeout-minutes``, so each calling step keeps its own, with its ``shell:``, +``if:``, ``env:`` and ``continue-on-error``. Each script's docstring says what it +does; each has tests under :file:`tests/project/` that run it. + +.. list-table:: + :header-rows: 1 + + * - Script + - Run by + * - :file:`util/leg_summary.py` + - ``KTLO summary`` in ``unit-tests.yml``'s ``test``, ``integration`` and + ``pypcap-parity`` (and so their KTLO twins) and ``python-compatibility.yml``'s + ``compatibility``, as ``leg_summary.py ktlo``; ``Experimental summary`` in + ``experimental.yml``'s ``experimental``, as ``leg_summary.py experimental``. + * - :file:`util/classify_changes.py` + - ``Classify the diff`` in the ``changes`` job of ``unit-tests.yml`` and of + ``experimental.yml``. + * - :file:`util/coverage_report.py` + - ``Report coverage`` in ``unit-tests.yml``'s ``test``, on the coverage leg. + +The summaries run on the image's ``/usr/bin/python3`` rather than the leg's +interpreter, which may have failed to set up or be too broken to run anything, so +they need only the standard library, as ``classify_changes.py`` does on the +runner's ``python3``. Both ``changes`` jobs check out on every event, since the +script is in the tree. ``create-release.yml``'s ``pypi`` job keeps its own copy of +the KTLO summary's shell until its shell moves to scripts too; a project pin runs +that copy and the script on the same steps and requires the same output. + .. _Action Pins: Action Pins (`#1968 `__) diff --git a/tests/project/test_coverage_data_guard.py b/tests/project/test_coverage_data_guard.py index 8c69fb0c4b..9e9e45d877 100644 --- a/tests/project/test_coverage_data_guard.py +++ b/tests/project/test_coverage_data_guard.py @@ -6,13 +6,14 @@ process that dies before coverage's atexit save leaves no ``.coverage.*`` file, and ``coverage combine`` reports what is left as the whole. So the ``Run unit tests`` step of :file:`.github/workflows/unit-tests.yml` logs every measured -process (``COVERAGE_DEBUG=pid,process``), and ``Report coverage`` runs a check -before ``coverage combine`` that fails the step, posting no total, unless the -controller and each of its direct children -- the xdist workers -- saved a file. +process (``COVERAGE_DEBUG=pid,process``), and ``Report coverage`` runs +:file:`util/coverage_report.py` (#1967), whose check, before ``coverage combine``, +fails the step, posting no total, unless the controller and each of its direct +children -- the xdist workers -- saved a file. -The check is extracted from the workflow and run here, both on hand-written -logs and on one written by a real ``coverage run``, so the format it parses is -the one coverage emits rather than one assumed. +The script's ``check`` stage is run here, both on hand-written logs and on one +written by a real ``coverage run``, so the format it parses is the one coverage +emits rather than one assumed. """ @@ -25,7 +26,6 @@ import subprocess # nosec: B404 import sys import tempfile -import textwrap import types import typing import unittest @@ -37,6 +37,7 @@ ROOT = pathlib.Path(__file__).resolve().parents[2] WORKFLOW = ROOT / '.github' / 'workflows' / 'unit-tests.yml' RCFILE = ROOT / '.github' / 'coverage.toml' +SCRIPT = ROOT / 'util' / 'coverage_report.py' LOG = 'coverage-processes.log' CONTROLLER_CMD = ("['/x/coverage/__main__.py', 'run', '--rcfile=.github/coverage.toml', " "'-m', 'pytest', '-q', '-n', 'auto']") @@ -50,12 +51,6 @@ def _report_step() -> str: return text.split('- name: Report coverage', 1)[1].split('\n - name:', 1)[0] -def _guard() -> str: - """The Python source of the check, the first heredoc in ``Report coverage``.""" - body = _report_step().split("python - <<'PY'\n", 1)[1].split('\n PY\n', 1)[0] - return textwrap.dedent(body) - - def _has_coverage() -> bool: try: import coverage # pylint: disable=import-outside-toplevel @@ -80,9 +75,9 @@ def setUp(self) -> None: self.summary = self.tempdir / 'summary.md' self.summary.touch() - def run_guard(self) -> 'subprocess.CompletedProcess[str]': + def run_guard(self, stage: str = 'check') -> 'subprocess.CompletedProcess[str]': env = dict(os.environ, GITHUB_STEP_SUMMARY=str(self.summary)) - return subprocess.run([sys.executable, '-c', _guard()], cwd=self.tempdir, env=env, # nosec: B603 + return subprocess.run([sys.executable, str(SCRIPT), stage], cwd=self.tempdir, env=env, # nosec: B603 capture_output=True, text=True, check=False, timeout=60) def write(self, log: str, pids: 'list[int]') -> None: @@ -98,8 +93,15 @@ def assert_fails(self, why: str) -> None: def test_the_check_runs_before_combine(self) -> None: """``combine`` deletes the data files it merges, so the check must come first.""" - step = _report_step() - self.assertLess(step.index('#1387'), step.index('coverage combine')) + self.assertIn('run: python util/coverage_report.py\n', _report_step()) + log = _entry(100, 1, CONTROLLER_CMD) + _entry(101, 100, "['-c']") + _entry(102, 100, "['-c']") + self.write(log, [100, 101]) + result = self.run_guard('all') + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("::error::Coverage data is incomplete: no data file from pid ['102']", result.stdout) + # Nothing was combined: both data files are where the tests left them. + self.assertEqual(len(list(self.tempdir.glob('.coverage.*'))), 2) + self.assertFalse((self.tempdir / '.coverage').exists()) tests = WORKFLOW.read_text(encoding='utf-8').split('- name: Run unit tests', 1)[1] self.assertRegex(tests.split('- name:', 1)[0], r'COVERAGE_DEBUG=pid,process COVERAGE_DEBUG_FILE="\$PWD/' + re.escape(LOG)) diff --git a/tests/project/test_ktlo_legs_stay_green_1672.py b/tests/project/test_ktlo_legs_stay_green_1672.py index e8c6ae8199..0765471f55 100644 --- a/tests/project/test_ktlo_legs_stay_green_1672.py +++ b/tests/project/test_ktlo_legs_stay_green_1672.py @@ -18,7 +18,9 @@ ``continue-on-error`` step's failed ``outcome`` concluding ``success`` -- so each step's failure is tried on every leg instead of read off the text. Expressions go through :mod:`tests.project.test_release_gates`'s evaluator, and the scripts that -write annotations are executed. +write annotations are executed. Both summaries run :file:`util/leg_summary.py` +(#1967), and :file:`create-release.yml`'s ``pypi`` keeps an inline copy of its +shell, which is run against the script on the same steps. #1660 added an experimental tier, free-threaded CPython and GraalPy, as legs of ``test`` only; #1732 moved them to a job of their own, ``experimental``, which @@ -40,6 +42,7 @@ import re import shutil import subprocess +import sys import tempfile import unittest from typing import TYPE_CHECKING, NamedTuple @@ -51,6 +54,7 @@ workflow_commands) from tests.project.test_python_versions_1660 import ROOT, live_outputs, prose, resolve from tests.project.test_release_gates import NULL, STATUS, evaluate_condition, evaluate_expression +from util.leg_summary import REASONS if TYPE_CHECKING: from typing import Any, Optional @@ -118,6 +122,15 @@ #: Why each summary's warning says the job still succeeds; the scripts differ in nothing else. KTLO_REASON = 'KTLO legs never block (#1633), so this job still succeeds (#1672)' EXPERIMENTAL_REASON = 'Experimental legs never block (#1660), so this job still succeeds' +#: Each summary's command (#1967): the image's interpreter, not the leg's, which may +#: have failed to set up. +IMAGE_PYTHON = '/usr/bin/python3' +SUMMARY_RUN = f'{IMAGE_PYTHON} util/leg_summary.py ktlo' +EXPERIMENTAL_SUMMARY_RUN = f'{IMAGE_PYTHON} util/leg_summary.py experimental' +#: The ``KTLO summary`` that keeps its shell inline until #1967's ``create-release.yml`` step. +INLINE_SUMMARY = ('create-release.yml', 'pypi') +#: A ``util/`` script a ``run:`` invokes, whose own annotations are the step's. +UTIL_SCRIPT = re.compile(r'\butil/[\w-]+\.py\b') #: ``experimental``'s steps with a ``timeout-minutes`` of their own (#1660), by ``id``: #: setup-python, the install, the watcher's setup-python and requests (#1938) and the #: tests. #1732: the tests' is GraalPy's unit tier less the round-trip sweeps, 88-113 @@ -272,8 +285,16 @@ def scenarios(name: 'str', job_id: 'str', spec: 'dict[str, Any]', def commands(script: 'str') -> 'set[str]': - """The workflow commands ``script`` writes.""" - return set(COMMAND.findall(script)) + """The workflow commands ``script`` writes, and each ``util/`` script it runs (#1967).""" + found = set(COMMAND.findall(script)) + for path in set(UTIL_SCRIPT.findall(script)): + found |= set(COMMAND.findall((ROOT / path).read_text(encoding='utf-8'))) + return found + + +def run_summary(script: 'str', env: 'dict[str, str]') -> 'subprocess.CompletedProcess[str]': + """A summary step's ``run:`` in the checkout, on this interpreter in place of the image's.""" + return bash(script.replace(IMAGE_PYTHON, sys.executable), env, ROOT) def bash(script: 'str', env: 'dict[str, str]', cwd: 'pathlib.Path') -> 'subprocess.CompletedProcess[str]': @@ -329,7 +350,6 @@ def test_every_failure_it_masks_reaches_the_summary(self) -> None: KTLO's to report. """ - scripts = set() for name, job_id in KTLO_JOBS: spec = job(name, job_id) summary = summary_step(spec) @@ -339,7 +359,8 @@ def test_every_failure_it_masks_reaches_the_summary(self) -> None: self.assertEqual(summary.get('continue-on-error'), KTLO_GUARD) self.assertEqual(summary.get('shell'), 'bash') self.assertEqual(summary.get('env'), SUMMARY_ENV) - scripts.add(summary['run']) + if (name, job_id) != INLINE_SUMMARY: + self.assertEqual(summary['run'], SUMMARY_RUN) blocking = BLOCKS_EVERY_LEG.get((name, job_id), ()) for leg in filter(is_ktlo, legs(name, job_id)): for index, _, record in scenarios(name, job_id, spec, leg): @@ -350,7 +371,6 @@ def test_every_failure_it_masks_reaches_the_summary(self) -> None: self.assertNotEqual(record[-1].outcome, 'skipped') if step.get('continue-on-error') is not True: self.assertIn('id', step) - self.assertEqual(len(scripts), 1, f'the copies of {SUMMARY!r} have drifted apart') class TestAKTLOFailureIsAWarning(unittest.TestCase): @@ -397,16 +417,18 @@ def test_no_step_a_ktlo_leg_runs_writes_an_error(self) -> None: def summary(self, steps: 'dict[str, str]') -> 'tuple[subprocess.CompletedProcess[str], str]': """``KTLO summary`` on 3.7, given each step's ``outcome``; its result and job summary.""" - if shutil.which('jq') is None: # pragma: no cover - self.skipTest('jq, on every Ubuntu runner image, is needed to run KTLO summary') - script = summary_step(job('unit-tests.yml', 'test'))['run'] + return self.run_summary(summary_step(job('unit-tests.yml', 'test'))['run'], steps) + + def run_summary(self, script: 'str', steps: 'dict[str, str]', + inline: 'bool' = False) -> 'tuple[subprocess.CompletedProcess[str], str]': + """``script``, a ``KTLO summary``'s ``run:``, given each step's ``outcome``.""" context = {key: {'outputs': {}, 'outcome': outcome, 'conclusion': 'success' if outcome == 'failure' else outcome} for key, outcome in steps.items()} page = self.tmp / 'summary.md' page.write_text('', encoding='utf-8') - result = bash(script, {'LEG': LEG_37, 'STEPS': json.dumps(context, indent=2), - 'GITHUB_STEP_SUMMARY': str(page)}, self.tmp) + env = {'LEG': LEG_37, 'STEPS': json.dumps(context, indent=2), 'GITHUB_STEP_SUMMARY': str(page)} + result = bash(script, env, self.tmp) if inline else run_summary(script, env) return result, page.read_text(encoding='utf-8') def test_the_summary_warns_once_per_failed_step(self) -> None: @@ -436,6 +458,21 @@ def test_the_summary_is_quiet_when_nothing_failed(self) -> None: self.assertEqual(page, '') self.assertIn(f'{LEG_37}: no step failed.', result.stdout) + def test_the_inline_release_copy_reports_as_the_script_does(self) -> None: + """``pypi``'s ``KTLO summary`` keeps its shell (#1967), so it must not drift from the script.""" + if shutil.which('jq') is None: # pragma: no cover + self.skipTest('jq, on every Ubuntu runner image, is needed to run the inline KTLO summary') + inline = summary_step(job(*INLINE_SUMMARY))['run'] + self.assertNotIn('leg_summary.py', inline) + for steps in ({'checkout': 'success', 'python': 'success', 'tests': 'success'}, + {'checkout': 'success', 'python': 'success', 'install': 'failure', 'tests': 'skipped'}, + {'checkout': 'failure', 'python': 'failure', 'install': 'skipped'}): + with self.subTest(steps=steps): + script, script_page = self.run_summary(SUMMARY_RUN, steps) + shell, shell_page = self.run_summary(inline, steps, inline=True) + self.assertEqual((script.returncode, script.stdout, script_page), + (shell.returncode, shell.stdout, shell_page)) + def engine_cell(self, py: 'str', level: 'Optional[str]') -> 'subprocess.CompletedProcess[str]': """``engine-tests``'s cell script for Scapy on ``py``, whose extra fails to install. @@ -592,10 +629,10 @@ def test_every_failure_it_masks_reaches_the_experimental_summary(self) -> None: self.assertNotEqual(record[self.position].outcome, 'skipped') if step.get('continue-on-error') is not True: self.assertIn('id', step) - # One script with the KTLO summary's, so a fix to either reaches both. - self.assertIn(EXPERIMENTAL_REASON, self.summary['run']) - self.assertEqual(self.summary['run'].replace(EXPERIMENTAL_REASON, KTLO_REASON), - summary_step(job('unit-tests.yml', 'test'))['run']) + # One script with the KTLO summary's, so a fix to either reaches both (#1967). + self.assertEqual(self.summary['run'], EXPERIMENTAL_SUMMARY_RUN) + self.assertEqual(summary_step(job('unit-tests.yml', 'test'))['run'], SUMMARY_RUN) + self.assertEqual(REASONS, {'ktlo': KTLO_REASON, 'experimental': EXPERIMENTAL_REASON}) def test_each_step_names_the_leg_by_its_tier(self) -> None: """``LEG`` in an annotation's title: ``(experimental, non-blocking)``.""" @@ -620,8 +657,8 @@ def test_no_step_an_experimental_leg_runs_writes_an_error(self) -> None: self.assertLessEqual(commands(step.get('run', '')), KTLO_COMMANDS) def test_the_summary_warns_once_per_failed_step(self) -> None: - if shutil.which('bash') is None or shutil.which('jq') is None: # pragma: no cover - self.skipTest('bash and jq, on every Ubuntu runner image, are needed to run the summary') + if shutil.which('bash') is None: # pragma: no cover + self.skipTest('bash is needed to run the summary step') with tempfile.TemporaryDirectory(prefix='pcapkit-1660-') as tmp: for leg in self.legs: name = leg_label(leg) @@ -634,8 +671,8 @@ def test_the_summary_warns_once_per_failed_step(self) -> None: for key, outcome in steps.items()} with self.subTest(python=leg['python-version']): self.assertEqual(interpolate(self.summary['env']['LEG'], {'matrix': leg}), name) - result = bash(self.summary['run'], {'LEG': name, 'STEPS': json.dumps(context), - 'GITHUB_STEP_SUMMARY': str(page)}, pathlib.Path(tmp)) + result = run_summary(self.summary['run'], {'LEG': name, 'STEPS': json.dumps(context), + 'GITHUB_STEP_SUMMARY': str(page)}) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(workflow_commands(result.stdout), [ f'::warning title=install failed on {title}::{EXPERIMENTAL_REASON}; ' diff --git a/tests/project/test_unit_tests_queue.py b/tests/project/test_unit_tests_queue.py index 7fa0b4e85a..7aa05abd68 100644 --- a/tests/project/test_unit_tests_queue.py +++ b/tests/project/test_unit_tests_queue.py @@ -15,9 +15,9 @@ ``engine-tests``'s KTLO cells stay non-blocking through a step-level ``continue-on-error`` rather than a job-level one (#1633), and the job has no ``DPKT`` cell (#1538). -* The classifier, the engine cell loops and the apt retry are executed too: the - classifier against a scratch merge commit, the others with their commands - stubbed on ``PATH``. +* The classifier, :file:`util/classify_changes.py` (#1967), the engine cell loops + and the apt retry are executed too: the classifier against a scratch merge + commit, the others with their commands stubbed on ``PATH``. * Every test outside :file:`tests/project` that reads ``docs/`` or Markdown is run by ``project-tests``, so a docs-only PR cannot break it unseen. @@ -30,11 +30,13 @@ from __future__ import annotations import ast +import importlib.util import os import pathlib import re import shutil import subprocess +import sys import tempfile import textwrap import unittest @@ -47,6 +49,10 @@ ROOT = pathlib.Path(__file__).resolve().parents[2] WORKFLOW = ROOT / '.github' / 'workflows' / 'unit-tests.yml' +#: The ``changes`` classifier, which ``Classify the diff`` runs (#1967). +CLASSIFIER = ROOT / 'util' / 'classify_changes.py' +#: ``Classify the diff``'s command, as the job's checkout root runs it. +CLASSIFY_RUN = ' run: python3 util/classify_changes.py\n' #: The legs a docs-only pull request may skip. LEGS = ('test', 'integration', 'engine-tests', 'pypcap-parity') @@ -156,15 +162,14 @@ def test_an_unclassified_diff_skips_nothing(self) -> None: class TestChangesClassifier(unittest.TestCase): - """``is_docs`` from the ``changes`` job, executed on sample paths.""" + """``is_docs`` from the ``changes`` job's script, executed on sample paths.""" def setUp(self) -> None: - script = run_block(job('changes')) - source = textwrap.dedent(script.split("<<'PY'\n", 1)[1].split("\ncode = 'true'", 1)[0]) - namespace = {} # type: dict[str, object] - exec(compile(source, f'{WORKFLOW}:changes', 'exec'), namespace) # pylint: disable=exec-used - is_docs = namespace['is_docs'] - self.is_docs = is_docs # type: Callable[[str], bool] # type: ignore[assignment] + spec = importlib.util.spec_from_file_location('classify_changes', CLASSIFIER) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + self.is_docs = module.is_docs # type: Callable[[str], bool] def test_docs_and_markdown_are_docs(self) -> None: """``docs/**`` and top-level Markdown are docs.""" @@ -185,9 +190,17 @@ def test_everything_else_is_code(self) -> None: def test_only_this_workflows_own_pull_request_is_classified(self) -> None: """Other events, and callers, always run everything.""" section = job('changes') - self.assertIn("github.event_name == 'pull_request' && github.workflow == 'Unit Tests'", + self.assertIn("OWN_PR: ${{ github.event_name == 'pull_request' && github.workflow == 'Unit Tests' }}", section) - self.assertIn("'--no-renames'", section) + self.assertIn("'--no-renames'", CLASSIFIER.read_text(encoding='utf-8')) + + def test_the_step_runs_the_script_in_a_checkout(self) -> None: + """#1967: the script is in the tree, so every event checks out, with the merge's first parent.""" + section = job('changes') + self.assertEqual(section.count(CLASSIFY_RUN), 1) + checkout = section.split(' - uses: actions/checkout@', 1)[1].split('\n - ', 1)[0] + self.assertNotIn('if:', checkout) + self.assertIn('fetch-depth: 2', checkout) class TestJobGating(unittest.TestCase): @@ -332,7 +345,7 @@ def setUp(self) -> None: super().setUp() if shutil.which('git') is None: # pragma: no cover self.skipTest('git is not available') - self.script = run_block(job('changes')) + self.assertIn(CLASSIFY_RUN, job('changes')) self.repo = self.tmp / 'repo' self.repo.mkdir() self.git('init', '-q', '-b', 'main') @@ -372,8 +385,9 @@ def pr(self, *files: 'str') -> None: def classify(self, own_pr: 'str' = 'true', cwd: 'Optional[pathlib.Path]' = None) -> 'str': """Run the classifier and return the line it wrote to ``GITHUB_OUTPUT``.""" output = self.tmp / 'output' - status = bash(self.script, {'OWN_PR': own_pr, 'GITHUB_OUTPUT': str(output)}, - cwd=str(cwd or self.repo)) + status = subprocess.run([sys.executable, str(CLASSIFIER)], + env=dict(os.environ, OWN_PR=own_pr, GITHUB_OUTPUT=str(output)), + cwd=str(cwd or self.repo), check=False, capture_output=True, text=True).returncode self.assertEqual(status, 0) return output.read_text(encoding='utf-8').strip() diff --git a/tests/project/test_workflow_scripts_1967.py b/tests/project/test_workflow_scripts_1967.py new file mode 100644 index 0000000000..236873a08d --- /dev/null +++ b/tests/project/test_workflow_scripts_1967.py @@ -0,0 +1,233 @@ +# -*- coding: utf-8 -*- +"""#1967: the ``util/`` scripts that replace shell the workflows repeated, run as the steps run them. + +* :file:`util/leg_summary.py`, every ``KTLO summary`` and the ``Experimental + summary``: :class:`TestLegSummary`. The steps themselves are pinned in + :mod:`tests.project.test_ktlo_legs_stay_green_1672`. +* :file:`util/classify_changes.py`, ``Classify the diff`` in both ``changes`` jobs: + :class:`TestClassifyChanges`. Its classification against a real merge commit is + :mod:`tests.project.test_unit_tests_queue`'s. +* :file:`util/coverage_report.py`, ``Report coverage``: :class:`TestCoverageReport`. + Its data check is :mod:`tests.project.test_coverage_data_guard`'s. + +""" + +from __future__ import annotations + +import json +import os +import pathlib +import re +import shutil +import subprocess # nosec: B404 +import sys +import tempfile +import unittest + +from tests._workflows import steps + +ROOT = pathlib.Path(__file__).resolve().parents[2] +UTIL = ROOT / 'util' +#: A leg name holding a comma, which an annotation's title must escape. +LEG = 'Python 3.9 (KTLO, non-blocking)' + + +def steps_json(**outcomes: 'str') -> 'str': + """``toJSON(steps)`` for steps with these outcomes, in this order.""" + return json.dumps({key: {'outputs': {}, 'outcome': outcome, + 'conclusion': 'success' if outcome == 'failure' else outcome} + for key, outcome in outcomes.items()}) + + +def run(script: 'str', *args: 'str', env: 'dict[str, str]', + cwd: 'pathlib.Path') -> 'subprocess.CompletedProcess[str]': + """``python util/