diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 12e26011ad..721ce88ffe 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -508,33 +508,14 @@ jobs: continue-on-error: ${{ matrix.ktlo == true }} run: python -m pip install -e '.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]' - # `shell: bash` rather than the default, because it runs with -o pipefail: - # without it the `tee` below would swallow the generator's exit status. + # util/regenerate_samples.py annotates a failure as a fixture failure, not + # a test one, and warns when the PCAP-NG captures are stand-ins. - name: Regenerate sample captures id: samples if: ${{ steps.install.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} shell: bash - run: | - # Keep a failure here legible as what it is -- the fixtures could not - # be built, so no test has run yet and nothing is wrong with the code - # under test. - if ! python examples/generators/make_samples.py 2>&1 | tee "$RUNNER_TEMP/make-samples.log"; then - echo "::${ANNOTATION_LEVEL} title=Sample fixture generation failed::examples/generators/make_samples.py could not rebuild examples/captures/. This is a fixture-generation failure, not a test failure -- the test suite has not run." - exit 1 - fi - - # examples/generators/pcapng.py fetches captures from the Wireshark - # repository and synthesises stand-ins when the download fails. The - # stand-ins parse, and tests/protocols/test_pcapng_regression.py only - # asserts that extraction succeeds, so a degraded run goes green and - # is otherwise indistinguishable from a clean one. Say which happened. - if grep -q '(download unavailable)' "$RUNNER_TEMP/make-samples.log"; then - echo "::warning title=Sample fixtures degraded::The upstream Wireshark captures were unreachable, so synthesised stand-ins were used. The PCAP-NG tier ran, but not against the upstream bytes." - grep '(download unavailable)' "$RUNNER_TEMP/make-samples.log" - else - echo "Upstream Wireshark captures were used, matching their pinned SHA-256 digests." - fi + run: python util/regenerate_samples.py # See the `test` job above for why this step exists. - name: Report available parallelism @@ -1323,25 +1304,13 @@ jobs: python -m pip install -U pip setuptools wheel python -m pip install -e '.[test,Scapy,DPKT,cli,PyShark,PyPCAP,PyPCAPFile]' - # See the `integration` job above for why this step is shaped the way it - # is. + # See the `integration` job above. - name: Regenerate sample captures id: samples if: ${{ steps.install.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} shell: bash - run: | - if ! python examples/generators/make_samples.py 2>&1 | tee "$RUNNER_TEMP/make-samples.log"; then - echo "::${ANNOTATION_LEVEL} title=Sample fixture generation failed::examples/generators/make_samples.py could not rebuild examples/captures/. This is a fixture-generation failure, not a test failure -- the test suite has not run." - exit 1 - fi - - if grep -q '(download unavailable)' "$RUNNER_TEMP/make-samples.log"; then - echo "::warning title=Sample fixtures degraded::The upstream Wireshark captures were unreachable, so synthesised stand-ins were used. The PCAP-NG tier ran, but not against the upstream bytes." - grep '(download unavailable)' "$RUNNER_TEMP/make-samples.log" - else - echo "Upstream Wireshark captures were used, matching their pinned SHA-256 digests." - fi + run: python util/regenerate_samples.py # See the `test` job above for why this step exists. - name: Report available parallelism @@ -1835,21 +1804,13 @@ jobs: python -m pip install -U pip setuptools wheel python -m pip install -e '.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]' - # See the integration job above for why this step is shaped the way it is. + # See the `integration` job above. This job has no KTLO leg, so a + # failure is always an error. - name: Regenerate sample captures shell: bash - run: | - if ! python examples/generators/make_samples.py 2>&1 | tee "$RUNNER_TEMP/make-samples.log"; then - echo "::error title=Sample fixture generation failed::examples/generators/make_samples.py could not rebuild examples/captures/. This is a fixture-generation failure, not a test failure -- the test suite has not run." - exit 1 - fi - - if grep -q '(download unavailable)' "$RUNNER_TEMP/make-samples.log"; then - echo "::warning title=Sample fixtures degraded::The upstream Wireshark captures were unreachable, so synthesised stand-ins were used. The PCAP-NG tier ran, but not against the upstream bytes." - grep '(download unavailable)' "$RUNNER_TEMP/make-samples.log" - else - echo "Upstream Wireshark captures were used, matching their pinned SHA-256 digests." - fi + env: + ANNOTATION_LEVEL: error + run: python util/regenerate_samples.py # See the `test` job above for why this step exists. - name: Report available parallelism diff --git a/docs/source/contributing/releasing.rst b/docs/source/contributing/releasing.rst index dc219b9953..1e7caccdae 100644 --- a/docs/source/contributing/releasing.rst +++ b/docs/source/contributing/releasing.rst @@ -52,7 +52,7 @@ Editing ``__version__`` by hand without also moving :file:`CITATION.cff`'s asserts the two agree, and ``create-release.yml``'s ``unit-tests`` job calls ``unit-tests.yml`` with ``gate-only: true``, which runs the **full** suite rather than the tiered -subset an ordinary push runs (its ``gate`` job, ``unit-tests.yml:1798-1873``). +subset an ordinary push runs (its ``gate`` job). That gate runs whenever a release will -- a moved ``__version__`` is exactly what makes ``version_check``'s evidence read "not yet published" -- and every publishing job's ``needs:`` requires it to have succeeded, so a @@ -62,8 +62,8 @@ Either run the script, or move both fields by hand in the same commit. This is the **only** edit a person makes to get a release started -- the tag, the Release, and every upload are the workflow's job from here. Other checks run unconditionally on the same path, though none is a step in *this* -process: the ``changelog`` job (``unit-tests.yml:1641-1667``) fails outright if -``CHANGELOG.md`` has drifted from its source entry under +process: the ``changelog`` job's "Check CHANGELOG.md is in step with its source +entry" step fails outright if ``CHANGELOG.md`` has drifted from its source entry under :file:`docs/source/changelog/`, and ``github``'s "Select release body" step warns, without failing, if that entry's heading still reads "unreleased". Each enforces what the changelog convention diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 8a7a6ae37d..ad3eeb6b6a 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -375,8 +375,8 @@ the same commit several times over. It also skips ``changes``, `Project Pins`_ below), and ``required-checks`` (see `Required Status Checks`_ below), which are gated out by their own ``if:`` rather than by having already run. ``changelog`` carries no -``if:`` at all and runs on every path regardless -- deliberately, per its own -comment (``unit-tests.yml:1629-1635``): ``create-release.yml`` feeds +``if:`` at all and runs on every path regardless -- deliberately, per the comment +above the job: ``create-release.yml`` feeds ``CHANGELOG.md`` to the GitHub Release body, so the release path is exactly where a drifted file must not go unchecked. Confirmed on run `36210743295 `__ (a @@ -688,7 +688,7 @@ grepping every workflow file for its name: - Where * - ``Required checks passed`` - job ``required-checks`` - - ``unit-tests.yml:1964`` + - ``unit-tests.yml`` * - ``Compat Python 3.11`` - job ``compatibility``, matrix leg ``3.11`` - ``python-compatibility.yml``, ``.github/python-versions.toml:20`` @@ -709,23 +709,22 @@ That is the list after `#1633 `__), and in ``unit-tests.yml``'s ``test`` and ``test-ktlo``, on PyPy's legs (`#2005 `__). + * - :file:`util/regenerate_samples.py` + - ``Regenerate sample captures`` in ``unit-tests.yml``'s ``integration`` and + ``pypcap-parity`` (and so their KTLO twins), at the job's ``ANNOTATION_LEVEL``, + and ``gate``, at ``error``. The summaries run on the image's ``/usr/bin/python3`` rather than the leg's interpreter, which may have failed to set up or be too broken to run anything, so @@ -1211,7 +1214,9 @@ they need only the standard library, as ``classify_changes.py`` does on the runner's ``python3``. :file:`util/pytest_shards.py` runs on ``/usr/bin/python3`` too, standard library only, so on GraalPy's leg the leg's interpreter runs pytest alone; on PyPy's legs the leg's interpreter runs the script as well, under the -file-size cap every test command of ``unit-tests.yml`` takes. Both ``changes`` jobs check out on every event, since the script is in the +file-size cap every test command of ``unit-tests.yml`` takes. +:file:`util/regenerate_samples.py` is standard library only but runs on the leg's +interpreter, since the generator it runs imports ``pcapkit``. Both ``changes`` jobs check out on every event, since the script is in the tree. ``create-release.yml``'s ``pypi`` job keeps its own copy of the KTLO summary's shell until its shell moves to scripts too; a project pin runs that copy and the script on the same steps and requires the same output. diff --git a/tests/project/test_regenerate_samples_1967.py b/tests/project/test_regenerate_samples_1967.py new file mode 100644 index 0000000000..89ab25c4e2 --- /dev/null +++ b/tests/project/test_regenerate_samples_1967.py @@ -0,0 +1,246 @@ +# -*- coding: utf-8 -*- +"""#1967: :file:`util/regenerate_samples.py`, the ``Regenerate sample captures`` step, run as the step runs it. + +* :class:`TestRegenerateSamples` runs the script against a stand-in generator: a + clean run, a run on stand-in captures, a failed run at either level, and a bad + level. +* :class:`TestTheFormerShellAgrees` runs the shell the steps held before the + script against the same stand-ins and requires the same output, log and status. +* :class:`TestTheStepsRunTheScript` pins the step in the ``integration``, + ``pypcap-parity`` and ``gate`` jobs to the script, and keeps the old inline + shell out of every workflow. + +""" + +from __future__ import annotations + +import os +import pathlib +import shutil +import subprocess # nosec: B404 +import sys +import tempfile +import textwrap +import unittest + +from tests._workflows import WORKFLOWS, job, load, steps + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SCRIPT = ROOT / 'util' / 'regenerate_samples.py' +STEP = 'Regenerate sample captures' +RUN = 'python util/regenerate_samples.py' +#: The jobs whose step runs the script, and the level each takes from where. +LEVELLED_JOBS = ('integration', 'pypcap-parity') +GATE = 'gate' +#: Text only the old inline step held, which no workflow may hold now. +INLINE = ('make_samples.py', 'make-samples.log', '(download unavailable)', 'Sample fixture generation failed') + +FAILED = ('title=Sample fixture generation failed::examples/generators/make_samples.py could not rebuild ' + 'examples/captures/. This is a fixture-generation failure, not a test failure -- the test suite ' + 'has not run.') +DEGRADED = ('::warning title=Sample fixtures degraded::The upstream Wireshark captures were unreachable, so ' + 'synthesised stand-ins were used. The PCAP-NG tier ran, but not against the upstream bytes.') +UPSTREAM = 'Upstream Wireshark captures were used, matching their pinned SHA-256 digests.' + +#: A stand-in generator: its output alternates streams, flushed, so the merged order is fixed. +GENERATOR = textwrap.dedent('''\ + import pathlib, sys + pathlib.Path('ran').write_text('yes') + for stream, text in {lines!r}: + (sys.stderr if stream == 'err' else sys.stdout).write(text + '\\n') + (sys.stderr if stream == 'err' else sys.stdout).flush() + sys.exit({status}) +''') +CLEAN = (('out', 'wrote examples/captures/a.pcap'), ('err', 'a warning on stderr'), + ('out', 'wrote examples/captures/b.pcapng (upstream)')) +DEGRADED_RUN = (('out', 'wrote examples/captures/a.pcap'), + ('out', 'test.pcapng: synthesised (download unavailable)'), + ('err', 'urlopen failed'), + ('out', 'other.pcapng: synthesised (download unavailable)')) + +#: The ``run:`` the ``integration`` and ``pypcap-parity`` steps held before the +#: script; ``gate``'s differed only in a literal ``error`` for ``${ANNOTATION_LEVEL}``. +FORMER_SHELL = textwrap.dedent('''\ + if ! python examples/generators/make_samples.py 2>&1 | tee "$RUNNER_TEMP/make-samples.log"; then + echo "::${ANNOTATION_LEVEL} title=Sample fixture generation failed::examples/generators/make_samples.py could not rebuild examples/captures/. This is a fixture-generation failure, not a test failure -- the test suite has not run." + exit 1 + fi + + if grep -q '(download unavailable)' "$RUNNER_TEMP/make-samples.log"; then + echo "::warning title=Sample fixtures degraded::The upstream Wireshark captures were unreachable, so synthesised stand-ins were used. The PCAP-NG tier ran, but not against the upstream bytes." + grep '(download unavailable)' "$RUNNER_TEMP/make-samples.log" + else + echo "Upstream Wireshark captures were used, matching their pinned SHA-256 digests." + fi +''') # noqa: E501 + + +def merged(lines: 'tuple[tuple[str, str], ...]') -> 'str': + """The generator's standard output and error, merged as ``2>&1`` merges them.""" + return ''.join(text + '\n' for _, text in lines) + + +class _Scratch(unittest.TestCase): + """A scratch checkout root holding a stand-in generator, and a ``RUNNER_TEMP``.""" + + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-1967-samples-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = pathlib.Path(tmp.name) + (self.tmp / 'examples' / 'generators').mkdir(parents=True) + self.runner_temp = self.tmp / 'runner-temp' + self.runner_temp.mkdir() + self.log = self.runner_temp / 'make-samples.log' + + def generator(self, lines: 'tuple[tuple[str, str], ...]', status: 'int' = 0) -> 'None': + (self.tmp / 'examples' / 'generators' / 'make_samples.py').write_text( + GENERATOR.format(lines=lines, status=status), encoding='utf-8') + + def env(self, level: 'str') -> 'dict[str, str]': + env = {key: value for key, value in os.environ.items() if key not in ('ANNOTATION_LEVEL', 'RUNNER_TEMP')} + env.update(ANNOTATION_LEVEL=level, RUNNER_TEMP=str(self.runner_temp), PYTHONIOENCODING='utf-8') + return env + + def script(self, level: 'str' = 'error') -> 'subprocess.CompletedProcess[str]': + """``python util/regenerate_samples.py`` from the scratch root, as the step runs it.""" + return subprocess.run([sys.executable, str(SCRIPT)], cwd=self.tmp, env=self.env(level), # nosec: B603 + capture_output=True, text=True, check=False, timeout=120) + + +class TestRegenerateSamples(_Scratch): + """The generator's output on standard output and in the log, then one verdict.""" + + def test_a_clean_run_says_upstream_was_used(self) -> None: + self.generator(CLEAN) + result = self.script() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, merged(CLEAN) + UPSTREAM + '\n') + self.assertEqual(self.log.read_text(encoding='utf-8'), merged(CLEAN)) + + def test_a_run_on_stand_ins_warns_and_lists_them(self) -> None: + self.generator(DEGRADED_RUN) + result = self.script() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, merged(DEGRADED_RUN) + DEGRADED + '\n' + 'test.pcapng: synthesised (download unavailable)\n' + 'other.pcapng: synthesised (download unavailable)\n') + self.assertEqual(self.log.read_text(encoding='utf-8'), merged(DEGRADED_RUN)) + + def test_a_failed_run_annotates_at_the_level_and_fails(self) -> None: + """A failure ends the step before the degraded check, even with stand-ins in the output.""" + self.generator(DEGRADED_RUN, status=3) + for level in ('error', 'warning'): + with self.subTest(level=level): + result = self.script(level) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(result.stdout, merged(DEGRADED_RUN) + f'::{level} {FAILED}\n') + self.assertEqual(self.log.read_text(encoding='utf-8'), merged(DEGRADED_RUN)) + + def test_a_ktlo_leg_writes_no_error(self) -> None: + """At ``warning``, a KTLO leg's level, no outcome writes an ``::error``.""" + for lines, status in ((CLEAN, 0), (DEGRADED_RUN, 0), (CLEAN, 1)): + with self.subTest(status=status, degraded=lines is DEGRADED_RUN): + self.generator(lines, status) + self.assertNotIn('::error', self.script('warning').stdout) + + def test_a_bad_level_fails_before_the_generator_runs(self) -> None: + self.generator(CLEAN) + for level in ('', 'notice', 'ERROR'): + with self.subTest(level=level): + result = self.script(level) + self.assertEqual(result.returncode, 2) + self.assertEqual(result.stdout, '') + self.assertIn('ANNOTATION_LEVEL', result.stderr) + self.assertFalse((self.tmp / 'ran').exists()) + self.assertFalse(self.log.exists()) + + def test_no_runner_temp_fails_the_step(self) -> None: + self.generator(CLEAN) + env = self.env('error') + del env['RUNNER_TEMP'] + result = subprocess.run([sys.executable, str(SCRIPT)], cwd=self.tmp, env=env, # nosec: B603 + capture_output=True, text=True, check=False, timeout=120) + self.assertNotEqual(result.returncode, 0) + self.assertIn('RUNNER_TEMP', result.stderr) + + def test_it_takes_no_arguments(self) -> None: + result = subprocess.run([sys.executable, str(SCRIPT), 'extra'], cwd=self.tmp, # nosec: B603 + env=self.env('error'), capture_output=True, text=True, check=False, timeout=120) + self.assertEqual(result.returncode, 2) + self.assertFalse((self.tmp / 'ran').exists()) + + +@unittest.skipIf(shutil.which('bash') is None or shutil.which('tee') is None or shutil.which('grep') is None, + 'needs bash, tee and grep') +class TestTheFormerShellAgrees(_Scratch): + """The script prints, logs and exits as the steps' former shell did, under ``shell: bash``.""" + + def former(self, level: 'str') -> 'subprocess.CompletedProcess[str]': + # `python` is the leg's interpreter on the runner; here, this one's. + bin_dir = self.tmp / 'bin' + bin_dir.mkdir(exist_ok=True) + python = bin_dir / 'python' + if not python.exists(): + python.symlink_to(sys.executable) + env = self.env(level) + env['PATH'] = f'{bin_dir}{os.pathsep}{env.get("PATH", "")}' + return subprocess.run(['bash', '--noprofile', '--norc', '-eo', 'pipefail', '-c', FORMER_SHELL], # nosec + cwd=self.tmp, env=env, capture_output=True, text=True, check=False, timeout=120) + + def test_every_outcome_matches(self) -> None: + for lines, status, level in ((CLEAN, 0, 'error'), (DEGRADED_RUN, 0, 'error'), + (DEGRADED_RUN, 3, 'error'), (CLEAN, 1, 'warning')): + with self.subTest(status=status, level=level, degraded=lines is DEGRADED_RUN): + self.generator(lines, status) + old = self.former(level) + old_log = self.log.read_text(encoding='utf-8') + self.log.unlink() + new = self.script(level) + self.assertEqual((new.returncode, new.stdout), (old.returncode, old.stdout)) + self.assertEqual(self.log.read_text(encoding='utf-8'), old_log) + + +class TestTheStepsRunTheScript(unittest.TestCase): + """Each job's ``Regenerate sample captures`` is one call of the script.""" + + def step(self, job_id: 'str') -> 'dict[str, object]': + found = [step for step in steps('unit-tests.yml', job_id) if step.get('name') == STEP] + self.assertEqual(len(found), 1, job_id) + return found[0] + + def test_each_job_runs_the_script(self) -> None: + for job_id in (*LEVELLED_JOBS, GATE): + with self.subTest(job=job_id): + step = self.step(job_id) + self.assertEqual(step['run'], RUN) + self.assertEqual(step['shell'], 'bash') + + def test_the_levelled_jobs_keep_their_guards_and_level(self) -> None: + for job_id in LEVELLED_JOBS: + with self.subTest(job=job_id): + step = self.step(job_id) + self.assertEqual(step['id'], 'samples') + self.assertEqual(step['if'], "${{ steps.install.outcome == 'success' }}") + self.assertEqual(step['continue-on-error'], '${{ matrix.ktlo == true }}') + self.assertNotIn('env', step) + self.assertEqual(job('unit-tests.yml', job_id)['env']['ANNOTATION_LEVEL'], + "${{ matrix.ktlo && 'warning' || 'error' }}") + + def test_gate_keeps_its_error_level(self) -> None: + step = self.step(GATE) + self.assertEqual(step['env'], {'ANNOTATION_LEVEL': 'error'}) + self.assertNotIn('ANNOTATION_LEVEL', job('unit-tests.yml', GATE).get('env') or {}) + self.assertFalse({'id', 'if', 'continue-on-error'} & set(step)) + + def test_no_workflow_keeps_the_inline_shell(self) -> None: + found = [] + for path in sorted(WORKFLOWS.glob('*.yml')): + for job_id, body in load(path).get('jobs', {}).items(): + for step in body.get('steps', []): + found.extend(f'{path.name} {job_id} {step.get("name")!r}: {text}' + for text in INLINE if text in step.get('run', '')) + self.assertEqual(found, []) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_unit_tests_queue.py b/tests/project/test_unit_tests_queue.py index a3af58fd21..ef4aa4c3ab 100644 --- a/tests/project/test_unit_tests_queue.py +++ b/tests/project/test_unit_tests_queue.py @@ -605,30 +605,10 @@ def test_the_scan_is_not_blind(self) -> None: self.assertIn('tests.corekit.test_sentinel_exports_unit', docs_readers()) -#: Every ``unit-tests.yml:N`` or ``unit-tests.yml:N-M`` citation in these documents, -#: as (text the cited first line must contain, text the last line must contain). -#: A citation the table does not account for fails, and so does a table entry no -#: citation matches -- so moving a job without renumbering the prose fails here. -CITATIONS: 'dict[str, list[tuple[str, Optional[str]]]]' = { - 'docs/source/contributing/releasing.rst': [ - (' gate:', 'run: python util/cap_file_size.py python -m pytest -q -n auto --dist load'), - (' changelog:', 'fi'), - ], - 'docs/source/contributing/workflows.rst': [ - ('# Deliberately *not* gated on ``gate-only``', '# this is neither expensive'), - ('name: Required checks passed', None), - ('and a `Compat Python` one per supported version', None), - ("# Ruleset 23497679's required_status_checks", '# them.'), - (' required-checks:', 'echo "All required jobs reported an accepted result."'), - ('if: ${{ always() && inputs.gate-only != true }}', None), - ('# The KTLO tier (#1633)', '`ubuntu-latest` moves to 26.04 from 2026-10-19.'), - ], -} - - #: The workflows the prose cites by job and step name, never by line number: #: trimming a comment in one moves every line below it (#1967). -BY_NAME = ('create-release.yml', 'python-compatibility.yml', 'coverage-comment.yml') +BY_NAME = ('unit-tests.yml', 'create-release.yml', 'python-compatibility.yml', + 'coverage-comment.yml') #: A document that names one of :data:`BY_NAME` may still cite another file bare, #: as ``:N``; each such citation is listed here with the file it does cite, and @@ -676,31 +656,5 @@ def test_no_document_cites_them_by_line_number(self) -> None: self.assertEqual(elsewhere - bare, set(), f'{document} no longer cites these bare') -class TestDocCitationsResolve(unittest.TestCase): - """The prose's ``unit-tests.yml`` line numbers land where it says.""" - - def test_every_citation_lands_on_its_anchor(self) -> None: - """Each ``unit-tests.yml`` citation's range starts and ends on the line its anchor names.""" - lines = WORKFLOW.read_text(encoding='utf-8').splitlines() - for document in documents(): - text = (ROOT / document).read_text(encoding='utf-8') - cited = [(int(m.group(1)), int(m.group(2) or m.group(1))) - for m in re.finditer(r'unit-tests\.yml:(\d+)(?:-(\d+))?', text)] - anchors = CITATIONS.get(document, []) - unmatched = list(anchors) - for first, last in cited: - with self.subTest(document=document, citation=f'{first}-{last}'): - self.assertLessEqual(last, len(lines)) - hit = next((a for a in anchors if a[0] in lines[first - 1] - and (a[1] is None or a[1] in lines[last - 1])), None) - self.assertIsNotNone( - hit, f'{document} cites unit-tests.yml:{first}-{last}, which is ' - f'{lines[first - 1].strip()!r} .. {lines[last - 1].strip()!r}') - if hit in unmatched: - unmatched.remove(hit) - with self.subTest(document=document): - self.assertFalse(unmatched, f'{document}: no citation lands on {unmatched}') - - if __name__ == '__main__': unittest.main() diff --git a/util/regenerate_samples.py b/util/regenerate_samples.py new file mode 100644 index 0000000000..40b9d0ed10 --- /dev/null +++ b/util/regenerate_samples.py @@ -0,0 +1,116 @@ +# -*- coding: utf-8 -*- +"""Rebuild :file:`examples/captures/`, and say whether the fixtures are upstream's (#1967). + +The ``Regenerate sample captures`` step of :file:`.github/workflows/unit-tests.yml`'s +``integration`` and ``pypcap-parity`` jobs (and so their KTLO twins) and of its +``gate`` job runs, from the checkout's root and on the leg's own interpreter:: + + python util/regenerate_samples.py + +with ``ANNOTATION_LEVEL``, ``error`` or ``warning``, and ``RUNNER_TEMP`` in the +environment. It runs :file:`examples/generators/make_samples.py` on the same +interpreter, its standard error merged into its standard output, and copies that +output, as it arrives, both to its own standard output and to +:file:`$RUNNER_TEMP/make-samples.log`. Then: + +* **The generator failed**: one ``ANNOTATION_LEVEL`` annotation saying the + fixtures could not be built, so no test has run and nothing is wrong with the + code under test, and exit 1. +* **It succeeded on stand-ins**: :file:`examples/generators/pcapng.py` fetches + captures from the Wireshark repository and synthesises stand-ins when the + download fails, marking each such line ``(download unavailable)``. The + stand-ins parse, and :file:`tests/protocols/test_pcapng_regression.py` only + asserts that extraction succeeds, so a degraded run goes green and is otherwise + indistinguishable from a clean one. One ``::warning`` says which happened, + followed by each marked line, and the exit status is 0. +* **It succeeded on upstream's captures**: one line saying so, and exit 0. + +An ``ANNOTATION_LEVEL`` other than ``error`` or ``warning`` exits 2 before the +generator runs. The script is standard library only. + +""" + +from __future__ import annotations + +import argparse +import os +import pathlib +import subprocess # nosec: B404 +import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Optional + +__all__ = ['GENERATOR', 'LEVELS', 'MARKER', 'LOG_NAME', 'FAILED', 'DEGRADED', 'UPSTREAM', 'main'] + +#: The generator, relative to the checkout's root. +GENERATOR = 'examples/generators/make_samples.py' +#: The annotation levels ``ANNOTATION_LEVEL`` may hold. +LEVELS = ('error', 'warning') +#: What :file:`examples/generators/pcapng.py` writes on each stand-in's line. +MARKER = b'(download unavailable)' +#: The generator's output, under ``$RUNNER_TEMP``. +LOG_NAME = 'make-samples.log' +#: The annotation of a failed run, after ``:: ``. +FAILED = ('title=Sample fixture generation failed::examples/generators/make_samples.py could not rebuild ' + 'examples/captures/. This is a fixture-generation failure, not a test failure -- the test suite ' + 'has not run.') +#: The warning of a run on stand-ins. +DEGRADED = ('::warning title=Sample fixtures degraded::The upstream Wireshark captures were unreachable, so ' + 'synthesised stand-ins were used. The PCAP-NG tier ran, but not against the upstream bytes.') +#: The line of a run on upstream's captures. +UPSTREAM = 'Upstream Wireshark captures were used, matching their pinned SHA-256 digests.' + + +def emit(data: 'bytes') -> 'None': + """Write ``data`` to standard output as it is, at once.""" + sys.stdout.buffer.write(data) + sys.stdout.buffer.flush() + + +def say(text: 'str') -> 'None': + """Write ``text`` and a newline to standard output, at once.""" + emit(text.encode('utf-8') + b'\n') + + +def main(argv: 'Optional[list[str]]' = None) -> 'int': + """Run the generator, copy its output to the log, and report on it; the exit status.""" + parser = argparse.ArgumentParser(description=__doc__.split('\n', 1)[0]) + parser.parse_args(argv) + + level = os.environ['ANNOTATION_LEVEL'] + if level not in LEVELS: + print(f'ANNOTATION_LEVEL is {level!r}, not one of {", ".join(LEVELS)}.', file=sys.stderr) + return 2 + + output = bytearray() + with open(pathlib.Path(os.environ['RUNNER_TEMP']) / LOG_NAME, 'wb') as log: + with subprocess.Popen([sys.executable, GENERATOR], # nosec: B603 + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as process: + assert process.stdout is not None # nosec: B101 + for line in process.stdout: + emit(line) + log.write(line) + log.flush() + output.extend(line) + status = process.returncode + + # Keep a failure here legible as what it is: the fixtures could not be + # built, so no test has run yet and nothing is wrong with the code under test. + if status != 0: + say(f'::{level} {FAILED}') + return 1 + + marked = [line for line in bytes(output).split(b'\n') if MARKER in line] + if marked: + say(DEGRADED) + for line in marked: + emit(line + b'\n') + else: + say(UPSTREAM) + return 0 + + +if __name__ == '__main__': + sys.exit(main())