diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 6c78f329b1..ce792ec4f5 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -673,7 +673,8 @@ jobs: timeout-minutes: 30 env: ENGINE: ${{ matrix.engine }} - # Read by the apt plan step; see the note above it. + # Read by util/apt_packages.py's plan and install; see the note above + # the plan step. APT_PACKAGES: ${{ matrix.engine == 'PyShark' && 'tshark' || matrix.engine == 'PCAP_CT' && 'libpcap0.8' || matrix.engine == 'PyPCAP' && 'build-essential libpcap-dev' || '' }} APT_SET: ${{ matrix.engine }} APT_CACHE_SAVE: 'true' @@ -768,34 +769,19 @@ jobs: # on a hit, so a real install failure still fails; a cache-service error # is only a warning from actions/cache. The 180s bound is unchanged: with # the above it binds only on a cold miss, and TestAptRetry in - # tests/project/test_unit_tests_queue.py pins it. APT_PACKAGES (the job's - # env, read by the plan step) must name what the install step installs, - # or the key describes the wrong set -- a lost cache, not a wrong install. + # tests/project/test_unit_tests_queue.py pins it. APT_PACKAGES, the job's + # env, is what both the plan and the install read, so the key always + # describes the set installed. # - # `pypcap-parity` runs the same steps; every body but the install's is - # byte-identical to these. + # The plan, seed, install and collect steps run util/apt_packages.py + # (#1967), which runs the commands above under `sudo` and writes the same + # annotations; its docstring says what each subcommand does. `pypcap-parity` + # runs the same steps; every body but the install's is identical to these. - name: Plan this engine's system packages id: apt-plan if: env.APT_PACKAGES != '' timeout-minutes: 4 - run: | - set -eu - for attempt in 1 2; do - if sudo timeout 60 apt-get update; then - break - fi - echo "::warning title=apt-get update attempt $attempt of 2 failed (${APT_SET})::apt-get update failed or exceeded its 60s bound." - [ "$attempt" = 1 ] || exit 1 - done - # shellcheck disable=SC2086 - sim=$(apt-get install -s --no-install-recommends $APT_PACKAGES) - plan=$(printf '%s\n' "$sim" | grep '^Inst ' | sort || true) - printf 'To download:\n%s\n' "${plan:-(nothing)}" - if [ -n "$plan" ]; then - prefix="apt-debs-${ImageOS:-unknown}-${RUNNER_ARCH}-${APT_SET}-" - echo "restore-prefix=$prefix" >> "$GITHUB_OUTPUT" - echo "key=${prefix}$(printf '%s\n' "$plan" | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT" - fi + run: /usr/bin/python3 util/apt_packages.py plan - name: Restore this engine's cached .deb files id: apt-cache @@ -808,40 +794,11 @@ jobs: - name: Seed apt's archive with this engine's cached .deb files if: steps.apt-cache.outputs.cache-matched-key != '' - run: | - set -eu - sudo cp "$RUNNER_TEMP"/apt-debs/*.deb /var/cache/apt/archives/ - echo "Seeded $(find "$RUNNER_TEMP/apt-debs" -name '*.deb' | wc -l) .deb files" + run: /usr/bin/python3 util/apt_packages.py seed - name: Install this engine's system packages timeout-minutes: 10 - run: | - set -eu - mirrors=/etc/apt/apt-mirrors.txt - apt_install() { - for attempt in 1 2; do - if sudo timeout 60 apt-get update \ - && sudo DEBIAN_FRONTEND=noninteractive timeout 180 apt-get install -y --no-install-recommends "$@"; then - return 0 - fi - echo "::warning title=apt attempt $attempt of 2 failed (${ENGINE})::apt-get update or install failed or exceeded its per-attempt bound; retrying once." - sudo dpkg --configure -a || true - if [ "$attempt" = 1 ] && [ -f "$mirrors" ] \ - && grep -v 'azure\.archive\.ubuntu\.com' "$mirrors" | grep -q '^http' \ - && sudo sed -i '/azure\.archive\.ubuntu\.com/d' "$mirrors"; then - echo "::warning title=apt retry on a fallback mirror (${ENGINE})::Retrying against $(head -n 1 "$mirrors" | cut -f 1) instead of azure.archive.ubuntu.com, resuming the partial download." - fi - done - return 1 - } - if [ "$ENGINE" = "PyShark" ]; then - echo "wireshark-common wireshark-common/install-setuid boolean false" | sudo debconf-set-selections - apt_install tshark - elif [ "$ENGINE" = "PCAP_CT" ]; then - apt_install libpcap0.8 - elif [ "$ENGINE" = "PyPCAP" ]; then - apt_install build-essential libpcap-dev - fi + run: /usr/bin/python3 util/apt_packages.py install --name "$ENGINE" # #2019: from Ubuntu 25.04, apparmor 4.1 ships an enforcing profile for # /usr/bin/tshark (/etc/apparmor.d/tshark) whose own file rules read only @@ -867,22 +824,7 @@ jobs: if: >- env.APT_CACHE_SAVE == 'true' && steps.apt-plan.outputs.key != '' && steps.apt-cache.outputs.cache-hit != 'true' - run: | - set -eu - out="$RUNNER_TEMP/apt-debs" - rm -rf "$out" - mkdir -p "$out" - for deb in /var/cache/apt/archives/*.deb; do - [ -e "$deb" ] || continue - pkg=$(dpkg-deb -f "$deb" Package) - ver=$(dpkg-deb -f "$deb" Version) - arch=$(dpkg-deb -f "$deb" Architecture) - # shellcheck disable=SC2016 - if [ "$(dpkg-query -W -f '${Version}' "$pkg:$arch" 2>/dev/null || true)" = "$ver" ]; then - cp "$deb" "$out/" - fi - done - echo "Collected $(find "$out" -name '*.deb' | wc -l) .deb files" + run: /usr/bin/python3 util/apt_packages.py collect - name: Save this engine's .deb files to the cache if: >- @@ -1117,7 +1059,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 env: - # Read by the apt plan step; see `engine-tests`'s note on it. + # Read by util/apt_packages.py's plan and install; see `engine-tests`'s + # note on it. APT_PACKAGES: build-essential libpcap-dev tshark APT_SET: pypcap-parity # Every cell installs the same set under the same key; only one saves it, @@ -1155,11 +1098,10 @@ jobs: # *_runtime.py wholesale. Without tshark, PyShark.unsupported_reason() # declines the engine, the extractor silently falls back to the default # parser, and the test's assertions used to pass on that fallback's - # output -- a vacuous pass, invisible in skip counts. The debconf - # pre-seed and DEBIAN_FRONTEND mirror `engine-tests`'s "Install this - # engine's system packages" step exactly, for the same reason: tshark's - # postinst otherwise blocks on the "allow non-superusers to capture - # packets" prompt. + # output -- a vacuous pass, invisible in skip counts. The install + # pre-seeds debconf and sets DEBIAN_FRONTEND as `engine-tests`'s does, + # for the same reason: tshark's postinst otherwise blocks on the "allow + # non-superusers to capture packets" prompt. # # The apt steps -- plan, restore, seed, install, collect, save -- are # `engine-tests`'s, for the reasons in the note above its plan step: @@ -1174,24 +1116,7 @@ jobs: if: env.APT_PACKAGES != '' continue-on-error: ${{ matrix.ktlo == true }} timeout-minutes: 4 - run: | - set -eu - for attempt in 1 2; do - if sudo timeout 60 apt-get update; then - break - fi - echo "::warning title=apt-get update attempt $attempt of 2 failed (${APT_SET})::apt-get update failed or exceeded its 60s bound." - [ "$attempt" = 1 ] || exit 1 - done - # shellcheck disable=SC2086 - sim=$(apt-get install -s --no-install-recommends $APT_PACKAGES) - plan=$(printf '%s\n' "$sim" | grep '^Inst ' | sort || true) - printf 'To download:\n%s\n' "${plan:-(nothing)}" - if [ -n "$plan" ]; then - prefix="apt-debs-${ImageOS:-unknown}-${RUNNER_ARCH}-${APT_SET}-" - echo "restore-prefix=$prefix" >> "$GITHUB_OUTPUT" - echo "key=${prefix}$(printf '%s\n' "$plan" | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT" - fi + run: /usr/bin/python3 util/apt_packages.py plan - name: Restore the cached .deb files id: apt-cache @@ -1207,33 +1132,13 @@ jobs: id: apt-seed if: steps.apt-cache.outputs.cache-matched-key != '' continue-on-error: ${{ matrix.ktlo == true }} - run: | - set -eu - sudo cp "$RUNNER_TEMP"/apt-debs/*.deb /var/cache/apt/archives/ - echo "Seeded $(find "$RUNNER_TEMP/apt-debs" -name '*.deb' | wc -l) .deb files" + run: /usr/bin/python3 util/apt_packages.py seed - name: Install libpcap headers, a C toolchain, and tshark id: apt-install continue-on-error: ${{ matrix.ktlo == true }} timeout-minutes: 10 - run: | - set -eu - echo "wireshark-common wireshark-common/install-setuid boolean false" | sudo debconf-set-selections - mirrors=/etc/apt/apt-mirrors.txt - for attempt in 1 2; do - if sudo timeout 60 apt-get update \ - && sudo DEBIAN_FRONTEND=noninteractive timeout 180 apt-get install -y --no-install-recommends build-essential libpcap-dev tshark; then - exit 0 - fi - echo "::warning title=apt attempt $attempt of 2 failed::apt-get update or install failed or exceeded its per-attempt bound; retrying once." - sudo dpkg --configure -a || true - if [ "$attempt" = 1 ] && [ -f "$mirrors" ] \ - && grep -v 'azure\.archive\.ubuntu\.com' "$mirrors" | grep -q '^http' \ - && sudo sed -i '/azure\.archive\.ubuntu\.com/d' "$mirrors"; then - echo "::warning title=apt retry on a fallback mirror::Retrying against $(head -n 1 "$mirrors" | cut -f 1) instead of azure.archive.ubuntu.com, resuming the partial download." - fi - done - exit 1 + run: /usr/bin/python3 util/apt_packages.py install # #2019: tshark's AppArmor profile on Ubuntu 25.04 and later; see # `engine-tests`'s step of the same name, whose body this is. @@ -1252,22 +1157,7 @@ jobs: if: >- env.APT_CACHE_SAVE == 'true' && steps.apt-plan.outputs.key != '' && steps.apt-cache.outputs.cache-hit != 'true' - run: | - set -eu - out="$RUNNER_TEMP/apt-debs" - rm -rf "$out" - mkdir -p "$out" - for deb in /var/cache/apt/archives/*.deb; do - [ -e "$deb" ] || continue - pkg=$(dpkg-deb -f "$deb" Package) - ver=$(dpkg-deb -f "$deb" Version) - arch=$(dpkg-deb -f "$deb" Architecture) - # shellcheck disable=SC2016 - if [ "$(dpkg-query -W -f '${Version}' "$pkg:$arch" 2>/dev/null || true)" = "$ver" ]; then - cp "$deb" "$out/" - fi - done - echo "Collected $(find "$out" -name '*.deb' | wc -l) .deb files" + run: /usr/bin/python3 util/apt_packages.py collect - name: Save the .deb files to the cache if: >- diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index 45981de385..6afe5eca93 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -1218,6 +1218,12 @@ does; each has tests under :file:`tests/project/` that run it. * - :file:`util/discussion_report.py` - ``Post the report to Discussions`` in the ``discussion`` job of ``ci-timing.yml`` and of ``cron-vendor.yml`` (see `CI Reports`_). + * - :file:`util/apt_packages.py` + - The apt steps of ``unit-tests.yml``'s ``engine-tests`` and ``pypcap-parity`` + (and so its KTLO twin): the plan, seed, install and collect steps, as + ``apt_packages.py plan``, ``seed``, ``install`` and ``collect``. The + ``actions/cache`` steps and the AppArmor step (`#2019 + `__) stay as they were. The summaries run on the image's ``/usr/bin/python3`` rather than the leg's interpreter, which may have failed to set up or be too broken to run anything, so @@ -1229,7 +1235,15 @@ file-size cap every test command of ``unit-tests.yml`` takes. :file:`util/regenerate_samples.py` is standard library only but runs on the leg's interpreter, since the generator it runs imports ``pcapkit``, and so does :file:`util/pip_install.py`, since it installs into that interpreter; it runs on -every version the matrices hold, 3.7 and the experimental tier included. Both ``changes`` jobs check out on every event, since the script is in the +every version the matrices hold, 3.7 and the experimental tier included. :file:`util/apt_packages.py` runs on +``/usr/bin/python3``, as the apt shell needed no interpreter of the leg's, and runs +under ``sudo`` the commands that shell ran. Its ``plan`` and ``install`` both read +the job's ``APT_PACKAGES``, so the cache key always describes the set installed. +Its bounds are still ``timeout`` run as root inside ``sudo``, given no option, so +uutils' ``timeout`` behaves as GNU's (`#2020 +`__), and the bound reaches +apt's own children, which a timeout in the script, running as the runner's user, +could not signal. Both ``changes`` jobs check out on every event, since the script is in the tree. ``create-release.yml``'s ``pypi`` job keeps its own copy of the KTLO summary's shell until its shell moves to scripts too; a project pin runs that copy and the script on the same steps and requires the same output. diff --git a/tests/project/test_apt_packages_1967.py b/tests/project/test_apt_packages_1967.py new file mode 100644 index 0000000000..ef1317f449 --- /dev/null +++ b/tests/project/test_apt_packages_1967.py @@ -0,0 +1,810 @@ +# -*- coding: utf-8 -*- +"""#1967: :file:`util/apt_packages.py`, the apt steps of ``engine-tests`` and ``pypcap-parity``. + +* :class:`TestPlan`, :class:`TestSeed`, :class:`TestInstall` and :class:`TestCollect` + run each subcommand in this process with :func:`subprocess.run` stubbed, so every + command it would run, and every failure of one, is checked without ``sudo`` or apt. +* :class:`TestTheFormerShellAgrees` runs the shell the steps held before the script, + and then the steps' own ``run:``, against the same stand-in ``sudo``, ``timeout``, + ``apt-get``, ``dpkg``, ``dpkg-deb``, ``dpkg-query`` and ``debconf-set-selections``, + and requires the same status, output, commands and files. +* :class:`TestTheStepsRunTheScript` pins each apt step to the script with its + ``id``, ``if:``, ``continue-on-error`` and ``timeout-minutes``, the KTLO twin to + the same steps, and the AppArmor step (#2019) to its inline shell; no other step + in any workflow keeps apt shell of its own, and #1672's scan of what a KTLO leg + may write sees the script's warnings. + +The retry, the fallback mirror and the cache key are also run through the steps by +:mod:`tests.project.test_unit_tests_queue` and :mod:`tests.project.test_workflow_apt_cache`. + +""" + +from __future__ import annotations + +import contextlib +import hashlib +import io +import os +import pathlib +import shlex +import shutil +import subprocess # nosec: B404 +import sys +import tempfile +import textwrap +import unittest +from typing import TYPE_CHECKING +from unittest import mock + +from tests._workflows import WORKFLOWS, load, steps +from tests.project.test_ktlo_legs_stay_green_1672 import KTLO_COMMANDS, commands +from tests.project.test_unit_tests_queue import IMAGE_PYTHON +from util import apt_packages + +if TYPE_CHECKING: + from typing import Any, Callable, Iterator, Optional + +ROOT = pathlib.Path(__file__).resolve().parents[2] +WORKFLOW = 'unit-tests.yml' +SCRIPT = 'util/apt_packages.py' + +#: ``apt-get install -s`` output: noise, ``Conf`` lines and unsorted ``Inst`` lines. +SIMULATION = (b'NOTE: This is only a simulation!\n' + b'Inst tshark (4.2.2-1.1build3 Ubuntu:24.04/noble [amd64])\n' + b'Conf tshark (4.2.2-1.1build3 Ubuntu:24.04/noble [amd64])\n' + b'Inst libwireshark17t64 (4.2.2-1.1build3 Ubuntu:24.04/noble [amd64])\n') +INST = (b'Inst libwireshark17t64 (4.2.2-1.1build3 Ubuntu:24.04/noble [amd64])\n' + b'Inst tshark (4.2.2-1.1build3 Ubuntu:24.04/noble [amd64])') +#: The runner image's mirror list, as configure-apt-sources.sh writes it. +MIRRORS = ('http://azure.archive.ubuntu.com/ubuntu/\tpriority:1\n' + 'https://archive.ubuntu.com/ubuntu/\tpriority:2\n' + 'https://security.ubuntu.com/ubuntu/\tpriority:3\n') + +UPDATE = ['sudo', 'timeout', '60', 'apt-get', 'update'] +DPKG = ['sudo', 'dpkg', '--configure', '-a'] +DEBCONF = ['sudo', 'debconf-set-selections'] + + +def install_command(*packages: 'str') -> 'list[str]': + return ['sudo', 'DEBIAN_FRONTEND=noninteractive', 'timeout', '180', 'apt-get', 'install', '-y', + '--no-install-recommends', *packages] + + +class Runner: + """A stand-in :func:`subprocess.run`: records each command, answers from ``respond``.""" + + def __init__(self, respond: 'Callable[[list[str], dict[str, Any]], tuple[int, bytes]]') -> None: + self.respond = respond + self.calls = [] # type: list[tuple[list[str], dict[str, Any]]] + + def __call__(self, command: 'list[str]', check: 'bool' = False, + **kwargs: 'Any') -> 'subprocess.CompletedProcess[bytes]': + assert check is False # nosec: B101 + self.calls.append((list(command), kwargs)) + status, stdout = self.respond(list(command), kwargs) + return subprocess.CompletedProcess(command, status, stdout if kwargs.get('stdout') else None) + + @property + def commands(self) -> 'list[list[str]]': + return [command for command, _ in self.calls] + + +class _InProcess(unittest.TestCase): + """A scratch directory, the script's environment, and a way to run :func:`apt_packages.main`.""" + + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-1967-apt-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = pathlib.Path(tmp.name) + self.runner_temp = self.tmp / 'runner-temp' + self.runner_temp.mkdir() + self.output = self.tmp / 'output' + self.output.write_text('earlier=1\n', encoding='utf-8') + self.archives = self.tmp / 'archives' + self.archives.mkdir() + self.mirrors = self.tmp / 'apt-mirrors.txt' + self.environ = {'APT_PACKAGES': 'tshark', 'APT_SET': 'PyShark', 'ImageOS': 'ubuntu24', + 'RUNNER_ARCH': 'X64', 'GITHUB_OUTPUT': str(self.output), + 'RUNNER_TEMP': str(self.runner_temp)} + + def main(self, args: 'list[str]', + respond: 'Callable[[list[str], dict[str, Any]], tuple[int, bytes]]') -> 'tuple[int, str, Runner]': + """``apt_packages.main(args)`` with ``respond`` for every command; status, output, runner.""" + runner = Runner(respond) + out = io.BytesIO() + stdout = io.TextIOWrapper(out, encoding='utf-8', write_through=True) + with mock.patch.dict(os.environ, self.environ, clear=True), \ + mock.patch.object(apt_packages.subprocess, 'run', runner), \ + mock.patch.object(apt_packages.sys, 'stdout', stdout): + status = apt_packages.main(args) + return status, out.getvalue().decode('utf-8'), runner + + +def answers(**by_command: 'Any') -> 'Callable[[list[str], dict[str, Any]], tuple[int, bytes]]': + """A ``respond`` keyed on a command's name: ``update``, ``simulate``, ``install`` or the program. + + Each value is an ``(status, stdout)`` pair, or a list of them taken in turn. + + """ + queues = {key: list(value) if isinstance(value, list) else [value] for key, value in by_command.items()} + + def respond(command: 'list[str]', _: 'dict[str, Any]') -> 'tuple[int, bytes]': + if command[:2] == ['apt-get', 'install']: + key = 'simulate' + elif 'apt-get' in command: + key = command[command.index('apt-get') + 1] + else: + key = command[1] if command[0] == 'sudo' else command[0] + queue = queues.get(key.replace('-', '_'), [(0, b'')]) + return queue.pop(0) if len(queue) > 1 else queue[0] + return respond + + +class TestPlan(_InProcess): + """Up to two updates, then the key of the sorted ``Inst`` lines of a simulation.""" + + def key(self, prefix: 'str') -> 'str': + return prefix + hashlib.sha256(INST + b'\n').hexdigest()[:16] + + def test_a_clean_plan_writes_the_key(self) -> None: + status, output, runner = self.main(['plan'], answers(simulate=(0, SIMULATION))) + self.assertEqual(status, 0) + self.assertEqual(runner.commands, [UPDATE, ['apt-get', 'install', '-s', '--no-install-recommends', 'tshark']]) + self.assertEqual(runner.calls[1][1], {'stdout': subprocess.PIPE}) + self.assertEqual(output, 'To download:\n' + INST.decode() + '\n') + prefix = 'apt-debs-ubuntu24-X64-PyShark-' + self.assertEqual(self.output.read_text(encoding='utf-8'), + f'earlier=1\nrestore-prefix={prefix}\nkey={self.key(prefix)}\n') + + def test_every_package_is_simulated(self) -> None: + self.environ['APT_PACKAGES'] = ' build-essential\tlibpcap-dev tshark ' + _, _, runner = self.main(['plan'], answers(simulate=(0, b''))) + self.assertEqual(runner.commands[1][4:], ['build-essential', 'libpcap-dev', 'tshark']) + + def test_image_os_falls_back_to_unknown(self) -> None: + for value in (None, ''): + with self.subTest(ImageOS=value): + self.output.write_text('', encoding='utf-8') + if value is None: + del self.environ['ImageOS'] + else: + self.environ['ImageOS'] = value + self.assertEqual(self.main(['plan'], answers(simulate=(0, SIMULATION)))[0], 0) + self.assertIn('restore-prefix=apt-debs-unknown-X64-PyShark-\n', + self.output.read_text(encoding='utf-8')) + + def test_nothing_to_download_writes_no_key(self) -> None: + for simulation in (b'', b'NOTE: This is only a simulation!\n', b'Conf tshark (1)\n Inst indented\n'): + with self.subTest(simulation=simulation): + status, output, _ = self.main(['plan'], answers(simulate=(0, simulation))) + self.assertEqual((status, output), (0, 'To download:\n(nothing)\n')) + self.assertEqual(self.output.read_text(encoding='utf-8'), 'earlier=1\n') + + def test_one_failed_update_is_retried(self) -> None: + status, output, runner = self.main(['plan'], answers(update=[(124, b''), (0, b'')], + simulate=(0, SIMULATION))) + self.assertEqual(status, 0) + self.assertEqual(runner.commands[:2], [UPDATE, UPDATE]) + self.assertTrue(output.startswith('::warning title=apt-get update attempt 1 of 2 failed (PyShark)::' + 'apt-get update failed or exceeded its 60s bound.\nTo download:\n'), output) + + def test_two_failed_updates_fail_before_the_simulation(self) -> None: + status, output, runner = self.main(['plan'], answers(update=(100, b''))) + self.assertEqual(status, 1) + self.assertEqual(runner.commands, [UPDATE, UPDATE]) + self.assertEqual(output.splitlines(), [ + f'::warning title=apt-get update attempt {n} of 2 failed (PyShark)::apt-get update failed or ' + 'exceeded its 60s bound.' for n in (1, 2)]) + self.assertEqual(self.output.read_text(encoding='utf-8'), 'earlier=1\n') + + def test_a_failed_simulation_ends_with_its_status(self) -> None: + for code, expected in ((100, 100), (-9, 137)): + with self.subTest(code=code): + status, output, _ = self.main(['plan'], answers(simulate=(code, SIMULATION))) + self.assertEqual((status, output), (expected, '')) + self.assertEqual(self.output.read_text(encoding='utf-8'), 'earlier=1\n') + + def test_a_missing_variable_is_an_error(self) -> None: + for name in ('APT_PACKAGES', 'RUNNER_ARCH', 'GITHUB_OUTPUT'): + with self.subTest(variable=name): + del self.environ[name] + with self.assertRaises(KeyError): + self.main(['plan'], answers(simulate=(0, SIMULATION))) + self.environ[name] = 'restored' if name != 'GITHUB_OUTPUT' else str(self.output) + + +class TestSeed(_InProcess): + """The restored files into apt's archive with ``sudo cp``, then their count.""" + + def test_the_files_are_copied_and_counted(self) -> None: + cache = self.runner_temp / 'apt-debs' + (cache / 'nested').mkdir(parents=True) + for name in ('b.deb', 'a.deb', 'nested/c.deb', 'notes.txt'): + (cache / name).write_bytes(b'') + status, output, runner = self.main(['seed', '--archives', str(self.archives)], answers()) + self.assertEqual(status, 0) + self.assertEqual(runner.commands, [['sudo', 'cp', str(cache / 'a.deb'), str(cache / 'b.deb'), + f'{self.archives}/']]) + self.assertEqual(output, 'Seeded 3 .deb files\n') + + def test_the_default_archive_is_apts(self) -> None: + (self.runner_temp / 'apt-debs').mkdir() + (self.runner_temp / 'apt-debs' / 'a.deb').write_bytes(b'') + _, _, runner = self.main(['seed'], answers()) + self.assertEqual(runner.commands[0][-1], '/var/cache/apt/archives/') + + def test_nothing_restored_fails_on_the_pattern(self) -> None: + for code, expected in ((1, 1), (-15, 143)): + with self.subTest(code=code): + status, output, runner = self.main(['seed', '--archives', str(self.archives)], answers(cp=(code, b''))) + self.assertEqual((status, output), (expected, '')) + self.assertEqual(runner.commands, [['sudo', 'cp', str(self.runner_temp / 'apt-debs' / '*.deb'), + f'{self.archives}/']]) + + +class TestInstall(_InProcess): + """debconf for tshark, then up to two update-and-install attempts, the second on a fallback mirror.""" + + def install(self, *args: 'str', **by_command: 'Any') -> 'tuple[int, str, Runner]': + return self.main(['install', '--mirrors', str(self.mirrors), *args], answers(**by_command)) + + def test_nothing_to_install_runs_nothing(self) -> None: + for packages in ('', ' '): + with self.subTest(packages=packages): + self.environ['APT_PACKAGES'] = packages + status, output, runner = self.install() + self.assertEqual((status, output, runner.commands), (0, '', [])) + + def test_a_clean_install(self) -> None: + self.mirrors.write_text(MIRRORS, encoding='utf-8') + status, output, runner = self.install() + self.assertEqual((status, output), (0, '')) + self.assertEqual(runner.commands, [DEBCONF, UPDATE, install_command('tshark')]) + self.assertEqual(runner.calls[0][1], {'input': apt_packages.DEBCONF}) + self.assertEqual(self.mirrors.read_text(encoding='utf-8'), MIRRORS) + + def test_debconf_only_for_tshark(self) -> None: + for packages, debconf in (('libpcap0.8', False), ('build-essential libpcap-dev', False), + ('build-essential libpcap-dev tshark', True), ('tshark-extra', False)): + with self.subTest(packages=packages): + self.environ['APT_PACKAGES'] = packages + _, _, runner = self.install() + self.assertEqual(runner.commands[0] == DEBCONF, debconf) + self.assertEqual(runner.commands[-1], install_command(*packages.split())) + + def test_a_failed_debconf_ends_with_its_status(self) -> None: + status, output, runner = self.install(debconf_set_selections=(3, b'')) + self.assertEqual((status, output, runner.commands), (3, '', [DEBCONF])) + + def test_a_failed_first_attempt_retries_on_the_fallback_mirror(self) -> None: + self.mirrors.write_text(MIRRORS, encoding='utf-8') + for failing in ('update', 'install'): + with self.subTest(failing=failing): + status, output, runner = self.install('--name', 'PyShark', **{failing: [(124, b''), (0, b'')]}) + self.assertEqual(status, 0) + sed = ['sudo', 'sed', '-i', r'/azure\.archive\.ubuntu\.com/d', str(self.mirrors)] + first = [UPDATE] if failing == 'update' else [UPDATE, install_command('tshark')] + self.assertEqual(runner.commands, [DEBCONF, *first, DPKG, sed, UPDATE, install_command('tshark')]) + # The stand-in sed changes nothing, so the list's first line is still azure's. + self.assertEqual(output.splitlines(), [ + '::warning title=apt attempt 1 of 2 failed (PyShark)::apt-get update or install failed or ' + 'exceeded its per-attempt bound; retrying once.', + '::warning title=apt retry on a fallback mirror (PyShark)::Retrying against ' + 'http://azure.archive.ubuntu.com/ubuntu/ instead of azure.archive.ubuntu.com, resuming the ' + 'partial download.']) + + def test_the_mirror_named_is_the_first_line_after_the_edit(self) -> None: + self.mirrors.write_text(MIRRORS, encoding='utf-8') + edited = ''.join(MIRRORS.splitlines(True)[1:]) + otherwise = answers(update=[(1, b''), (0, b'')]) + + def respond(command: 'list[str]', kwargs: 'dict[str, Any]') -> 'tuple[int, bytes]': + if 'sed' in command: + self.mirrors.write_text(edited, encoding='utf-8') + return otherwise(command, kwargs) + status, output, _ = self.main(['install', '--mirrors', str(self.mirrors)], respond) + self.assertEqual(status, 0) + self.assertIn('::warning title=apt retry on a fallback mirror::Retrying against ' + 'https://archive.ubuntu.com/ubuntu/ instead', output) + + def test_no_fallback_without_another_http_mirror(self) -> None: + for mirrors in (None, '', 'http://azure.archive.ubuntu.com/ubuntu/\tpriority:1\n', + 'ftp://elsewhere/\n http://indented/\n'): + with self.subTest(mirrors=mirrors): + if mirrors is None: + if self.mirrors.exists(): + self.mirrors.unlink() + else: + self.mirrors.write_text(mirrors, encoding='utf-8') + status, output, runner = self.install(update=[(1, b''), (0, b'')]) + self.assertEqual(status, 0) + self.assertFalse([command for command in runner.commands if 'sed' in command]) + self.assertNotIn('fallback mirror', output) + + def test_a_directory_is_not_a_mirror_list(self) -> None: + self.mirrors.mkdir() + status, _, runner = self.install(update=[(1, b''), (0, b'')]) + self.assertEqual(status, 0) + self.assertFalse([command for command in runner.commands if 'sed' in command]) + + def test_a_failed_edit_is_not_announced(self) -> None: + self.mirrors.write_text(MIRRORS, encoding='utf-8') + status, output, runner = self.install(update=[(1, b''), (0, b'')], sed=(4, b'')) + self.assertEqual(status, 0) + self.assertEqual(sum(1 for command in runner.commands if 'sed' in command), 1) + self.assertNotIn('fallback mirror', output) + + def test_two_failures_fail_after_one_edit_and_two_repairs(self) -> None: + self.mirrors.write_text(MIRRORS, encoding='utf-8') + status, output, runner = self.install(update=(124, b''), dpkg=(1, b'')) + self.assertEqual(status, 1) + self.assertEqual(runner.commands.count(UPDATE), 2) + self.assertEqual(runner.commands.count(DPKG), 2) + self.assertEqual(sum(1 for command in runner.commands if 'sed' in command), 1) + self.assertEqual([line.split('::')[1] for line in output.splitlines()], [ + 'warning title=apt attempt 1 of 2 failed', 'warning title=apt retry on a fallback mirror', + 'warning title=apt attempt 2 of 2 failed']) + + def test_the_default_mirror_list_is_the_images(self) -> None: + self.assertEqual(apt_packages.MIRRORS, '/etc/apt/apt-mirrors.txt') + with mock.patch.object(apt_packages, 'fallback') as fallback: + self.main(['install'], answers(update=[(1, b''), (0, b'')])) + fallback.assert_called_once_with('/etc/apt/apt-mirrors.txt', '') + + +class TestCollect(_InProcess): + """Each archived ``.deb`` at its installed version into a fresh cache directory.""" + + def deb(self, name: 'str', package: 'str', version: 'str', arch: 'str' = 'amd64') -> 'pathlib.Path': + path = self.archives / name + path.write_text(f'Package={package}\nVersion={version}\nArchitecture={arch}\n', encoding='utf-8') + return path + + def respond(self, installed: 'dict[str, str]', broken: 'tuple[str, ...]' = ()) -> 'Callable[..., Any]': + def respond(command: 'list[str]', _: 'dict[str, Any]') -> 'tuple[int, bytes]': + if command[0] == 'dpkg-deb': + if pathlib.Path(command[2]).name in broken: + return 2, b'' + fields = dict(line.split('=', 1) for line in pathlib.Path(command[2]).read_text().splitlines()) + return 0, fields[command[3]].encode() + b'\n\n' + if command[0] == 'dpkg-query': + found = installed.get(command[-1]) + return (0, found.encode()) if found is not None else (1, b'') + raise AssertionError(command) + return respond + + def test_only_the_installed_versions_are_kept(self) -> None: + self.deb('tshark_4.2.deb', 'tshark', '4.2') + self.deb('old_1.deb', 'old', '1') + self.deb('gone_1.deb', 'gone', '1') + self.deb('i386_1.deb', 'lib', '1', 'i386') + (self.archives / 'broken.deb').symlink_to(self.archives / 'nowhere') + (self.archives / 'partial').mkdir() + out = self.runner_temp / 'apt-debs' + out.mkdir() + (out / 'stale.deb').write_bytes(b'') + status, output, runner = self.main(['collect', '--archives', str(self.archives)], self.respond( + {'tshark:amd64': '4.2', 'old:amd64': '2', 'lib:amd64': '1'})) + self.assertEqual((status, output), (0, 'Collected 1 .deb files\n')) + self.assertEqual(sorted(path.name for path in out.iterdir()), ['tshark_4.2.deb']) + self.assertEqual(runner.commands[:4], [ + ['dpkg-deb', '-f', str(self.archives / 'gone_1.deb'), 'Package'], + ['dpkg-deb', '-f', str(self.archives / 'gone_1.deb'), 'Version'], + ['dpkg-deb', '-f', str(self.archives / 'gone_1.deb'), 'Architecture'], + ['dpkg-query', '-W', '-f', '${Version}', 'gone:amd64']]) + self.assertEqual(runner.calls[3][1], {'stdout': subprocess.PIPE, 'stderr': subprocess.DEVNULL}) + self.assertFalse([command for command in runner.commands if str(self.archives / 'broken.deb') in command]) + + def test_an_empty_archive_collects_nothing(self) -> None: + out = self.runner_temp / 'apt-debs' + out.write_text('a file in the way', encoding='utf-8') + status, output, runner = self.main(['collect', '--archives', str(self.archives)], self.respond({})) + self.assertEqual((status, output, runner.commands), (0, 'Collected 0 .deb files\n', [])) + self.assertTrue(out.is_dir()) + + def test_a_failed_dpkg_deb_ends_with_its_status(self) -> None: + self.deb('a.deb', 'a', '1') + self.deb('b.deb', 'b', '1') + status, output, runner = self.main(['collect', '--archives', str(self.archives)], + self.respond({'a:amd64': '1'}, broken=('b.deb',))) + self.assertEqual((status, output), (2, '')) + self.assertEqual(runner.commands[-1], ['dpkg-deb', '-f', str(self.archives / 'b.deb'), 'Package']) + self.assertEqual([path.name for path in (self.runner_temp / 'apt-debs').iterdir()], ['a.deb']) + + def test_the_default_archive_is_apts(self) -> None: + with mock.patch.object(apt_packages, 'debs', return_value=[]) as debs: + self.main(['collect'], self.respond({})) + debs.assert_called_once_with('/var/cache/apt/archives') + + +class TestArguments(_InProcess): + """One subcommand, and only its own options.""" + + def test_refused(self) -> None: + for args in ([], ['update'], ['plan', 'extra'], ['seed', '--mirrors', 'x'], ['install', 'tshark'], + ['collect', '--name', 'x']): + with self.subTest(args=args), contextlib.redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as raised: + self.main(args, answers()) + self.assertEqual(raised.exception.code, 2) + + +#: The steps' shell before the script, each with the runner paths it named. +FORMER_PLAN = textwrap.dedent(r''' + set -eu + for attempt in 1 2; do + if sudo timeout 60 apt-get update; then + break + fi + echo "::warning title=apt-get update attempt $attempt of 2 failed (${APT_SET})::apt-get update failed or exceeded its 60s bound." + [ "$attempt" = 1 ] || exit 1 + done + # shellcheck disable=SC2086 + sim=$(apt-get install -s --no-install-recommends $APT_PACKAGES) + plan=$(printf '%s\n' "$sim" | grep '^Inst ' | sort || true) + printf 'To download:\n%s\n' "${plan:-(nothing)}" + if [ -n "$plan" ]; then + prefix="apt-debs-${ImageOS:-unknown}-${RUNNER_ARCH}-${APT_SET}-" + echo "restore-prefix=$prefix" >> "$GITHUB_OUTPUT" + echo "key=${prefix}$(printf '%s\n' "$plan" | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT" + fi +''').lstrip() # noqa: E501 +FORMER_SEED = textwrap.dedent(r''' + set -eu + sudo cp "$RUNNER_TEMP"/apt-debs/*.deb /var/cache/apt/archives/ + echo "Seeded $(find "$RUNNER_TEMP/apt-debs" -name '*.deb' | wc -l) .deb files" +''').lstrip() +FORMER_RETRY = textwrap.dedent(r''' + for attempt in 1 2; do + if sudo timeout 60 apt-get update \ + && sudo DEBIAN_FRONTEND=noninteractive timeout 180 apt-get install -y --no-install-recommends PACKAGES; then + SUCCEED + fi + echo "::warning title=apt attempt $attempt of 2 failed SUFFIX::apt-get update or install failed or exceeded its per-attempt bound; retrying once." + sudo dpkg --configure -a || true + if [ "$attempt" = 1 ] && [ -f "$mirrors" ] \ + && grep -v 'azure\.archive\.ubuntu\.com' "$mirrors" | grep -q '^http' \ + && sudo sed -i '/azure\.archive\.ubuntu\.com/d' "$mirrors"; then + echo "::warning title=apt retry on a fallback mirror SUFFIX::Retrying against $(head -n 1 "$mirrors" | cut -f 1) instead of azure.archive.ubuntu.com, resuming the partial download." + fi + done +''') # noqa: E501 +FORMER_ENGINE_INSTALL = ('set -eu\nmirrors=/etc/apt/apt-mirrors.txt\napt_install() {\n' + + textwrap.indent(FORMER_RETRY.lstrip().replace('PACKAGES', '"$@"') + .replace('SUCCEED', 'return 0').replace(' SUFFIX', ' (${ENGINE})'), ' ') + + ' return 1\n}\n' + textwrap.dedent('''\ + if [ "$ENGINE" = "PyShark" ]; then + echo "wireshark-common wireshark-common/install-setuid boolean false" | sudo debconf-set-selections + apt_install tshark + elif [ "$ENGINE" = "PCAP_CT" ]; then + apt_install libpcap0.8 + elif [ "$ENGINE" = "PyPCAP" ]; then + apt_install build-essential libpcap-dev + fi +''')) +FORMER_PARITY_INSTALL = ('set -eu\n' + 'echo "wireshark-common wireshark-common/install-setuid boolean false" ' + '| sudo debconf-set-selections\nmirrors=/etc/apt/apt-mirrors.txt\n' + + FORMER_RETRY.lstrip().replace('PACKAGES', 'build-essential libpcap-dev tshark') + .replace('SUCCEED', 'exit 0').replace(' SUFFIX', '') + 'exit 1\n') +FORMER_COLLECT = textwrap.dedent(r''' + set -eu + out="$RUNNER_TEMP/apt-debs" + rm -rf "$out" + mkdir -p "$out" + for deb in /var/cache/apt/archives/*.deb; do + [ -e "$deb" ] || continue + pkg=$(dpkg-deb -f "$deb" Package) + ver=$(dpkg-deb -f "$deb" Version) + arch=$(dpkg-deb -f "$deb" Architecture) + # shellcheck disable=SC2016 + if [ "$(dpkg-query -W -f '${Version}' "$pkg:$arch" 2>/dev/null || true)" = "$ver" ]; then + cp "$deb" "$out/" + fi + done + echo "Collected $(find "$out" -name '*.deb' | wc -l) .deb files" +''').lstrip() + +#: ``engine-tests``'s and ``pypcap-parity``'s apt steps, ``(name, id, if, continue-on-error, +#: timeout-minutes, run)``, in order, the AppArmor step (#2019) between install and collect. +ENGINE_STEPS = ( + ("Plan this engine's system packages", 'apt-plan', "env.APT_PACKAGES != ''", None, 4, + f'{IMAGE_PYTHON} {SCRIPT} plan'), + ("Seed apt's archive with this engine's cached .deb files", None, + "steps.apt-cache.outputs.cache-matched-key != ''", None, None, f'{IMAGE_PYTHON} {SCRIPT} seed'), + ("Install this engine's system packages", None, None, None, 10, + f'{IMAGE_PYTHON} {SCRIPT} install --name "$ENGINE"'), + ("Collect this engine's installed .deb files for the cache", None, + "env.APT_CACHE_SAVE == 'true' && steps.apt-plan.outputs.key != '' && steps.apt-cache.outputs.cache-hit != 'true'", + None, None, f'{IMAGE_PYTHON} {SCRIPT} collect'), +) +KTLO = '${{ matrix.ktlo == true }}' +PARITY_STEPS = ( + ('Plan the system packages', 'apt-plan', "env.APT_PACKAGES != ''", KTLO, 4, f'{IMAGE_PYTHON} {SCRIPT} plan'), + ("Seed apt's archive with the cached .deb files", 'apt-seed', "steps.apt-cache.outputs.cache-matched-key != ''", + KTLO, None, f'{IMAGE_PYTHON} {SCRIPT} seed'), + ('Install libpcap headers, a C toolchain, and tshark', 'apt-install', None, KTLO, 10, + f'{IMAGE_PYTHON} {SCRIPT} install'), + ('Collect the installed .deb files for the cache', None, ENGINE_STEPS[3][2], None, None, + f'{IMAGE_PYTHON} {SCRIPT} collect'), +) +SITES = {'engine-tests': ENGINE_STEPS, 'pypcap-parity': PARITY_STEPS} +#: Each step's former shell, by the subcommand that replaced it. +FORMER = {'plan': FORMER_PLAN, 'seed': FORMER_SEED, 'collect': FORMER_COLLECT} +APPARMOR = 'Let tshark read the checkout past its AppArmor profile' +#: Text only the apt steps' former shell held. +INLINE = ('apt-get', 'dpkg', 'debconf-set-selections', 'apt-mirrors', '/var/cache/apt', 'apt_install') + +#: Stand-ins, each logging its call to ``$CALLS``. ``apt-get``'s first ``$FAIL_UPDATES`` +#: updates and ``$FAIL_INSTALLS`` installs fail; ``install -s`` prints ``$STUB_SIM`` and +#: exits ``$SIM_STATUS``. ``dpkg-deb -f`` reads ``Field=value`` lines from the archive +#: itself; ``dpkg-query`` reads ``pkg:arch version`` lines from ``$INSTALLED``. +STUBS = { + 'sudo': 'echo "sudo $*" >> "$CALLS"\nexec env "$@"\n', + 'timeout': 'echo "timeout $1" >> "$CALLS"\nshift\nexec "$@"\n', + 'apt-get': textwrap.dedent('''\ + echo "apt-get $*" >> "$CALLS" + if [ "$1" = update ]; then + n=$(grep -c '^apt-get update' "$CALLS") + echo "Hit:$n http://mirror noble InRelease" + [ "$n" -gt "${FAIL_UPDATES:-0}" ] || { echo "E: update $n failed" >&2; exit 100; } + elif [ "$2" = -s ]; then + cat "$STUB_SIM" + echo "W: simulated" >&2 + exit "${SIM_STATUS:-0}" + else + n=$(grep -c '^apt-get install -y' "$CALLS") + echo "Setting up $n" + [ "$n" -gt "${FAIL_INSTALLS:-0}" ] || exit 124 + fi + '''), + 'dpkg': 'echo "dpkg $*" >> "$CALLS"\necho "dpkg: configured"\nexit "${DPKG_STATUS:-0}"\n', + 'debconf-set-selections': 'echo "debconf $(cat)" >> "$CALLS"\nexit "${DEBCONF_STATUS:-0}"\n', + 'dpkg-deb': textwrap.dedent('''\ + echo "dpkg-deb $*" >> "$CALLS" + grep -q '^Package=' "$2" 2>/dev/null || { echo "dpkg-deb: error: '$2' is not a Debian format archive" >&2; exit 2; } + sed -n "s/^$3=//p" "$2" + '''), # noqa: E501 + 'dpkg-query': textwrap.dedent('''\ + echo "dpkg-query $*" >> "$CALLS" + v=$(sed -n "s/^$4 //p" "$INSTALLED") + [ -n "$v" ] || { echo "dpkg-query: no packages found matching $4" >&2; exit 1; } + printf '%s' "$v" + '''), +} + + +@unittest.skipIf(any(shutil.which(tool) is None for tool in ('bash', 'sed', 'grep', 'sort', 'sha256sum', 'cut', + 'find', 'wc', 'head', 'cp', 'env')), + 'needs bash and the coreutils the former shell ran') +class TestTheFormerShellAgrees(unittest.TestCase): + """Each step's ``run:`` writes, runs and leaves what its former shell did, on the same stand-ins.""" + + def setUp(self) -> None: + steps(WORKFLOW, 'engine-tests') # skips the test, not each subtest, without PyYAML + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-1967-apt-parity-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = pathlib.Path(tmp.name) + self.bin = self.tmp / 'bin' + self.bin.mkdir() + for name, body in STUBS.items(): + path = self.bin / name + path.write_text('#!/bin/bash\n' + body, encoding='utf-8') + path.chmod(0o755) + self.mirrors = self.tmp / 'apt-mirrors.txt' + self.archives = self.tmp / 'archives' + self.runner_temp = self.tmp / 'runner-temp' + self.output = self.tmp / 'output' + self.calls = self.tmp / 'calls' + + def env(self, **extra: 'str') -> 'dict[str, str]': + env = {key: value for key, value in os.environ.items() + if key not in ('APT_PACKAGES', 'APT_SET', 'ENGINE', 'ImageOS', 'RUNNER_ARCH', 'RUNNER_TEMP', + 'GITHUB_OUTPUT', 'PYTHONPATH', 'PYTHONSAFEPATH') and not key.startswith('LC_')} + env.update(PATH=f'{self.bin}{os.pathsep}{os.environ["PATH"]}', CALLS=str(self.calls), LC_ALL='C', + RUNNER_TEMP=str(self.runner_temp), GITHUB_OUTPUT=str(self.output), + STUB_SIM=str(self.tmp / 'sim'), INSTALLED=str(self.tmp / 'installed'), RUNNER_ARCH='X64', + APT_SET='PyShark', ImageOS='ubuntu24', PYTHONIOENCODING='utf-8') + env.update(extra) + return env + + def state(self) -> 'tuple[object, ...]': + """What a run leaves: its commands, the outputs file, the mirror list, the cache directory.""" + def read(path: 'pathlib.Path') -> 'Optional[str]': + return path.read_text(encoding='utf-8') if path.is_file() else None + cache = self.runner_temp / 'apt-debs' + listing = sorted(str(path.relative_to(cache)) for path in cache.rglob('*')) if cache.is_dir() else None + return read(self.calls), read(self.output), read(self.mirrors), listing + + def both(self, job: 'str', step: 'str', former: 'str', prepare: 'Callable[[], None]', + **env: 'str') -> 'tuple[tuple[object, ...], tuple[object, ...]]': + """``(status, output, *state)`` of the former shell, then of the step's ``run:``, each from ``prepare``.""" + run = next(item for item in steps(WORKFLOW, job) if item.get('name') == step)['run'] + command = run.split(f'{SCRIPT} ', 1)[1].split()[0] + option = {'install': '--mirrors', 'plan': None}.get(command, '--archives') + target = self.mirrors if command == 'install' else self.archives + script = run.replace(IMAGE_PYTHON, shlex.quote(sys.executable)) + if option is not None: + script = script.replace(f'{SCRIPT} {command}', f'{SCRIPT} {command} {option} {shlex.quote(str(target))}') + shell = former.replace('/etc/apt/apt-mirrors.txt', str(self.mirrors)) \ + .replace('/var/cache/apt/archives', str(self.archives)) + found = [] + for text in (shell, script): + self.reset() + prepare() + result = subprocess.run(['bash', '-e', '-c', text], cwd=ROOT, env=self.env(**env), # nosec: B603 + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False, timeout=120) + found.append((result.returncode, result.stdout.decode('utf-8', 'replace'), *self.state())) + return found[0], found[1] + + def reset(self) -> None: + for path in (self.mirrors, self.archives, self.runner_temp, self.output, self.calls, + self.tmp / 'sim', self.tmp / 'installed'): + if path.is_dir(): + shutil.rmtree(path) + elif path.exists(): + path.unlink() + self.archives.mkdir() + self.runner_temp.mkdir() + self.output.write_text('earlier=1\n', encoding='utf-8') + self.calls.write_text('', encoding='utf-8') + + def scenarios(self) -> 'Iterator[tuple[str, str, str, dict[str, str]]]': + """``(job, step, former shell, env)`` for every apt step but the installs.""" + for job, sites in SITES.items(): + for name, _, _, _, _, run in sites: + command = run.split(f'{SCRIPT} ', 1)[1].split()[0] + if command in FORMER: + yield job, name, FORMER[command], {} + + def test_plan(self) -> None: + simulations = (SIMULATION, b'', b'NOTE: This is only a simulation!\n', SIMULATION[::-1], + b'Inst b (1)\nInst a (2)\nInst B (3)\nInst a-b (1)\nInst a.b (1)\n') + keyed = 0 + for job, step, former, _ in self.scenarios(): + if former is not FORMER_PLAN: + continue + for simulation in simulations: + for env in ({}, {'FAIL_UPDATES': '1'}, {'FAIL_UPDATES': '2'}, {'SIM_STATUS': '100'}, + {'ImageOS': ''}, {'APT_PACKAGES': 'build-essential libpcap-dev\ttshark'}): + with self.subTest(job=job, simulation=simulation, env=env): + env = {'APT_PACKAGES': 'tshark', **env} + old, new = self.both(job, step, former, + lambda: (self.tmp / 'sim').write_bytes(simulation), **env) + self.assertEqual(new, old) + keyed += 'key=' in str(old[3]) + self.assertTrue(keyed) + + def test_seed(self) -> None: + def nothing() -> None: + pass + + def restored() -> None: + (self.runner_temp / 'apt-debs' / 'deep').mkdir(parents=True) + for name in ('a.deb', 'b.deb', 'deep/c.deb', 'readme'): + (self.runner_temp / 'apt-debs' / name).write_text(name, encoding='utf-8') + + for job, step, former, _ in self.scenarios(): + if former is not FORMER_SEED: + continue + for prepare in (restored, nothing): + with self.subTest(job=job, restored=prepare is restored): + old, new = self.both(job, step, former, prepare) + self.assertEqual(new, old) + self.assertEqual(sorted(path.name for path in self.archives.iterdir()), + ['a.deb', 'b.deb'] if prepare is restored else []) + + def test_collect(self) -> None: + def archive(broken: 'bool') -> 'Callable[[], None]': + def prepare() -> None: + for name, fields in (('tshark.deb', 'tshark 4.2 amd64'), ('old.deb', 'old 1 amd64'), + ('gone.deb', 'gone 1 amd64'), ('i386.deb', 'lib 1 i386')): + package, version, arch = fields.split() + (self.archives / name).write_text( + f'Package={package}\nVersion={version}\nArchitecture={arch}\n', encoding='utf-8') + (self.archives / 'dangling.deb').symlink_to(self.archives / 'nowhere') + if broken: + (self.archives / 'm-broken.deb').write_text('not a deb', encoding='utf-8') + (self.tmp / 'installed').write_text('tshark:amd64 4.2\nold:amd64 2\nlib:amd64 1\n', encoding='utf-8') + (self.runner_temp / 'apt-debs').mkdir() + (self.runner_temp / 'apt-debs' / 'stale.deb').write_text('', encoding='utf-8') + return prepare + + for job, step, former, _ in self.scenarios(): + if former is not FORMER_COLLECT: + continue + for broken in (False, True): + with self.subTest(job=job, broken=broken): + old, new = self.both(job, step, former, archive(broken)) + self.assertEqual(new, old) + self.assertEqual(old[0], 2 if broken else 0) + + def test_install(self) -> None: + engines = {'Scapy': '', 'PyShark': 'tshark', 'PyPCAPFile': '', 'PyPCAP': 'build-essential libpcap-dev', + 'PCAP_CT': 'libpcap0.8'} + cases = [('engine-tests', FORMER_ENGINE_INSTALL, {'ENGINE': engine, 'APT_PACKAGES': packages}) + for engine, packages in engines.items()] + cases.append(('pypcap-parity', FORMER_PARITY_INSTALL, {'APT_PACKAGES': 'build-essential libpcap-dev tshark'})) + step = {job: sites[2][0] for job, sites in SITES.items()} + seen = set() # type: set[str] + for job, former, base in cases: + for mirrors in (MIRRORS, MIRRORS.splitlines(True)[0], None, ''): + for failures in ({}, {'FAIL_UPDATES': '1'}, {'FAIL_INSTALLS': '1'}, {'FAIL_UPDATES': '2'}, + {'FAIL_UPDATES': '1', 'FAIL_INSTALLS': '1'}, + {'FAIL_INSTALLS': '2', 'DPKG_STATUS': '1'}, {'DEBCONF_STATUS': '3'}): + def prepare(mirrors: 'Optional[str]' = mirrors) -> None: + if mirrors is not None: + self.mirrors.write_text(mirrors, encoding='utf-8') + with self.subTest(job=job, env=base, mirrors=mirrors, failures=failures): + old, new = self.both(job, step[job], former, prepare, **base, **failures) + self.assertEqual(new, old) + seen.update(piece for piece in ('fallback mirror', 'attempt 2 of 2', 'debconf', 'Setting up') + if piece in str(old)) + # Every path ran: the retry, the fallback, the second failure, debconf and an install. + self.assertEqual(seen, {'fallback mirror', 'attempt 2 of 2', 'debconf', 'Setting up'}) + + +class TestTheStepsRunTheScript(unittest.TestCase): + """Each apt step is one call of the script with its guards; no step keeps apt shell.""" + + def test_each_site_runs_the_script_with_its_guards(self) -> None: + for job, sites in {**SITES, 'pypcap-parity-ktlo': PARITY_STEPS}.items(): + found = steps(WORKFLOW, job) + names = [item.get('name') for item in found] + for name, step_id, condition, masked, cap, run in sites: + with self.subTest(job=job, step=name): + item = found[names.index(name)] + self.assertEqual(item['run'], run) + self.assertEqual((item.get('id'), item.get('if'), item.get('continue-on-error'), + item.get('timeout-minutes')), (step_id, condition, masked, cap)) + self.assertFalse(set(item) - {'name', 'id', 'if', 'continue-on-error', 'timeout-minutes', 'run'}) + positions = [names.index(name) for name, *_ in sites] + with self.subTest(job=job, order=True): + self.assertEqual(positions, sorted(positions)) + self.assertEqual(names[positions[2] + 1], APPARMOR) + self.assertEqual(positions[3], positions[2] + 2) + + def test_a_ktlo_leg_reads_only_warnings_from_it(self) -> None: + """#1672's scan of a step's commands sees the script's, which are warnings alone.""" + for name, *_, run in PARITY_STEPS: + with self.subTest(step=name): + self.assertEqual(commands(run), {'warning'}) + self.assertLessEqual(commands(run), KTLO_COMMANDS) + + def test_the_ktlo_twin_runs_the_same_steps(self) -> None: + self.assertEqual(steps(WORKFLOW, 'pypcap-parity-ktlo'), steps(WORKFLOW, 'pypcap-parity')) + + def test_the_apparmor_step_stays_inline(self) -> None: + for job in ('engine-tests', 'pypcap-parity'): + with self.subTest(job=job): + item = next(item for item in steps(WORKFLOW, job) if item.get('name') == APPARMOR) + self.assertNotIn('util/', item['run']) + self.assertIn('sudo apparmor_parser -r /etc/apparmor.d/tshark', item['run']) + + def test_the_packages_come_from_the_job(self) -> None: + """Plan and install read one ``APT_PACKAGES``, so the key is of the set installed.""" + jobs = load(WORKFLOW)['jobs'] + for job in (*SITES, 'pypcap-parity-ktlo'): + with self.subTest(job=job): + self.assertIn('APT_PACKAGES', jobs[job]['env']) + self.assertEqual(jobs['pypcap-parity']['env']['APT_PACKAGES'], 'build-essential libpcap-dev tshark') + self.assertEqual(jobs['pypcap-parity-ktlo']['env']['APT_PACKAGES'], + jobs['pypcap-parity']['env']['APT_PACKAGES']) + + def test_no_step_keeps_apt_shell(self) -> None: + found = [] + covered = {name for sites in SITES.values() for name, *_ in sites} + for path in sorted(WORKFLOWS.glob('*.yml')): + for job, spec in (load(path).get('jobs') or {}).items(): + for item in spec.get('steps') or (): + run = '\n'.join(line for line in str(item.get('run', '')).splitlines() + if not line.lstrip().startswith('#')) + found.extend(f'{path.name} {job} {item.get("name")!r}: {piece}' + for piece in INLINE if piece in run) + if SCRIPT in run and (path.name, item.get('name')) not in {(WORKFLOW, name) for name in covered}: + found.append(f'{path.name} {job} {item.get("name")!r}: unlisted') + self.assertEqual(found, []) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_gnu_timeout_2020.py b/tests/project/test_gnu_timeout_2020.py index 0e67d4f8f3..d07fd44355 100644 --- a/tests/project/test_gnu_timeout_2020.py +++ b/tests/project/test_gnu_timeout_2020.py @@ -12,12 +12,17 @@ A ``timeout`` with no option, ``timeout 60 apt-get update`` as the apt steps run it, behaves alike under both in the same image: exit 124, and its child's own children end with it. So the rule here is that a ``timeout`` given any option -runs through that resolution, in every workflow. +runs through that resolution, in every workflow. Since #1967 the steps' shell +moves into ``util/`` scripts, so the rule covers what those run too: no string in +one is an optioned ``timeout`` command, and no command one builds as a sequence, +``['sudo', 'timeout', '60', 'apt-get', 'update']`` as :file:`util/apt_packages.py` +does, gives ``timeout`` an option. """ from __future__ import annotations +import ast import os import pathlib import re @@ -40,6 +45,10 @@ OPTIONED = re.compile(r'(? 'Iterator[tuple[str, str, str]]': @@ -52,6 +61,55 @@ def scripts() -> 'Iterator[tuple[str, str, str]]': yield path.name, name, COMMENT.sub(r'\1', entry['run']) +def optioned_commands(source: 'str') -> 'Iterator[tuple[int, str]]': + """Each ``(line, text)`` in Python ``source`` that runs ``timeout`` with an option. + + A string holding an optioned ``timeout`` command, as a shell line would, or a + list or tuple display whose element after a :data:`TIMEOUTS` one is a string + starting with ``-`` or a starred, so unknown, expression. + + """ + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Constant) and isinstance(node.value, str) and OPTIONED.search(node.value): + yield node.lineno, node.value + elif isinstance(node, (ast.List, ast.Tuple)): + for here, after in zip(node.elts, node.elts[1:]): + if isinstance(here, ast.Constant) and here.value in TIMEOUTS and ( + isinstance(after, ast.Starred) + or isinstance(after, ast.Constant) and str(after.value).startswith('-')): + yield node.lineno, ast.dump(node) + + +def timeout_commands(source: 'str') -> 'int': + """How many list or tuple displays in ``source`` run one of :data:`TIMEOUTS`.""" + return sum(1 for node in ast.walk(ast.parse(source)) if isinstance(node, (ast.List, ast.Tuple)) + and any(isinstance(item, ast.Constant) and item.value in TIMEOUTS for item in node.elts)) + + +class TestNoUtilScriptRunsAnOptionedTimeout(unittest.TestCase): + """#1967: the scripts the steps run keep to the same rule as the steps' shell.""" + + def test_the_detector(self) -> None: + for source in ("call(['sudo', 'timeout', '-k', '3', '60', 'x'])", "('timeout', '--signal=INT', '1')", + "['timeout', *options, '60']", "script = 'sudo timeout -s INT 1 x'"): + with self.subTest(source=source): + self.assertTrue(list(optioned_commands(source))) + for source in ("call(['sudo', 'timeout', str(BOUND), 'apt-get', 'update'])", "['timeout', '60', '-x']", + "'sudo timeout 60 apt-get update'", "['gnutimeout', '-k', '1', '1']"): + with self.subTest(source=source): + self.assertEqual(list(optioned_commands(source)), []) + + def test_every_util_script(self) -> None: + found = 0 + for path in sorted(UTIL.glob('*.py')): + source = path.read_text(encoding='utf-8') + found += timeout_commands(source) + with self.subTest(script=path.name): + self.assertEqual(list(optioned_commands(source)), []) + # util/apt_packages.py's update and install bounds, so the scan is not blind. + self.assertGreaterEqual(found, 2) + + class TestNoWorkflowRunsAnOptionedTimeoutDirectly(unittest.TestCase): def test_the_pattern(self) -> None: diff --git a/tests/project/test_ktlo_legs_stay_green_1672.py b/tests/project/test_ktlo_legs_stay_green_1672.py index 0765471f55..ad1f97aea3 100644 --- a/tests/project/test_ktlo_legs_stay_green_1672.py +++ b/tests/project/test_ktlo_legs_stay_green_1672.py @@ -96,10 +96,13 @@ #: interpreters, which reads :data:`ENGINE_KTLO_OUTPUT` (#1660). ENGINE_KTLO = ('Run the KTLO cells (non-blocking)',) ENGINE_KTLO_OUTPUT = '${{ needs.python-versions.outputs.engine-ktlo }}' +#: The job's level, as a step's shell writes it. +LEVEL = '${ANNOTATION_LEVEL}' #: The workflow commands a step a KTLO leg runs may write. -KTLO_COMMANDS = frozenset({'${ANNOTATION_LEVEL}', 'warning', 'notice', 'group', 'endgroup'}) -#: A workflow command a script writes, by its name. -COMMAND = re.compile(r'''(?:echo|printf|print\(f?)\s*["']::([^\s:"']+)''') +KTLO_COMMANDS = frozenset({LEVEL, 'warning', 'notice', 'group', 'endgroup'}) +#: A workflow command a script writes, by its name: ``say`` is the printer of +#: util/apt_packages.py and util/regenerate_samples.py (#1967). +COMMAND = re.compile(r'''(?:echo|printf|(?:print|say)\(f?)\s*["']::([^\s:"']+)''') #: A status-check function: an ``if:`` holding none gets ``success() &&`` prepended. STATUS_FUNCTION = re.compile(r'\b(?:success|failure|cancelled|always)\(\)') #: A ``needs`` output a step's ``if:`` reads, such as the tier file's (#1660). @@ -285,10 +288,17 @@ def scenarios(name: 'str', job_id: 'str', spec: 'dict[str, Any]', def commands(script: 'str') -> 'set[str]': - """The workflow commands ``script`` writes, and each ``util/`` script it runs (#1967).""" + """The workflow commands ``script`` writes, and each ``util/`` script it runs (#1967). + + A script's ``::{level}``, where it reads ``ANNOTATION_LEVEL`` into ``level`` as + :file:`util/regenerate_samples.py` does, is the shell's ``${ANNOTATION_LEVEL}``. + + """ found = set(COMMAND.findall(script)) for path in set(UTIL_SCRIPT.findall(script)): - found |= set(COMMAND.findall((ROOT / path).read_text(encoding='utf-8'))) + text = (ROOT / path).read_text(encoding='utf-8') + levelled = "level = os.environ['ANNOTATION_LEVEL']" in text + found |= {LEVEL if name == '{level}' and levelled else name for name in COMMAND.findall(text)} return found @@ -412,7 +422,7 @@ def test_no_step_a_ktlo_leg_runs_writes_an_error(self) -> None: self.assertEqual(step.get('if'), unreachable) continue self.assertLessEqual(commands(step.get('run', '')), KTLO_COMMANDS) - if '${ANNOTATION_LEVEL}' in step.get('run', ''): + if LEVEL in commands(step.get('run', '')): self.assertIn((name, job_id), LEVELLED) def summary(self, steps: 'dict[str, str]') -> 'tuple[subprocess.CompletedProcess[str], str]': diff --git a/tests/project/test_tshark_apparmor_2019.py b/tests/project/test_tshark_apparmor_2019.py index 5fffedf7a0..1773926a16 100644 --- a/tests/project/test_tshark_apparmor_2019.py +++ b/tests/project/test_tshark_apparmor_2019.py @@ -63,13 +63,21 @@ def test_the_step_directly_follows_each_install(self) -> None: self.assertEqual(names[names.index(install) + 1], NAME) def test_every_job_installing_tshark_is_covered(self) -> None: - """A new tshark install elsewhere in the workflows needs the step too.""" + """A new tshark install elsewhere in the workflows needs the step too. + + Since #1967 an install runs :file:`util/apt_packages.py install`, which installs + the job's ``APT_PACKAGES``, so a job counts when that names tshark. + + """ found = set() for path in sorted(WORKFLOWS.glob('*.yml')): for job_id, spec in (load(path).get('jobs') or {}).items(): + packages = str((spec.get('env') or {}).get('APT_PACKAGES', '')) for item in spec.get('steps') or (): - if re.search(r'apt-get install[^\n]*\btshark\b|apt_install tshark\b', - str(item.get('run', ''))): + run = str(item.get('run', '')) + if re.search(r'apt-get install[^\n]*\btshark\b|apt_install tshark\b', run) or ( + re.search(r'\butil/apt_packages\.py install\b', run) + and re.search(r'\btshark\b', packages)): found.add((path.name, job_id)) self.assertEqual(found, {(WORKFLOW, job_id) for job_id, _ in SITES}) diff --git a/tests/project/test_unit_tests_queue.py b/tests/project/test_unit_tests_queue.py index ef4aa4c3ab..14fb191ef9 100644 --- a/tests/project/test_unit_tests_queue.py +++ b/tests/project/test_unit_tests_queue.py @@ -16,8 +16,9 @@ ``continue-on-error`` rather than a job-level one (#1633), and the job has no ``DPKT`` cell (#1538). * The classifier, :file:`util/classify_changes.py` (#1967), the engine cell loops - and the apt retry are executed too: the classifier against a scratch merge - commit, the others with their commands stubbed on ``PATH``. + and the apt retry, :file:`util/apt_packages.py`'s since #1967, are executed + too: the classifier against a scratch merge commit, the others with their + commands stubbed on ``PATH``. * Every test outside :file:`tests/project` that reads ``docs/`` or Markdown is run by ``project-tests``, so a docs-only PR cannot break it unseen. @@ -34,6 +35,7 @@ import os import pathlib import re +import shlex import shutil import subprocess import sys @@ -295,9 +297,17 @@ def step(section: 'str', name: 'str') -> 'str': return match.group(1) +#: The interpreter the steps run a ``util/`` script on, the image's (#1967). +IMAGE_PYTHON = '/usr/bin/python3' + + def step_script(section: 'str', name: 'str') -> 'str': - """The dedented ``run: |`` script of step ``name``.""" - body = step(section, name).split(' run: |\n', 1)[1] + """The dedented script of step ``name``: its ``run: |`` block, or its one-line ``run:``.""" + text = step(section, name) + line = re.search(r'(?m)^ run: (?!\|$)(.+)$', text) + if line is not None: + return line.group(1) + '\n' + body = text.split(' run: |\n', 1)[1] lines = [] for line in body.splitlines(): if line.strip() and not line.startswith(' ' * 10): @@ -320,6 +330,16 @@ def bash(script: 'str', env: 'dict[str, str]', cwd: 'Optional[str]' = None) -> ' check=False, capture_output=True, text=True).returncode +def checkout_script(script: 'str', mirrors: 'Optional[pathlib.Path]' = None) -> 'str': + """``script`` runnable from the checkout here (#1967): this interpreter for the + image's and, given ``mirrors``, that file for the runner's mirror list.""" + script = script.replace(IMAGE_PYTHON, shlex.quote(sys.executable)) + if mirrors is not None: + script = script.replace('util/apt_packages.py install', + f'util/apt_packages.py install --mirrors {shlex.quote(str(mirrors))}') + return script + + class _Scratch(unittest.TestCase): """A temporary directory per test, and ``bash`` required.""" @@ -476,10 +496,11 @@ def test_no_ktlo_cell_is_a_passing_no_op(self) -> None: self.assertEqual(status, 0) -#: The two apt steps, as (job, step name, extra env). +#: The two apt steps, as (job, step name, extra env), each with its job's ``APT_PACKAGES``. APT_STEPS: 'tuple[tuple[str, str, dict[str, str]], ...]' = ( - ('engine-tests', "Install this engine's system packages", {'ENGINE': 'PyShark'}), - ('pypcap-parity', 'Install libpcap headers, a C toolchain, and tshark', {}), + ('engine-tests', "Install this engine's system packages", {'ENGINE': 'PyShark', 'APT_PACKAGES': 'tshark'}), + ('pypcap-parity', 'Install libpcap headers, a C toolchain, and tshark', + {'APT_PACKAGES': 'build-essential libpcap-dev tshark'}), ) #: Stubs for the commands the apt steps run, put first on PATH. @@ -497,7 +518,7 @@ def test_no_ktlo_cell_is_a_passing_no_op(self) -> None: class TestAptRetry(_Scratch): - """Both apt steps, with ``sudo``/``timeout``/``apt-get`` stubbed.""" + """Both apt install steps, :file:`util/apt_packages.py` (#1967), with ``sudo``/``timeout``/``apt-get`` stubbed.""" def run_apt(self, name: 'str', job_name: 'str', extra: 'dict[str, str]', fail_updates: 'int') -> 'tuple[int, list[str], list[int]]': @@ -511,9 +532,9 @@ def run_apt(self, name: 'str', job_name: 'str', extra: 'dict[str, str]', log = self.tmp / 'stub' for suffix in ('.apt', '.timeouts'): pathlib.Path(f'{log}{suffix}').write_text('', encoding='utf-8') - status = bash(step_script(job(job_name), name), + status = bash(checkout_script(step_script(job(job_name), name), self.tmp / 'no-mirror-list'), dict(extra, PATH=f'{bin_dir}{os.pathsep}{os.environ["PATH"]}', - STUB_LOG=str(log), FAIL_UPDATES=str(fail_updates))) + STUB_LOG=str(log), FAIL_UPDATES=str(fail_updates)), str(ROOT)) apt = pathlib.Path(f'{log}.apt').read_text(encoding='utf-8').split() timeouts = pathlib.Path(f'{log}.timeouts').read_text(encoding='utf-8') bounds = [int(n) for n in timeouts.split()] diff --git a/tests/project/test_workflow_apt_cache.py b/tests/project/test_workflow_apt_cache.py index d6dd5dbb6d..40c72d8565 100644 --- a/tests/project/test_workflow_apt_cache.py +++ b/tests/project/test_workflow_apt_cache.py @@ -8,15 +8,15 @@ computes a cache key, ``actions/cache`` restore/seed/collect/save steps, and a retry that drops azure from the runner's ``mirror+file:`` list. -Each property below is exercised by running the step's real ``run:`` script -under ``bash -e`` with ``sudo``/``timeout``/``apt-get`` stubbed, the way -``TestAptRetry`` does, or by reading the step text where only the Actions runner -could execute it: +Each property below is exercised by running the step's real ``run:`` script, which +runs :file:`util/apt_packages.py` since #1967, under ``bash -e`` with +``sudo``/``timeout``/``apt-get`` stubbed, the way ``TestAptRetry`` does, or by +reading the step text where only the Actions runner could execute it: * the azure line is dropped before attempt 2 only, and only when another ``http`` mirror would remain; -* each job's ``APT_PACKAGES`` (which the key is computed from) names exactly - what its install step installs, for every engine in the matrix; +* each job's ``APT_PACKAGES`` (which the key is computed from) is what its + install step installs, for every engine in the matrix, since both read it; * the key is ``apt-debs----`` of the sorted ``Inst`` lines, and ``restore-keys`` is that same-image, same-set prefix only; * only one job per key saves, so ``pypcap-parity``'s two cells do not race to @@ -38,8 +38,9 @@ from typing import TYPE_CHECKING from tests import _workflows -from tests.project.test_python_versions_1660 import legs, matrix -from tests.project.test_unit_tests_queue import job, step_script +from tests.project.test_python_versions_1660 import ROOT, legs, matrix +from tests.project.test_unit_tests_queue import checkout_script, job, step_script +from util.apt_packages import MIRRORS as MIRRORS_DEFAULT if TYPE_CHECKING: from typing import Optional @@ -50,6 +51,8 @@ 'https://security.ubuntu.com/ubuntu/\tpriority:3\n') #: The path the install steps edit, replaced by a scratch file in these tests. MIRRORS_PATH = '/etc/apt/apt-mirrors.txt' +#: The install steps' command (#1967), which takes ``--mirrors`` for that scratch file. +INSTALL = 'util/apt_packages.py install' #: (job, plan step, install step) for both apt sites. SITES = ( @@ -107,11 +110,18 @@ def reset(self) -> None: for suffix in ('.apt', '.pkgs'): pathlib.Path(f'{self.log}{suffix}').write_text('', encoding='utf-8') - def run_script(self, script: 'str', env: 'dict[str, str]') -> 'tuple[int, str]': - """Run ``script`` under ``bash -e``; return its status and combined output.""" + def run_script(self, script: 'str', env: 'dict[str, str]', + mirrors: 'Optional[pathlib.Path]' = None) -> 'tuple[int, str]': + """Run ``script`` under ``bash -e`` from the checkout; return its status and combined output. + + :file:`util/apt_packages.py install` reads ``mirrors`` for the runner's mirror + list, a missing scratch file by default, so no test reads the host's. + + """ full = dict(os.environ, **env, PATH=f'{self.bin}{os.pathsep}{os.environ["PATH"]}', STUB_LOG=str(self.log)) - proc = subprocess.run(['bash', '-e', '-c', script], env=full, check=False, + script = checkout_script(script, mirrors or self.tmp / 'no-mirror-list') + proc = subprocess.run(['bash', '-e', '-c', script], env=full, check=False, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) return proc.returncode, proc.stdout @@ -131,8 +141,8 @@ def run_install(self, job_name: 'str', name: 'str', mirrors: 'Optional[str]', fail_updates: 'int') -> 'tuple[int, str, list[str], Optional[str]]': """Run ``job_name``'s install step against a scratch mirror list.""" script = step_script(job(job_name), name) - self.assertEqual(script.count(MIRRORS_PATH), 1, - f'{name!r} no longer names {MIRRORS_PATH} exactly once') + self.assertEqual(script.count(INSTALL), 1, f'{name!r} no longer runs {INSTALL} exactly once') + self.assertEqual(MIRRORS_DEFAULT, MIRRORS_PATH) self.reset() fake = self.tmp / 'apt-mirrors.txt' if fake.exists(): @@ -140,8 +150,8 @@ def run_install(self, job_name: 'str', name: 'str', mirrors: 'Optional[str]', if mirrors is not None: fake.write_text(mirrors, encoding='utf-8') status, output = self.run_script( - script.replace(MIRRORS_PATH, str(fake)), - {'ENGINE': 'PyShark', 'MIRRORS': str(fake), 'FAIL_UPDATES': str(fail_updates)}) + script, {'ENGINE': 'PyShark', 'APT_PACKAGES': 'tshark', 'MIRRORS': str(fake), + 'FAIL_UPDATES': str(fail_updates)}, fake) after = fake.read_text(encoding='utf-8') if mirrors is not None else None return status, output, self.apt_calls(), after @@ -235,19 +245,21 @@ def test_every_engine_named_in_the_expression_is_in_the_matrix(self) -> None: self.assertLessEqual(named, set(matrix_engines())) def test_engine_tests_installs_what_it_plans(self) -> None: - """Per engine, the install step's apt-get operands equal APT_PACKAGES.""" + """Per engine, the install step's apt-get operands are APT_PACKAGES, whatever ENGINE says.""" script = step_script(job('engine-tests'), SITES[0][2]) - for engine, packages in engine_packages().items(): - with self.subTest(engine=engine): - self.reset() - status, output = self.run_script(script, {'ENGINE': engine}) - self.assertEqual(status, 0, output) - self.assertEqual(self.installed(), packages.split()) + engines = engine_packages() + for engine, packages in engines.items(): + for named in (engine, *[other for other in engines if other != engine][:1]): + with self.subTest(engine=engine, ENGINE=named): + self.reset() + status, output = self.run_script(script, {'ENGINE': named, 'APT_PACKAGES': packages}) + self.assertEqual(status, 0, output) + self.assertEqual(self.installed(), packages.split()) def test_pypcap_parity_installs_what_it_plans(self) -> None: - """The parity job's literal list equals its install step's operands.""" + """The parity job's literal list is its install step's operands.""" packages = job_env('pypcap-parity', 'APT_PACKAGES') - status, output = self.run_script(step_script(job('pypcap-parity'), SITES[1][2]), {}) + status, output = self.run_script(step_script(job('pypcap-parity'), SITES[1][2]), {'APT_PACKAGES': packages}) self.assertEqual(status, 0, output) self.assertEqual(self.installed(), packages.split()) self.assertTrue(self.installed()) diff --git a/tests/project/test_workflow_apt_timeouts.py b/tests/project/test_workflow_apt_timeouts.py index bbeb96b2d2..eba15ad815 100644 --- a/tests/project/test_workflow_apt_timeouts.py +++ b/tests/project/test_workflow_apt_timeouts.py @@ -47,6 +47,9 @@ assertion that must run everywhere does not need the dependency, and the one that needs it guards the substitute. +Since #1967 the steps run apt through :file:`util/apt_packages.py`, so a step running +its ``plan`` or ``install`` counts as invoking apt, as one naming ``apt-get`` does. + Comments are stripped before the ``apt-get`` marker is looked for. Without that, prose *about* apt -- including the long note #974 added above the very step this module guards -- would register as an invocation, so a step could be flagged for @@ -86,12 +89,18 @@ #: it cannot invent a second one. Adding a different installer to a workflow #: means adding its marker here. APT_MARKER = 'apt-get' +#: The commands that invoke apt for a step since #1967: :file:`util/apt_packages.py`'s +#: ``plan`` and ``install``, which run ``apt-get``. Its ``seed`` and ``collect`` run +#: none, and so need no bound of their own, as before. +APT_SCRIPT = re.compile(r'\butil/apt_packages\.py (?:plan|install)\b') #: The steps known to invoke apt, as ``(workflow name, step name)``. Quoted #: so :class:`TestTheScannerIsNotBlind` can tell "the rule holds everywhere" from #: "the scanner found nothing to check", which look identical in a passing run. KNOWN_APT_STEPS = { + ('unit-tests.yml', "Plan this engine's system packages"), ('unit-tests.yml', "Install this engine's system packages"), + ('unit-tests.yml', 'Plan the system packages'), ('unit-tests.yml', 'Install libpcap headers, a C toolchain, and tshark'), } @@ -119,6 +128,11 @@ def strip_comments(text: 'str') -> 'str': if not line.strip().startswith('#')) +def invokes_apt(script: 'str', marker: 'str' = APT_MARKER) -> 'bool': + """Whether ``script`` holds ``marker`` or a command of :data:`APT_SCRIPT`.""" + return marker in script or APT_SCRIPT.search(script) is not None + + class Step: """One step of one workflow: its name, its script, and the job it belongs to.""" @@ -137,8 +151,9 @@ def timeout(self) -> 'Optional[int]': return None if match is None else int(match.group('minutes')) def invokes(self, marker: 'str') -> 'bool': - """Whether the step's script, comments stripped, contains ``marker``.""" - return marker in strip_comments(self.block) + """Whether the step's script, comments stripped, contains ``marker`` or runs apt through + :data:`APT_SCRIPT`.""" + return invokes_apt(strip_comments(self.block), marker) def __repr__(self) -> 'str': return f'' @@ -294,7 +309,7 @@ def test_yaml_finds_the_same_apt_steps(self) -> None: for step in job.get('steps') or []: if not isinstance(step, dict): continue - if APT_MARKER in strip_comments(step.get('run') or ''): + if invokes_apt(strip_comments(step.get('run') or '')): parsed.add((path.name, step.get('name'))) scanned = {(step.workflow, step.name) for step in apt_steps()} @@ -318,7 +333,7 @@ def test_yaml_agrees_on_the_declared_timeouts(self) -> None: for step in job.get('steps') or []: if not isinstance(step, dict): continue - if APT_MARKER in strip_comments(step.get('run') or ''): + if invokes_apt(strip_comments(step.get('run') or '')): parsed[(path.name, step.get('name'))] = step.get('timeout-minutes') scanned = {(step.workflow, step.name): step.timeout for step in apt_steps()} diff --git a/util/apt_packages.py b/util/apt_packages.py new file mode 100644 index 0000000000..c3288a4f94 --- /dev/null +++ b/util/apt_packages.py @@ -0,0 +1,268 @@ +# -*- coding: utf-8 -*- +"""Plan, seed, install and collect a job's system packages (#1967). + +The apt steps of :file:`.github/workflows/unit-tests.yml`'s ``engine-tests`` and +``pypcap-parity`` jobs (and so ``pypcap-parity-ktlo``) run, from the checkout's +root and on the image's ``/usr/bin/python3``:: + + /usr/bin/python3 util/apt_packages.py plan + /usr/bin/python3 util/apt_packages.py seed + /usr/bin/python3 util/apt_packages.py install [--name NAME] + /usr/bin/python3 util/apt_packages.py collect + +``APT_PACKAGES``, the job's environment, names the packages, separated by +whitespace, for both ``plan`` and ``install``, so the cache key cannot describe a +set other than the one installed. Each subcommand runs the commands the steps' +shell ran, ``sudo`` included, and writes the same lines and annotations: + +``plan`` + ``sudo timeout 60 apt-get update``, up to twice, with a ``::warning`` titled by + ``APT_SET`` after each failure and exit 1 after the second. Then ``apt-get + install -s --no-install-recommends APT_PACKAGES``, whose failure ends the + script with its status. Its ``Inst`` lines, sorted, are printed under ``To + download:``, or ``(nothing)``; when there are some, ``restore-prefix``, + ``apt-debs----``, and ``key``, that prefix and the + first 16 hex digits of the lines' SHA-256, are appended to ``$GITHUB_OUTPUT``. + ``ImageOS`` is ``unknown`` when unset or empty. +``seed`` + ``sudo cp $RUNNER_TEMP/apt-debs/*.deb /var/cache/apt/archives/``, its status the + script's on a failure, then how many ``.deb`` files that directory holds. +``install`` + Nothing when ``APT_PACKAGES`` is empty. Otherwise, when it names ``tshark``, + pre-seeds debconf against its "non-superusers may capture" prompt, then up to + two attempts of ``sudo timeout 60 apt-get update`` and ``sudo + DEBIAN_FRONTEND=noninteractive timeout 180 apt-get install -y + --no-install-recommends APT_PACKAGES``. After a failed attempt: a ``::warning``, + ``sudo dpkg --configure -a``, its failure ignored, and, after the first, azure's + line is dropped from :file:`/etc/apt/apt-mirrors.txt` with ``sudo sed`` when + another ``http`` mirror would remain, with a second ``::warning`` naming the + mirror now first (#1432). Exit 1 after the second. ``--name`` appends `` (NAME)`` + to both titles. +``collect`` + Empties :file:`$RUNNER_TEMP/apt-debs`, copies into it each + :file:`/var/cache/apt/archives/*.deb` whose package and architecture are + installed at its version, as ``dpkg-deb -f`` and ``dpkg-query -W`` report them, + and says how many it holds. A failed ``dpkg-deb`` ends the script with its status. + +The bounds stay GNU's ``timeout(1)`` run under ``sudo``, as in the shell, rather than +a :mod:`subprocess` timeout: apt runs as root, so this process could signal +``sudo`` alone and not apt's own children, its download methods, which ``timeout``, +running as root in a process group of its own, ends with apt. It is given no +option, so uutils' ``timeout`` on Ubuntu 26.04 behaves as GNU's (#2020). The +step's ``timeout-minutes`` stays the hang detector. The script is standard library +only, and runs on every interpreter the matrices hold, 3.7 included. + +""" + +from __future__ import annotations + +import argparse +import glob +import hashlib +import os +import shutil +import subprocess # nosec: B404 +import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Callable, Optional, Sequence + +__all__ = ['ATTEMPTS', 'UPDATE_BOUND', 'INSTALL_BOUND', 'MIRRORS', 'ARCHIVES', 'CACHE_DIR', 'AZURE', + 'DEBCONF', 'main'] + +#: How many times ``plan`` updates, and ``install`` updates and installs. +ATTEMPTS = 2 +#: ``timeout``'s bound, in seconds, on each ``apt-get update`` (#974). +UPDATE_BOUND = 60 +#: ``timeout``'s bound, in seconds, on each ``apt-get install`` (#1306). +INSTALL_BOUND = 180 +#: The runner image's ``mirror+file:`` list, which ``install`` edits before its retry. +MIRRORS = '/etc/apt/apt-mirrors.txt' +#: apt's archive, which ``seed`` fills and ``collect`` reads. +ARCHIVES = '/var/cache/apt/archives' +#: The cached ``.deb`` files, under ``$RUNNER_TEMP``. +CACHE_DIR = 'apt-debs' +#: The mirror that serves slowly rather than failing, so apt never leaves it (#1432). +AZURE = b'azure.archive.ubuntu.com' +#: The debconf answer that stops tshark's postinst prompting. +DEBCONF = b'wireshark-common wireshark-common/install-setuid boolean false\n' + + +def say(text: 'str') -> 'None': + """Write ``text`` and a newline to standard output at once, ahead of any command's.""" + sys.stdout.buffer.write(text.encode('utf-8', 'surrogateescape') + b'\n') + sys.stdout.buffer.flush() + + +def status_of(code: 'int') -> 'int': + """``code`` as the shell reports it: a signal ``N`` as ``128 + N``.""" + return code if code >= 0 else 128 - code + + +def call(command: 'Sequence[str]', **kwargs: 'object') -> 'subprocess.CompletedProcess[bytes]': + """Run ``command`` to its end, its streams this process's unless ``kwargs`` says otherwise.""" + return subprocess.run(list(command), check=False, **kwargs) # type: ignore[call-overload] # nosec: B603 + + +def packages() -> 'list[str]': + """``APT_PACKAGES``, split on whitespace as the shell split it.""" + return os.environ['APT_PACKAGES'].split() + + +def count_debs(path: 'str') -> 'int': + """How many entries under ``path`` match ``*.deb``, as ``find PATH -name '*.deb' | wc -l`` counts.""" + found = 0 + for _, dirs, files in os.walk(path): + found += sum(1 for name in dirs + files if name.endswith('.deb')) + return found + + +def debs(directory: 'str') -> 'list[str]': + """``directory/*.deb`` as the shell expands it, sorted, hidden files left out.""" + return sorted(glob.glob(os.path.join(glob.escape(directory), '*.deb'))) + + +def plan() -> 'int': + """Update apt's indexes, then write the cache key of what this image still lacks.""" + for attempt in range(1, ATTEMPTS + 1): + if call(['sudo', 'timeout', str(UPDATE_BOUND), 'apt-get', 'update']).returncode == 0: + break + say(f'::warning title=apt-get update attempt {attempt} of {ATTEMPTS} failed ({os.environ["APT_SET"]})' + f'::apt-get update failed or exceeded its {UPDATE_BOUND}s bound.') + if attempt == ATTEMPTS: + return 1 + + simulated = call(['apt-get', 'install', '-s', '--no-install-recommends', *packages()], stdout=subprocess.PIPE) + if simulated.returncode != 0: + return status_of(simulated.returncode) + lines = sorted(line for line in simulated.stdout.rstrip(b'\n').split(b'\n') if line.startswith(b'Inst ')) + listed = b'\n'.join(lines) + sys.stdout.buffer.write(b'To download:\n' + (listed or b'(nothing)') + b'\n') + sys.stdout.buffer.flush() + if lines: + prefix = f'apt-debs-{os.environ.get("ImageOS") or "unknown"}-{os.environ["RUNNER_ARCH"]}-' \ + f'{os.environ["APT_SET"]}-' + digest = hashlib.sha256(listed + b'\n').hexdigest()[:16] + with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as output: + # Concatenated: vermin reads `f'key={prefix}'` as 3.8's `{prefix=}`. + output.write('restore-prefix=' + prefix + '\n') + output.write('key=' + prefix + digest + '\n') + return 0 + + +def seed(archives: 'str') -> 'int': + """Copy the restored ``.deb`` files into apt's archive.""" + cache = os.path.join(os.environ['RUNNER_TEMP'], CACHE_DIR) + # An unmatched glob stays the pattern, as the shell's does, so `cp` fails on it. + copied = call(['sudo', 'cp', *(debs(cache) or [os.path.join(cache, '*.deb')]), archives + '/']) + if copied.returncode != 0: + return status_of(copied.returncode) + say(f'Seeded {count_debs(cache)} .deb files') + return 0 + + +def fallback(mirrors: 'str', suffix: 'str') -> 'None': + """Drop azure from ``mirrors`` when another ``http`` mirror would remain, and say so.""" + if not os.path.isfile(mirrors): + return + try: + with open(mirrors, 'rb') as file: + lines = file.read().split(b'\n') + except OSError: + return + if not any(line.startswith(b'http') for line in lines if AZURE not in line): + return + if call(['sudo', 'sed', '-i', r'/azure\.archive\.ubuntu\.com/d', mirrors]).returncode != 0: + return + try: + with open(mirrors, 'rb') as file: + first = file.read().split(b'\n', 1)[0].split(b'\t', 1)[0] + except OSError: + first = b'' + say(f'::warning title=apt retry on a fallback mirror{suffix}::Retrying against ' + f'{first.decode("utf-8", "surrogateescape")} instead of azure.archive.ubuntu.com, ' + 'resuming the partial download.') + + +def install(mirrors: 'str', name: 'Optional[str]') -> 'int': + """Install ``APT_PACKAGES``, retrying once, on a fallback mirror when there is one.""" + wanted = packages() + if not wanted: + return 0 + suffix = '' if name is None else f' ({name})' + if 'tshark' in wanted: + answered = call(['sudo', 'debconf-set-selections'], input=DEBCONF) + if answered.returncode != 0: + return status_of(answered.returncode) + for attempt in range(1, ATTEMPTS + 1): + if call(['sudo', 'timeout', str(UPDATE_BOUND), 'apt-get', 'update']).returncode == 0 \ + and call(['sudo', 'DEBIAN_FRONTEND=noninteractive', 'timeout', str(INSTALL_BOUND), + 'apt-get', 'install', '-y', '--no-install-recommends', *wanted]).returncode == 0: + return 0 + say(f'::warning title=apt attempt {attempt} of {ATTEMPTS} failed{suffix}::apt-get update or install ' + 'failed or exceeded its per-attempt bound; retrying once.') + call(['sudo', 'dpkg', '--configure', '-a']) + if attempt == 1: + fallback(mirrors, suffix) + return 1 + + +def field(deb: 'str', name: 'str') -> 'tuple[int, str]': + """``dpkg-deb -f DEB NAME``'s status and output, trailing newlines stripped.""" + result = call(['dpkg-deb', '-f', deb, name], stdout=subprocess.PIPE) + return status_of(result.returncode), result.stdout.decode('utf-8', 'surrogateescape').rstrip('\n') + + +def collect(archives: 'str') -> 'int': + """Copy into the cache each archived ``.deb`` whose version is the one installed.""" + out = os.path.join(os.environ['RUNNER_TEMP'], CACHE_DIR) + # `rm -rf`, then `mkdir -p`: a failure of either ends the script. + if os.path.isdir(out) and not os.path.islink(out): + shutil.rmtree(out) + elif os.path.lexists(out): + os.remove(out) + os.makedirs(out, exist_ok=True) + for deb in debs(archives): + if not os.path.exists(deb): + continue + fields = {} # type: dict[str, str] + for name in ('Package', 'Version', 'Architecture'): + status, fields[name] = field(deb, name) + if status != 0: + return status + installed = call(['dpkg-query', '-W', '-f', '${Version}', f'{fields["Package"]}:{fields["Architecture"]}'], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + if installed.stdout.decode('utf-8', 'surrogateescape').rstrip('\n') == fields['Version']: + shutil.copy(deb, out) + say(f'Collected {count_debs(out)} .deb files') + return 0 + + +def main(argv: 'Optional[list[str]]' = None) -> 'int': + """Run one subcommand; its exit status.""" + parser = argparse.ArgumentParser(description=__doc__.split('\n', 1)[0]) + commands = parser.add_subparsers(dest='command', metavar='COMMAND') + commands.required = True + commands.add_parser('plan', help='update the indexes and write the cache key') + for command, text in (('seed', 'copy the restored .deb files into apt\'s archive'), + ('collect', 'copy the installed versions\' .deb files into the cache')): + sub = commands.add_parser(command, help=text) + sub.add_argument('--archives', default=ARCHIVES, help=f'apt\'s archive (default: {ARCHIVES})') + sub = commands.add_parser('install', help='install APT_PACKAGES, retrying once') + sub.add_argument('--name', help='what the annotation titles name, in parentheses') + sub.add_argument('--mirrors', default=MIRRORS, help=f'the mirror list (default: {MIRRORS})') + args = parser.parse_args(argv) + + run = { + 'plan': plan, + 'seed': lambda: seed(args.archives), + 'install': lambda: install(args.mirrors, args.name), + 'collect': lambda: collect(args.archives), + }[args.command] # type: Callable[[], int] + sys.stdout.flush() + return run() + + +if __name__ == '__main__': + sys.exit(main())