From 0fac0e47d94c48e5f5146dbe62b4ec2a1dc83614 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Wed, 21 Oct 2026 22:51:34 -0400 Subject: [PATCH] ci: install through util/pip_install.py (#1967) - util/pip_install.py: upgrades pip, setuptools and wheel, then installs the given editable spec, teeing its merged output to $RUNNER_TEMP/pip-install.log; a failed install writes the same ANNOTATION_LEVEL annotation naming pip's last ERROR line and exits 1. Standard library only, 3.7-compatible. - unit-tests.yml: the install step of test, integration, pypcap-parity, unittest-ordering, project-tests and gate runs the script. The last four gain the annotation; the latest-only jobs set ANNOTATION_LEVEL: error and LEG on the step, pypcap-parity sets LEG. id, if, continue-on-error and shell are unchanged. - tests/_dependency_gates.py: EDITABLE_INSTALL, the shared extras pattern for both install forms, used by every reader of install lines. - tests: test_pip_install_1967.py covers the script, its parity with the former shell, and pins every site; the 1633 install tests stub pip instead of python; the tier-guard doctorings use the new form. - workflows.rst: the script joins the Workflow Scripts table. --- .github/workflows/unit-tests.yml | 67 ++-- docs/source/contributing/workflows.rst | 8 +- tests/_dependency_gates.py | 14 +- tests/project/test_pip_install_1967.py | 372 ++++++++++++++++++ .../project/test_python_matrix_tiers_1633.py | 37 +- .../test_sweeps_on_latest_only_1732.py | 2 +- .../test_whole_suite_on_latest_1958.py | 3 +- tests/test_tier_guard.py | 20 +- util/pip_install.py | 123 ++++++ 9 files changed, 577 insertions(+), 69 deletions(-) create mode 100644 tests/project/test_pip_install_1967.py create mode 100644 util/pip_install.py diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 721ce88ffe..6c78f329b1 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -174,13 +174,12 @@ jobs: # why. # # #1633: a KTLO leg that cannot install -- such as 3.7 -- fails fast and - # once: one annotation names pip's last ERROR line (its title's `,` as - # `%2C`, as the runner splits properties on `,`), and every later step - # needs `steps.install.outcome == 'success'`, so it is skipped rather - # than failing against a missing package. The install needs its own - # interpreter: if setup-python failed, the image's `python` would be - # tested under this leg's name. On a supported leg either fails the job; - # on a KTLO leg the annotation is a warning (#1672). + # once: util/pip_install.py writes one annotation naming pip's last ERROR + # line, and every later step needs `steps.install.outcome == 'success'`, + # so it is skipped rather than failing against a missing package. The + # install needs its own interpreter: if setup-python failed, the image's + # `python` would be tested under this leg's name. On a supported leg + # either fails the job; on a KTLO leg the annotation is a warning (#1672). - name: Install package and test dependencies id: install if: ${{ steps.python.outcome == 'success' }} @@ -188,14 +187,7 @@ jobs: shell: bash env: LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} - run: | - python -m pip install -U pip setuptools wheel - if ! python -m pip install -e '.[test,DPKT,crypto,NGAP]' 2>&1 | tee "$RUNNER_TEMP/pip-install.log"; then - reason=$(sed -n 's/^ERROR: //p' "$RUNNER_TEMP/pip-install.log" | tail -n 1) - if [ -z "$reason" ]; then reason="pip failed without an ERROR line; see this step's log"; fi - echo "::${ANNOTATION_LEVEL} title=pcapkit did not install on ${LEG//,/%2C}::${reason//%/%25}. The rest of this leg is skipped." - exit 1 - fi + run: python util/pip_install.py -e '.[test,DPKT,crypto,NGAP]' # #1958, the owner's ruling E: latest CPython runs the whole suite, so its # leg installs everything the suite gates on that pip can install there -- @@ -491,14 +483,7 @@ jobs: shell: bash env: LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} - run: | - python -m pip install -U pip setuptools wheel - if ! python -m pip install -e '.[test,Scapy,DPKT,cli]' 2>&1 | tee "$RUNNER_TEMP/pip-install.log"; then - reason=$(sed -n 's/^ERROR: //p' "$RUNNER_TEMP/pip-install.log" | tail -n 1) - if [ -z "$reason" ]; then reason="pip failed without an ERROR line; see this step's log"; fi - echo "::${ANNOTATION_LEVEL} title=pcapkit did not install on ${LEG//,/%2C}::${reason//%/%25}. The rest of this leg is skipped." - exit 1 - fi + run: python util/pip_install.py -e '.[test,Scapy,DPKT,cli]' # #1958: the `test` job's step of the same name. On `latest` this job runs # the whole fixture tier, so it installs the whole suite's set too. @@ -1296,13 +1281,14 @@ jobs: # As in the `test` job (#1672): the install needs this leg's interpreter, # and the steps after it need the install, so a KTLO leg whose setup or # install failed stops there instead of testing the image's `python`. + # util/pip_install.py annotates a failed install, as in `test`. - name: Install package, test and generator dependencies id: install if: ${{ steps.python.outcome == 'success' }} continue-on-error: ${{ matrix.ktlo == true }} - run: | - python -m pip install -U pip setuptools wheel - python -m pip install -e '.[test,Scapy,DPKT,cli,PyShark,PyPCAP,PyPCAPFile]' + env: + LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }} + run: python util/pip_install.py -e '.[test,Scapy,DPKT,cli,PyShark,PyPCAP,PyPCAPFile]' # See the `integration` job above. - name: Regenerate sample captures @@ -1569,11 +1555,12 @@ jobs: # DPKT/crypto/NGAP join `test` for the same reason the `test` job above # installs them: this leg's root-level modules and its own directory's # modules both reach gates that would otherwise skip silently rather - # than run. + # than run. This job has no KTLO leg, so a failed install is an error. - name: Install package and test dependencies - run: | - python -m pip install -U pip setuptools wheel - python -m pip install -e '.[test,DPKT,crypto,NGAP]' + env: + ANNOTATION_LEVEL: error + LEG: Python ${{ steps.python-versions.outputs.latest }} + run: python util/pip_install.py -e '.[test,DPKT,crypto,NGAP]' - name: Report available parallelism run: | @@ -1711,11 +1698,13 @@ jobs: cache: pip # The whole suite's set, the line `test` adds on `latest` (#1958), so a - # gated pin runs here exactly as the rest of its tier does there. + # gated pin runs here exactly as the rest of its tier does there. No KTLO + # leg, so a failed install is an error. - name: Install package and test dependencies - run: | - python -m pip install -U pip setuptools wheel - python -m pip install -e '.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]' + env: + ANNOTATION_LEVEL: error + LEG: Python ${{ steps.python-versions.outputs.latest }} + run: python util/pip_install.py -e '.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]' # See the `test` job above for why this step exists. - name: Report available parallelism @@ -1798,11 +1787,13 @@ jobs: # #1958: `vendor`, `docs` and `PyShark` join them, so this is the set the # `latest` legs install (see `test`'s "Install the whole suite's # dependencies on latest" step), and the html5lib, Sphinx and PyShark - # gates run here as they do there. + # gates run here as they do there. No KTLO leg, so a failed install is an + # error. - name: Install package, test and generator dependencies - run: | - python -m pip install -U pip setuptools wheel - python -m pip install -e '.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]' + env: + ANNOTATION_LEVEL: error + LEG: Python ${{ needs.python-versions.outputs.latest }} + run: python util/pip_install.py -e '.[test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark]' # See the `integration` job above. This job has no KTLO leg, so a # failure is always an error. diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index ad3eeb6b6a..9d24160f69 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -1207,6 +1207,10 @@ does; each has tests under :file:`tests/project/` that run it. - ``Regenerate sample captures`` in ``unit-tests.yml``'s ``integration`` and ``pypcap-parity`` (and so their KTLO twins), at the job's ``ANNOTATION_LEVEL``, and ``gate``, at ``error``. + * - :file:`util/pip_install.py` + - The install step of ``unit-tests.yml``'s ``test``, ``integration`` and + ``pypcap-parity`` (and so their KTLO twins), at the job's ``ANNOTATION_LEVEL``, + and of ``unittest-ordering``, ``project-tests`` and ``gate``, at ``error``. The summaries run on the image's ``/usr/bin/python3`` rather than the leg's interpreter, which may have failed to set up or be too broken to run anything, so @@ -1216,7 +1220,9 @@ too, standard library only, so on GraalPy's leg the leg's interpreter runs pytes alone; on PyPy's legs the leg's interpreter runs the script as well, under the file-size cap every test command of ``unit-tests.yml`` takes. :file:`util/regenerate_samples.py` is standard library only but runs on the leg's -interpreter, since the generator it runs imports ``pcapkit``. Both ``changes`` jobs check out on every event, since the script is in the +interpreter, since the generator it runs imports ``pcapkit``, and so does +:file:`util/pip_install.py`, since it installs into that interpreter; it runs on +every version the matrices hold, 3.7 included. Both ``changes`` jobs check out on every event, since the script is in the tree. ``create-release.yml``'s ``pypi`` job keeps its own copy of the KTLO summary's shell until its shell moves to scripts too; a project pin runs that copy and the script on the same steps and requires the same output. diff --git a/tests/_dependency_gates.py b/tests/_dependency_gates.py index b85952d5d4..034c5190ea 100644 --- a/tests/_dependency_gates.py +++ b/tests/_dependency_gates.py @@ -103,8 +103,8 @@ from typing import Iterator, Optional __all__ = [ - 'WORKFLOW', 'PYPROJECT', 'LATEST_LEG_CONDITION', 'CORE', 'MODULE_PROVIDERS', 'NON_DISTRIBUTION_FLAGS', - 'MUTUALLY_EXCLUSIVE_IMPORTS', 'DEPENDENCY_GATE_EXCLUSIONS', + 'WORKFLOW', 'PYPROJECT', 'LATEST_LEG_CONDITION', 'EDITABLE_INSTALL', 'CORE', 'MODULE_PROVIDERS', + 'NON_DISTRIBUTION_FLAGS', 'MUTUALLY_EXCLUSIVE_IMPORTS', 'DEPENDENCY_GATE_EXCLUSIONS', 'Requirement', 'Gate', 'Job', 'Gap', 'Exclusion', 'AmbiguousProvider', 'requirement_key', 'provided_by', 'declared_requirements', 'extras_providing', 'module_providers', 'contested_imports', @@ -136,6 +136,10 @@ #: The ``if:`` of a step that runs on the ``latest`` leg alone (#1958), which is #: how :func:`pytest_jobs` tells that leg's install line from every leg's. LATEST_LEG_CONDITION = 'matrix.python-version == needs.python-versions.outputs.latest' +#: An editable install of the checkout, its one group the extras: pip's own +#: ``pip install -e '.[...]'``, or ``util/pip_install.py -e '.[...]'``, which runs +#: it for most install steps (#1967). Every reader of an install line uses it. +EDITABLE_INSTALL = re.compile(r"(?:pip install|util/pip_install\.py) -e '\.\[([^]]*)\]'") #: Stands in for ``[project] dependencies`` where an extra name is expected: #: what ``pip install -e .`` installs with no extra asked for at all. Spelled @@ -1245,7 +1249,7 @@ def _engine_matrix_variants(name: 'str', section: 'str') -> 'Optional[tuple[Job, ) declared_engines = frozenset(re.findall(r'-[ \t]*(\w+)', declared_match.group(1))) - base_installs = re.findall(r"pip install -e '\.\[([^]]*)\]'", code) + base_installs = EDITABLE_INSTALL.findall(code) if len(base_installs) != 1: raise AssertionError( f'the {name!r} job varies by matrix.engine but has {len(base_installs)} ' @@ -1339,14 +1343,14 @@ def pytest_jobs(workflow: 'Optional[pathlib.Path]' = None) -> 'tuple[Job, ...]': # install line, read apart from the one every leg runs. latest_steps = [step for step in steps if LATEST_LEG_CONDITION in step.partition('run:')[0]] latest_installs = [extras for step in latest_steps - for extras in re.findall(r"pip install -e '\.\[([^]]*)\]'", step)] + for extras in EDITABLE_INSTALL.findall(step)] if len(latest_installs) > 1: raise AssertionError( f'the {name!r} job has {len(latest_installs)} latest-only ' f"\"pip install -e '.[...]'\" lines, and this guard cannot tell which " f'extras that leg would have' ) - installs = re.findall(r"pip install -e '\.\[([^]]*)\]'", section) + installs = EDITABLE_INSTALL.findall(section) installs = [extras for extras in installs if extras not in latest_installs] \ if latest_installs else installs if len(installs) != 1: diff --git a/tests/project/test_pip_install_1967.py b/tests/project/test_pip_install_1967.py new file mode 100644 index 0000000000..349529fca1 --- /dev/null +++ b/tests/project/test_pip_install_1967.py @@ -0,0 +1,372 @@ +# -*- coding: utf-8 -*- +"""#1967: :file:`util/pip_install.py`, the install step of most ``unit-tests.yml`` jobs, run as the steps run it. + +* :class:`TestPipInstall` runs the script against a stand-in ``pip``: a clean + install, a failed one with and without an ``ERROR:`` line, a failed upgrade, + and the arguments and levels it refuses. +* :class:`TestTheFormerShellAgrees` runs the shell the steps held before the + script against the same stand-in and requires the same output, log and status; + for the steps that had no annotation, the same except for the annotation a + failed install now writes. +* :class:`TestTheStepsRunTheScript` pins each covered step to the script, with + its guards and environment, and every other ``pip install`` in + ``unit-tests.yml`` to a listed step that keeps its shell. + +""" + +from __future__ import annotations + +import json +import os +import pathlib +import re +import shutil +import signal +import subprocess # nosec: B404 +import sys +import tempfile +import textwrap +import unittest + +from tests._dependency_gates import EDITABLE_INSTALL +from tests._workflows import job, load, steps + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SCRIPT = ROOT / 'util' / 'pip_install.py' +WORKFLOW = ROOT / '.github' / 'workflows' / 'unit-tests.yml' +LOG = 'pip-install.log' +SPEC = '.[test,DPKT]' + +#: A leg's name holding a comma, which the title must escape, and a ``%``, which it must not. +LEG = 'Python 3.7 (KTLO, non-blocking) 100%' +TITLE = 'pcapkit did not install on Python 3.7 (KTLO%2C non-blocking) 100%' +SKIPPED = '. The rest of this leg is skipped.' +NO_REASON = "pip failed without an ERROR line; see this step's log" + +#: The KTLO-capable sites' ``LEG``, and their jobs' ``ANNOTATION_LEVEL``. +MATRIX_LEG = "Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }}" +MATRIX_LEVEL = "${{ matrix.ktlo && 'warning' || 'error' }}" +#: Each covered step, as ``(job, step name)``, and the extras it installs. +MATRIX_SITES = { + ('test', 'Install package and test dependencies'): 'test,DPKT,crypto,NGAP', + ('integration', 'Install package, test and generator dependencies'): 'test,Scapy,DPKT,cli', + ('pypcap-parity', 'Install package, test and generator dependencies'): + 'test,Scapy,DPKT,cli,PyShark,PyPCAP,PyPCAPFile', +} +#: The jobs on ``latest`` alone, which have no KTLO leg; each step sets ``error``. +LATEST_SITES = { + ('unittest-ordering', 'Install package and test dependencies'): 'test,DPKT,crypto,NGAP', + ('project-tests', 'Install package and test dependencies'): 'test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark', + ('gate', 'Install package, test and generator dependencies'): 'test,Scapy,DPKT,crypto,cli,NGAP,vendor,docs,PyShark', +} +#: The ``pip install`` steps that keep their shell: the ``latest`` leg's addition, which +#: upgrades nothing and annotates nothing, coverage, which is not the package, and +#: ``engine-tests``' cells, whose installs run in a venv and may fail by design. +LEFT_INLINE = frozenset({ + ('test', "Install the whole suite's dependencies on latest"), + ('integration', "Install the whole suite's dependencies on latest"), + ('test', 'Install coverage'), + ('engine-tests', 'Write the per-cell script'), +}) +#: The KTLO jobs, which run their twin's steps by alias (#1738). +TWINS = {'test-ktlo': 'test', 'integration-ktlo': 'integration', 'pypcap-parity-ktlo': 'pypcap-parity'} +#: Text only the former inline install held. +INLINE = ('pip-install.log', 'did not install on', "s/^ERROR: //p") + +#: A stand-in ``pip``: it appends its arguments to ``calls``, then does what +#: ``PIP_`` says, ``STEP`` ``UPGRADE`` or ``INSTALL``: ``OUTPUT`` lines, each +#: ``out:`` or ``err:``, flushed in turn, then exit ``STATUS``, or a signal as ``-N``. +PIP = textwrap.dedent('''\ + import json, os, signal, sys + with open('calls', 'a', encoding='utf-8') as calls: + calls.write(json.dumps(sys.argv[1:]) + '\\n') + step = 'UPGRADE' if sys.argv[1:3] == ['install', '-U'] else 'INSTALL' + for line in json.loads(os.environ.get('PIP_' + step + '_OUTPUT', '[]')): + stream = sys.stderr if line.startswith('err:') else sys.stdout + stream.write(line[4:] + '\\n') + stream.flush() + status = int(os.environ.get('PIP_' + step + '_STATUS', '0')) + if status < 0: + os.kill(os.getpid(), -status) + sys.exit(status) +''') + +#: The ``run:`` of ``test``'s and ``integration``'s install before the script, +#: under ``shell: bash``; ``SPEC`` stands for each job's extras. +FORMER_ANNOTATED = textwrap.dedent('''\ + python -m pip install -U pip setuptools wheel + if ! python -m pip install -e 'SPEC' 2>&1 | tee "$RUNNER_TEMP/pip-install.log"; then + reason=$(sed -n 's/^ERROR: //p' "$RUNNER_TEMP/pip-install.log" | tail -n 1) + if [ -z "$reason" ]; then reason="pip failed without an ERROR line; see this step's log"; fi + echo "::${ANNOTATION_LEVEL} title=pcapkit did not install on ${LEG//,/%2C}::${reason//%/%25}. The rest of this leg is skipped." + exit 1 + fi +''').replace('SPEC', SPEC) # noqa: E501 +#: The ``run:`` of the other covered steps before the script, under the default shell. +FORMER_BARE = textwrap.dedent('''\ + python -m pip install -U pip setuptools wheel + python -m pip install -e 'SPEC' +''').replace('SPEC', SPEC) + +CLEAN = ['out:Obtaining file:///checkout', 'err:a deprecation warning', 'out:Successfully installed pypcapkit-1.5.0'] +FAILED = ['out:Collecting pytest>=8', 'err:ERROR: Ignored the following versions: 8.0.0 Requires-Python >=3.8', + 'out:INFO: pip is looking at multiple versions', 'err:ERROR: No matching distribution found for pytest>=8'] +#: Every ``(install output, install status)`` an install step can meet. +OUTCOMES = ( + (CLEAN, 0), + (FAILED, 1), + (['out:Killed'], 137), + (['err:ERROR: 100% of the index timed out'], 2), + (['err:ERROR: the first reason', 'err:ERROR: '], 1), + (['out:ERROR:no space after the colon'], 1), + (FAILED, -signal.SIGTERM), +) + + +def text(lines: 'list[str]') -> 'str': + """What the stand-in writes for ``lines``, its two streams merged.""" + return ''.join(line[4:] + '\n' for line in lines) + + +class _Scratch(unittest.TestCase): + """A scratch checkout root, a stand-in ``pip`` on ``PYTHONPATH`` and a ``RUNNER_TEMP``.""" + + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-1967-pip-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = pathlib.Path(tmp.name) + (self.tmp / 'stub' / 'pip').mkdir(parents=True) + (self.tmp / 'stub' / 'pip' / '__init__.py').write_text('', encoding='utf-8') + (self.tmp / 'stub' / 'pip' / '__main__.py').write_text(PIP, encoding='utf-8') + self.runner_temp = self.tmp / 'runner-temp' + self.runner_temp.mkdir() + self.log = self.runner_temp / LOG + + def env(self, level: 'str' = 'warning', **pip: 'object') -> 'dict[str, str]': + env = {key: value for key, value in os.environ.items() + if key not in ('ANNOTATION_LEVEL', 'LEG', 'RUNNER_TEMP', 'PYTHONPATH', 'PYTHONSAFEPATH') + and not key.startswith('PIP_')} + env.update(ANNOTATION_LEVEL=level, LEG=LEG, RUNNER_TEMP=str(self.runner_temp), + PYTHONPATH=str(self.tmp / 'stub'), PYTHONIOENCODING='utf-8') + for key, value in pip.items(): + env[f'PIP_{key.upper()}'] = json.dumps(value) if isinstance(value, list) else str(value) + return env + + def calls(self) -> 'list[list[str]]': + path = self.tmp / 'calls' + if not path.exists(): + return [] + calls = [json.loads(line) for line in path.read_text(encoding='utf-8').splitlines()] + path.unlink() + return calls + + def script(self, *args: 'str', env: 'dict[str, str]') -> 'subprocess.CompletedProcess[str]': + """``python util/pip_install.py ARGS`` from the scratch root, as the step runs it.""" + return subprocess.run([sys.executable, str(SCRIPT), *args], cwd=self.tmp, env=env, # nosec: B603 + capture_output=True, text=True, check=False, timeout=120) + + +class TestPipInstall(_Scratch): + """The upgrade, then the install on standard output and in the log, then one verdict.""" + + def test_a_clean_install_writes_no_annotation(self) -> None: + result = self.script('-e', SPEC, env=self.env(install_output=CLEAN, upgrade_output=['out:upgraded'])) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, 'upgraded\n' + text(CLEAN)) + self.assertEqual(self.log.read_text(encoding='utf-8'), text(CLEAN)) + self.assertEqual(self.calls(), [['install', '-U', 'pip', 'setuptools', 'wheel'], ['install', '-e', SPEC]]) + + def test_a_failed_install_names_pips_last_error_at_the_level(self) -> None: + for level in ('error', 'warning'): + with self.subTest(level=level): + result = self.script('-e', SPEC, env=self.env(level, install_output=FAILED, install_status=1)) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(result.stdout, text(FAILED) + f'::{level} title={TITLE}::No matching ' + f'distribution found for pytest>=8{SKIPPED}\n') + self.assertEqual(self.log.read_text(encoding='utf-8'), text(FAILED)) + + def test_a_failure_without_a_reason_is_still_named(self) -> None: + """No ``ERROR:`` line, or an empty last one, falls back; a ``%`` in the reason is escaped.""" + for output, reason in ((['out:Killed'], NO_REASON), (['err:ERROR: first', 'err:ERROR: '], NO_REASON), + (['err:ERROR: 100% of the index timed out'], '100%25 of the index timed out')): + with self.subTest(output=output): + result = self.script('-e', SPEC, env=self.env(install_output=output, install_status=1)) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertTrue(result.stdout.endswith(f'::warning title={TITLE}::{reason}{SKIPPED}\n'), + result.stdout) + + def test_a_signal_is_a_failed_install(self) -> None: + result = self.script('-e', SPEC, env=self.env(install_output=FAILED, install_status=-signal.SIGTERM)) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertEqual(len([line for line in result.stdout.splitlines() if line.startswith('::')]), 1) + + def test_a_failed_upgrade_ends_with_pips_status_and_no_annotation(self) -> None: + for status, expected in ((3, 3), (-signal.SIGTERM, 128 + signal.SIGTERM)): + with self.subTest(status=status): + result = self.script('-e', SPEC, env=self.env(upgrade_output=['err:no index'], upgrade_status=status)) + self.assertEqual(result.returncode, expected, result.stderr) + self.assertEqual(result.stdout, '') + self.assertEqual(result.stderr, 'no index\n') + self.assertFalse(self.log.exists()) + self.assertEqual(self.calls(), [['install', '-U', 'pip', 'setuptools', 'wheel']]) + + def test_a_bad_level_fails_before_pip_runs(self) -> None: + for level in ('', 'notice', 'ERROR'): + with self.subTest(level=level): + result = self.script('-e', SPEC, env=self.env(level)) + self.assertEqual(result.returncode, 2) + self.assertEqual(result.stdout, '') + self.assertIn('ANNOTATION_LEVEL', result.stderr) + self.assertEqual(self.calls(), []) + + def test_it_takes_one_editable_spec_and_nothing_else(self) -> None: + for args in ((), ('-e',), (SPEC,), ('-e', SPEC, 'extra'), ('-r', 'requirements.txt')): + with self.subTest(args=args): + result = self.script(*args, env=self.env()) + self.assertEqual(result.returncode, 2) + self.assertEqual(self.calls(), []) + + def test_a_missing_variable_fails_before_pip_runs(self) -> None: + for name in ('ANNOTATION_LEVEL', 'LEG', 'RUNNER_TEMP'): + with self.subTest(variable=name): + env = self.env() + del env[name] + result = self.script('-e', SPEC, env=env) + self.assertNotEqual(result.returncode, 0) + self.assertIn(name, result.stderr) + self.assertEqual(self.calls(), []) + + +@unittest.skipIf(any(shutil.which(tool) is None for tool in ('bash', 'tee', 'sed', 'tail')), + 'needs bash, tee, sed and tail') +class TestTheFormerShellAgrees(_Scratch): + """The script prints, logs and exits as the steps' former shell did.""" + + def former(self, shell: 'str', env: 'dict[str, str]') -> 'subprocess.CompletedProcess[str]': + """``shell`` as the step ran it: ``shell: bash`` for the annotated steps, ``bash -e`` for the rest.""" + bin_dir = self.tmp / 'bin' + bin_dir.mkdir(exist_ok=True) + python = bin_dir / 'python' + if not python.exists(): + python.symlink_to(sys.executable) # the leg's `python`; here, this interpreter + env = dict(env, PATH=f'{bin_dir}{os.pathsep}{env.get("PATH", "")}') + command = (['bash', '--noprofile', '--norc', '-eo', 'pipefail', '-c', shell] if shell is FORMER_ANNOTATED + else ['bash', '-e', '-c', shell]) + return subprocess.run(command, cwd=self.tmp, env=env, # nosec: B603 + capture_output=True, text=True, check=False, timeout=120) + + def both(self, shell: 'str', level: 'str', **pip: 'object') -> 'tuple[tuple[object, ...], tuple[object, ...]]': + """``(status, stdout, log, pip's calls)`` of the former shell, then of the script, each with its stderr.""" + found = [] + for run in (lambda env: self.former(shell, env), lambda env: self.script('-e', SPEC, env=env)): + result = run(self.env(level, **pip)) + log = self.log.read_text(encoding='utf-8') if self.log.exists() else None + if self.log.exists(): + self.log.unlink() + found.append(((result.returncode, result.stdout, log, self.calls()), result.stderr)) + return found[0], found[1] + + def test_every_annotated_outcome_matches(self) -> None: + for output, status in OUTCOMES: + for level in ('error', 'warning'): + with self.subTest(output=output, status=status, level=level): + (old, _), (new, _) = self.both(FORMER_ANNOTATED, level, install_output=output, + install_status=status) + self.assertEqual(new, old) + + def test_a_failed_upgrade_matches_on_both_forms(self) -> None: + for shell in (FORMER_ANNOTATED, FORMER_BARE): + for status in (1, 3, -signal.SIGTERM): + with self.subTest(annotated=shell is FORMER_ANNOTATED, status=status): + (old, _), (new, _) = self.both(shell, 'error', upgrade_output=['err:no index'], + upgrade_status=status) + self.assertEqual(new, old) + + def test_the_bare_steps_gain_the_annotation(self) -> None: + """Formerly pip's status, its two streams apart and no annotation; now its streams merged + into standard output and the log, and on a failure status 1 and the annotation.""" + for output, status in OUTCOMES: + with self.subTest(output=output, status=status): + (old, old_stderr), (new, new_stderr) = self.both(FORMER_BARE, 'error', install_output=output, + install_status=status) + self.assertEqual(new[3], old[3]) + self.assertEqual((old[1], old[2]), (text([line for line in output if line.startswith('out:')]), None)) + self.assertTrue(old_stderr.startswith(text([line for line in output if line.startswith('err:')]))) + self.assertEqual((new[2], new_stderr), (text(output), '')) + if status == 0: + self.assertEqual((new[0], new[1]), (0, text(output))) + continue + self.assertNotEqual(old[0], 0) + self.assertEqual(new[0], 1) + self.assertTrue(str(new[1]).startswith(f'{text(output)}::error title={TITLE}::'), new[1]) + self.assertEqual(str(new[1]).count('\n::'), 1) + + +class TestTheStepsRunTheScript(unittest.TestCase): + """Each covered step is one call of the script, with its guards; no other keeps the shell.""" + + def step(self, job_id: 'str', name: 'str') -> 'dict[str, object]': + found = [entry for entry in steps(WORKFLOW, job_id) if entry.get('name') == name] + self.assertEqual(len(found), 1, f'{job_id} {name!r}') + return found[0] + + def test_each_site_runs_the_script(self) -> None: + for (job_id, name), extras in {**MATRIX_SITES, **LATEST_SITES}.items(): + with self.subTest(job=job_id): + run = self.step(job_id, name)['run'] + self.assertEqual(run, f"python util/pip_install.py -e '.[{extras}]'") + self.assertEqual(EDITABLE_INSTALL.findall(str(run)), [extras]) + + def test_the_matrix_sites_keep_their_guards_and_level(self) -> None: + for job_id, name in MATRIX_SITES: + with self.subTest(job=job_id): + step = self.step(job_id, name) + self.assertEqual(step['id'], 'install') + self.assertEqual(step['if'], "${{ steps.python.outcome == 'success' }}") + self.assertEqual(step['continue-on-error'], '${{ matrix.ktlo == true }}') + self.assertEqual(step['env'], {'LEG': MATRIX_LEG}) + self.assertEqual(step.get('shell'), None if job_id == 'pypcap-parity' else 'bash') + self.assertNotIn('timeout-minutes', step) + self.assertEqual(job(WORKFLOW, job_id)['env']['ANNOTATION_LEVEL'], MATRIX_LEVEL) + + def test_the_latest_sites_annotate_at_error_on_their_own_version(self) -> None: + for job_id, name in LATEST_SITES: + with self.subTest(job=job_id): + step = self.step(job_id, name) + setup = next(entry for entry in steps(WORKFLOW, job_id) + if str(entry.get('uses', '')).startswith('actions/setup-python@')) + self.assertEqual(step['env'], {'ANNOTATION_LEVEL': 'error', + 'LEG': f"Python {setup['with']['python-version']}"}) + self.assertFalse({'id', 'if', 'continue-on-error', 'shell', 'timeout-minutes'} & set(step)) + self.assertNotIn('ANNOTATION_LEVEL', job(WORKFLOW, job_id).get('env') or {}) + self.assertFalse([leg for leg in (job(WORKFLOW, job_id).get('strategy') or {}).get('matrix', {}) + .get('include', []) if leg.get('ktlo')]) + + def test_the_ktlo_twins_run_the_same_step(self) -> None: + for twin, job_id in TWINS.items(): + name = next(name for site, name in MATRIX_SITES if site == job_id) + with self.subTest(job=twin): + self.assertEqual(self.step(twin, name), self.step(job_id, name)) + + def test_every_other_pip_install_is_a_listed_one(self) -> None: + sites = {*MATRIX_SITES, *LATEST_SITES} + found = [] + for job_id, body in load(WORKFLOW)['jobs'].items(): + for step in body.get('steps', []): + where = (TWINS.get(job_id, job_id), step.get('name')) + run = str(step.get('run', '')) + if re.search(r'pip install|pip_install\.py', run) and where not in sites | LEFT_INLINE: + found.append(f'{job_id} {step.get("name")!r}') + if where not in LEFT_INLINE: + found.extend(f'{job_id} {step.get("name")!r}: {piece}' for piece in INLINE if piece in run) + self.assertEqual(found, []) + + def test_the_listed_steps_still_install(self) -> None: + """So :data:`LEFT_INLINE` names nothing stale.""" + for job_id, name in LEFT_INLINE: + with self.subTest(job=job_id, step=name): + self.assertIn('pip install', str(self.step(job_id, name)['run'])) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/project/test_python_matrix_tiers_1633.py b/tests/project/test_python_matrix_tiers_1633.py index 33965dc129..056e29b14c 100644 --- a/tests/project/test_python_matrix_tiers_1633.py +++ b/tests/project/test_python_matrix_tiers_1633.py @@ -48,6 +48,7 @@ import re import shutil import subprocess +import sys import tempfile import unittest from typing import TYPE_CHECKING @@ -454,15 +455,17 @@ def test_no_conda_leg_is_ktlo_and_315_leaves_base_python_alone(self) -> None: "${{ matrix.python-version != '3.15' && matrix.python-version || '' }}") -#: ``python`` for an install step: ``pip install -U`` passes, ``pip install -e`` prints +#: ``pip`` for an install step, which runs util/pip_install.py on the real +#: interpreter (#1967): ``pip install -U`` passes, ``pip install -e`` prints #: ``$PIP_OUTPUT`` and exits ``$PIP_STATUS``. PIP_STUB = '''\ -case "$*" in - "-m pip install -U pip setuptools wheel") exit 0 ;; - "-m pip install -e "*) printf '%s\\n' "$PIP_OUTPUT"; exit "$PIP_STATUS" ;; -esac -echo "unexpected python $*" >&2 -exit 99 +import os, sys +if sys.argv[1:] == ['install', '-U', 'pip', 'setuptools', 'wheel']: + sys.exit(0) +if sys.argv[1:3] == ['install', '-e']: + print(os.environ['PIP_OUTPUT']) + sys.exit(int(os.environ['PIP_STATUS'])) +sys.exit('unexpected pip ' + ' '.join(sys.argv[1:])) ''' #: ``python`` for ``Verify package``: compileall and the import, each scripted by env. VERIFY_STUB = '''\ @@ -573,25 +576,35 @@ def run_step(self, name: 'str', job_id: 'str', step_name: 'str', stub: 'str', """Step ``step_name`` on ``leg`` (3.7's by default) under ``shell: bash``, ``python`` stubbed. ``LEG`` and the job's ``ANNOTATION_LEVEL`` get the values their expressions - take on that leg, and ``-eo pipefail`` is what ``shell: bash`` runs. + take on that leg, and ``-eo pipefail`` is what ``shell: bash`` runs. With + :data:`PIP_STUB`, ``python`` is this interpreter and the stub is its ``pip``. """ leg = KTLO_37 if leg is None else leg spec = job(name, job_id) step = next(step for step in spec['steps'] if step.get('name') == step_name) - self.assertEqual(step.get('shell'), 'bash', 'pipefail is what keeps `tee` from hiding a failure') + self.assertEqual(step.get('shell'), 'bash') self.assertEqual(step['env']['LEG'], leg_template(name, job_id)) self.assertEqual(spec['env']['ANNOTATION_LEVEL'], annotation_level(name, job_id)) env = dict(env, LEG=leg_label(leg), ANNOTATION_LEVEL='warning' if is_nonblocking(leg) else 'error') bin_dir = self.tmp / 'bin' bin_dir.mkdir(exist_ok=True) python = bin_dir / 'python' - python.write_text('#!/bin/bash\n' + stub, encoding='utf-8') - python.chmod(0o755) + if python.is_symlink() or python.exists(): + python.unlink() # never write a stub through the link to this interpreter + if stub is PIP_STUB: + python.symlink_to(sys.executable) + (self.tmp / 'stub' / 'pip').mkdir(parents=True, exist_ok=True) + (self.tmp / 'stub' / 'pip' / '__init__.py').write_text('', encoding='utf-8') + (self.tmp / 'stub' / 'pip' / '__main__.py').write_text(stub, encoding='utf-8') + env = dict(env, PYTHONPATH=str(self.tmp / 'stub')) + else: + python.write_text('#!/bin/bash\n' + stub, encoding='utf-8') + python.chmod(0o755) return subprocess.run(['bash', '--noprofile', '--norc', '-eo', 'pipefail', '-c', step['run']], env=dict(os.environ, PATH=f'{bin_dir}{os.pathsep}{os.environ["PATH"]}', RUNNER_TEMP=str(self.tmp), **env), - capture_output=True, text=True, check=False, timeout=60) + cwd=ROOT, capture_output=True, text=True, check=False, timeout=60) def test_37_is_a_ktlo_leg_of_exactly_the_three_test_matrices(self) -> None: """On ubuntu-22.04; not engine-tests or pypcap-parity (``ubuntu-latest`` only), nor a release leg.""" diff --git a/tests/project/test_sweeps_on_latest_only_1732.py b/tests/project/test_sweeps_on_latest_only_1732.py index a783830eaf..7bd03a736b 100644 --- a/tests/project/test_sweeps_on_latest_only_1732.py +++ b/tests/project/test_sweeps_on_latest_only_1732.py @@ -536,7 +536,7 @@ def test_it_installs_what_test_installs(self) -> None: installs = {} for where in (JOB, EXPERIMENTAL): # #1958: less the step `test` runs on `latest` alone, which no experimental leg is. - found = [re.findall(r"pip install -e '\.\[([^]]*)\]'", str(entry.get('run', ''))) + found = [_dependency_gates.EDITABLE_INSTALL.findall(str(entry.get('run', ''))) for entry in job(*where)['steps'] if _dependency_gates.LATEST_LEG_CONDITION not in str(entry.get('if', ''))] installs[where] = [extra for entries in found for extra in entries] diff --git a/tests/project/test_whole_suite_on_latest_1958.py b/tests/project/test_whole_suite_on_latest_1958.py index 9ff50e3d6a..332b887c6b 100644 --- a/tests/project/test_whole_suite_on_latest_1958.py +++ b/tests/project/test_whole_suite_on_latest_1958.py @@ -15,7 +15,6 @@ from __future__ import annotations import pathlib -import re import tempfile import unittest @@ -67,7 +66,7 @@ def step(job_id: 'str', name: 'str') -> 'dict[str, object]': def extras(run: 'object') -> 'frozenset[str]': - found = re.findall(r"pip install -e '\.\[([^]]*)\]'", str(run)) + found = _dependency_gates.EDITABLE_INSTALL.findall(str(run)) if len(found) != 1: raise AssertionError(f'expected one editable install in {run!r}, found {len(found)}') return frozenset(extra.strip() for extra in found[0].split(',')) diff --git a/tests/test_tier_guard.py b/tests/test_tier_guard.py index 6ec1c19279..b64150f731 100644 --- a/tests/test_tier_guard.py +++ b/tests/test_tier_guard.py @@ -1869,9 +1869,9 @@ def test_two_install_lines_in_one_pytest_job_is_refused(self) -> None: """Ambiguity fails loudly instead of the first line winning.""" doctored = doctored_workflow( self, - "python -m pip install -e '.[test,DPKT,crypto,NGAP]'", + "python util/pip_install.py -e '.[test,DPKT,crypto,NGAP]'", "python -m pip install -e '.[test,DPKT,crypto,NGAP]'\n" - " python -m pip install -e '.[Scapy]'", + " python util/pip_install.py -e '.[Scapy]'", ) with self.assertRaises(AssertionError) as caught: _dependency_gates.pytest_jobs(doctored) @@ -2572,8 +2572,8 @@ def test_swapping_pypcap_parity_to_pcap_ct_is_an_ambiguous_satisfaction(self) -> """ doctored = doctored_workflow( self, - "python -m pip install -e '.[test,Scapy,DPKT,cli,PyShark,PyPCAP,PyPCAPFile]'", - "python -m pip install -e '.[test,Scapy,DPKT,cli,PyShark,PCAP_CT,PyPCAPFile]'", + "python util/pip_install.py -e '.[test,Scapy,DPKT,cli,PyShark,PyPCAP,PyPCAPFile]'", + "python util/pip_install.py -e '.[test,Scapy,DPKT,cli,PyShark,PCAP_CT,PyPCAPFile]'", ) gaps = {(gap.flag, gap.job) for gap in _dependency_gates.dependency_gate_gaps(doctored)} @@ -2649,8 +2649,8 @@ def test_gaining_the_vendor_extra_is_not_an_ambiguous_satisfaction(self) -> None """ doctored = doctored_workflow( self, - "python -m pip install -e '.[test,DPKT,crypto,NGAP]'", - "python -m pip install -e '.[test,DPKT,crypto,NGAP,vendor]'", + "python util/pip_install.py -e '.[test,DPKT,crypto,NGAP]'", + "python util/pip_install.py -e '.[test,DPKT,crypto,NGAP,vendor]'", ) gaps = {(gap.flag, gap.job) for gap in _dependency_gates.dependency_gate_gaps(doctored)} @@ -2678,8 +2678,8 @@ def test_removing_the_negation_or_the_exact_path_reopens_762(self) -> None: """ doctored_pypcap_parity = doctored_workflow( self, - "python -m pip install -e '.[test,Scapy,DPKT,cli,PyShark,PyPCAP,PyPCAPFile]'", - "python -m pip install -e '.[test,Scapy,DPKT,cli,PyShark,PCAP_CT,PyPCAPFile]'", + "python util/pip_install.py -e '.[test,Scapy,DPKT,cli,PyShark,PyPCAP,PyPCAPFile]'", + "python util/pip_install.py -e '.[test,Scapy,DPKT,cli,PyShark,PCAP_CT,PyPCAPFile]'", ) doctored_engine_tests = doctored_workflow( self, @@ -2747,8 +2747,8 @@ def test_a_non_distribution_flag_is_not_reported_even_when_doctored_ambiguous(se """ doctored = doctored_workflow( self, - "python -m pip install -e '.[test,Scapy,DPKT,cli,PyShark,PyPCAP,PyPCAPFile]'", - "python -m pip install -e '.[test,Scapy,DPKT,cli,PyShark,PCAP_CT,PyPCAPFile]'", + "python util/pip_install.py -e '.[test,Scapy,DPKT,cli,PyShark,PyPCAP,PyPCAPFile]'", + "python util/pip_install.py -e '.[test,Scapy,DPKT,cli,PyShark,PCAP_CT,PyPCAPFile]'", ) findings = {(f.job, f.flag) for f in _dependency_gates.ambiguous_satisfactions(doctored)} diff --git a/util/pip_install.py b/util/pip_install.py new file mode 100644 index 0000000000..3a35311acf --- /dev/null +++ b/util/pip_install.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- +"""Install the checkout and its extras, and say why when it fails (#1967). + +The install step of each :file:`.github/workflows/unit-tests.yml` job that installs +the package -- ``test``, ``integration``, ``pypcap-parity`` (and so their KTLO +twins), ``unittest-ordering``, ``project-tests`` and ``gate`` -- runs, from the +checkout's root and on the leg's own interpreter:: + + python util/pip_install.py -e '.[test,DPKT,crypto,NGAP]' + +with ``ANNOTATION_LEVEL``, ``error`` or ``warning``, ``LEG``, the leg's name, and +``RUNNER_TEMP`` in the environment. It runs, on the same interpreter: + +1. ``python -m pip install -U pip setuptools wheel``, its output passed through. A + failure ends the script with pip's status and no annotation, as ``set -e`` ended + the step. +2. ``python -m pip install -e SPEC``, its standard error merged into its standard + output, which it copies, as it arrives, both to its own standard output and to + :file:`$RUNNER_TEMP/pip-install.log`. On a failure it writes one + ``ANNOTATION_LEVEL`` annotation, ``pcapkit did not install on LEG``, whose + message is pip's last ``ERROR:`` line without that prefix, or a fallback when + there is none, and exits 1. The title escapes ``,`` as ``%2C``, since the runner + splits an annotation's properties on ``,`` (#1633), and the message ``%`` as + ``%25``. + +On a KTLO leg the level is ``warning`` and the step carries ``continue-on-error``, +so every later step, which waits on the install's ``outcome``, is skipped. An +``ANNOTATION_LEVEL`` other than ``error`` or ``warning`` exits 2 before pip runs. +The script is standard library only, and runs on every interpreter the matrices +hold, 3.7 included. + +""" + +from __future__ import annotations + +import argparse +import os +import pathlib +import subprocess # nosec: B404 +import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Optional + +__all__ = ['LEVELS', 'UPGRADE', 'LOG_NAME', 'PREFIX', 'NO_REASON', 'TITLE', 'SKIPPED', 'main'] + +#: The annotation levels ``ANNOTATION_LEVEL`` may hold. +LEVELS = ('error', 'warning') +#: What the first pip run installs. +UPGRADE = ('install', '-U', 'pip', 'setuptools', 'wheel') +#: The second pip run's output, under ``$RUNNER_TEMP``. +LOG_NAME = 'pip-install.log' +#: How pip starts each line of an error. +PREFIX = b'ERROR: ' +#: The annotation's message when pip wrote no ``ERROR:`` line. +NO_REASON = b"pip failed without an ERROR line; see this step's log" +#: The annotation's title, before the leg's name. +TITLE = 'pcapkit did not install on ' +#: The end of the annotation's message. +SKIPPED = b'. The rest of this leg is skipped.' + + +def emit(data: 'bytes') -> 'None': + """Write ``data`` to standard output as it is, at once.""" + sys.stdout.buffer.write(data) + sys.stdout.buffer.flush() + + +def status_of(code: 'int') -> 'int': + """``code`` as the shell reports it: a signal ``N`` as ``128 + N``.""" + return code if code >= 0 else 128 - code + + +def reason(output: 'bytes') -> 'bytes': + """Pip's last ``ERROR:`` line in ``output``, less the prefix, or :data:`NO_REASON`.""" + found = [line[len(PREFIX):] for line in output.split(b'\n') if line.startswith(PREFIX)] + return found[-1] if found and found[-1] else NO_REASON + + +def main(argv: 'Optional[list[str]]' = None) -> 'int': + """Upgrade pip, install ``SPEC`` editable, and annotate a failed install; the exit status.""" + parser = argparse.ArgumentParser(description=__doc__.split('\n', 1)[0]) + parser.add_argument('-e', '--editable', required=True, metavar='SPEC', + help="what pip installs editable, such as '.[test,DPKT]'") + args = parser.parse_args(argv) + + level = os.environ['ANNOTATION_LEVEL'] + if level not in LEVELS: + print(f'ANNOTATION_LEVEL is {level!r}, not one of {", ".join(LEVELS)}.', file=sys.stderr) + return 2 + leg = os.environ['LEG'] + log_path = pathlib.Path(os.environ['RUNNER_TEMP']) / LOG_NAME + + sys.stdout.flush() + upgraded = subprocess.run([sys.executable, '-m', 'pip', *UPGRADE], check=False) # nosec: B603 + if upgraded.returncode != 0: + return status_of(upgraded.returncode) + + output = bytearray() + with open(log_path, 'wb') as log: + with subprocess.Popen([sys.executable, '-m', 'pip', 'install', '-e', args.editable], # nosec: B603 + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as process: + assert process.stdout is not None # nosec: B101 + for line in process.stdout: + emit(line) + log.write(line) + log.flush() + output.extend(line) + status = process.returncode + if status == 0: + return 0 + + # The runner splits the properties on `,` before it un-escapes `%2C`, so the + # leg's `(KTLO, non-blocking)` would end the title at "(KTLO" (#1633). + title = (TITLE + leg.replace(',', '%2C')).encode('utf-8', 'surrogateescape') + message = reason(bytes(output)).replace(b'%', b'%25') + SKIPPED + emit(b'::' + level.encode('utf-8') + b' title=' + title + b'::' + message + b'\n') + return 1 + + +if __name__ == '__main__': + sys.exit(main())