ci: install through util/pip_install.py in experimental and Compat (#1967)

.github/workflows/experimental.yml

@@ -231,7 +231,8 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: pip
 
-      # As unit-tests.yml `test`'s install, for the same reasons.
+      # As unit-tests.yml `test`'s install, for the same reasons: util/pip_install.py
+      # annotates a failed install, at the job's `warning` (#1967).
       - name: Install package and test dependencies
         id: install
         if: ${{ steps.python.outcome == 'success' }}
@@ -240,14 +241,7 @@ jobs:
         shell: bash
         env:
           LEG: Python ${{ matrix.python-version }} (experimental, non-blocking)
-        run: |
-          python -m pip install -U pip setuptools wheel
-          if ! python -m pip install -e '.[test,DPKT,crypto,NGAP]' 2>&1 | tee "$RUNNER_TEMP/pip-install.log"; then
-            reason=$(sed -n 's/^ERROR: //p' "$RUNNER_TEMP/pip-install.log" | tail -n 1)
-            if [ -z "$reason" ]; then reason="pip failed without an ERROR line; see this step's log"; fi
-            echo "::${ANNOTATION_LEVEL} title=pcapkit did not install on ${LEG//,/%2C}::${reason//%/%25}. The rest of this leg is skipped."
-            exit 1
-          fi
+        run: python util/pip_install.py -e '.[test,DPKT,crypto,NGAP]'
 
       - name: Report available parallelism
         id: parallelism

.github/workflows/python-compatibility.yml

@@ -78,13 +78,14 @@ jobs:
       # The install needs this leg's interpreter, and the check below needs the
       # install, as in unit-tests.yml's `test` job: on a KTLO leg a failed step
       # does not stop the job, so each one waits on the `outcome` it needs.
+      # util/pip_install.py annotates a failed install, as in `test` (#1967).
       - name: Install package
         id: install
         if: ${{ steps.python.outcome == 'success' }}
         continue-on-error: ${{ matrix.ktlo == true }}
-        run: |
-          python -m pip install -U pip setuptools wheel
-          python -m pip install -e .
+        env:
+          LEG: Python ${{ matrix.python-version }}${{ matrix.ktlo && ' (KTLO, non-blocking)' || '' }}
+        run: python util/pip_install.py -e .
 
       # One annotation saying why, so a failed leg -- such as 3.7's, where the
       # install succeeds and the sources do not compile -- is legible from the

docs/source/contributing/workflows.rst

@@ -1209,8 +1209,10 @@ does; each has tests under :file:`tests/project/` that run it.
        and ``gate``, at ``error``.
    * - :file:`util/pip_install.py`
      - The install step of ``unit-tests.yml``'s ``test``, ``integration`` and
-       ``pypcap-parity`` (and so their KTLO twins), at the job's ``ANNOTATION_LEVEL``,
-       and of ``unittest-ordering``, ``project-tests`` and ``gate``, at ``error``.
+       ``pypcap-parity`` (and so their KTLO twins), ``experimental.yml``'s
+       ``experimental`` and ``python-compatibility.yml``'s ``compatibility``, at the
+       job's ``ANNOTATION_LEVEL``, and of ``unittest-ordering``, ``project-tests`` and
+       ``gate``, at ``error``.
 
 The summaries run on the image's ``/usr/bin/python3`` rather than the leg's
 interpreter, which may have failed to set up or be too broken to run anything, so
@@ -1222,7 +1224,7 @@ file-size cap every test command of ``unit-tests.yml`` takes.
 :file:`util/regenerate_samples.py` is standard library only but runs on the leg's
 interpreter, since the generator it runs imports ``pcapkit``, and so does
 :file:`util/pip_install.py`, since it installs into that interpreter; it runs on
-every version the matrices hold, 3.7 included. Both ``changes`` jobs check out on every event, since the script is in the
+every version the matrices hold, 3.7 and the experimental tier included. Both ``changes`` jobs check out on every event, since the script is in the
 tree. ``create-release.yml``'s ``pypi`` job keeps its own copy of
 the KTLO summary's shell until its shell moves to scripts too; a project pin runs
 that copy and the script on the same steps and requires the same output.

tests/project/test_pip_install_1967.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-"""#1967: :file:`util/pip_install.py`, the install step of most ``unit-tests.yml`` jobs, run as the steps run it.
+"""#1967: :file:`util/pip_install.py`, the install step of most test jobs, run as the steps run it.
 
 * :class:`TestPipInstall` runs the script against a stand-in ``pip``: a clean
   install, a failed one with and without an ``ERROR:`` line, a failed upgrade,
@@ -11,6 +11,9 @@
 * :class:`TestTheStepsRunTheScript` pins each covered step to the script, with
   its guards and environment, and every other ``pip install`` in
   ``unit-tests.yml`` to a listed step that keeps its shell.
+* :class:`TestTheOtherWorkflowsRunTheScript` does the same for the install of
+  ``experimental.yml``'s ``experimental`` and ``python-compatibility.yml``'s
+  ``compatibility``.
 
 """
 
@@ -29,11 +32,11 @@
 import unittest
 
 from tests._dependency_gates import EDITABLE_INSTALL
-from tests._workflows import job, load, steps
+from tests._workflows import WORKFLOWS, job, load, steps
 
 ROOT = pathlib.Path(__file__).resolve().parents[2]
 SCRIPT = ROOT / 'util' / 'pip_install.py'
-WORKFLOW = ROOT / '.github' / 'workflows' / 'unit-tests.yml'
+WORKFLOW = WORKFLOWS / 'unit-tests.yml'
 LOG = 'pip-install.log'
 SPEC = '.[test,DPKT]'
 
@@ -72,6 +75,36 @@
 TWINS = {'test-ktlo': 'test', 'integration-ktlo': 'integration', 'pypcap-parity-ktlo': 'pypcap-parity'}
 #: Text only the former inline install held.
 INLINE = ('pip-install.log', 'did not install on', "s/^ERROR: //p")
+#: The covered steps outside ``unit-tests.yml``, as ``(workflow, job)``, each step whole,
+#: and its job's ``ANNOTATION_LEVEL``. The experimental legs are never blocking, so
+#: always ``warning``; Compat's KTLO legs take ``warning`` as ``test``'s do. Compat
+#: installs no extra, and its step keeps the default shell.
+ELSEWHERE = {
+    ('experimental.yml', 'experimental'): ({
+        'name': 'Install package and test dependencies',
+        'id': 'install',
+        'if': "${{ steps.python.outcome == 'success' }}",
+        'continue-on-error': True,
+        'timeout-minutes': 10,
+        'shell': 'bash',
+        'env': {'LEG': 'Python ${{ matrix.python-version }} (experimental, non-blocking)'},
+        'run': "python util/pip_install.py -e '.[test,DPKT,crypto,NGAP]'",
+    }, 'warning'),
+    ('python-compatibility.yml', 'compatibility'): ({
+        'name': 'Install package',
+        'id': 'install',
+        'if': "${{ steps.python.outcome == 'success' }}",
+        'continue-on-error': '${{ matrix.ktlo == true }}',
+        'env': {'LEG': MATRIX_LEG},
+        'run': 'python util/pip_install.py -e .',
+    }, MATRIX_LEVEL),
+}
+#: The other ``pip install`` steps of those workflows, which install ``requests`` for
+#: util/superseded.py on an interpreter of their own, never the package.
+ELSEWHERE_INLINE = frozenset({
+    ('experimental.yml', 'current', 'Install requests'),
+    ('experimental.yml', 'experimental', "Install the watcher's requests"),
+})
 
 #: A stand-in ``pip``: it appends its arguments to ``calls``, then does what
 #: ``PIP_<STEP>`` says, ``STEP`` ``UPGRADE`` or ``INSTALL``: ``OUTPUT`` lines, each
@@ -368,5 +401,41 @@ def test_the_listed_steps_still_install(self) -> None:
                 self.assertIn('pip install', str(self.step(job_id, name)['run']))
 
 
+class TestTheOtherWorkflowsRunTheScript(unittest.TestCase):
+    """The experimental and Compat installs run the script, guards unchanged; nothing else there keeps the shell."""
+
+    def test_each_site_is_the_script_with_its_guards(self) -> None:
+        for (name, job_id), (expected, _) in ELSEWHERE.items():
+            with self.subTest(workflow=name, job=job_id):
+                found = [entry for entry in steps(WORKFLOWS / name, job_id) if entry.get('name') == expected['name']]
+                self.assertEqual(found, [expected])
+
+    def test_each_job_sets_the_level(self) -> None:
+        for (name, job_id), (_, level) in ELSEWHERE.items():
+            with self.subTest(workflow=name, job=job_id):
+                self.assertEqual(job(WORKFLOWS / name, job_id)['env']['ANNOTATION_LEVEL'], level)
+
+    def test_every_other_pip_install_is_a_listed_one(self) -> None:
+        sites = {(name, job_id, expected['name']) for (name, job_id), (expected, _) in ELSEWHERE.items()}
+        found = []
+        for name in sorted({name for name, _ in ELSEWHERE}):
+            for job_id, body in load(WORKFLOWS / name)['jobs'].items():
+                for step in body.get('steps', []):
+                    where = (name, job_id, step.get('name'))
+                    run = str(step.get('run', ''))
+                    if re.search(r'pip install|pip_install\.py', run) and where not in sites | ELSEWHERE_INLINE:
+                        found.append(f'{name} {job_id} {step.get("name")!r}')
+                    found.extend(f'{name} {job_id} {step.get("name")!r}: {piece}' for piece in INLINE if piece in run)
+        self.assertEqual(found, [])
+
+    def test_the_listed_steps_still_install(self) -> None:
+        """So :data:`ELSEWHERE_INLINE` names nothing stale."""
+        for name, job_id, step_name in ELSEWHERE_INLINE:
+            with self.subTest(workflow=name, job=job_id, step=step_name):
+                found = [entry for entry in steps(WORKFLOWS / name, job_id) if entry.get('name') == step_name]
+                self.assertEqual(len(found), 1)
+                self.assertIn('pip install', str(found[0]['run']))
+
+
 if __name__ == '__main__':
     unittest.main()

tests/project/test_python_matrix_tiers_1633.py

@@ -509,6 +509,8 @@ def test_no_conda_leg_is_ktlo_and_315_leaves_base_python_alone(self) -> None:
     ('unit-tests.yml', 'pypcap-parity', 'Install package, test and generator dependencies'),
     ('python-compatibility.yml', 'compatibility', 'Install package'),
 )
+#: The install steps of the 3.7 legs, each running util/pip_install.py (#1967).
+INSTALLS_37 = (LEGACY_37[0], LEGACY_37[1], INSTALLS[3])
 
 
 def workflow_commands(output: 'str') -> 'list[str]':
@@ -573,17 +575,19 @@ def setUp(self) -> None:
     def run_step(self, name: 'str', job_id: 'str', step_name: 'str', stub: 'str',
                  env: 'dict[str, str]', leg: 'Optional[dict[str, Any]]' = None,
                  ) -> 'subprocess.CompletedProcess[str]':
-        """Step ``step_name`` on ``leg`` (3.7's by default) under ``shell: bash``, ``python`` stubbed.
+        """Step ``step_name`` on ``leg`` (3.7's by default) under its shell, ``python`` stubbed.
 
         ``LEG`` and the job's ``ANNOTATION_LEVEL`` get the values their expressions
-        take on that leg, and ``-eo pipefail`` is what ``shell: bash`` runs. With
-        :data:`PIP_STUB`, ``python`` is this interpreter and the stub is its ``pip``.
+        take on that leg; ``-eo pipefail`` is what ``shell: bash`` runs, and ``-e``
+        the default shell. With :data:`PIP_STUB`, ``python`` is this interpreter and
+        the stub is its ``pip``.
 
         """
         leg = KTLO_37 if leg is None else leg
         spec = job(name, job_id)
         step = next(step for step in spec['steps'] if step.get('name') == step_name)
-        self.assertEqual(step.get('shell'), 'bash')
+        self.assertIn(step.get('shell'), ('bash', None))
+        flags = ['--noprofile', '--norc', '-eo', 'pipefail'] if step.get('shell') == 'bash' else ['-e']
         self.assertEqual(step['env']['LEG'], leg_template(name, job_id))
         self.assertEqual(spec['env']['ANNOTATION_LEVEL'], annotation_level(name, job_id))
         env = dict(env, LEG=leg_label(leg), ANNOTATION_LEVEL='warning' if is_nonblocking(leg) else 'error')
@@ -601,7 +605,7 @@ def run_step(self, name: 'str', job_id: 'str', step_name: 'str', stub: 'str',
         else:
             python.write_text('#!/bin/bash\n' + stub, encoding='utf-8')
             python.chmod(0o755)
-        return subprocess.run(['bash', '--noprofile', '--norc', '-eo', 'pipefail', '-c', step['run']],
+        return subprocess.run(['bash', *flags, '-c', step['run']],
                               env=dict(os.environ, PATH=f'{bin_dir}{os.pathsep}{os.environ["PATH"]}',
                                        RUNNER_TEMP=str(self.tmp), **env),
                               cwd=ROOT, capture_output=True, text=True, check=False, timeout=60)
@@ -648,7 +652,7 @@ def test_a_leg_that_cannot_install_stops_there(self) -> None:
                     self.assertIn("steps.install.outcome == 'success'", condition)
 
     def test_a_failed_install_fails_once_naming_pips_reason(self) -> None:
-        for name, job_id, install in LEGACY_37[:2]:
+        for name, job_id, install in INSTALLS_37:
             with self.subTest(job=job_id):
                 result = self.run_step(name, job_id, install, PIP_STUB, {'PIP_OUTPUT': PIP_37, 'PIP_STATUS': '1'})
                 self.assertNotEqual(result.returncode, 0)
@@ -670,7 +674,7 @@ def test_an_install_failure_without_an_error_line_is_still_named(self) -> None:
                     'The rest of this leg is skipped.'])
 
     def test_a_clean_install_passes_without_an_annotation(self) -> None:
-        for name, job_id, install in LEGACY_37[:2]:
+        for name, job_id, install in INSTALLS_37:
             with self.subTest(job=job_id):
                 result = self.run_step(name, job_id, install, PIP_STUB,
                                        {'PIP_OUTPUT': 'Successfully installed pypcapkit-1.5.0', 'PIP_STATUS': '0'})
@@ -702,7 +706,7 @@ def test_a_supported_leg_fails_the_same_steps_at_error_level(self) -> None:
         cases = [(name, job_id, install, PIP_STUB, {'PIP_OUTPUT': PIP_37, 'PIP_STATUS': '1'},
                   '::error title=pcapkit did not install on Python 3.12::No matching distribution found '
                   'for pytest>=8; extra == "test". The rest of this leg is skipped.')
-                 for name, job_id, install in LEGACY_37[:2]]
+                 for name, job_id, install in INSTALLS_37]
         cases.append(('python-compatibility.yml', 'compatibility', 'Verify package', VERIFY_STUB,
                       {'COMPILE_OUTPUT': '', 'COMPILE_STATUS': '0', 'IMPORT_STATUS': '1'},
                       "::error title=pcapkit does not import on Python 3.12::See the traceback in this step's log."))

util/pip_install.py

@@ -3,8 +3,10 @@
 
 The install step of each :file:`.github/workflows/unit-tests.yml` job that installs
 the package -- ``test``, ``integration``, ``pypcap-parity`` (and so their KTLO
-twins), ``unittest-ordering``, ``project-tests`` and ``gate`` -- runs, from the
-checkout's root and on the leg's own interpreter::
+twins), ``unittest-ordering``, ``project-tests`` and ``gate`` -- and of the
+``experimental`` job of :file:`.github/workflows/experimental.yml` and the
+``compatibility`` job of :file:`.github/workflows/python-compatibility.yml` runs,
+from the checkout's root and on the leg's own interpreter::
 
     python util/pip_install.py -e '.[test,DPKT,crypto,NGAP]'
 
@@ -27,7 +29,7 @@
 so every later step, which waits on the install's ``outcome``, is skipped. An
 ``ANNOTATION_LEVEL`` other than ``error`` or ``warning`` exits 2 before pip runs.
 The script is standard library only, and runs on every interpreter the matrices
-hold, 3.7 included.
+hold, 3.7 and the experimental tier included.
 
 """