refactor(util): share one stdlib-only Actions helper across the CI scripts (#2047)

docs/source/contributing/workflows.rst

@@ -1228,6 +1228,10 @@ does; each has tests under :file:`tests/project/` that run it.
        ``apt_packages.py plan``, ``seed``, ``install`` and ``collect``. The
        ``actions/cache`` steps and the AppArmor step (`#2019
        <https://github.com/JarryShaw/PyPCAPKit/issues/2019>`__) stay as they were.
+   * - :file:`util/_actions.py`
+     - No step: it is imported by the scripts that share its annotations, step
+       outputs, job summary lines and logged runs (`#2047
+       <https://github.com/JarryShaw/PyPCAPKit/issues/2047>`__).
 
 The summaries run on the image's ``/usr/bin/python3`` rather than the leg's
 interpreter, which may have failed to set up or be too broken to run anything, so
@@ -1251,6 +1255,8 @@ could not signal. Both ``changes`` jobs check out on every event, since the scri
 tree. ``create-release.yml``'s ``pypi`` job keeps its own copy of
 the KTLO summary's shell until its shell moves to scripts too; a project pin runs
 that copy and the script on the same steps and requires the same output.
+:file:`util/_actions.py` is standard library only, and keeps to 3.7's syntax,
+since the scripts on ``/usr/bin/python3`` and on every leg's interpreter import it.
 
 .. _CI Reports:
 

util/_actions.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+"""The GitHub Actions plumbing the :file:`util/` scripts share (#2047).
+
+Workflow commands, ``$GITHUB_OUTPUT`` and ``$GITHUB_STEP_SUMMARY`` writes, and
+the run that copies a command's output to the log and to a file, written once
+here instead of in each script. A script that the steps run as ``python
+util/<script>.py`` has :file:`util/` first on :data:`sys.path`, so it imports
+this module as ``_actions``; imported as ``util.<script>``, or loaded by path,
+from the repository root, it imports ``util._actions``.
+
+The scripts run on the image's ``/usr/bin/python3``, or on a leg's own
+interpreter, which may be 3.7, GraalPy or a free-threaded build, and
+:file:`util/pip_install.py` is the step that installs anything at all. So this
+module is standard library only and keeps to 3.7's syntax; a project test,
+:file:`tests/project/test_actions_helper_2047.py`, holds it to both.
+
+The escaping is GitHub's, from the "Workflow commands for GitHub Actions"
+reference (https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands)
+and ``escapeData`` and ``escapeProperty`` in ``@actions/core``'s ``command.ts``:
+a command's message escapes ``%``, ``\\r`` and ``\\n``, and each property's value
+escapes ``:`` and ``,`` as well.
+
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import subprocess  # nosec: B404
+import sys
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from os import PathLike
+    from typing import IO, Iterator, Mapping, Optional, Sequence, Tuple, Union
+
+__all__ = ['LEVELS', 'escape_data', 'escape_property', 'annotation', 'annotate', 'group',
+           'set_output', 'summary', 'open_summary', 'emit', 'say', 'exit_status', 'run']
+
+#: The annotation commands, by level.
+LEVELS = ('error', 'warning', 'notice')
+
+
+def escape_data(text: 'str') -> 'str':
+    """``text`` as a workflow command's message must hold it."""
+    return text.replace('%', '%25').replace('\r', '%0D').replace('\n', '%0A')
+
+
+def escape_property(text: 'str') -> 'str':
+    """``text`` as the value of a workflow command's property must hold it."""
+    return escape_data(text).replace(':', '%3A').replace(',', '%2C')
+
+
+def annotation(level: 'str', message: 'str', *, title: 'Optional[str]' = None, file: 'Optional[str]' = None,
+               line: 'Optional[int]' = None, escape: 'bool' = True) -> 'str':
+    """The ``::<level>`` command, without its newline, of ``message`` and the properties given.
+
+    The properties are ``title``, ``file`` and ``line``, in that order, each left
+    out where it is :data:`None`. ``escape=False`` writes ``message`` and the
+    properties as they are, for a caller that escapes them its own way.
+
+    """
+    if level not in LEVELS:
+        raise ValueError(f'{level!r} is not one of {", ".join(LEVELS)}')
+    data = escape_data if escape else str
+    value = escape_property if escape else str
+    properties = ','.join(name + '=' + value(str(setting)) for name, setting
+                          in (('title', title), ('file', file), ('line', line)) if setting is not None)
+    return '::' + level + (' ' + properties if properties else '') + '::' + data(message)
+
+
+def annotate(level: 'str', message: 'str', *, title: 'Optional[str]' = None, file: 'Optional[str]' = None,
+             line: 'Optional[int]' = None, escape: 'bool' = True, stream: 'Optional[IO[str]]' = None) -> 'None':
+    """Print :func:`annotation` to ``stream``, standard output by default, at once."""
+    print(annotation(level, message, title=title, file=file, line=line, escape=escape),
+          file=sys.stdout if stream is None else stream, flush=True)
+
+
+@contextlib.contextmanager
+def group(title: 'str', *, escape: 'bool' = True) -> 'Iterator[None]':
+    """Fold what the block prints into one ``::group::`` of the log, titled ``title``."""
+    print('::group::' + (escape_data(title) if escape else title), flush=True)
+    try:
+        yield
+    finally:
+        print('::endgroup::', flush=True)
+
+
+def _path(variable: 'str', required: 'bool', environ: 'Optional[Mapping[str, str]]') -> 'Optional[str]':
+    """The file ``$variable`` names: :exc:`KeyError` where it is unset and ``required``, else :data:`None`."""
+    environ = os.environ if environ is None else environ
+    if required:
+        return environ[variable]
+    return environ.get(variable) or None
+
+
+def set_output(name: 'str', value: 'str', *, required: 'bool' = True,
+               environ: 'Optional[Mapping[str, str]]' = None) -> 'bool':
+    """Append the step output ``name`` to ``$GITHUB_OUTPUT``; whether it was written.
+
+    A ``value`` holding a line break, ``\\n`` or ``\\r``, is written under a fresh
+    delimiter, ``name<<EOF_<hex>``, as GitHub's multiline form requires, since the
+    runner reads the file a line at a time. Where ``$GITHUB_OUTPUT`` is unset, a
+    ``required`` write raises :exc:`KeyError` and any other writes nothing.
+
+    """
+    path = _path('GITHUB_OUTPUT', required, environ)
+    if path is None:
+        return False
+    if '\n' in value or '\r' in value:
+        import uuid  # pylint: disable=import-outside-toplevel
+        delimiter = f'EOF_{uuid.uuid4().hex}'
+        text = f'{name}<<{delimiter}\n{value}\n{delimiter}\n'
+    else:
+        text = name + '=' + value + '\n'
+    with open(path, 'a', encoding='utf-8') as file:
+        file.write(text)
+    return True
+
+
+def open_summary(*, environ: 'Optional[Mapping[str, str]]' = None) -> 'IO[str]':
+    """``$GITHUB_STEP_SUMMARY``, open to append to, in UTF-8; :exc:`KeyError` where it is unset."""
+    return open((os.environ if environ is None else environ)['GITHUB_STEP_SUMMARY'], 'a', encoding='utf-8')
+
+
+def summary(*lines: 'str', required: 'bool' = True, environ: 'Optional[Mapping[str, str]]' = None) -> 'bool':
+    """Append each of ``lines``, and a newline, to ``$GITHUB_STEP_SUMMARY``; whether it was written.
+
+    Where ``$GITHUB_STEP_SUMMARY`` is unset, a ``required`` write raises
+    :exc:`KeyError` and any other writes nothing.
+
+    """
+    path = _path('GITHUB_STEP_SUMMARY', required, environ)
+    if path is None:
+        return False
+    with open(path, 'a', encoding='utf-8') as file:
+        for text in lines:
+            file.write(text + '\n')
+    return True
+
+
+def emit(data: 'bytes') -> 'None':
+    """Write ``data`` to standard output as it is, at once, ahead of any command's."""
+    sys.stdout.buffer.write(data)
+    sys.stdout.buffer.flush()
+
+
+def say(text: 'str') -> 'None':
+    """Write ``text`` and a newline to standard output at once, a lone surrogate as the byte it stands for."""
+    emit(text.encode('utf-8', 'surrogateescape') + b'\n')
+
+
+def exit_status(code: 'int') -> 'int':
+    """``code`` as the shell reports it: a signal ``N`` as ``128 + N``."""
+    return code if code >= 0 else 128 - code
+
+
+def run(command: 'Sequence[str]', log_path: 'Union[str, PathLike[str]]') -> 'Tuple[int, bytes]':
+    """Run ``command``, its standard error with its output, copying each line to standard output and the log.
+
+    Each line goes out as it arrives, through :func:`emit`, and is flushed to
+    the file ``log_path``, which is truncated first. The result is the command's
+    return code, negative for a signal as :mod:`subprocess` gives it, and the
+    whole of its output.
+
+    """
+    output = bytearray()
+    with open(log_path, 'wb') as log:
+        with subprocess.Popen(list(command), stdout=subprocess.PIPE,  # nosec: B603
+                              stderr=subprocess.STDOUT) as process:
+            assert process.stdout is not None  # nosec: B101
+            for line in process.stdout:
+                emit(line)
+                log.write(line)
+                log.flush()
+                output.extend(line)
+        status = process.returncode
+    return status, bytes(output)

util/apt_packages.py

@@ -65,6 +65,11 @@
 import sys
 from typing import TYPE_CHECKING
 
+try:  # #2047: run as `python util/<script>.py`, with util/ first on sys.path
+    import _actions
+except ImportError:  # imported as util.<script>, or loaded by path, from the repository root
+    from util import _actions
+
 if TYPE_CHECKING:
     from typing import Callable, Optional, Sequence
 
@@ -89,17 +94,6 @@
 DEBCONF = b'wireshark-common wireshark-common/install-setuid boolean false\n'
 
 
-def say(text: 'str') -> 'None':
-    """Write ``text`` and a newline to standard output at once, ahead of any command's."""
-    sys.stdout.buffer.write(text.encode('utf-8', 'surrogateescape') + b'\n')
-    sys.stdout.buffer.flush()
-
-
-def status_of(code: 'int') -> 'int':
-    """``code`` as the shell reports it: a signal ``N`` as ``128 + N``."""
-    return code if code >= 0 else 128 - code
-
-
 def call(command: 'Sequence[str]', **kwargs: 'object') -> 'subprocess.CompletedProcess[bytes]':
     """Run ``command`` to its end, its streams this process's unless ``kwargs`` says otherwise."""
     return subprocess.run(list(command), check=False, **kwargs)  # type: ignore[call-overload] # nosec: B603
@@ -128,14 +122,15 @@ def plan() -> 'int':
     for attempt in range(1, ATTEMPTS + 1):
         if call(['sudo', 'timeout', str(UPDATE_BOUND), 'apt-get', 'update']).returncode == 0:
             break
-        say(f'::warning title=apt-get update attempt {attempt} of {ATTEMPTS} failed ({os.environ["APT_SET"]})'
-            f'::apt-get update failed or exceeded its {UPDATE_BOUND}s bound.')
+        # #2047: each command stays literal, where #1672's scan of a KTLO step's scripts reads it.
+        _actions.say(f'::warning title=apt-get update attempt {attempt} of {ATTEMPTS} failed '
+                     f'({os.environ["APT_SET"]})::apt-get update failed or exceeded its {UPDATE_BOUND}s bound.')
         if attempt == ATTEMPTS:
             return 1
 
     simulated = call(['apt-get', 'install', '-s', '--no-install-recommends', *packages()], stdout=subprocess.PIPE)
     if simulated.returncode != 0:
-        return status_of(simulated.returncode)
+        return _actions.exit_status(simulated.returncode)
     lines = sorted(line for line in simulated.stdout.rstrip(b'\n').split(b'\n') if line.startswith(b'Inst '))
     listed = b'\n'.join(lines)
     sys.stdout.buffer.write(b'To download:\n' + (listed or b'(nothing)') + b'\n')
@@ -144,10 +139,8 @@ def plan() -> 'int':
         prefix = f'apt-debs-{os.environ.get("ImageOS") or "unknown"}-{os.environ["RUNNER_ARCH"]}-' \
                  f'{os.environ["APT_SET"]}-'
         digest = hashlib.sha256(listed + b'\n').hexdigest()[:16]
-        with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as output:
-            # Concatenated: vermin reads `f'key={prefix}'` as 3.8's `{prefix=}`.
-            output.write('restore-prefix=' + prefix + '\n')
-            output.write('key=' + prefix + digest + '\n')
+        _actions.set_output('restore-prefix', prefix)
+        _actions.set_output('key', prefix + digest)
     return 0
 
 
@@ -157,8 +150,8 @@ def seed(archives: 'str') -> 'int':
     # An unmatched glob stays the pattern, as the shell's does, so `cp` fails on it.
     copied = call(['sudo', 'cp', *(debs(cache) or [os.path.join(cache, '*.deb')]), archives + '/'])
     if copied.returncode != 0:
-        return status_of(copied.returncode)
-    say(f'Seeded {count_debs(cache)} .deb files')
+        return _actions.exit_status(copied.returncode)
+    _actions.say(f'Seeded {count_debs(cache)} .deb files')
     return 0
 
 
@@ -180,9 +173,9 @@ def fallback(mirrors: 'str', suffix: 'str') -> 'None':
             first = file.read().split(b'\n', 1)[0].split(b'\t', 1)[0]
     except OSError:
         first = b''
-    say(f'::warning title=apt retry on a fallback mirror{suffix}::Retrying against '
-        f'{first.decode("utf-8", "surrogateescape")} instead of azure.archive.ubuntu.com, '
-        'resuming the partial download.')
+    _actions.say(f'::warning title=apt retry on a fallback mirror{suffix}::Retrying against '
+                 f'{first.decode("utf-8", "surrogateescape")} instead of azure.archive.ubuntu.com, '
+                 'resuming the partial download.')
 
 
 def install(mirrors: 'str', name: 'Optional[str]') -> 'int':
@@ -194,14 +187,14 @@ def install(mirrors: 'str', name: 'Optional[str]') -> 'int':
     if 'tshark' in wanted:
         answered = call(['sudo', 'debconf-set-selections'], input=DEBCONF)
         if answered.returncode != 0:
-            return status_of(answered.returncode)
+            return _actions.exit_status(answered.returncode)
     for attempt in range(1, ATTEMPTS + 1):
         if call(['sudo', 'timeout', str(UPDATE_BOUND), 'apt-get', 'update']).returncode == 0 \
                 and call(['sudo', 'DEBIAN_FRONTEND=noninteractive', 'timeout', str(INSTALL_BOUND),
                           'apt-get', 'install', '-y', '--no-install-recommends', *wanted]).returncode == 0:
             return 0
-        say(f'::warning title=apt attempt {attempt} of {ATTEMPTS} failed{suffix}::apt-get update or install '
-            'failed or exceeded its per-attempt bound; retrying once.')
+        _actions.say(f'::warning title=apt attempt {attempt} of {ATTEMPTS} failed{suffix}::apt-get update or '
+                     'install failed or exceeded its per-attempt bound; retrying once.')
         call(['sudo', 'dpkg', '--configure', '-a'])
         if attempt == 1:
             fallback(mirrors, suffix)
@@ -211,7 +204,8 @@ def install(mirrors: 'str', name: 'Optional[str]') -> 'int':
 def field(deb: 'str', name: 'str') -> 'tuple[int, str]':
     """``dpkg-deb -f DEB NAME``'s status and output, trailing newlines stripped."""
     result = call(['dpkg-deb', '-f', deb, name], stdout=subprocess.PIPE)
-    return status_of(result.returncode), result.stdout.decode('utf-8', 'surrogateescape').rstrip('\n')
+    status = _actions.exit_status(result.returncode)
+    return status, result.stdout.decode('utf-8', 'surrogateescape').rstrip('\n')
 
 
 def collect(archives: 'str') -> 'int':
@@ -235,7 +229,7 @@ def collect(archives: 'str') -> 'int':
                          stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
         if installed.stdout.decode('utf-8', 'surrogateescape').rstrip('\n') == fields['Version']:
             shutil.copy(deb, out)
-    say(f'Collected {count_debs(out)} .deb files')
+    _actions.say(f'Collected {count_debs(out)} .deb files')
     return 0
 
 

util/classify_changes.py

@@ -32,6 +32,11 @@
 import subprocess  # nosec B404
 import sys
 
+try:  # #2047: run as `python util/<script>.py`, with util/ first on sys.path
+    import _actions
+except ImportError:  # imported as util.<script>, or loaded by path, from the repository root
+    from util import _actions
+
 __all__ = ['CODE_DIRS', 'is_docs', 'main']
 
 #: Directories whose every file is code, a Markdown one included.
@@ -57,8 +62,10 @@ def main() -> 'int':
                               capture_output=True, text=True, check=False)
         files = [path for path in diff.stdout.split('\0') if path]
         if diff.returncode != 0 or not files:
-            print(f'::warning title=Could not list the changed files::git diff exited {diff.returncode} '
-                  f'with {len(files)} files: {diff.stderr.strip()}. Treating the diff as code.')
+            # #2047: unescaped, as before the helper, so git's stderr keeps its lines.
+            _actions.annotate('warning', f'git diff exited {diff.returncode} with {len(files)} files: '
+                              f'{diff.stderr.strip()}. Treating the diff as code.',
+                              title='Could not list the changed files', escape=False)
         else:
             code_files = [path for path in files if not is_docs(path)]
             print(f'{len(files)} changed files, {len(code_files)} of them code:')
@@ -67,8 +74,7 @@ def main() -> 'int':
             if not code_files:
                 code = 'false'
     print(f'code={code}')
-    with open(os.environ['GITHUB_OUTPUT'], 'a', encoding='utf-8') as file:
-        print(f'code={code}', file=file)
+    _actions.set_output('code', code)
     return 0
 
 

util/coverage_report.py

@@ -39,6 +39,11 @@
 import sys
 from typing import TYPE_CHECKING
 
+try:  # #2047: run as `python util/<script>.py`, with util/ first on sys.path
+    import _actions
+except ImportError:  # imported as util.<script>, or loaded by path, from the repository root
+    from util import _actions
+
 if TYPE_CHECKING:
     from typing import Optional, TextIO
 
@@ -87,8 +92,8 @@ def check(log: 'str' = LOG) -> 'int':
         print(f'  pid {pid}: {atexit_.get(pid, "no atexit/sigterm entry -- died before saving")}')
     if len(root) != 1 or not workers or missing:
         why = f'no data file from pid {missing}' if missing else 'the process log is missing or unreadable'
-        with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as page:  # pylint: disable=unspecified-encoding
-            print(f'**Coverage not reported:** {why} (#1387); a total would under-count.', file=page)
+        _actions.summary(f'**Coverage not reported:** {why} (#1387); a total would under-count.')
+        # #2047: the command stays literal, where #1672's scan of a KTLO step's scripts reads it.
         print(f'::error::Coverage data is incomplete: {why}.')
         return 1
     return 0
@@ -129,7 +134,7 @@ def summary(page: 'TextIO') -> 'int':
 
 
 def _summary_to_job_page() -> 'int':
-    with open(os.environ['GITHUB_STEP_SUMMARY'], 'a', encoding='utf-8') as page:
+    with _actions.open_summary() as page:
         return summary(page)