ci(experimental): count raised MemoryErrors, not mentions of one (#1962)

.github/workflows/experimental.yml

@@ -426,8 +426,11 @@ jobs:
       #   way the tests run the one argv, in the scope or not.
       # `tests-status` holds the exit status, the scope and the probe's error,
       # the seconds the tests took, the kernel's OOM kills while they ran
-      # (/proc/vmstat `oom_kill`), the MemoryError count (`-w`, so Java's
-      # `OutOfMemoryError` and the flag are not counted) and the workers that
+      # (/proc/vmstat `oom_kill`), the MemoryErrors raised, as the lines that
+      # `MemoryError` heads, alone or before a `:`, at the line's start or after
+      # pytest's `E`, an `INTERNALERROR>` or an exception group's `|` (#1962: a
+      # mention is not one, such as a docstring a traceback prints, and nor are
+      # Java's `OutOfMemoryError` and the flag), and the workers that
       # crashed, by xdist's `[gwN] node down:` lines (under `-q` it prints no
       # `replacing crashed worker`, xdist/dsession.py:78-80 at v3.8.0), for
       # `Containment summary`.
@@ -516,7 +519,7 @@ jobs:
           set -e
           elapsed=$((SECONDS - started))
           after=$(oom_kills)
-          memory_errors=$(grep -cw MemoryError "$RUNNER_TEMP/pytest.log" || true)
+          memory_errors=$(grep -cE '^(INTERNALERROR> *)?(E +)?( *\| *)*MemoryError(:|[[:space:]]*$)' "$RUNNER_TEMP/pytest.log" || true)
           crashed_workers=$(grep -c '\] node down: ' "$RUNNER_TEMP/pytest.log" || true)
           printf 'exit=%s\nscope=%s\nprobe=%s\nelapsed=%s\noom_kills=%s\nmemory_errors=%s\ncrashed_workers=%s\n' \
             "$status" "$scope" "$probe" "$elapsed" "$(( ${after:-0} - ${before:-0} ))" "$memory_errors" \

docs/source/contributing/workflows.rst

@@ -891,7 +891,8 @@ warnings:
 
 The step writes ``$RUNNER_TEMP/tests-status``: the exit status, the scope and the
 probe's error, the seconds the tests took, the OOM kills while they ran
-(``/proc/vmstat``), the ``MemoryError`` lines in their output and xdist's ``node
+(``/proc/vmstat``), the lines in their output that report a raised ``MemoryError`` (#1962),
+not those that mention one, such as a docstring a traceback prints, and xdist's ``node
 down`` lines. From it and the monitor's last sample, ``Containment summary`` warns
 ``No memory cap``, ``tests lost a process to the OOM killer``, ``tests timed out at 110
 minutes``, ``tests raised MemoryError`` or ``tests lost an xdist worker``, each a

tests/project/test_graalpy_worker_heap_1753.py

@@ -109,7 +109,7 @@ def test_every_worker_that_went_down_is_counted(self) -> None:
         self.assertEqual(status['memory_errors'], '0')
 
     def test_a_java_out_of_memory_line_is_no_memory_error(self) -> None:
-        """``-w``: the flag, and Java's ``OutOfMemoryError``, are not Python's ``MemoryError``."""
+        """The flag, and Java's ``OutOfMemoryError``, are not Python's ``MemoryError`` (#1962)."""
         status = self.status('echo "heap $GRAAL_PYTHON_VM_ARGS"\necho "java.lang.OutOfMemoryError: Java heap space"\n'
                              'echo "E   MemoryError"\necho "MemoryError: out of memory"\nexit 1\n')
         self.assertEqual(status['memory_errors'], '2')

tests/project/test_memory_error_count_1962.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+"""#1962: ``Run unit tests`` counts the ``MemoryError`` raised, not a mention of one.
+
+The count behind ``Containment summary``'s ``tests raised MemoryError`` warning
+was ``grep -cw MemoryError``, which also matched a test's docstring that a
+traceback printed (``fail with :exc:`MemoryError```), so every GraalPy run that
+timed out warned of a ``MemoryError`` none of its tests raised (job
+118149195212). The step now counts the lines that ``MemoryError`` heads, as an
+exception's own line: at the line's start, or after pytest's ``E``, an
+``INTERNALERROR>`` or an exception group's ``|``. The lines below are the
+experimental logs' own, the raised one from job 116172356218 (#1753).
+
+"""
+
+from __future__ import annotations
+
+import os
+import pathlib
+import shutil
+import sys
+import tempfile
+import unittest
+
+from tests.project.test_python_versions_1660 import ROOT, prose
+from tests.project.test_resource_monitor_1740 import DEFAULT, TESTS, sh, step
+
+#: The docstring line job 118149195212 printed in a traceback, which ``-w`` counted.
+DOCSTRING = '        much and either hang or fail with :exc:`MemoryError` rather than'
+#: The line that reported #1753's ``MemoryError`` in pytest's own failure report.
+INTERNAL = 'INTERNALERROR> E             MemoryError'
+#: Lines that mention ``MemoryError`` but report none raised.
+MENTIONS = (
+    DOCSTRING,
+    '    MemoryError: if the heap is spent',
+    'heap --vm.Xmx2g --vm.XX:+ExitOnOutOfMemoryError',
+    'java.lang.OutOfMemoryError: Java heap space',
+    '>       raise MemoryError()',
+)
+#: Lines that report a raised ``MemoryError``, each in a form pytest or Python prints.
+RAISED = (
+    INTERNAL,
+    'E   MemoryError',
+    'E       MemoryError: java.lang.OutOfMemoryError: Java heap space',
+    'MemoryError',
+    'MemoryError: out of memory',
+    '    | MemoryError',
+)
+
+
+@unittest.skipUnless(sys.platform.startswith('linux') and all(map(shutil.which, ('bash', 'timeout', 'prlimit'))),
+                     'the tests step runs coreutils timeout and util-linux prlimit, as on the Ubuntu runners')
+class TestTheMemoryErrorsAreCounted(unittest.TestCase):
+    """The tests step, with ``sudo`` and ``python`` stubbed on ``PATH``, and pytest's output a file."""
+
+    def setUp(self) -> None:
+        tmp = tempfile.TemporaryDirectory(prefix='pcapkit-1962-')  # pylint: disable=consider-using-with
+        self.addCleanup(tmp.cleanup)
+        self.tmp = pathlib.Path(tmp.name)
+        self.temp = self.tmp / 'temp'
+        self.temp.mkdir()
+        self.bindir = self.tmp / 'bin'
+        self.bindir.mkdir()
+        (self.temp / 'resource-monitor.log').write_text('', encoding='utf-8')
+        self.vmstat = self.tmp / 'vmstat'
+        self.vmstat.write_text('oom_kill 0\n', encoding='utf-8')
+        self.output = self.tmp / 'pytest-output'
+        self.env = dict(os.environ, RUNNER_TEMP=str(self.temp), GITHUB_WORKSPACE=str(ROOT), WORKERS='3',
+                        GRAAL_PYTHON_VM_ARGS='', PATH=str(self.bindir) + os.pathsep + os.environ.get('PATH', ''))
+        # No scope: the probe fails, so the tests run the same argv uncapped.
+        self.stub('sudo', 'exit 1\n')
+        self.stub('python', f"cat '{self.output}'\nexit 1\n")
+
+    def stub(self, name: 'str', body: 'str') -> 'None':
+        path = self.bindir / name
+        path.write_text('#!/bin/sh\n' + body, encoding='utf-8')
+        path.chmod(0o755)
+
+    def count(self, *lines: 'str') -> 'int':
+        """The ``memory_errors`` the step writes where pytest prints ``lines``."""
+        self.output.write_text(''.join(line + '\n' for line in lines), encoding='utf-8')
+        self.assertEqual(shutil.which('sudo', path=self.env['PATH']), str(self.bindir / 'sudo'))
+        result = sh(step(TESTS)['run'].replace('/proc/vmstat', str(self.vmstat)), self.env, DEFAULT)
+        self.assertEqual(result.returncode, 1, result.stderr)
+        text = (self.temp / 'tests-status').read_text(encoding='utf-8')
+        return int(dict(line.split('=', 1) for line in text.splitlines())['memory_errors'])
+
+    def test_a_docstring_mention_is_no_memory_error(self) -> None:
+        self.assertEqual(self.count('F.', DOCSTRING, 'KeyboardInterrupt'), 0)
+
+    def test_no_mention_is_counted(self) -> None:
+        for line in MENTIONS:
+            with self.subTest(line=line):
+                self.assertEqual(self.count(line), 0)
+
+    def test_a_raised_memory_error_is_counted(self) -> None:
+        self.assertEqual(self.count(DOCSTRING, INTERNAL, 'INTERNALERROR> E           assert False'), 1)
+
+    def test_every_raised_form_is_counted(self) -> None:
+        for line in RAISED:
+            with self.subTest(line=line):
+                self.assertEqual(self.count(line), 1)
+        self.assertEqual(self.count(*RAISED, *MENTIONS), len(RAISED))
+
+
+class TestTheDocsSayIt(unittest.TestCase):
+
+    def test_the_workflows_page_says_what_is_counted(self) -> None:
+        page = prose(ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst')
+        self.assertIn('raised ``MemoryError`` (#1962)', page)
+
+
+if __name__ == '__main__':
+    unittest.main()