From 22bd359d8b3bcdf008d2a3ecc300cb9a672c6b99 Mon Sep 17 00:00:00 2001 From: Jarry Shaw Date: Mon, 19 Oct 2026 22:11:55 -0400 Subject: [PATCH] ci(experimental): count raised MemoryErrors, not mentions of one (#1962) - Run unit tests: count the lines that MemoryError heads, at the line's start or after pytest's E, an INTERNALERROR> or an exception group's |, instead of every word match, which counted a test docstring a traceback printed and warned of a MemoryError on every GraalPy timeout. - Pin it: a docstring, source line or Java OutOfMemoryError mention is not counted; a raised MemoryError, #1753's included, is. - Say so in the step's comment and the workflows page. Closes #1962 --- .github/workflows/experimental.yml | 9 +- docs/source/contributing/workflows.rst | 3 +- .../project/test_graalpy_worker_heap_1753.py | 2 +- tests/project/test_memory_error_count_1962.py | 113 ++++++++++++++++++ 4 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 tests/project/test_memory_error_count_1962.py diff --git a/.github/workflows/experimental.yml b/.github/workflows/experimental.yml index 27ab9a7207..9cdd8b8190 100644 --- a/.github/workflows/experimental.yml +++ b/.github/workflows/experimental.yml @@ -426,8 +426,11 @@ jobs: # way the tests run the one argv, in the scope or not. # `tests-status` holds the exit status, the scope and the probe's error, # the seconds the tests took, the kernel's OOM kills while they ran - # (/proc/vmstat `oom_kill`), the MemoryError count (`-w`, so Java's - # `OutOfMemoryError` and the flag are not counted) and the workers that + # (/proc/vmstat `oom_kill`), the MemoryErrors raised, as the lines that + # `MemoryError` heads, alone or before a `:`, at the line's start or after + # pytest's `E`, an `INTERNALERROR>` or an exception group's `|` (#1962: a + # mention is not one, such as a docstring a traceback prints, and nor are + # Java's `OutOfMemoryError` and the flag), and the workers that # crashed, by xdist's `[gwN] node down:` lines (under `-q` it prints no # `replacing crashed worker`, xdist/dsession.py:78-80 at v3.8.0), for # `Containment summary`. @@ -516,7 +519,7 @@ jobs: set -e elapsed=$((SECONDS - started)) after=$(oom_kills) - memory_errors=$(grep -cw MemoryError "$RUNNER_TEMP/pytest.log" || true) + memory_errors=$(grep -cE '^(INTERNALERROR> *)?(E +)?( *\| *)*MemoryError(:|[[:space:]]*$)' "$RUNNER_TEMP/pytest.log" || true) crashed_workers=$(grep -c '\] node down: ' "$RUNNER_TEMP/pytest.log" || true) printf 'exit=%s\nscope=%s\nprobe=%s\nelapsed=%s\noom_kills=%s\nmemory_errors=%s\ncrashed_workers=%s\n' \ "$status" "$scope" "$probe" "$elapsed" "$(( ${after:-0} - ${before:-0} ))" "$memory_errors" \ diff --git a/docs/source/contributing/workflows.rst b/docs/source/contributing/workflows.rst index f15c25fc59..c8916e7a75 100644 --- a/docs/source/contributing/workflows.rst +++ b/docs/source/contributing/workflows.rst @@ -891,7 +891,8 @@ warnings: The step writes ``$RUNNER_TEMP/tests-status``: the exit status, the scope and the probe's error, the seconds the tests took, the OOM kills while they ran -(``/proc/vmstat``), the ``MemoryError`` lines in their output and xdist's ``node +(``/proc/vmstat``), the lines in their output that report a raised ``MemoryError`` (#1962), +not those that mention one, such as a docstring a traceback prints, and xdist's ``node down`` lines. From it and the monitor's last sample, ``Containment summary`` warns ``No memory cap``, ``tests lost a process to the OOM killer``, ``tests timed out at 110 minutes``, ``tests raised MemoryError`` or ``tests lost an xdist worker``, each a diff --git a/tests/project/test_graalpy_worker_heap_1753.py b/tests/project/test_graalpy_worker_heap_1753.py index dcb6d59764..64b3ee59ac 100644 --- a/tests/project/test_graalpy_worker_heap_1753.py +++ b/tests/project/test_graalpy_worker_heap_1753.py @@ -109,7 +109,7 @@ def test_every_worker_that_went_down_is_counted(self) -> None: self.assertEqual(status['memory_errors'], '0') def test_a_java_out_of_memory_line_is_no_memory_error(self) -> None: - """``-w``: the flag, and Java's ``OutOfMemoryError``, are not Python's ``MemoryError``.""" + """The flag, and Java's ``OutOfMemoryError``, are not Python's ``MemoryError`` (#1962).""" status = self.status('echo "heap $GRAAL_PYTHON_VM_ARGS"\necho "java.lang.OutOfMemoryError: Java heap space"\n' 'echo "E MemoryError"\necho "MemoryError: out of memory"\nexit 1\n') self.assertEqual(status['memory_errors'], '2') diff --git a/tests/project/test_memory_error_count_1962.py b/tests/project/test_memory_error_count_1962.py new file mode 100644 index 0000000000..857569ebe8 --- /dev/null +++ b/tests/project/test_memory_error_count_1962.py @@ -0,0 +1,113 @@ +# -*- coding: utf-8 -*- +"""#1962: ``Run unit tests`` counts the ``MemoryError`` raised, not a mention of one. + +The count behind ``Containment summary``'s ``tests raised MemoryError`` warning +was ``grep -cw MemoryError``, which also matched a test's docstring that a +traceback printed (``fail with :exc:`MemoryError```), so every GraalPy run that +timed out warned of a ``MemoryError`` none of its tests raised (job +118149195212). The step now counts the lines that ``MemoryError`` heads, as an +exception's own line: at the line's start, or after pytest's ``E``, an +``INTERNALERROR>`` or an exception group's ``|``. The lines below are the +experimental logs' own, the raised one from job 116172356218 (#1753). + +""" + +from __future__ import annotations + +import os +import pathlib +import shutil +import sys +import tempfile +import unittest + +from tests.project.test_python_versions_1660 import ROOT, prose +from tests.project.test_resource_monitor_1740 import DEFAULT, TESTS, sh, step + +#: The docstring line job 118149195212 printed in a traceback, which ``-w`` counted. +DOCSTRING = ' much and either hang or fail with :exc:`MemoryError` rather than' +#: The line that reported #1753's ``MemoryError`` in pytest's own failure report. +INTERNAL = 'INTERNALERROR> E MemoryError' +#: Lines that mention ``MemoryError`` but report none raised. +MENTIONS = ( + DOCSTRING, + ' MemoryError: if the heap is spent', + 'heap --vm.Xmx2g --vm.XX:+ExitOnOutOfMemoryError', + 'java.lang.OutOfMemoryError: Java heap space', + '> raise MemoryError()', +) +#: Lines that report a raised ``MemoryError``, each in a form pytest or Python prints. +RAISED = ( + INTERNAL, + 'E MemoryError', + 'E MemoryError: java.lang.OutOfMemoryError: Java heap space', + 'MemoryError', + 'MemoryError: out of memory', + ' | MemoryError', +) + + +@unittest.skipUnless(sys.platform.startswith('linux') and all(map(shutil.which, ('bash', 'timeout', 'prlimit'))), + 'the tests step runs coreutils timeout and util-linux prlimit, as on the Ubuntu runners') +class TestTheMemoryErrorsAreCounted(unittest.TestCase): + """The tests step, with ``sudo`` and ``python`` stubbed on ``PATH``, and pytest's output a file.""" + + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory(prefix='pcapkit-1962-') # pylint: disable=consider-using-with + self.addCleanup(tmp.cleanup) + self.tmp = pathlib.Path(tmp.name) + self.temp = self.tmp / 'temp' + self.temp.mkdir() + self.bindir = self.tmp / 'bin' + self.bindir.mkdir() + (self.temp / 'resource-monitor.log').write_text('', encoding='utf-8') + self.vmstat = self.tmp / 'vmstat' + self.vmstat.write_text('oom_kill 0\n', encoding='utf-8') + self.output = self.tmp / 'pytest-output' + self.env = dict(os.environ, RUNNER_TEMP=str(self.temp), GITHUB_WORKSPACE=str(ROOT), WORKERS='3', + GRAAL_PYTHON_VM_ARGS='', PATH=str(self.bindir) + os.pathsep + os.environ.get('PATH', '')) + # No scope: the probe fails, so the tests run the same argv uncapped. + self.stub('sudo', 'exit 1\n') + self.stub('python', f"cat '{self.output}'\nexit 1\n") + + def stub(self, name: 'str', body: 'str') -> 'None': + path = self.bindir / name + path.write_text('#!/bin/sh\n' + body, encoding='utf-8') + path.chmod(0o755) + + def count(self, *lines: 'str') -> 'int': + """The ``memory_errors`` the step writes where pytest prints ``lines``.""" + self.output.write_text(''.join(line + '\n' for line in lines), encoding='utf-8') + self.assertEqual(shutil.which('sudo', path=self.env['PATH']), str(self.bindir / 'sudo')) + result = sh(step(TESTS)['run'].replace('/proc/vmstat', str(self.vmstat)), self.env, DEFAULT) + self.assertEqual(result.returncode, 1, result.stderr) + text = (self.temp / 'tests-status').read_text(encoding='utf-8') + return int(dict(line.split('=', 1) for line in text.splitlines())['memory_errors']) + + def test_a_docstring_mention_is_no_memory_error(self) -> None: + self.assertEqual(self.count('F.', DOCSTRING, 'KeyboardInterrupt'), 0) + + def test_no_mention_is_counted(self) -> None: + for line in MENTIONS: + with self.subTest(line=line): + self.assertEqual(self.count(line), 0) + + def test_a_raised_memory_error_is_counted(self) -> None: + self.assertEqual(self.count(DOCSTRING, INTERNAL, 'INTERNALERROR> E assert False'), 1) + + def test_every_raised_form_is_counted(self) -> None: + for line in RAISED: + with self.subTest(line=line): + self.assertEqual(self.count(line), 1) + self.assertEqual(self.count(*RAISED, *MENTIONS), len(RAISED)) + + +class TestTheDocsSayIt(unittest.TestCase): + + def test_the_workflows_page_says_what_is_counted(self) -> None: + page = prose(ROOT / 'docs' / 'source' / 'contributing' / 'workflows.rst') + self.assertIn('raised ``MemoryError`` (#1962)', page) + + +if __name__ == '__main__': + unittest.main()