runtest.py runs processes in parallel (#196)

For example, on a laptop with 10 CPUs, testing 14 Python versions now
takes 40.8 seconds instead of 119.6 seconds: 3x faster!

Add --jobs option to runtests.py.

tests/test_pythoncapi_compat.py now builds C and C++ extensions in a
temporary build directory.

Fix some ruff check warnings.

Author: vstinner

.gitignore

@@ -1,5 +1,3 @@
-build
 *.swp
-tests/build/
 *.py[cod]
 __pycache__

docs/changelog.rst

@@ -2,7 +2,8 @@ Changelog
 =========
 
 * 2026-10-10: Add ``PyObject_GenericHash()`` and
-  ``PyUnicodeWriter_WriteUCS4()`` functions.
+  ``PyUnicodeWriter_WriteUCS4()`` functions. runtests.py now runs tests in
+  parallel.
 * 2026-10-08: Add support for the limited C API 3.2.
 * 2026-10-08: Add functions:
 

runtests.py

@@ -12,12 +12,11 @@
 import os.path
 import shutil
 import sys
+import tempfile
 import time
-from shutil import which
-
-
-from tests.utils import run_command
+from concurrent.futures import ThreadPoolExecutor
 
+from tests.utils import get_output, run_command
 
 TEST_DIR = os.path.join(os.path.dirname(__file__), 'tests')
 TEST_COMPAT = os.path.join(TEST_DIR, "test_pythoncapi_compat.py")
@@ -53,26 +52,13 @@
 )
 
 
-def run_tests_exe(executable, verbose, tested):
-    tested_key = os.path.realpath(executable)
-    if tested_key in tested:
-        return
-
+def get_test_command(executable, verbose, build_dir):
     # Don't use realpath() for the executed command to support virtual
     # environments
-    cmd = [executable, TEST_COMPAT]
+    cmd = [executable, "-u", TEST_COMPAT, build_dir]
     if verbose:
         cmd.append('-v')
-    run_command(cmd)
-    tested.add(tested_key)
-
-
-def run_tests(python, verbose, tested):
-    executable = which(python)
-    if not executable:
-        print("Ignore missing Python executable: %s" % python)
-        return
-    run_tests_exe(executable, verbose, tested)
+    return cmd
 
 
 def parse_args():
@@ -82,39 +68,88 @@ def parse_args():
     parser.add_argument('-c', '--current', action="store_true",
                         help="Only test the current Python executable "
                              "(don't test multiple Python versions)")
+    parser.add_argument('-j', '--jobs', type=int,
+                        help="Number of jobs run in parallel")
     return parser.parse_args()
 
 
-def main():
-    start_time = time.perf_counter()
-    args = parse_args()
+def run_tests_parallel(args):
+    jobs = []
+
+    tested = set()
+    tested_key = os.path.realpath(sys.executable)
+    tested.add(tested_key)
+    jobs.append(sys.executable)
+
+    for python in PYTHONS:
+        executable = shutil.which(python)
+        if not executable:
+            print(f"Ignore missing Python executable: {python}")
+            continue
+        tested_key = os.path.realpath(executable)
+        if tested_key in tested:
+            continue
+        tested.add(tested_key)
+        jobs.append(executable)
+
+    def worker(executable):
+        with tempfile.TemporaryDirectory() as build_dir:
+            cmd = get_test_command(executable, args.verbose, build_dir)
+            return get_output(cmd)
+
+    max_workers = args.jobs
+    if not max_workers:
+        if hasattr(os, 'process_cpu_count'):
+            max_workers = os.process_cpu_count()
+        else:
+            max_workers = os.cpu_count()
+
+    print()
+    print(f"Run {len(jobs)} jobs with {max_workers} workers (threads)")
+    print()
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        exitcode = None
+        for exitcode, stdout in executor.map(worker, jobs):
+            print(stdout, end='', flush=True)
+            if exitcode:
+                break
 
-    path = os.path.join(TEST_DIR, 'build')
-    if os.path.exists(path):
-        shutil.rmtree(path)
+        if exitcode:
+            executor.shutdown(wait=True, cancel_futures=True)
+            sys.exit(exitcode)
 
+    print()
+    print(f"Tested: {len(jobs)} Python executables")
+
+
+def test_upgrade_pythoncapi(args):
     # upgrade_pythoncapi.py requires Python 3.6 or newer
-    if sys.version_info >= (3, 6):
-        print("Run %s" % TEST_UPGRADE)
-        cmd = [sys.executable, TEST_UPGRADE]
-        if args.verbose:
-            cmd.append('-v')
+    print(f"Run {TEST_UPGRADE}")
+    cmd = [sys.executable, "-u", TEST_UPGRADE]
+    if args.verbose:
+        cmd.append('-v')
+    run_command(cmd)
+    print()
+
+
+def test_current_version(args):
+    with tempfile.TemporaryDirectory() as build_dir:
+        cmd = get_test_command(sys.executable, args.verbose, build_dir)
         run_command(cmd)
-    else:
-        print("Don't test upgrade_pythoncapi.py: it requires Python 3.6")
     print()
 
-    tested = set()
-    if not args.current:
-        for python in PYTHONS:
-            run_tests(python, args.verbose, tested)
-        run_tests_exe(sys.executable, args.verbose, tested)
 
-        print()
-        print("Tested: %s Python executables" % len(tested))
+def main():
+    start_time = time.perf_counter()
+    args = parse_args()
+
+    test_upgrade_pythoncapi(args)
+
+    if not args.current:
+        run_tests_parallel(args)
     else:
-        run_tests_exe(sys.executable, args.verbose, tested)
-        print()
+        test_current_version(args)
 
     dt = time.perf_counter() - start_time
     print(f"Total time: {dt:.1f} seconds")

tests/setup.py

@@ -28,7 +28,7 @@
 else:
     TEST_OLD_LIMITED_C_API = False
 
-OLD_LIMITED_CAPI = f'-DPy_LIMITED_API=0x3020000'
+OLD_LIMITED_CAPI = '-DPy_LIMITED_API=0x3020000'
 
 SRC_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
 LIMITED_SUFFIX = "_limited"
@@ -136,9 +136,9 @@ def cxx_extension_name(std):
 
 def main():
     try:
-        from setuptools import setup, Extension
+        from setuptools import Extension, setup
     except ImportError:
-        from distutils.core import setup, Extension
+        from distutils.core import Extension, setup
 
     cflags = list(CFLAGS)
     cxxflags = list(CXXFLAGS)

tests/test_pythoncapi_compat.py

@@ -7,6 +7,7 @@
     python3 run_tests.py
     python3 run_tests.py -v # verbose mode
 """
+import argparse
 import faulthandler
 import gc
 import os.path
@@ -17,7 +18,7 @@
 
 # test.utils
 import setup
-from utils import run_command, command_stdout
+from utils import run_command, get_output
 
 
 # Windows uses MSVC compiler
@@ -62,26 +63,24 @@ def display_title(title):
     sys.stdout.flush()
 
 
-def build_ext():
+def build_ext(build_dir):
     display_title("Build test extensions")
-    if os.path.exists("build"):
-        shutil.rmtree("build")
-    cmd = [sys.executable, "setup.py", "build"]
+    cmd = [sys.executable, "-u", "setup.py", "build", "--build-base", build_dir]
     if VERBOSE:
         run_command(cmd)
         print()
     else:
-        exitcode, stdout = command_stdout(cmd, stderr=subprocess.STDOUT)
+        exitcode, stdout = get_output(cmd)
         if exitcode:
             print(stdout.rstrip())
             sys.exit(exitcode)
 
 
-def import_tests(module_name):
+def import_tests(build_dir, module_name):
     pythonpath = None
-    for name in os.listdir("build"):
+    for name in os.listdir(build_dir):
         if name.startswith('lib.'):
-            pythonpath = os.path.join("build", name)
+            pythonpath = os.path.join(build_dir, name)
 
     if not pythonpath:
         raise Exception("Failed to find the build directory")
@@ -152,7 +151,7 @@ def python_version():
     return "%s %s (%s build)" % (python_impl, pyver, build)
 
 
-def run_tests(module_name, std):
+def run_tests(build_dir, module_name, std):
     lang = std.upper() if std else None
     if VERBOSE:
         print("")
@@ -162,7 +161,7 @@ def run_tests(module_name, std):
         titlte = f"{title} ({lang})"
     display_title(title)
 
-    testmod = import_tests(module_name)
+    testmod = import_tests(build_dir, module_name)
 
     if VERBOSE:
         empty_line = False
@@ -203,27 +202,36 @@ def test_func():
         msg += " (no reference leak detected)"
     print(msg)
 
+    # Unload the extension module
+    testmod = None
+    del sys.modules[module_name]
 
-def main():
-    global VERBOSE
-    VERBOSE = ("-v" in sys.argv[1:] or "--verbose" in sys.argv[1:])
 
-    if (3, 13) <= sys.version_info <= (3, 13, 0, 'alpha', 4):
-        print("SKIP Python 3.13 alpha 1..4: not supported!")
-        return
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-v', '--verbose',
+                        action='store_true')  # on/off flag
+    parser.add_argument('build_dir')
+    return parser.parse_args()
+
 
-    if faulthandler is not None:
-        faulthandler.enable()
+def main():
+    faulthandler.enable()
+
+    global VERBOSE
+    args = parse_args()
+    VERBOSE = args.verbose
+    build_dir = args.build_dir
 
     src_dir = os.path.dirname(__file__)
     if src_dir:
         os.chdir(src_dir)
 
-    build_ext()
+    build_ext(build_dir)
 
     tests = setup.C_TESTS + setup.CXX_TESTS
     for module_name, std, limited in tests:
-        run_tests(module_name, std)
+        run_tests(build_dir, module_name, std)
 
 
 if __name__ == "__main__":

tests/utils.py

@@ -2,37 +2,37 @@
 import sys
 
 
-def run_command(cmd, **kw):
+def _run_command(cmd, **kw):
+    sys.stdout.flush()
+    sys.stderr.flush()
+
+    kw['shell'] = False
     if hasattr(subprocess, 'run'):
         proc = subprocess.run(cmd, **kw)
     else:
-        kw['shell'] = False
         proc = subprocess.Popen(cmd, **kw)
         try:
+            proc.communicate()
             proc.wait()
         except:
             proc.kill()
             proc.wait()
             raise
+    return proc
 
+
+def run_command(cmd, **kw):
+    proc = _run_command(cmd, **kw)
     exitcode = proc.returncode
     if exitcode:
         sys.exit(exitcode)
 
 
-def command_stdout(cmd, **kw):
-    kw['stdout'] = subprocess.PIPE
+def get_output(cmd, **kw):
+    # Legacy for text=True
     kw['universal_newlines'] = True
-    if hasattr(subprocess, 'run'):
-        proc = subprocess.run(cmd, **kw)
-        return (proc.returncode, proc.stdout)
-    else:
-        kw['shell'] = False
-        proc = subprocess.Popen(cmd, **kw)
-        try:
-            stdout = proc.communicate()[0]
-        except:
-            proc.kill()
-            proc.wait()
-            raise
-        return (proc.returncode, stdout)
+    proc = _run_command(cmd,
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.STDOUT,
+                        **kw)
+    return (proc.returncode, proc.stdout)