From 8da39c78a0420387968cd3ccadb2ba4faaf5a11a Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 4 Nov 2026 16:36:10 -0700 Subject: [PATCH 1/6] GH-161367: Add `test` and `buildbottest` test subcommands to `Platforms/WASI` --- .github/workflows/reusable-wasi.yml | 4 +- ...-11-04-16-34-48.gh-issue-161367.bO5MGy.rst | 2 + Platforms/WASI/README.md | 31 +++++++++- Platforms/WASI/__main__.py | 30 ++++++++- Platforms/WASI/_build.py | 6 +- Platforms/WASI/_package.py | 61 +++++++----------- Platforms/WASI/_shared.py | 28 +++++++++ Platforms/WASI/_test.py | 62 +++++++++++++++++++ 8 files changed, 175 insertions(+), 49 deletions(-) create mode 100644 Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst create mode 100644 Platforms/WASI/_test.py diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 12dd790897ad392..5b2fda1fe4f82dc 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -64,6 +64,4 @@ jobs: - name: "Display build info of the host/WASI Python" run: python3 Platforms/WASI pythoninfo-host - name: "Test" - run: | - CROSS_BUILD_WASI=$(python3 Platforms/WASI path) - make --directory "${CROSS_BUILD_WASI}" test + run: python3 Platforms/WASI test diff --git a/Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst b/Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst new file mode 100644 index 000000000000000..8bb54b9dfff15db --- /dev/null +++ b/Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst @@ -0,0 +1,2 @@ +Add ``test`` and ``buildbottest`` commands to :file:`Platforms/WASI` for CI +and Buildbot use, respectively. diff --git a/Platforms/WASI/README.md b/Platforms/WASI/README.md index 5fe610615aec336..ff35e69b4f72d5b 100644 --- a/Platforms/WASI/README.md +++ b/Platforms/WASI/README.md @@ -199,7 +199,36 @@ python3 Platforms/WASI clean ### Testing -To find out where the WASI build directory is, you can run: +After building, you can run the CI tests with: + +```shell +python3 Platforms/WASI test +``` + +This runs `make test` in the WASI build directory. + +After running `gather` or `package`, you can run the buildbot tests against the +distribution with: + +```shell +python3 Platforms/WASI buildbottest +``` + +This uses `make buildbottest` appropriately with the binary for releasing. + +To test the contents of the release archive instead of the gathered files, use: + +```shell +python3 Platforms/WASI buildbottest --unpack +``` + +This requires an archive created by `package` or `release`. It deletes the +selected versioned distribution directory inside `dist/`, then restores it +from the matching `.tar.xz` archive before running `make buildbottest`. +The archive and other distributions in `dist/` are left untouched. + +You can also tests more directly. To find out where the WASI build directory +is, you can run: ```shell python3 Platforms/WASI path diff --git a/Platforms/WASI/__main__.py b/Platforms/WASI/__main__.py index 0d504d43283297a..3000cbc53b72bc1 100644 --- a/Platforms/WASI/__main__.py +++ b/Platforms/WASI/__main__.py @@ -5,6 +5,7 @@ "_build", "_package", "_shared", + "_test", ] import argparse @@ -14,6 +15,7 @@ import _build import _package import _shared +import _test def main(): @@ -76,6 +78,15 @@ def main(): release = subcommands.add_parser( "release", help="Build and package for a release" ) + test = subcommands.add_parser("test", help="Run the CI tests") + buildbottest = subcommands.add_parser( + "buildbottest", help="Run the buildbot tests against the distribution" + ) + buildbottest.add_argument( + "--unpack", + action="store_true", + help="Replace the distribution directory with the release archive's contents", + ) path = subcommands.add_parser( "path", help="Print the path to a build or distribution directory" ) @@ -100,6 +111,8 @@ def main(): pythoninfo_host, build_host, release, + test, + buildbottest, ): subcommand.add_argument( "--quiet", @@ -129,6 +142,13 @@ def main(): dest="clean", help="Delete any relevant directories before building", ) + for subcommand in build, configure_build, configure_host: + subcommand.add_argument( + "--release", + help="Configure build for release mode", + action="store_true", + default=False, + ) for subcommand in ( build, configure_build, @@ -137,7 +157,7 @@ def main(): build_host, ): subcommand.add_argument( - "args", nargs="*", help="Extra arguments to pass to `configure`" + "_args", nargs="*", help="Extra arguments to pass to `configure`" ) for subcommand in build, configure_host, build_host, release: subcommand.add_argument( @@ -167,6 +187,8 @@ def main(): package, path, release, + test, + buildbottest, ): subcommand.add_argument( "--host-triple", @@ -219,7 +241,7 @@ def main(): _package.gather(context) _package.archive(context) case "release": - context.args = ["--without-static-libpython"] + context.release = True _build.clean_contents(context) _build.configure_build_python(context) _build.make_build_python(context) @@ -227,6 +249,10 @@ def main(): _build.make_wasi_python(context) _package.gather(context) _package.archive(context) + case "test": + _test.test(context) + case "buildbottest": + _test.buildbottest(context) case "path": paths = { "build-python": "build_python_path", diff --git a/Platforms/WASI/_build.py b/Platforms/WASI/_build.py index 9ea92a58ef5bb34..5efd435950d8c31 100644 --- a/Platforms/WASI/_build.py +++ b/Platforms/WASI/_build.py @@ -144,8 +144,7 @@ def call(command, *, context=None, quiet=False, **kwargs): def configure_build_python(context, working_dir): """Configure the build/host Python.""" configure = [os.path.relpath(context.checkout / "configure", working_dir)] - if context.args: - configure.extend(context.args) + configure.extend(context.configure_args) call(configure, context=context) @@ -247,8 +246,7 @@ def configure_wasi_python(context, working_dir): ] if context.is_debug: configure.append("--with-pydebug") - if context.args: - configure.extend(context.args) + configure.extend(context.configure_args) call( configure, env=updated_env(env_additions | wasi_sdk_env(context)), diff --git a/Platforms/WASI/_package.py b/Platforms/WASI/_package.py index 862ffb30130d048..6c47930b344e1fc 100644 --- a/Platforms/WASI/_package.py +++ b/Platforms/WASI/_package.py @@ -26,32 +26,18 @@ def wasmtime_script(context): script_dir=$(CDPATH= cd "$(dirname "$0")" && pwd -P) root=$(CDPATH= cd "$script_dir/.." && pwd -P) -wasm_file="$root/lib/python{python_version(context)}/lib-wasm/python{python_version(context, debug_ok=True)}.wasm" +wasm_file="$root/lib/python{context.python_version()}/lib-wasm/python{context.python_version(debug_ok=True)}.wasm" exec wasmtime run \\ --argv0 "$wasm_file" \\ - --config "$root/etc/python{python_version(context)}/wasmtime.toml" \\ + --config "$root/etc/python{context.python_version()}/wasmtime.toml" \\ --dir "/" \\ "$wasm_file" "$@" """ -def python_version(context, debug_ok=False): - """Calculate the M.N part of Python's version. - - If *debug_ok* is True, then "d" is appended as appropriate. - """ - details = context.wasi_build_details - major = details["language"]["version_info"]["major"] - minor = details["language"]["version_info"]["minor"] - version = f"{major}.{minor}" - if debug_ok and context.is_debug: - version += "d" - return version - - def lib_python(context): - return pathlib.PurePath("lib") / f"python{python_version(context)}" + return pathlib.PurePath("lib") / f"python{context.python_version()}" def license_file(context): @@ -101,12 +87,12 @@ def pkgconfig_files(context): pkgconfig = pathlib.PurePath("lib") / "pkgconfig" return [ ( - pkgconfig / f"python-{python_version(context, debug_ok=True)}.pc", + pkgconfig / f"python-{context.python_version(debug_ok=True)}.pc", misc_dir / "python.pc", ), ( pkgconfig - / f"python-{python_version(context, debug_ok=True)}-embed.pc", + / f"python-{context.python_version(debug_ok=True)}-embed.pc", misc_dir / "python-embed.pc", ), ] @@ -127,11 +113,11 @@ def pkgconfig_symlinks(pkgconfig_files, context): paths += [ ( embed_path.parent - / f"python-{python_version(context)}-embed.pc", + / f"python-{context.python_version()}-embed.pc", embed_path, ), ( - plain_path.parent / f"python-{python_version(context)}.pc", + plain_path.parent / f"python-{context.python_version()}.pc", plain_path, ), ] @@ -143,7 +129,7 @@ def pyconfig_file(context): """Have /pyconfig.h end up in include/pythonXYd?/pyconfig.h.""" return ( pathlib.PurePath("include") - / f"python{python_version(context, debug_ok=True)}" + / f"python{context.python_version(debug_ok=True)}" / "pyconfig.h", context.wasi_build_path / "pyconfig.h", ) @@ -158,7 +144,7 @@ def header_files(context): file_path = pathlib.Path(root) / filename details = ( pathlib.PurePath("include") - / f"python{python_version(context, debug_ok=True)}" + / f"python{context.python_version(debug_ok=True)}" / file_path.relative_to(include_dir), file_path, ) @@ -171,7 +157,7 @@ def man_file(context): man_dir = pathlib.PurePath("share", "man", "man1") man_file = context.checkout / "Misc" / "python.man" return ( - man_dir / f"python{python_version(context)}.1", + man_dir / f"python{context.python_version()}.1", man_file, ) @@ -187,7 +173,7 @@ def wasmtime_config_file(context): config = context.checkout / "Platforms" / "WASI" / "wasmtime.toml" return ( pathlib.PurePath("etc") - / f"python{python_version(context)}" + / f"python{context.python_version()}" / "wasmtime.toml", config, ) @@ -198,17 +184,15 @@ def wasm_file(context): return ( lib_python(context) / "lib-wasm" - / f"python{python_version(context, debug_ok=True)}.wasm", + / f"python{context.python_version(debug_ok=True)}.wasm", context.wasi_build_path / "python.wasm", ) -def python_wasmtime_script(base, context): +def python_wasmtime_script(context): """Create bin/pythonN.Md?.wasmtime.""" script = wasmtime_script(context) - path = ( - base / f"bin/python{python_version(context, debug_ok=True)}.wasmtime" - ) + path = context.dist_python_interpreter path.parent.mkdir(parents=True, exist_ok=True) with open(path, "w") as f: f.write(script) @@ -221,7 +205,7 @@ def python_wasmtime_symlink(path, context): """Symlink bin/pythonN.Md?.wasmtime. - bin/pythonN.M.wasmtime (if debug build) - - bin/pythonN.wasmtime (if debug build) + - bin/pythonN.wasmtime """ symlinks = [ path.parent / f"python{context.wasi_build_version['major']}.wasmtime" @@ -229,7 +213,7 @@ def python_wasmtime_symlink(path, context): if context.is_debug: # The file already has the debug name, so the missing symlink is the non-debug name. symlinks.append( - path.parent / f"python{python_version(context)}.wasmtime" + path.parent / f"python{context.python_version()}.wasmtime" ) return [(symlink, path) for symlink in symlinks] @@ -239,7 +223,7 @@ def config_file(context): """Have Misc/config.sh end up at bin/pythonN.Md?.config.""" return ( pathlib.PurePath("bin") - / f"python{python_version(context, debug_ok=True)}-config", + / f"python{context.python_version(debug_ok=True)}-config", context.wasi_build_path / "Misc" / "python-config.sh", ) @@ -258,7 +242,7 @@ def config_symlink(config_path, context): if context.is_debug: # The file already has the debug name, so the missing symlink is the non-debug name. symlinks.append( - config_path.parent / f"python{python_version(context)}-config" + config_path.parent / f"python{context.python_version()}-config" ) return [(symlink, config_path) for symlink in symlinks] @@ -281,8 +265,8 @@ def symlink_files(files, base): def gather(context): - py_version = python_version(context) - py_d_version = python_version(context, debug_ok=True) + py_version = context.python_version() + py_d_version = context.python_version(debug_ok=True) dist = context.dist_path if dist.exists(): @@ -300,7 +284,7 @@ def gather(context): symlink_files(config_symlink(config_location[0], context), base) _shared.log("๐Ÿ“„", f"python{py_d_version}.wasmtime", spacing=indent * 3) - script_path = python_wasmtime_script(base, context) + script_path = python_wasmtime_script(context) symlink_files(python_wasmtime_symlink(script_path, context), base) _shared.log("๐Ÿ“", "etc/", spacing=indent * 2) @@ -353,8 +337,7 @@ def gather(context): def archive(context): - file_name = f"{context.archive_stem}.tar.xz" - file_path = context.dist_path / file_name + file_path = context.archive_path if file_path.exists(): _shared.log("๐Ÿงน", f"Deleting {file_path} ...") file_path.unlink() diff --git a/Platforms/WASI/_shared.py b/Platforms/WASI/_shared.py index 306aabd1f1c0977..e8829436f8e73e1 100644 --- a/Platforms/WASI/_shared.py +++ b/Platforms/WASI/_shared.py @@ -178,6 +178,13 @@ def wasi_sdk_path(self): return wasi_sdk_path + @functools.cached_property + def configure_args(self): + args = getattr(self, "_args", []) + if self.release: + args.extend(["--without-static-libpython"]) + return args + @functools.cached_property def log_path(self): if self._log_path is not None: @@ -206,6 +213,27 @@ def archive_stem(self): def archive_dir(self): return self.dist_path / self.archive_stem + @functools.cached_property + def archive_path(self): + return self.dist_path / f"{self.archive_stem}.tar.xz" + + @functools.cached_property + def dist_python_interpreter(self): + version = self.python_version(debug_ok=True) + return self.archive_dir / "bin" / f"python{version}.wasmtime" + + def python_version(self, debug_ok=False): + """Calculate the M.N part of Python's version. + + If *debug_ok* is True, then "d" is appended as appropriate. + """ + major = self.wasi_build_version["major"] + minor = self.wasi_build_version["minor"] + version = f"{major}.{minor}" + if debug_ok and "d" in self.wasi_build_details["abi"]["flags"]: + version += "d" + return version + def log(emoji, message, *, spacing=None): """Print a notification with an emoji. diff --git a/Platforms/WASI/_test.py b/Platforms/WASI/_test.py new file mode 100644 index 000000000000000..030d18196d5b0fa --- /dev/null +++ b/Platforms/WASI/_test.py @@ -0,0 +1,62 @@ +__lazy_modules__ = [ + "shlex", + "shutil", + "_shared", +] + +import shlex +import shutil + +import _build +import _shared + + +@_build.subdir("wasi_build_path") +def test(context, _working_dir): + _build.call(["make", "test"], env=_build.updated_env(), context=context) + + +@_build.subdir("wasi_build_path") +def buildbottest(context, _working_dir): + if context.unpack: + archive = context.archive_path + if not archive.is_file(): + raise FileNotFoundError( + f"Release archive not found at {archive}; " + "run `package` or `release` first" + ) + if context.archive_dir.exists(): + _shared.log("๐Ÿงน", f"Deleting {context.archive_dir} ...") + shutil.rmtree(context.archive_dir) + _build.call( + [ + "tar", + "--extract", + "--file", + archive, + "--directory", + context.dist_path, + "--", + context.archive_dir.name, + ], + context=context, + ) + + runner = context.dist_python_interpreter + if not runner.is_file(): + raise FileNotFoundError( + f"Packaged WASI interpreter not found at {runner}; " + "run `gather`, `package`, or `release` first" + ) + + test_opts = shlex.join(["--python", shlex.quote(str(runner))]) + _build.call( + [ + "make", + "buildbottest", + "TESTPYTHON=$(RUNSHARED) $(PYTHON_FOR_FREEZE) $(TESTPYTHONOPTS)", + f"TESTOPTS=$(EXTRATESTOPTS) {test_opts}", + ], + env=_build.updated_env(), + context=context, + ) From 00eb20a4ce3939b4fd0c0057b283f4406d7fb261 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 5 Nov 2026 10:18:29 -0700 Subject: [PATCH 2/6] Generalize the `test` command --- ...-11-04-16-34-48.gh-issue-161367.bO5MGy.rst | 5 +- Platforms/WASI/README.md | 19 +-- Platforms/WASI/__main__.py | 23 +-- Platforms/WASI/_build.py | 151 ++---------------- Platforms/WASI/_shared.py | 117 ++++++++++++++ Platforms/WASI/_test.py | 95 ++++++----- 6 files changed, 210 insertions(+), 200 deletions(-) diff --git a/Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst b/Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst index 8bb54b9dfff15db..e40d572c4bfb83f 100644 --- a/Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst +++ b/Misc/NEWS.d/next/WASI/2026-11-04-16-34-48.gh-issue-161367.bO5MGy.rst @@ -1,2 +1,3 @@ -Add ``test`` and ``buildbottest`` commands to :file:`Platforms/WASI` for CI -and Buildbot use, respectively. +Add a ``test`` command to :file:`Platforms/WASI` with ``--speed`` to select +fast CI or slow Buildbot tests and ``--target`` to test the build, gathered +distribution, or release archive. diff --git a/Platforms/WASI/README.md b/Platforms/WASI/README.md index ff35e69b4f72d5b..1578a7dffa00a69 100644 --- a/Platforms/WASI/README.md +++ b/Platforms/WASI/README.md @@ -205,26 +205,23 @@ After building, you can run the CI tests with: python3 Platforms/WASI test ``` -This runs `make test` in the WASI build directory. +The `--speed` flag selects `fast` (the default) for `make test` or `slow` for +`make buildbottest`. The `--target` flag selects the files to test: -After running `gather` or `package`, you can run the buildbot tests against the -distribution with: +- `build` (the default): the WASI build in `cross-build/`. +- `dist`: the distribution files gathered in `dist/`. +- `archive`: the contents of the release archive, unpacked into `dist/`. -```shell -python3 Platforms/WASI buildbottest -``` - -This uses `make buildbottest` appropriately with the binary for releasing. -To test the contents of the release archive instead of the gathered files, use: +To thoroughly test the contents of the release archive, use: ```shell -python3 Platforms/WASI buildbottest --unpack +python3 Platforms/WASI test --speed slow --target archive ``` This requires an archive created by `package` or `release`. It deletes the selected versioned distribution directory inside `dist/`, then restores it -from the matching `.tar.xz` archive before running `make buildbottest`. +from the matching `.tar.xz` archive before running the tests. The archive and other distributions in `dist/` are left untouched. You can also tests more directly. To find out where the WASI build directory diff --git a/Platforms/WASI/__main__.py b/Platforms/WASI/__main__.py index 3000cbc53b72bc1..e42698a42314a7d 100644 --- a/Platforms/WASI/__main__.py +++ b/Platforms/WASI/__main__.py @@ -78,14 +78,19 @@ def main(): release = subcommands.add_parser( "release", help="Build and package for a release" ) - test = subcommands.add_parser("test", help="Run the CI tests") - buildbottest = subcommands.add_parser( - "buildbottest", help="Run the buildbot tests against the distribution" + test = subcommands.add_parser("test", help="Run the tests") + test.add_argument( + "--speed", + choices={"fast", "slow"}, + default="fast", + help="Run `make test` (fast, the default) or `make buildbottest` (slow)", ) - buildbottest.add_argument( - "--unpack", - action="store_true", - help="Replace the distribution directory with the release archive's contents", + test.add_argument( + "--target", + choices={"build", "dist", "archive"}, + default="build", + help="Test the build (the default), gathered distribution, or release archive; " + "'archive' replaces the distribution directory with the archive's contents", ) path = subcommands.add_parser( "path", help="Print the path to a build or distribution directory" @@ -112,7 +117,6 @@ def main(): build_host, release, test, - buildbottest, ): subcommand.add_argument( "--quiet", @@ -188,7 +192,6 @@ def main(): path, release, test, - buildbottest, ): subcommand.add_argument( "--host-triple", @@ -251,8 +254,6 @@ def main(): _package.archive(context) case "test": _test.test(context) - case "buildbottest": - _test.buildbottest(context) case "path": paths = { "build-python": "build_python_path", diff --git a/Platforms/WASI/_build.py b/Platforms/WASI/_build.py index 5efd435950d8c31..8efda324c4b048a 100644 --- a/Platforms/WASI/_build.py +++ b/Platforms/WASI/_build.py @@ -1,22 +1,16 @@ __lazy_modules__ = [ - "contextlib", - "functools", "os", "pathlib", "shutil", "subprocess", "sys", - "tempfile", ] -import contextlib -import functools import os import pathlib import shutil import subprocess import sys -import tempfile try: from os import process_cpu_count as cpu_count @@ -29,130 +23,19 @@ WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" -def separator(): - """Print a separator line across the terminal width.""" - try: - tput_output = subprocess.check_output( - ["tput", "cols"], encoding="utf-8" - ) - except subprocess.CalledProcessError: - terminal_width = 80 - else: - terminal_width = int(tput_output.strip()) - print("โŽฏ" * terminal_width) - - -def updated_env(updates={}): - """Create a new dict representing the environment to use. - - The changes made to the execution environment are printed out. - """ - env_defaults = {} - # https://reproducible-builds.org/docs/source-date-epoch/ - git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] - try: - epoch = subprocess.check_output( - git_epoch_cmd, encoding="utf-8" - ).strip() - env_defaults["SOURCE_DATE_EPOCH"] = epoch - except subprocess.CalledProcessError: - pass # Might be building from a tarball. - # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. - environment = env_defaults | os.environ | updates - - env_diff = {} - for key, value in environment.items(): - if os.environ.get(key) != value: - env_diff[key] = value - - env_vars = [ - f"\n {key}={item}" for key, item in sorted(env_diff.items()) - ] - _shared.log("๐ŸŒŽ", f"Environment changes:{''.join(env_vars)}") - - return environment - - -def subdir(context_attr, *, clean_ok=False): - """Decorator to change to a working directory.""" - - def decorator(func): - @functools.wraps(func) - def wrapper(context): - nonlocal context_attr - - working_dir = getattr(context, context_attr) - separator() - _shared.log("๐Ÿ“", os.fsdecode(working_dir)) - if clean_ok and context.clean and working_dir.exists(): - _shared.log("๐Ÿšฎ", "Deleting directory (--clean)...") - shutil.rmtree(working_dir) - - working_dir.mkdir(parents=True, exist_ok=True) - - with contextlib.chdir(working_dir): - return func(context, working_dir) - - return wrapper - - return decorator - - -def call(command, *, context=None, quiet=False, **kwargs): - """Execute a command. - - If 'quiet' is true, then redirect stdout and stderr to a temporary file. - """ - if context is not None: - quiet = context.quiet - - _shared.log("โฏ", " ".join(map(str, command)), spacing=" ") - if not quiet: - stdout = None - stderr = None - else: - if (log_path := getattr(context, "log_path", None)) is None: - log_path = pathlib.Path(tempfile.gettempdir()) - log_path.mkdir(parents=True, exist_ok=True) - stdout = tempfile.NamedTemporaryFile( - "w", - encoding="utf-8", - delete=False, - dir=log_path, - prefix="cpython-wasi-", - suffix=".log", - ) - stderr = subprocess.STDOUT - _shared.log("๐Ÿ“", f"Logging output to {stdout.name} (--quiet)...") - - try: - subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) - except subprocess.CalledProcessError as error: - if quiet: - _shared.log("โŒ", f"Exit code {error.returncode}") - separator() - with open(stdout.name, encoding="utf-8") as file: - lines = file.readlines() - # Inefficient, but the log shouldn't be dramatically large. - print("".join(lines[-10:]), end="") - if not lines[-1].endswith("\n"): - print() - sys.exit(error.returncode) - - -@subdir("build_python_path", clean_ok=True) +@_shared.subdir("build_python_path", clean_ok=True) def configure_build_python(context, working_dir): """Configure the build/host Python.""" configure = [os.path.relpath(context.checkout / "configure", working_dir)] configure.extend(context.configure_args) - call(configure, context=context) + context.call(configure) -@subdir("build_python_path") +@_shared.subdir("build_python_path") def make_build_python(context, _working_dir): """Make/build the build Python.""" - call(["make", "--jobs", str(cpu_count()), "all"], context=context) + context.call(["make", "--jobs", str(cpu_count()), "all"]) binary = context.build_python_interpreter cmd = [ @@ -205,7 +88,7 @@ def wasi_sdk_env(context): return env -@subdir("wasi_build_path", clean_ok=True) +@_shared.subdir("wasi_build_path", clean_ok=True) def configure_wasi_python(context, working_dir): """Configure the WASI/host build.""" config_site = os.fsdecode(context.here / "config.site-wasm32-wasi") @@ -247,10 +130,9 @@ def configure_wasi_python(context, working_dir): if context.is_debug: configure.append("--with-pydebug") configure.extend(context.configure_args) - call( + context.call( configure, - env=updated_env(env_additions | wasi_sdk_env(context)), - context=context, + env=_shared.updated_env(env_additions | wasi_sdk_env(context)), ) python_wasm = working_dir / "python.wasm" @@ -262,17 +144,18 @@ def configure_wasi_python(context, working_dir): sys.stdout.flush() -@subdir("wasi_build_path") +@_shared.subdir("wasi_build_path") def make_wasi_python(context, working_dir): """Run `make` for the WASI/host build.""" - call( + context.call( ["make", "--jobs", str(cpu_count()), "all"], - env=updated_env(), - context=context, + env=_shared.updated_env(), ) exec_script = working_dir / "python.sh" - call([exec_script, "-c", "import sys; print(sys.version)"], quiet=False) + context.call( + [exec_script, "-c", "import sys; print(sys.version)"], quiet=False + ) _shared.log( "๐ŸŽ‰", f"Use `{exec_script.relative_to(pathlib.Path().absolute())}` " @@ -289,13 +172,13 @@ def clean_contents(context): shutil.rmtree(path) -@subdir("build_python_path") +@_shared.subdir("build_python_path") def pythoninfo_build_python(context, working_dir): """Display build info of the build Python.""" - call(["make", "pythoninfo"], context=context) + context.call(["make", "pythoninfo"]) -@subdir("wasi_build_path") +@_shared.subdir("wasi_build_path") def pythoninfo_wasi_python(context, working_dir): """Display build info of the host/WASI Python.""" - call(["make", "pythoninfo"], context=context) + context.call(["make", "pythoninfo"]) diff --git a/Platforms/WASI/_shared.py b/Platforms/WASI/_shared.py index e8829436f8e73e1..459a12f91563fcc 100644 --- a/Platforms/WASI/_shared.py +++ b/Platforms/WASI/_shared.py @@ -1,19 +1,25 @@ __lazy_modules__ = [ + "contextlib", "functools", "json", "os", "pathlib", + "shutil", "subprocess", + "sys", "sysconfig", "tempfile", "tomllib", ] +import contextlib import functools import json import os import pathlib +import shutil import subprocess +import sys import sysconfig import tempfile import tomllib @@ -234,6 +240,61 @@ def python_version(self, debug_ok=False): version += "d" return version + def call(self, command, *, env=None, quiet=None, **kwargs): + """Execute a command. + + Print changes to the execution environment if 'env' is provided. + If 'quiet' is None, use the context's setting. If true, redirect + stdout and stderr to a temporary file. + """ + if quiet is None: + quiet = self.quiet + + if env is not None: + env_diff = {} + for key, value in env.items(): + if os.environ.get(key) != value: + env_diff[key] = value + + env_vars = [ + f"\n {key}={item}" for key, item in sorted(env_diff.items()) + ] + log("๐ŸŒŽ", f"Environment changes:{''.join(env_vars)}") + + log("โฏ", " ".join(map(str, command)), spacing=" ") + if not quiet: + stdout = None + stderr = None + else: + log_path = self.log_path + log_path.mkdir(parents=True, exist_ok=True) + stdout = tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + dir=log_path, + prefix="cpython-wasi-", + suffix=".log", + ) + stderr = subprocess.STDOUT + log("๐Ÿ“", f"Logging output to {stdout.name} (--quiet)...") + + try: + subprocess.check_call( + command, **kwargs, env=env, stdout=stdout, stderr=stderr + ) + except subprocess.CalledProcessError as error: + if quiet: + log("โŒ", f"Exit code {error.returncode}") + separator() + with open(stdout.name, encoding="utf-8") as file: + lines = file.readlines() + # Inefficient, but the log shouldn't be dramatically large. + print("".join(lines[-10:]), end="") + if not lines[-1].endswith("\n"): + print() + sys.exit(error.returncode) + def log(emoji, message, *, spacing=None): """Print a notification with an emoji. @@ -244,3 +305,59 @@ def log(emoji, message, *, spacing=None): if spacing is None: spacing = " " if len(emoji) == 1 else " " print("".join([emoji, spacing, message])) + + +def separator(): + """Print a separator line across the terminal width.""" + try: + tput_output = subprocess.check_output( + ["tput", "cols"], encoding="utf-8" + ) + except subprocess.CalledProcessError: + terminal_width = 80 + else: + terminal_width = int(tput_output.strip()) + print("โŽฏ" * terminal_width) + + +def updated_env(updates={}): + """Create a new dict representing the environment to use.""" + env_defaults = {} + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. + # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. + environment = env_defaults | os.environ | updates + + return environment + + +def subdir(context_attr, *, clean_ok=False): + """Decorator to change to a working directory.""" + + def decorator(func): + @functools.wraps(func) + def wrapper(context): + nonlocal context_attr + + working_dir = getattr(context, context_attr) + separator() + log("๐Ÿ“", os.fsdecode(working_dir)) + if clean_ok and context.clean and working_dir.exists(): + log("๐Ÿšฎ", "Deleting directory (--clean)...") + shutil.rmtree(working_dir) + + working_dir.mkdir(parents=True, exist_ok=True) + + with contextlib.chdir(working_dir): + return func(context, working_dir) + + return wrapper + + return decorator diff --git a/Platforms/WASI/_test.py b/Platforms/WASI/_test.py index 030d18196d5b0fa..819cbee5c2daff1 100644 --- a/Platforms/WASI/_test.py +++ b/Platforms/WASI/_test.py @@ -7,41 +7,10 @@ import shlex import shutil -import _build import _shared -@_build.subdir("wasi_build_path") -def test(context, _working_dir): - _build.call(["make", "test"], env=_build.updated_env(), context=context) - - -@_build.subdir("wasi_build_path") -def buildbottest(context, _working_dir): - if context.unpack: - archive = context.archive_path - if not archive.is_file(): - raise FileNotFoundError( - f"Release archive not found at {archive}; " - "run `package` or `release` first" - ) - if context.archive_dir.exists(): - _shared.log("๐Ÿงน", f"Deleting {context.archive_dir} ...") - shutil.rmtree(context.archive_dir) - _build.call( - [ - "tar", - "--extract", - "--file", - archive, - "--directory", - context.dist_path, - "--", - context.archive_dir.name, - ], - context=context, - ) - +def dist_test_args(context): runner = context.dist_python_interpreter if not runner.is_file(): raise FileNotFoundError( @@ -50,13 +19,55 @@ def buildbottest(context, _working_dir): ) test_opts = shlex.join(["--python", shlex.quote(str(runner))]) - _build.call( - [ - "make", - "buildbottest", - "TESTPYTHON=$(RUNSHARED) $(PYTHON_FOR_FREEZE) $(TESTPYTHONOPTS)", - f"TESTOPTS=$(EXTRATESTOPTS) {test_opts}", - ], - env=_build.updated_env(), - context=context, - ) + return [ + "TESTPYTHON=$(RUNSHARED) $(PYTHON_FOR_FREEZE) $(TESTPYTHONOPTS)", + f"TESTOPTS=$(EXTRATESTOPTS) {test_opts}", + ] + + +@_shared.subdir("wasi_build_path") +def test(context, _working_dir): + match context.speed: + case "fast": + make_target = "test" + case "slow": + make_target = "buildbottest" + case _: + raise ValueError(f"Unknown speed {context.speed}") + command = ["make", make_target] + match context.target: + case "archive": + archive = context.archive_path + if not archive.is_file(): + raise FileNotFoundError( + f"Release archive not found at {archive}; " + "run `package` or `release` first" + ) + if context.archive_dir.exists(): + _shared.log("๐Ÿงน", f"Deleting {context.archive_dir} ...") + shutil.rmtree(context.archive_dir) + context.call( + [ + "tar", + "--extract", + "--file", + archive, + "--directory", + context.dist_path, + "--", + context.archive_dir.name, + ], + ) + command.extend(dist_test_args(context)) + case "dist": + if not context.dist_python_interpreter.is_file(): + raise FileNotFoundError( + f"Python not found at {context.dist_python_interpreter}" + ) + command.extend(dist_test_args(context)) + case "build": + pass + case _: + raise ValueError(f"Unknown target {context.target}") + + context.call(command, env=_shared.updated_env()) From 40e9c789ebed51e1f10e66dcf3ee91e35269ddf0 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 5 Nov 2026 10:23:01 -0700 Subject: [PATCH 3/6] Tweak the README --- Platforms/WASI/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Platforms/WASI/README.md b/Platforms/WASI/README.md index 1578a7dffa00a69..370d4b27a59d5bb 100644 --- a/Platforms/WASI/README.md +++ b/Platforms/WASI/README.md @@ -205,14 +205,17 @@ After building, you can run the CI tests with: python3 Platforms/WASI test ``` -The `--speed` flag selects `fast` (the default) for `make test` or `slow` for -`make buildbottest`. The `--target` flag selects the files to test: +The `--speed` flag can be: + +- `fast` (the default): `make test` +- `slow`: `make buildbottest` + +The `--target` flag selects the files to test: - `build` (the default): the WASI build in `cross-build/`. - `dist`: the distribution files gathered in `dist/`. - `archive`: the contents of the release archive, unpacked into `dist/`. - To thoroughly test the contents of the release archive, use: ```shell From 2cbe3bb5d32f7a0f5445273fd12abac8c5d52ae3 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 5 Nov 2026 10:34:46 -0700 Subject: [PATCH 4/6] `ruff format` --- Platforms/WASI/_shared.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Platforms/WASI/_shared.py b/Platforms/WASI/_shared.py index 459a12f91563fcc..8e5f13d03adc57a 100644 --- a/Platforms/WASI/_shared.py +++ b/Platforms/WASI/_shared.py @@ -257,7 +257,8 @@ def call(self, command, *, env=None, quiet=None, **kwargs): env_diff[key] = value env_vars = [ - f"\n {key}={item}" for key, item in sorted(env_diff.items()) + f"\n {key}={item}" + for key, item in sorted(env_diff.items()) ] log("๐ŸŒŽ", f"Environment changes:{''.join(env_vars)}") From 003c3ec12dbb138950d9e2aff4b38ff214cc2829 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 5 Nov 2026 11:04:13 -0700 Subject: [PATCH 5/6] Apply suggestion from @hoodmane Co-authored-by: Hood Chatham --- Platforms/WASI/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Platforms/WASI/README.md b/Platforms/WASI/README.md index 370d4b27a59d5bb..6ed417f2ef5cc46 100644 --- a/Platforms/WASI/README.md +++ b/Platforms/WASI/README.md @@ -227,7 +227,7 @@ selected versioned distribution directory inside `dist/`, then restores it from the matching `.tar.xz` archive before running the tests. The archive and other distributions in `dist/` are left untouched. -You can also tests more directly. To find out where the WASI build directory +You can also run tests more directly. To find out where the WASI build directory is, you can run: ```shell From 17ce38fc8563c72eaa4644b6b8e841d3a0b83f92 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 6 Nov 2026 14:34:08 -0700 Subject: [PATCH 6/6] Make sure to use the PAX tar format --- Platforms/WASI/_package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Platforms/WASI/_package.py b/Platforms/WASI/_package.py index 6c47930b344e1fc..403247d4734a3d6 100644 --- a/Platforms/WASI/_package.py +++ b/Platforms/WASI/_package.py @@ -382,6 +382,7 @@ def archive(context): "--owner=0", "--group=0", "--numeric-owner", + "--format=pax", "--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime", "--mode=go+u,go-w", # Explicitly using `-T` because if you don't compress with threads you can't