python: add use_host_python option; use env in shebangs

BUCK

@@ -3,14 +3,12 @@
 
 """Shared tine Python support, and the toolchain Buck2 looks up as `toolchains//`."""
 
-load("@prelude//toolchains:python.bzl", "python_bootstrap_toolchain")
 load("//python:defs.bzl", "python")
 
 # Declared here rather than in a `toolchains` cell of its own: Buck2 forbids nesting a cell inside an
 # external cell, so declaring it here is what lets a consuming project pull tine in as one.
-python_bootstrap_toolchain(
+python.bootstrap_toolchain(
     name = "python_bootstrap",
-    interpreter = read_config("tine", "python_interpreter", "tine//tools:python3"),
     visibility = ["PUBLIC"],
 )
 

README.md

@@ -20,8 +20,9 @@ The host needs only:
  [sysctl](https://www.freedesktop.org/software/systemd/man/latest/sysctl.d.html):
    `kernel.apparmor_restrict_unprivileged_userns=0`.
  * A host root that supports `pivot_root()`; some kernels cannot pivot an initramfs root
- * glibc ≥ 2.17, which the pinned Python that starts every sandbox links against
- * `/usr/bin/python3` ≥ 3.14, or ≥ 3.12 together with `zstd`
+ * glibc ≥ 2.17, which the pinned Python that starts every sandbox links against, unless the project
+   [uses the host's Python](#hosts-without-a-glibc-loader)
+ * `python3` ≥ 3.14 on `PATH`, or ≥ 3.12 together with `zstd`
  * `git`
  * `/bin/sh`, which runs the fetch of a Rust project's git dependencies
  * `/dev/kvm` to run VMs for testing; without it, you can still build everything
@@ -33,6 +34,24 @@ The first [`tine`](bin/tine) invocation that runs Buck fetches and verifies the
 `${XDG_CACHE_HOME:-~/.cache}/tine/buck2` (one directory per pin, safe to delete); everything else is fetched
 and cached by Buck itself.
 
+### Hosts without a glibc loader
+
+The pinned Python that runs tine's drivers on the host needs a glibc loader at the standard path, such as
+`/lib64/ld-linux-x86-64.so.2`. On a host without one, such as NixOS or a musl-based distribution, set this
+in the project's configuration, for example in `.buckconfig.local`:
+
+```ini
+[tine]
+use_host_python = true
+```
+
+`tine` then runs the drivers with the `python3` on `PATH`, which must be 3.14 or newer, and fails before
+the build if it is not. It runs that interpreter isolated from `PYTHONPATH`, `sitecustomize` and the user's
+site directory. A driver that runs in a box still uses the pinned Python, on the box's own loader. Every
+action names the host interpreter's path and build, so upgrading it reruns the actions. That also keys
+them apart from the pinned interpreter's, so a shared cache filled by builds with it does not serve them.
+The setting replaces `tine.python_interpreter`, and the two cannot be combined.
+
 ### HOME directory
 
 Buck2 refuses to start without `$HOME` (some CI environments don't set it), as it runs its daemon from

bin/tine

@@ -1,4 +1,4 @@
-#!/usr/bin/python3 -SIB
+#!/usr/bin/env -S python3 -SIB
 
 # SPDX-FileCopyrightText: Amutable GmbH <https://amutable.com/>
 # SPDX-License-Identifier: MPL-2.0

bin/tine.py

@@ -63,14 +63,34 @@
 RE_CLIENT = "buck2_re_client"
 # Whether the configured cache takes uploads, which the execution platform reads back.
 CACHE_UPLOADS = "cache-uploads"
+# A project that sets USE_HOST_PYTHON gets the host's python3 resolved into HOST_PYTHON and
+# HOST_PYTHON_IDENTITY, which python/defs.bzl reads back.
+USE_HOST_PYTHON = "use_host_python"
+HOST_PYTHON = "host-python"
+HOST_PYTHON_IDENTITY = "host-python-identity"
+# The drivers decompress RPM payloads with compression.zstd, which arrived in 3.14, so importing it is
+# the version check. The digest of sys.version tells apart two builds of one version.
+_HOST_PYTHON_PROBE = """\
+import hashlib, os, sys
+import compression.zstd
+print(os.path.realpath(sys.executable))
+build = hashlib.sha256(sys.version.encode()).hexdigest()[:12]
+print("{}.{}.{}".format(*sys.version_info[:3]), build, sep="-")
+"""
+# python/defs.bzl runs the host's python3 with these flags, and so does the probe, to see what the
+# build sees.
+HOST_PYTHON_FLAGS = ("-E", "-s", "-S")
 
 PLATFORMS = ("Linux-aarch64", "Linux-riscv64", "Linux-x86_64")
 
 DETECTOR = "target_platform_detector_spec"
 DEFAULT_PLATFORM = f"{CELL}//platform:default"
 
 BLOCK_BEGIN = "# @generated by `tine`; rewritten on every command."
-BLOCK_NOTE = "# Buck reads the version, dev targets, mount ignores and cache settings; tine reads its TOML."
+BLOCK_NOTE = (
+    "# Buck reads the version, dev targets, mount ignores, cache settings and host python; "
+    "tine reads its TOML."
+)
 BLOCK_END = "# @end generated by `tine`; what follows is yours."
 
 SHELLS = ("bash", "fish", "zsh")
@@ -587,6 +607,34 @@ def dev_targets(root: Path, config: dict[str, dict[str, str]], mounts: list[str]
     return list(dict.fromkeys([entry for entry in configured if entry] + sorted(mounts)))
 
 
+def host_python(root: Path) -> dict[str, str]:
+    """Resolve the host's python3 for the bootstrap toolchain, if the project sets `use_host_python`.
+
+    A bare `python3` on a command line would let Buck reuse outputs across a host upgrade. Its
+    resolved path and build identity go on every command line instead, so an upgrade reruns the
+    actions.
+    """
+    if read_project_buckconfig(root).get(SECTION, {}).get(USE_HOST_PYTHON) != "true":
+        return {}
+    import shutil
+    import subprocess
+
+    setting = f"[{SECTION}] {USE_HOST_PYTHON}"
+    python = shutil.which("python3")
+    if python is None:
+        fail(f"{setting} is set, but there is no python3 on PATH")
+    proc = subprocess.run(
+        [python, *HOST_PYTHON_FLAGS, "-c", _HOST_PYTHON_PROBE], capture_output=True, text=True
+    )
+    if proc.returncode != 0:
+        fail(
+            f"{setting} is set, but {python} cannot run the build, which needs python 3.14 or newer "
+            f"with compression.zstd:\n{proc.stderr.strip()}"
+        )
+    path, identity = proc.stdout.splitlines()
+    return {HOST_PYTHON: path, HOST_PYTHON_IDENTITY: identity}
+
+
 def render_local_config_block(
     root: Path,
     ignores: list[str],
@@ -600,6 +648,7 @@ def render_local_config_block(
         lines.append(f"# no version components: {derived}")
     else:
         own |= {f"version-{key}": value for key, value in derived.items()}
+    own |= host_python(root)
     if dev:
         own[DEV] = ", ".join(dev)
     if cache is not None:

bin/tine_test.py

@@ -392,6 +392,73 @@ def test_the_address_reads_back_out_of_the_block(self) -> None:
         self.assertNotIn(tine.RE_CLIENT, tine.read_project_buckconfig(self.repo))
 
 
+class TestHostPython(unittest.TestCase):
+    """`use_host_python` resolves the python3 on PATH into the generated block."""
+
+    @override
+    def setUp(self) -> None:
+        self.root = scratch(self)
+        self.path = scratch(self, "tine-test-path.")
+        patched = unittest.mock.patch.dict(os.environ, {"PATH": str(self.path)})
+        patched.start()
+        self.addCleanup(patched.stop)
+
+    def use_host_python(self) -> None:
+        (self.root / tine.LOCAL).write_text(f"[{tine.SECTION}]\n{tine.USE_HOST_PYTHON} = true\n")
+
+    def python3(self, script: str | None = None) -> None:
+        """Put this test's interpreter on PATH as python3, or a shell script standing in for one."""
+        command = self.path / "python3"
+        if script is None:
+            command.symlink_to(sys.executable)
+        else:
+            command.write_text(f"#!/bin/sh\n{script}\n")
+            command.chmod(0o755)
+
+    def test_nothing_without_the_setting(self) -> None:
+        self.python3()
+        self.assertEqual(tine.host_python(self.root), {})
+
+    def test_resolves_the_interpreter_and_its_build(self) -> None:
+        self.use_host_python()
+        self.python3()
+        resolved = tine.host_python(self.root)
+        self.assertEqual(resolved[tine.HOST_PYTHON], os.path.realpath(sys.executable))
+        version = re.escape("{}.{}.{}".format(*sys.version_info[:3]))
+        self.assertRegex(resolved[tine.HOST_PYTHON_IDENTITY], rf"^{version}-[0-9a-f]{{12}}$")
+
+    def test_lands_in_the_generated_block(self) -> None:
+        self.use_host_python()
+        self.python3()
+        tine.refresh_local_buckconfig(self.root, [])
+        generated = read_generated_buckconfig(self.root / tine.LOCAL)[tine.SECTION]
+        self.assertEqual(generated[tine.HOST_PYTHON], os.path.realpath(sys.executable))
+        self.assertIn(tine.HOST_PYTHON_IDENTITY, generated)
+        # The setting itself stays the developer's.
+        project = tine.read_project_buckconfig(self.root)[tine.SECTION]
+        self.assertEqual(project, {tine.USE_HOST_PYTHON: "true"})
+
+    def test_no_python3_on_path(self) -> None:
+        self.use_host_python()
+        with self.assertRaisesRegex(SystemExit, "no python3 on PATH"):
+            tine.host_python(self.root)
+
+    def test_an_interpreter_without_compression_zstd_is_rejected(self) -> None:
+        self.use_host_python()
+        self.python3("echo \"ModuleNotFoundError: No module named 'compression'\" >&2; exit 1")
+        rejected = "needs python 3.14 or newer(.|\n)*No module named 'compression'"
+        with self.assertRaisesRegex(SystemExit, rejected):
+            tine.host_python(self.root)
+
+    def test_runs_the_probe_isolated_from_the_host(self) -> None:
+        # The build runs the interpreter with these flags, so the probe sees what the build will.
+        self.use_host_python()
+        self.python3()
+        with unittest.mock.patch.dict(os.environ, {"PYTHONPATH": str(scratch(self))}):
+            (Path(os.environ["PYTHONPATH"]) / "hashlib.py").write_text("raise ImportError('shadowed')\n")
+            self.assertIn(tine.HOST_PYTHON, tine.host_python(self.root))
+
+
 class TestMergeGeneratedConfigBlock(unittest.TestCase):
     """The block is this command's; the rest of the file stays the developer's."""
 

python/BUCK

@@ -16,7 +16,7 @@ python.test(
     deps = ["//archive:cpio"],
     env = {
         "PYTHON_TEST_INPUT": "$(location :test-input)",
-        "PYTHON_TEST_INTERPRETER": "$(location_exec //tools:python3)",
+        "PYTHON_TEST_INTERPRETER": python.bootstrap_interpreter(),
         "PYTHON_TEST_VALUE": "spaces 'quotes' \"double quotes\" $literal;",
     },
 )

python/defs.bzl

@@ -9,6 +9,7 @@ load("@prelude//decls:toolchains_common.bzl", "toolchains_common")
 load("@prelude//python_bootstrap:python_bootstrap.bzl", "PythonBootstrapSources")
 load("@prelude//python_bootstrap:toolchain.bzl", "PythonBootstrapToolchainInfo")
 load("@prelude//test:inject_test_run_info.bzl", "inject_test_run_info")
+load("@prelude//toolchains:python.bzl", "python_bootstrap_toolchain")
 load("@prelude//utils:selects.bzl", "selects")
 load("//box:defs.bzl", "box")
 
@@ -204,8 +205,58 @@ def _test(
     _python_test(name = name, box = box, srcs = srcs, deps = deps, **kwargs)
     ty_check(name = name + "-ty", srcs = srcs, deps = deps, boxes = selects.apply(box, lambda value: [] if value == None else [value]))
 
+def _host_python() -> list[str] | None:
+    """The command of the host's python3 that `tine.use_host_python` selects, or None for the pinned one.
+
+    bin/tine resolves the interpreter into the generated config, with an identity of its build.
+    """
+    if read_root_config("tine", "use_host_python") != "true":
+        return None
+    path = read_root_config("tine", "host-python")
+    identity = read_root_config("tine", "host-python-identity")
+    if path == None or identity == None:
+        fail("tine.use_host_python is set, but bin/tine has not resolved the host's python3; run Buck through `tine buck`")
+
+    # -E, -s and -S keep the host's PYTHON* variables, user site directory and sitecustomize out of
+    # the build. Python only records an -X option it does not know, so this one puts the
+    # interpreter's build on every command line and an upgrade reruns the actions.
+    return [path, "-B", "-E", "-s", "-S", "-X", "tine-host-python=" + identity]
+
+def _host_python_toolchain_impl(ctx: AnalysisContext) -> list[Provider]:
+    return [DefaultInfo(), PythonBootstrapToolchainInfo(interpreter = cmd_args(ctx.attrs.command))]
+
+_host_python_toolchain = rule(
+    impl = _host_python_toolchain_impl,
+    attrs = {"command": attrs.list(attrs.string())},
+    is_toolchain_rule = True,
+)
+
+def _bootstrap_toolchain(name: str, **kwargs) -> None:
+    """Declare the bootstrap toolchain on the pinned interpreter, or on the host's `python3` if asked to.
+
+    The pinned interpreter needs a glibc loader at the standard path. A host without one sets
+    `tine.use_host_python = true` in the project's config.
+    """
+    host = _host_python()
+    if host == None:
+        python_bootstrap_toolchain(name = name, interpreter = read_config("tine", "python_interpreter", "tine//tools:python3"), **kwargs)
+    elif read_config("tine", "python_interpreter") != None:
+        fail("tine.use_host_python and tine.python_interpreter both choose the bootstrap interpreter; set only one")
+    else:
+        _host_python_toolchain(name = name, command = host, **kwargs)
+
+def _bootstrap_interpreter() -> str:
+    """The path of the interpreter the bootstrap toolchain runs, for an attribute that expands macros.
+
+    ty only reads the pinned interpreter's tree and never runs it, so its checks keep that one.
+    """
+    host = _host_python()
+    return "$(location_exec tine//tools:python3)" if host == None else host[0]
+
 python = struct(
     binary = _binary,
+    bootstrap_interpreter = _bootstrap_interpreter,
+    bootstrap_toolchain = _bootstrap_toolchain,
     flat_tree = flat_tree,
     library = _library,
     test = _test,

tests/BUCK

@@ -40,6 +40,26 @@ python.binary(
     deps = ["//:util"],
 )
 
+# The bootstrap toolchain on the host's python3 (`tine.use_host_python`), in an isolation dir of its
+# own. It runs nested `tine buck` commands, so tools/ci.sh runs it as a group instead of as a test suite.
+python.binary(
+    name = "host-python-driver",
+    main = "host_python.py",
+    deps = ["//:util"],
+)
+
+command_alias(
+    name = "host-python",
+    args = ["--fallback", "$(location_exec //tools:python3)"],
+    exe = ":host-python-driver",
+)
+
+# What the driver above runs under the toolchain it switched.
+python.binary(
+    name = "host-python-probe",
+    main = "host_python_probe.py",
+)
+
 # Assert an artifact mentions what it should. Shared because "these things must appear in that
 # file" is the same test whether the file is an SBOM or a module manifest.
 export_file(