diff --git a/README.md b/README.md index ec0e9a12..3b2d84ec 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ SPDX-License-Identifier: MPL-2.0 tine composes operating-system images out of packages and pinned third-party repositories with [Buck2](https://buck2.build/). Every build action runs in an unprivileged sandbox without network access. -Filesystem images are stacks of overlay deltas. tine's primary output targets are [unified kernel +Each image layer describes a complete filesystem snapshot. tine's primary output targets are [unified kernel images](https://uapi-group.org/specifications/specs/unified_kernel_image/) and dm-verity protected GPT disks in various formats, and also additional artifacts like [SBOM](https://en.wikipedia.org/wiki/Software_supply_chain). @@ -25,6 +25,7 @@ The host needs only: * `git` * `/bin/sh`, which runs the fetch of a Rust project's git dependencies * `/dev/kvm` to run VMs for testing; without it, you can still build everything + * Read access to `/proc/sys/kernel/overflowuid` for directory exports tine uses the host's `python` and `git` to bootstrap itself and to read the project's git state, such as the version an image derives from it. A build also runs the host's `git` to fetch git checkouts, and the host's diff --git a/archive/BUCK b/archive/BUCK index 494734f3..643d606b 100644 --- a/archive/BUCK +++ b/archive/BUCK @@ -9,18 +9,19 @@ python.library( name = "cpio", srcs = ["cpio.py"], visibility = ["tine//..."], - deps = ["//:util"], + deps = ["//:util", "//uapi16:uapi16"], ) python.library( name = "tar", srcs = ["tar.py"], visibility = ["tine//..."], + deps = ["//uapi16:uapi16"], ) python.test( name = "test", box = "//tools:dev.box", srcs = ["cpio.py", "cpio_test.py", "tar.py", "tar_test.py"], - deps = ["//:util"], + deps = ["//:util", "//uapi16:uapi16"], ) diff --git a/archive/cpio.py b/archive/cpio.py index 1275a5bd..59ccb6ef 100644 --- a/archive/cpio.py +++ b/archive/cpio.py @@ -10,12 +10,15 @@ import mmap import os import stat -from collections.abc import Iterable, Iterator +from collections import Counter +from collections.abc import Callable, Iterable, Iterator, Sequence from pathlib import Path, PurePath, PurePosixPath from typing import NamedTuple, Self import util +import uapi16 + # What an archive is read from, spelled concretely because the reader slices it: an mmap and a # `bytes` both hand back `bytes`, where the `Buffer` protocol promises no indexing at all. type Source = mmap.mmap | bytes @@ -151,7 +154,7 @@ def _extract(source: Source, dest: Path, src_fd: int, start: int) -> int: elif fmt == 0o120000: # symlink (last-writer-wins across multi-archive extracts) target.parent.mkdir(parents=True, exist_ok=True) target.unlink(missing_ok=True) - target.symlink_to(e.target.decode()) + target.symlink_to(os.fsdecode(e.target)) elif fmt == 0o100000: # regular file _put_regular(target, e, src_fd) else: # device/fifo/socket @@ -163,10 +166,13 @@ def _extract(source: Source, dest: Path, src_fd: int, start: int) -> int: _hardlink(link, target) count += 1 for links in pending.values(): # a fully-empty hardlink set has no content carrier - for target, mode in links: - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(b"") - target.chmod(mode & 0o7777) + first, mode = links[0] + first.parent.mkdir(parents=True, exist_ok=True) + first.write_bytes(b"") + first.chmod(mode & 0o7777) + count += 1 + for target, _ in links[1:]: + _hardlink(target, first) count += 1 return count @@ -197,6 +203,7 @@ def __init__(self, path: Path, epoch: int, *, block_align: bool = True) -> None: self._epoch = epoch self._pos = 0 self._ino = 0 + self._links: dict[int, tuple[int, int]] = {} # Alignment pads the name field, and the kernel's early cpio reader (lib/earlycpio.c) refuses # an entry whose name field exceeds its 18-byte name buffer, so an archive it reads before # unpacking anything, microcode, is written without. @@ -221,8 +228,22 @@ def _pad(self, align: int) -> None: if pad := (-self._pos) % align: self._write(b"\x00" * pad) - def _header(self, name: str, mode: int, filesize: int, *, mtime: int, block_align: bool) -> None: + def _header( + self, + name: str, + mode: int, + filesize: int, + *, + mtime: int, + block_align: bool, + ino: int | None = None, + nlink: int = 1, + major: int = 0, + minor: int = 0, + ) -> None: self._ino += 1 + if name == TRAILER: + name = "./" + name raw = name.encode() namesize = len(raw) + 1 # name + its NUL terminator if block_align: @@ -231,21 +252,75 @@ def _header(self, name: str, mode: int, filesize: int, *, mtime: int, block_alig padded = _roundup(self._pos + _HEADER + namesize, _BLOCK) - (self._pos + _HEADER) if padded <= _PATH_MAX: namesize = padded - fields = (self._ino, mode, 0, 0, 1, min(mtime, self._epoch), filesize, 0, 0, 0, 0, namesize, 0) + fields = ( + self._ino if ino is None else ino, + mode, + 0, + 0, + nlink, + min(mtime, self._epoch), + filesize, + 0, + 0, + major, + minor, + namesize, + 0, + ) self._write(MAGIC + b"".join(b"%08x" % v for v in fields)) self._write(raw + b"\x00" * (namesize - len(raw))) self._pad(4) + def _link(self, link: tuple[int, int] | None) -> tuple[int | None, int, int]: + if link is None: + return None, 1, 1 + token, nlink = link + ino, remaining = self._links.get(token, (self._ino + 1, nlink)) + self._links[token] = (ino, remaining - 1) + return ino, nlink, remaining + def add_dir(self, name: str, *, mode: int, mtime: int) -> None: self._header(name, stat.S_IFDIR | (mode & 0o7777), 0, mtime=mtime, block_align=False) - def add_symlink(self, name: str, target: str, *, mtime: int) -> None: - data = target.encode() - self._header(name, stat.S_IFLNK | 0o777, len(data), mtime=mtime, block_align=False) + def add_symlink( + self, name: str, target: str, *, mtime: int, link: tuple[int, int] | None = None + ) -> None: + ino, nlink, _ = self._link(link) + data = os.fsencode(target) + self._header( + name, stat.S_IFLNK | 0o777, len(data), mtime=mtime, block_align=False, ino=ino, nlink=nlink + ) self._write(data) self._pad(4) - def add_file(self, name: str, src: Path, *, mode: int, mtime: int) -> None: + def add_node( + self, + name: str, + *, + mode: int, + mtime: int, + major: int = 0, + minor: int = 0, + link: tuple[int, int] | None = None, + ) -> None: + ino, nlink, _ = self._link(link) + self._header( + name, mode, 0, mtime=mtime, block_align=False, major=major, minor=minor, ino=ino, nlink=nlink + ) + + def add_file( + self, + name: str, + src: Path, + *, + mode: int, + mtime: int, + link: tuple[int, int] | None = None, + ) -> None: + ino, nlink, remaining = self._link(link) + if remaining > 1: + self._header(name, stat.S_IFREG | mode, 0, mtime=mtime, block_align=False, ino=ino, nlink=nlink) + return fd = os.open(src, os.O_RDONLY) try: size = os.fstat(fd).st_size @@ -255,6 +330,8 @@ def add_file(self, name: str, src: Path, *, mode: int, mtime: int) -> None: size, mtime=mtime, block_align=self._block_align and size >= _BLOCK, + ino=ino, + nlink=nlink, ) util.copy_range(self._fd, self._pos, fd, 0, size) self._pos += size @@ -274,31 +351,27 @@ def close(self) -> None: self._fd = -1 -def _add(w: Writer, tree: Path, path: Path) -> bool: +def _add(w: Writer, tree: Path, path: Path, link: tuple[int, int] | None = None) -> bool: """Add one path under `tree`; false for a type an initramfs has no use for.""" st = path.lstat() name = str(path.relative_to(tree)) mode = stat.S_IMODE(st.st_mode) - mtime = int(st.st_mtime) + mtime = st.st_mtime_ns // 10**9 if stat.S_ISDIR(st.st_mode): w.add_dir(name, mode=mode, mtime=mtime) elif stat.S_ISLNK(st.st_mode): w.add_symlink(name, os.readlink(path), mtime=mtime) elif stat.S_ISREG(st.st_mode): - w.add_file(name, path, mode=mode, mtime=mtime) + w.add_file(name, path, mode=mode, mtime=mtime, link=link) else: return False # devices/fifos/sockets: unneeded in an initramfs (systemd mounts /dev) return True -def pack_tree(tree: Path, out: Path, epoch: int, *, block_align: bool = True) -> int: - """Pack a whole tree into `out`.""" +def pack_tree(tree: Path, out: Path, epoch: int) -> int: + """Pack a whole tree into `out` without block alignment, which the kernel's early cpio reader needs.""" tree = Path(tree) - count = 0 - with Writer(out, epoch, block_align=block_align) as w: - for path in sorted(tree.rglob("*")): - count += _add(w, tree, path) - return count + return _pack_paths(tree, out, epoch, sorted(tree.rglob("*")), block_align=False) def pack_paths(tree: Path, out: Path, epoch: int, paths: Iterable[str | PurePath]) -> int: @@ -307,8 +380,54 @@ def pack_paths(tree: Path, out: Path, epoch: int, paths: Iterable[str | PurePath Sorting is what puts a directory before its contents, which the kernel's unpacker needs. """ tree = Path(tree) + return _pack_paths( + tree, out, epoch, [tree / rel for rel in sorted({PurePosixPath(path) for path in paths})] + ) + + +def _pack_paths( + tree: Path, out: Path, epoch: int, paths: Sequence[Path], *, block_align: bool = True +) -> int: + inodes = {path: path.lstat() for path in paths} + counts = Counter((st.st_dev, st.st_ino) for st in inodes.values() if stat.S_ISREG(st.st_mode)) + tokens: dict[tuple[int, int], int] = {} count = 0 - with Writer(out, epoch) as w: - for rel in sorted({PurePosixPath(path) for path in paths}): - count += _add(w, tree, tree / rel) + with Writer(out, epoch, block_align=block_align) as w: + for path, st in inodes.items(): + inode = st.st_dev, st.st_ino + link = None + if stat.S_ISREG(st.st_mode) and counts[inode] > 1: + link = (tokens.setdefault(inode, len(tokens) + 1), counts[inode]) + count += _add(w, tree, path, link) return count + + +def pack_manifest( + objects: Sequence[uapi16.Object], + content: Callable[[uapi16.Object], Path], + out: Path, + epoch: int, +) -> None: + counts = Counter(obj.get("inodeToken") for obj in objects if obj.get("inodeToken")) + kinds = {"fifo": stat.S_IFIFO, "chr": stat.S_IFCHR, "blk": stat.S_IFBLK, "sock": stat.S_IFSOCK} + with Writer(out, epoch) as w: + for obj in objects: + kind = obj["type"] + name, mode, mtime = obj.get("name", "."), obj.get("mode", 0o755), obj["mTimeNSec"] // 10**9 + token = obj.get("inodeToken") + link = (token, counts[token]) if token else None + if kind == "reg": + w.add_file(name, content(obj), mode=mode, mtime=mtime, link=link) + elif kind == "dir": + w.add_dir(name, mode=mode, mtime=mtime) + elif kind == "lnk": + w.add_symlink(name, uapi16.symlink_target(obj), mtime=mtime, link=link) + else: + w.add_node( + name, + mode=kinds[kind] | mode, + mtime=mtime, + major=obj.get("major", 0), + minor=obj.get("minor", 0), + link=link, + ) diff --git a/archive/cpio_test.py b/archive/cpio_test.py index 51f4a39e..92413d8d 100644 --- a/archive/cpio_test.py +++ b/archive/cpio_test.py @@ -28,6 +28,19 @@ def payload(data: bytes) -> bytes: class TestUnpack(unittest.TestCase): + def test_symlink_targets_need_not_be_utf8(self) -> None: + # A symlink target is an arbitrary byte string even when its filename is valid UTF-8. + # Packing and extracting the target must preserve its bytes without a Unicode error. + with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: + root = Path(scratch) + archive = root / "image.cpio" + target = b"target-\xff" + with cpio.Writer(archive, 0) as writer: + writer.add_symlink("link", os.fsdecode(target), mtime=0) + with archive.open("rb") as stream: + cpio.unpack(stream.fileno(), root / "unpacked") + self.assertEqual(os.fsencode(os.readlink(root / "unpacked/link")), target) + def test_extracts_an_ordinary_hardlink_set(self) -> None: with tempfile.TemporaryDirectory() as scratch: root = Path(scratch) @@ -133,5 +146,58 @@ def test_an_unaligned_archive_keeps_the_stock_name_field(self) -> None: self.assertEqual(int(header[6 + 11 * 8 : 6 + 12 * 8], 16), namesize) +class TestPackPaths(unittest.TestCase): + def test_a_literal_trailer_name_does_not_end_the_archive(self) -> None: + # A regular file named TRAILER!!! must be escaped with a relative prefix so the reader + # extracts it and the later file instead of treating it as the archive terminator. + with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: + root = Path(scratch) + tree = root / "tree" + tree.mkdir() + (tree / "TRAILER!!!").write_text("ordinary file") + (tree / "z-last").write_text("later file") + out = root / "image.cpio" + cpio.pack_tree(tree, out, 0) + self.assertEqual([obj.name for obj in cpio.read(out.read_bytes())], ["./TRAILER!!!", "z-last"]) + with out.open("rb") as stream: + cpio.unpack(stream.fileno(), root / "unpacked") + self.assertEqual((root / "unpacked/TRAILER!!!").read_text(), "ordinary file") + self.assertEqual((root / "unpacked/z-last").read_text(), "later file") + + def test_hardlinks_depend_on_selected_paths(self) -> None: + # A full selection preserves explicit links without merging equal files. Selecting one + # member of a group must emit a self-contained file with nlink=1 and its own payload. + with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: + root = Path(scratch) + tree = root / "tree" + tree.mkdir() + (tree / "a").write_text("same") + os.link(tree / "a", tree / "b") + (tree / "c").write_text("same") + out = root / "image.cpio" + cpio.pack_tree(tree, out, 0) + objects = {obj.name: obj for obj in cpio.read(out.read_bytes())} + self.assertEqual(objects["a"].ino, objects["b"].ino) + self.assertEqual(objects["a"].nlink, 2) + self.assertNotEqual(objects["a"].ino, objects["c"].ino) + cpio.pack_paths(tree, out, 0, ["a"]) + (obj,) = cpio.read(out.read_bytes()) + self.assertEqual(obj.nlink, 1) + self.assertEqual(obj.size, 4) + + def test_timestamp_truncation_does_not_round_up(self) -> None: + # A nanosecond below a second boundary must truncate to the preceding second, even when + # the floating-point stat timestamp rounds up to the boundary. + with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: + root = Path(scratch) + (root / "file").write_text("contents") + cutoff = 1000000000 + os.utime(root / "file", ns=(0, cutoff * 10**9 - 1)) + out = root / "image.cpio" + cpio.pack_paths(root, out, cutoff, ["file"]) + header = out.read_bytes()[: cpio._HEADER] + self.assertEqual(int(header[6 + 5 * 8 : 6 + 6 * 8], 16), cutoff - 1) + + if __name__ == "__main__": unittest.main() diff --git a/archive/tar.py b/archive/tar.py index 025d4a4f..c8d0e486 100644 --- a/archive/tar.py +++ b/archive/tar.py @@ -9,9 +9,11 @@ import os import tarfile -from collections.abc import Callable +from collections.abc import Callable, Sequence from pathlib import Path +import uapi16 + def _xattrs(path: Path) -> dict[str, str]: """Encode Linux xattrs using the convention understood by GNU tar and star.""" @@ -52,3 +54,51 @@ def pack_tree(tree: Path, out: Path, epoch: int) -> int: ) count += 1 return count + + +def pack_manifest( + objects: Sequence[uapi16.Object], + content: Callable[[uapi16.Object], Path], + out: Path, +) -> None: + kinds = { + "reg": tarfile.REGTYPE, + "dir": tarfile.DIRTYPE, + "lnk": tarfile.SYMTYPE, + "chr": tarfile.CHRTYPE, + "blk": tarfile.BLKTYPE, + "fifo": tarfile.FIFOTYPE, + } + links: dict[int, str] = {} + with tarfile.open(out, "w", format=tarfile.PAX_FORMAT) as archive: + for obj in objects: + kind = obj["type"] + if kind not in kinds: + raise ValueError(f"tar cannot represent inode type {kind!r} at {obj.get('name', '.')!r}") + info = tarfile.TarInfo("./" + obj["name"] if "name" in obj else ".") + info.type = kinds[kind] + info.mode = obj.get("mode", 0o777 if kind == "lnk" else 0o755) + headers: dict[str, str] = {} + ns = obj["mTimeNSec"] + info.mtime = ns // 10**9 + if ns % 10**9: + headers["mtime"] = f"{ns // 10**9}.{ns % 10**9:09}" + info.devmajor, info.devminor = obj.get("major", 0), obj.get("minor", 0) + for name, value in obj.get("extendedAttributes", {}).get("user", {}).items(): + data = uapi16.xattr_value(value) + headers["SCHILY.xattr.user." + name] = data.decode("utf-8", "surrogateescape") + info.pax_headers = headers + token = obj.get("inodeToken") + if token and token in links: + info.type, info.linkname = tarfile.LNKTYPE, links[token] + elif kind == "lnk": + info.linkname = uapi16.symlink_target(obj) + elif kind == "reg": + info.size = obj["size"] + if token: + links.setdefault(token, info.name) + if info.isreg(): + with content(obj).open("rb") as stream: + archive.addfile(info, stream) + else: + archive.addfile(info) diff --git a/box/build.bzl b/box/build.bzl index fc9cea09..d4936cc3 100644 --- a/box/build.bzl +++ b/box/build.bzl @@ -56,6 +56,8 @@ _BoxRootInfo = provider( # resolves its own layers. "resolver": provider_field(Dependency | None), "selected": provider_field(list[Dependency]), + # The snapshot stores of the box's layers, lowest first. + "snapshots": provider_field(list[Artifact]), "transaction": provider_field(Artifact), }, ) @@ -64,7 +66,7 @@ def _repository_labels(repositories: list[Dependency]) -> list[str]: return sorted([str(repository.label.raw_target()) for repository in repositories]) def _layer_dir(actions: AnalysisActions, name: str) -> Artifact: - """Declare a directory output that holds a box root or the overlay work directory for one.""" + """Declare a directory output for a box root or the snapshot store of one.""" # A box root takes several gigabytes. Buck keeps every content-based version of an output until a # clean, so each rebuild that changes the root adds another full copy. A rebuild that produces the @@ -132,10 +134,11 @@ def _install( arch: str, lower: list[Artifact], installed: Artifact | None, -) -> Artifact: - """Install `transaction` with `box`, which also verifies its packages, into a layer over `lower`. +) -> (Artifact, Artifact): + """Install `transaction` with `box`, which also verifies its packages, into a layer over the snapshots `lower`. - The layer leaves out the packages of `installed`. + The layer leaves out the packages of `installed`. Return the directory that the sandbox mounts and the + snapshot store of the layer. """ system = package_system[PackageSystemInfo] @@ -152,7 +155,7 @@ def _install( # Use the predecessor or bootstrapped root to install the box, or its layer over the parent. stage2 = _layer_dir(ctx.actions, "stage2") - work = _layer_dir(ctx.actions, "stage2.work") if lower else None + snapshot = _layer_dir(ctx.actions, "stage2.snapshot") ctx.actions.run( cmd_args( box_run( @@ -170,14 +173,14 @@ def _install( "langs": [], "lower": lower, "packages_dir": packages, + "snapshot": snapshot.as_output(), "target": stage2.as_output(), - "work": work.as_output() if work != None else None, }, ), ), category = "box", ) - return stage2 + return stage2, snapshot def _box_impl(ctx: AnalysisContext) -> list[Provider]: # A root box takes the repositories of its release rather than the release, because the release @@ -240,22 +243,24 @@ def _box_impl(ctx: AnalysisContext) -> list[Provider]: # See "Box layers" in docs/design/architecture.md. repository_labels = _repository_labels(repositories) lower = [] + snapshots = [] installed = None if parent != None and ctx.attrs.layered and _repository_labels(parent[_BoxRootInfo].selected) == repository_labels: lower = parent[BoxInfo].layers + snapshots = parent[_BoxRootInfo].snapshots installed = parent[_BoxRootInfo].transaction # With the resolver box as a dependency, the keyrings are anonymous targets. Every box that resolves # through the same resolver box shares them. A root box has no resolver box and verifies with its stage1 # root, which is no target. - stage2 = _install( + stage2, snapshot = _install( ctx, package_system, resolver_dep if resolver_dep != None else installer_box, repositories, transaction, ctx.attrs._arch, - lower, + snapshots, installed, ) info = BoxInfo( @@ -266,6 +271,7 @@ def _box_impl(ctx: AnalysisContext) -> list[Provider]: arch = ctx.attrs._arch, resolver = resolver_dep, selected = repositories, + snapshots = snapshots + [snapshot], transaction = transaction, ) sub_targets = { @@ -331,14 +337,14 @@ def _box_layer_impl(ctx: AnalysisContext) -> list[Provider]: resolver = root.resolver if root.resolver != None else parent names = _effective_packages(ctx.attrs.packages, parent, package_system) transaction = _resolve(ctx, package_system, resolver, _box_repositories(ctx, root.selected, package_system, None), names, root.arch) - layer = _install( + layer, _ = _install( ctx, package_system, resolver, root.selected, transaction, root.arch, - parent[BoxInfo].layers, + root.snapshots, root.transaction, ) return [DefaultInfo(default_output = layer)] diff --git a/box/isolation.py b/box/isolation.py index 7dc83f80..e43465b0 100644 --- a/box/isolation.py +++ b/box/isolation.py @@ -562,9 +562,17 @@ def _tmpfs(mode: str = _TMPFS_DEFAULT_MODE) -> int: return _fsmount(fs) -def _overlay(lowers: tuple[str, ...], upper: str | None, work: str | None, attributes: int = 0) -> int: +def _overlay( + lowers: tuple[str, ...], + upper: str | None, + work: str | None, + attributes: int = 0, + *, + datadirs: tuple[str, ...] = (), +) -> int: """Return a detached overlay. Without `upper` and `work`, the kernel makes it read-only.""" layers = [("lowerdir+", lower) for lower in lowers] + layers += [("datadir+", data) for data in datadirs] layers += [(key, path) for key, path in (("upperdir", upper), ("workdir", work)) if path is not None] with Close(_fsopen("overlay")) as fs: _fsconfig(fs, FSCONFIG_SET_STRING, "source", "overlayfs") @@ -991,26 +999,29 @@ def attach(self, root: int = AT_FDCWD) -> None: class Overlay(_Mount): def __init__( self, - lowerdirs: tuple[StrPath, ...], + lowerdir: StrPath, # Without an upperdir and a workdir, the kernel mounts the overlay read-only. upperdir: StrPath | None, workdir: StrPath | None, target: StrPath, *, + datadirs: tuple[StrPath, ...] = (), strict_unmount: bool = False, ) -> None: super().__init__(target, strict_unmount=strict_unmount) if (upperdir is None) != (workdir is None): raise ValueError("Overlay takes an upperdir and a workdir, or neither") - if upperdir is None and len(lowerdirs) < 2: - raise ValueError("a read-only Overlay needs at least two lowerdirs") - self.lowerdirs = tuple(os.fspath(path) for path in lowerdirs) + # overlayfs refuses a read-only mount of a single layer. + if upperdir is None and not datadirs: + raise ValueError("a read-only Overlay needs a datadir") + self.lowerdir = os.fspath(lowerdir) + self.datadirs = tuple(os.fspath(path) for path in datadirs) self.upperdir = None if upperdir is None else os.fspath(upperdir) self.workdir = None if workdir is None else os.fspath(workdir) @override def _open(self) -> int: - return _overlay(self.lowerdirs, self.upperdir, self.workdir) + return _overlay((self.lowerdir,), self.upperdir, self.workdir, datadirs=self.datadirs) Filesystem = Bind | File | Devices | Tmpfs | Symlink | Overlay diff --git a/box/isolation_test.py b/box/isolation_test.py index 833b004f..4d443d44 100644 --- a/box/isolation_test.py +++ b/box/isolation_test.py @@ -337,44 +337,85 @@ def test_overlay_paths_may_contain_option_separators(self) -> None: root = Path(directory) # The mount option parser splits options at `,`. overlayfs splits lowerdir at `:` and treats `\` # as an escape. overlayfs gets its layers as file descriptors and doesn't parse their paths. - top, bottom, upper, work = (root / name for name in ("to:p", "bot,tom", "up,per", "wo\\rk")) - for path in (top, bottom, upper, work): + lower, upper, work = (root / name for name in ("lo:w,er", "up,per", "wo\\rk")) + for path in (lower, upper, work): path.mkdir() - (top / "original").write_text("original") - (bottom / "underneath").write_text("underneath") - (top / "shadowed").write_text("from the top") - (bottom / "shadowed").write_text("from the bottom") + (lower / "original").write_text("original") target = root / "target" - with isolation.Overlay((top, bottom), upper, work, target) as mounted: + with isolation.Overlay(lower, upper, work, target) as mounted: mounted.mount() self.assertEqual((target / "original").read_text(), "original") - self.assertEqual((target / "underneath").read_text(), "underneath") - self.assertEqual((target / "shadowed").read_text(), "from the top") (target / "generated").write_text("output") self.assertEqual((upper / "generated").read_text(), "output") + def test_overlay_reads_data_layers_without_changing_blobs(self) -> None: + # Sparse lower files provide distinct metadata for shared data in two hidden data layers. + # Reading must not copy data up; chmod and writes must leave the data layers unchanged. + with tempfile.TemporaryDirectory(dir="/var/tmp") as directory: + root = Path(directory) + lower, first, second, upper, work = ( + root / name for name in ("lower", "data:one", "data,two\\", "upper", "work") + ) + for path in (lower, first, second, upper, work): + path.mkdir() + (first / "shared").write_bytes(b"first") + (second / "shared").write_bytes(b"other") + (second / "inherited").write_bytes(b"parent") + for name, redirect, size, mode, timestamp in ( + ("a", "shared", 5, 0o640, 111), + ("b", "shared", 5, 0o755, 222), + ("c", "inherited", 6, 0o644, 333), + ): + path = lower / name + with path.open("wb") as stream: + stream.truncate(size) + os.setxattr(path, "user.overlay.metacopy", b"") + os.setxattr(path, "user.overlay.redirect", ("/" + redirect).encode()) + path.chmod(mode) + os.utime(path, ns=(timestamp, timestamp)) + before = (first / "shared").stat() + target = root / "target" + with isolation.Overlay( + lower, upper, work, target, datadirs=(first, second), strict_unmount=True + ) as mounted: + mounted.mount() + self.assertEqual({path.name for path in target.iterdir()}, {"a", "b", "c"}) + self.assertEqual((target / "a").read_bytes(), b"first") + self.assertEqual((target / "b").read_bytes(), b"first") + self.assertEqual((target / "c").read_bytes(), b"parent") + self.assertFalse((target / "a").samefile(target / "b")) + self.assertEqual((target / "a").stat().st_mode & 0o777, 0o640) + self.assertEqual((target / "b").stat().st_mtime_ns, 222) + self.assertFalse((upper / "a").exists()) + (target / "a").chmod(0o600) + self.assertTrue((upper / "a").exists()) + self.assertEqual((upper / "a").read_bytes(), b"first") + (target / "a").write_bytes(b"changed") + self.assertEqual((target / "b").read_bytes(), b"first") + self.assertEqual((first / "shared").read_bytes(), b"first") + self.assertEqual((first / "shared").stat().st_mode, before.st_mode) + self.assertEqual((first / "shared").stat().st_mtime_ns, before.st_mtime_ns) + self.assertEqual((second / "shared").read_bytes(), b"other") + def test_overlay_without_an_upper_is_read_only(self) -> None: with tempfile.TemporaryDirectory(dir="/var/tmp") as directory: root = Path(directory) - top, bottom = root / "top", root / "bottom" - for path in (top, bottom): + lower, data = root / "lower", root / "data" + for path in (lower, data): path.mkdir() - (top / "shadowed").write_text("from the top") - (bottom / "shadowed").write_text("from the bottom") - (bottom / "underneath").write_text("underneath") + (lower / "underneath").write_text("underneath") target = root / "target" - with isolation.Overlay((top, bottom), None, None, target) as mounted: + with isolation.Overlay(lower, None, None, target, datadirs=(data,)) as mounted: mounted.mount() - self.assertEqual((target / "shadowed").read_text(), "from the top") self.assertEqual((target / "underneath").read_text(), "underneath") with self.assertRaises(OSError) as raised: (target / "generated").write_text("output") self.assertEqual(raised.exception.errno, errno.EROFS) - def test_overlay_needs_a_workdir_with_its_upper_and_two_lowers_without(self) -> None: - for lowers, upper, work in ((("lower",), "upper", None), (("lower",), None, None)): - with self.subTest(lowers=lowers, upper=upper, work=work), self.assertRaises(ValueError): - isolation.Overlay(lowers, upper, work, "target") + def test_overlay_needs_a_workdir_with_its_upper_and_a_datadir_without(self) -> None: + for upper, work in (("upper", None), (None, None)): + with self.subTest(upper=upper, work=work), self.assertRaises(ValueError): + isolation.Overlay("lower", upper, work, "target") def test_overlay_parent_keeps_its_mode_with_a_permissive_umask(self) -> None: with tempfile.TemporaryDirectory(dir="/var/tmp") as directory: @@ -387,7 +428,7 @@ def test_overlay_parent_keeps_its_mode_with_a_permissive_umask(self) -> None: with ExitStack() as stack: stack.callback(os.umask, os.umask(0)) stack.enter_context(mock.patch.object(isolation, "_move_mount")) - with isolation.Overlay((lower,), upper, work, target) as overlay: + with isolation.Overlay(lower, upper, work, target) as overlay: overlay.attach() self.assertEqual(target.parent.stat().st_mode & 0o777, 0o755) @@ -478,7 +519,7 @@ def test_mounts_unwind_on_success_and_failure(self) -> None: isolation.Tmpfs(target), isolation.Devices(target), # Strict, so that unwinding covers both unmount flavours. - isolation.Overlay((lower,), upper, work, target, strict_unmount=True), + isolation.Overlay(lower, upper, work, target, strict_unmount=True), ] for mount in mounts: for fail in (False, True): diff --git a/box/sandbox_test.py b/box/sandbox_test.py index b117f999..2dfb590e 100644 --- a/box/sandbox_test.py +++ b/box/sandbox_test.py @@ -616,7 +616,7 @@ def test_a_start_leaves_the_identity_of_running_boxes_alone(self) -> None: upper, work = f"/var/tmp/{name}.upper", f"/var/tmp/{name}.work" os.mkdir(upper) os.mkdir(work) - with Overlay((lower,), upper, work, merged) as overlay: + with Overlay(lower, upper, work, merged) as overlay: overlay.attach() with open("/var/tmp/merged/sub/file") as handle: print(handle.read()) diff --git a/docs/design/architecture.md b/docs/design/architecture.md index fb6680e6..ee0e5fe1 100644 --- a/docs/design/architecture.md +++ b/docs/design/architecture.md @@ -50,9 +50,11 @@ tine//package/ package-system-neutral providers and installation flow tine//package_system/ one directory per package system: its repository, resolver, installer, extractor, and optional indexer and builder tine//box/ box bootstrap and sandbox command construction -tine//rootfs/ bind/overlay mounting and stored-delta translation +tine//rootfs/ bind mounts and Buck-storable names tine//image/ layers, compositions, archive, directory, disk and boot artifact rules and drivers, and VM runners +tine//uapi16/ UAPI.16 manifests and the snapshot stores built on them +tine//archive/ tar and cpio readers and writers tine//git/ pinned Git repositories with local checkout overrides tine//cargo/ vendored crate trees and offline Rust source builds tine//go/ go.sum-verified module fetches and offline Go source builds @@ -542,16 +544,19 @@ The host contract is intentionally small; its short list of requirements is docu ### Box layers -If a box and its `parent` come from the same repositories, the box is a delta layer over its parent's +If a box and its `parent` come from the same repositories, the box is a layer over its parent's layers, like with images. It enforces addition only, and fails if its lock would update or remove parent packages. One likely cause is an outdated committed lock, which needs a catalog refresh. The other is that resolving both package lists together picks other packages than the parent got, for example `libcurl` for a box that needs it where the parent has `libcurl-minimal`; `layered = False` then installs the box on its own. -The sandbox merges the layers per top-level directory with a read-only overlay. Unlike `rootfs()`, it does -not decode stored whiteout markers, as that would walk every layer on every action, so the installer refuses -a box layer that deletes from the layers below. +Each box layer is a snapshot over the snapshots of its parent's layers, like an image layer. A child box +installs over its restored parent snapshot, so it sees the modes, hardlinks and extended attributes that the +package manager wrote. The installer also writes the objects that differ from the parent as a plain +directory with only what Buck stores: directories, symlinks and regular files with mode 0755 or 0644. The +sandbox merges these directories per top-level directory with a read-only overlay. A plain directory cannot +express a deletion, so the installer refuses a box layer that deletes from the layers below. If the parent has different repositories, the box only shares the parent's package list and is installed on its own. @@ -579,13 +584,14 @@ directory. Each chrooted `run` or `copy` mounts the project read-only under the the duration of that operation. Package scripts and remove globs cannot reach the project through that mount. apivfs covers `/run` with a tmpfs so the mount point is not captured into the image. -The sandbox only creates the execution environment. Drivers own their target-root layout through -`rootfs.rootfs()`: +The sandbox only creates the execution environment. Drivers restore their target root from snapshots and +bind it with `rootfs.rootfs()`: -- a fresh install or image layer binds an output directory at `/buildroot`; -- an incremental install or image layer mounts an ordered lower stack plus a persisted upper; -- a package build mounts its buildroot stack with an ephemeral upper and binds action scratch at `/build`; -- pack/disk operations merge a stack with an ephemeral upper so cleanup does not modify stored layers. +- an install or image layer restores its lower snapshots into a temporary root, binds it at `/buildroot` + and captures the result as a new snapshot; +- a package build restores its buildroot snapshots the same way and binds action scratch at `/build`; +- disk operations, and archives with authored tmpfiles rules, restore their snapshots into a temporary + root so cleanup does not modify stored layers. This division keeps one namespace boundary while letting each driver express the root it needs. Nesting a second sandbox inside a box would duplicate isolation, complicate mounts, and make remote execution @@ -603,10 +609,11 @@ actions never use relaxed mode. Native package installation has three phases shared by buildroots and images: 1. **Plan.** Run `PackageSystemInfo.plan` against each configured repository's materialized directory, - effective priority, and solver cache. Weak dependencies are disabled. Existing lower layers are mounted - read-only so installed packages can satisfy an incremental request. A request lists its packages in the - analysis, or in JSON files that an earlier action writes, such as the BuildRequires query of an rpm - package. A request with no packages installs nothing and leaves its layer empty. + effective priority, and solver cache. Weak dependencies are disabled. Existing lower snapshots are + restored into a temporary root so installed packages can satisfy an incremental request. A request lists + its packages in the analysis, or in JSON files that an earlier action writes, such as the BuildRequires + query of an rpm package. A request with no packages installs nothing, and its snapshot repeats the + manifest of the snapshot below it. 2. **Select.** Use the resulting transaction to select package files from repository pools, verified after download if the repository declares signing keys, named with the package system's declared suffix so its installer finds them. A local repository is materialized by @@ -618,9 +625,9 @@ Native package installation has three phases shared by buildroots and images: packages in it. Buildroots reject managers with local packages, because buildroot contents must come from the explicit, cycle-checked self-hosting locks. 3. **Install.** Run `PackageSystemInfo.install` over the exact package directory, whose file names carry - the package system's suffix. `install_packages()` owns a fresh root or incremental buildroot delta. An - image layer instead invokes the same installer against its already-mounted root so package and - filesystem operations have one output owner. + the package system's suffix. `install_packages()` writes a fresh root snapshot or an incremental + buildroot snapshot. An image layer instead invokes the same installer against its already-mounted root + so package and filesystem operations have one output owner. What a request looks like, rather than how one package system answers it, belongs to the neutral layer: `package/transaction.py` owns the transaction schema both planners write and Starlark reads back, and @@ -648,9 +655,9 @@ base. After installation, the installer parks the package database and scrubs the package-manager and ldconfig bookkeeping that would otherwise make identical roots differ; what parking takes is each package system's -own business. The action that owns the root then captures names and overlay metadata into Buck-storable -form. A fresh root receives the `uninitialized` machine-id marker systemd initializes on first boot; -incremental installs preserve any existing machine ID. +own business. The action that owns the root then captures it as a snapshot. A fresh root receives the +`uninitialized` machine-id marker systemd initializes on first boot; incremental installs preserve any +existing machine ID. `LocalPackageInfo` intentionally does not carry the producer's box. `local_repository` checks that its packages use one native package system and remains a box-independent declaration. An image or a buildroot @@ -720,13 +727,13 @@ An `rpm_package` action: read the contents of a spec, so an action stages the spec as the build does and runs `rpmbuild -bd` with the build's options in the base root and the delta. It writes the BuildRequires that are missing, and a second delta installs them. That delta is empty when the spec needs nothing more; -3. overlays the base and the deltas, stages its spec/sources in Buck action scratch, and runs `rpmbuild -ba`, - or `rpmbuild -bb --build-in-place` on a volatile overlay of a populated source tree. overlayfs shows the - directory under a mount rather than the mounted tree, so each directory mount below the source tree, - such as a `tine mount`, gets a volatile overlay of its own. The driver discovers directory mounts with - `listmount()` and `statmount()` using unique mount IDs. With `in_place_packaging`, a separate action - first fetches the dist-git commit that the tree pins, and the build takes its spec and Source/Patch files - from that commit. The project is read-only until the driver exits; +3. restores the base and the deltas into a temporary root, stages its spec/sources in Buck action scratch, + and runs `rpmbuild -ba`, or `rpmbuild -bb --build-in-place` on a volatile overlay of a populated source + tree. overlayfs shows the directory under a mount rather than the mounted tree, so each directory mount + below the source tree, such as a `tine mount`, gets a volatile overlay of its own. The driver discovers + directory mounts with `listmount()` and `statmount()` using unique mount IDs. With `in_place_packaging`, a + separate action first fetches the dist-git commit that the tree pins, and the build takes its spec and + Source/Patch files from that commit. The project is read-only until the driver exits; 4. freezes `%autorelease`, `_buildhost`, the dist tag, and the per-package source date epoch; 5. collects binary RPMs and the source RPM into one output directory, carrying its package-system identity in `LocalPackageInfo`; @@ -827,8 +834,6 @@ explicitly, because alpm rebases its system hook directory onto the target root administrator's, and the box's must not run against the image. Afterwards the driver replaces the wall-clock `%INSTALLDATE%` alpm records with the assembly epoch, and drops ldconfig's auxiliary cache, the sync databases and the transaction log. -It rewrites only the entries that actually change: alpm keeps one file per package, so rewriting an -entry a lower layer already parked would copy that whole database up into this layer's delta. The database stays at Arch's own `/var/lib/pacman`, since Arch has no usr-merged location for it. The package-database artifact is read from the assembled layer stack rather than from a terminal output, @@ -1110,22 +1115,59 @@ package's directory does not exist when the tests run. ### Filesystem layer representation -Buck directory artifacts cannot faithfully store overlay whiteout devices, opaque-directory xattrs, or a -backslash in a path component. tine stores filesystem deltas in a regular-file representation: - -- `.wh.` represents a whiteout; -- `.wh..wh..opq` represents an opaque directory; -- `.esc.` represents a path component Buck cannot store. - -`rootfs.capture()` translates a native overlay upper into this form after unmounting. Before a later mount, -`rootfs.rootfs()` constructs sparse sidecar layers that translate the stored markers back to native -overlayfs whiteouts/xattrs and thaw escaped names. Stored layers remain ordinary Buck artifacts and can -therefore move through its CAS. - -Directory modes are made traversable so Buck can materialize/delete them, and Buck's artifact model does -not preserve general ownership, capabilities, xattrs, or every mode bit. Authored tmpfiles snippets can -recreate paths, modes, and xattrs during terminal assembly. Ownership is deliberately normalized to uid/gid -zero rather than reconstructed. SELinux labels are not currently produced. +An image-producing action writes a complete UAPI.16 snapshot before Buck publishes its outputs. Logical +paths, inode types, modes, nanosecond modification times, user extended attributes, symlink targets and +explicit hardlink groups are in the manifest. Regular-file contents are immutable SHA-256 blobs. Each store +contains only blobs absent from its parents; the ordered store list remains an explicit input to every +consumer. Content references use `blobs/` in the combined store namespace, resolved in store order. +A store with parents therefore needs its declared parent stores to resolve its references. Standalone +directory exports include every referenced blob. + +Equal bytes share storage without sharing an inode. Reproducible `inodeToken` counters identify explicit +hardlink groups independently of content digests. A metadata-only change changes the manifest and therefore +Buck's input digest. Capture reads the visible merged filesystem after scriptlets and authored operations; +deletions disappear from the next snapshot, including native OverlayFS whiteouts, opaque directories and +file/directory replacements. Logical names never become the file names of a snapshot store. Capture clamps +modification times to the action's `SOURCE_DATE_EPOCH`, so the same install at another time writes the same +manifest. + +Reconstruction first reflinks one blob into the temporary directory. If that works, it clones every file into +a private root. Each independent file gets its own inode with shared data extents, and only declared hardlink +groups share an inode. Metadata restoration and later writes leave cached blobs unchanged. A read-only root +is a read-only bind mount of the private root. Reconstruction tests a clone instead of checking the +filesystem type. Separate btrfs filesystems cannot reflink between each other, and other filesystems support +reflinks too. + +If the blob cannot be reflinked, reconstruction uses OverlayFS. Each independent file has a sparse metadata +inode in the normal lower layer and reads contents from the blob stores as data-only lower layers. Explicit +hardlink groups start as private clones or copies in the upper, with aliases linked to that private inode. +`index=off` would break a lower hardlink group on copy-up. Metadata restoration finishes before mounting. +Directory times are restored after their contents, and symlink metadata is applied without following the +link. + +On the overlay path, metadata changes and data writes copy independent files into the upper. `metacopy=off` +copies file data even for chmod; the kernel attempts a reflink and falls back to a byte copy. An unprivileged +mount cannot enable inode indexing. A command that creates a hardlink to an inherited independent file can +see different inode numbers for the source and alias even though writes affect both. Snapshot capture scans +the visible tree and uses the upper inode to identify the actual hardlink group. The complete snapshot +retains the group when the root is reconstructed or exported. + +Read-only overlays use only metadata inodes in the normal lower, with aliases linked for declared groups. +Boot selection, UKI assembly, artifact extraction and SBOM scanning select between private reflinks and +read-only overlays. A terminal reader with authored tmpfiles rules gets a writable root, because the rules +write into it. Repart uses full private files because it reads inode identity without access to the upper. +Parent release inspection reads the os-release file from the parent snapshot without reconstructing a root. +Temporary roots are removed when the operation exits, including on failure. There is no persistent +materialized-root cache. + +Box layers are snapshots too, but the sandbox mounts the plain directory that the installer writes from +each snapshot. In that directory, `.esc.` encodes a name that Buck cannot store. +Ordinary box startup does not parse image manifests or reconstruct image roots. + +Ownership remains uid/gid zero. Terminal outputs apply the image permission policy before authored tmpfiles +rules and clamp timestamps to the assembly epoch. Permission normalization prepares the snapshot before +filesystem reconstruction, so a read-only consumer needs no chmod or copy-up. Security labels, capabilities +and named ACL entries are unsupported. ### Selecting a distribution @@ -1193,35 +1235,34 @@ whether the key is set or not, so CI does not depend on it. ### Image construction The `image.layer` rule creates either an initial image from a package manager, or a derived image from -`parent`. `image.ImageInfo` carries the package manager, the box the image runs in, an ordered stack of -filesystem deltas, deferred tmpfiles snippets, and the canonical lazy package-database and SBOM artifacts. A +`parent`. `image.ImageInfo` carries the package manager, the box the image runs in, an ordered list of +snapshot stores, deferred tmpfiles snippets, and the canonical lazy package-database and SBOM artifacts. A derived image inherits the construction configuration but declares fresh metadata for its completed stack, so one composition cannot silently switch package sources or tooling environments between stages. -Each `image.layer` call applies one ordered operation sequence in one action and persists exactly one overlay -upper. A layer's `packages` and `package_sets` are rule attributes rather than operations: it installs them -as one request before any operation runs, while `run`, `copy`, `mkdir`, `symlink`, and `remove` mutate the -same root afterwards. A package set resolves its symbolic name through the image's package manager during -analysis and joins the concrete names in the same request, deduplicated. Installation is a property of the -layer rather than a position in it because a layer resolves one closure at analysis time over the stack below -it: a second install could only ever be solved blind to what the first one added, so there is nothing an -ordering could mean. Making it an attribute is also what lets operation sequences compose, since two lists -that both need packages no longer collide, and what lets a sequence name a package set and extra packages -together, which a set alone cannot express because its members are known only during analysis. Installing -against the result of an earlier install is a second layer, which is where a closure resolved over that -result becomes available. `copy` introduces a declared Buck artifact at an absolute image path, preserving -its position relative to the other operations. `run` executes the image's own tools in a chroot by default; -`chroot = False` instead executes box tooling with the image available at `/buildroot`. Its `env` argument -overlays variables on the box or image environment for that command. Package installation runs outside the -chroot. `copy`, `merge_os_release`, `mkdir`, `symlink`, `write_file` and `remove` run inside it, so ordinary -absolute symlinks resolve relative to the image. The `image.layer()` declaration macro recursively flattens -operation lists, -allowing reusable helpers to return ordered groups of operations. Every rule that accepts operations is -wrapped in a declaration macro that flattens them the same way, so a helper returning an ordered group -works identically in `image.layer()` and in a composition; the underlying attribute stays a flat, typed -list so Buck can track every embedded source dependency. An `image.install()` target carries `packages` -beside its operations, so a reusable target declares what it needs and `image.ops.install_from()` folds that -into the installing layer's own request. +Each `image.layer` call applies one ordered operation sequence in one action and persists one complete +snapshot with its new blobs. A layer's `packages` and `package_sets` are rule attributes rather than +operations: it installs them as one request before any operation runs, while `run`, `copy`, `mkdir`, +`symlink`, and `remove` mutate the same root afterwards. A package set resolves its symbolic name through the +image's package manager during analysis and joins the concrete names in the same request, deduplicated. +Installation is a property of the layer rather than a position in it because a layer resolves one closure at +analysis time over the stack below it: a second install could only ever be solved blind to what the first one +added, so there is nothing an ordering could mean. Making it an attribute is also what lets operation +sequences compose, since two lists that both need packages no longer collide, and what lets a sequence name a +package set and extra packages together, which a set alone cannot express because its members are known only +during analysis. Installing against the result of an earlier install is a second layer, which is where a +closure resolved over that result becomes available. `copy` introduces a declared Buck artifact at an +absolute image path, preserving its position relative to the other operations. `run` executes the image's own +tools in a chroot by default; `chroot = False` instead executes box tooling with the image available at +`/buildroot`. Its `env` argument overlays variables on the box or image environment for that command. Package +installation runs outside the chroot. `copy`, `merge_os_release`, `mkdir`, `symlink`, `write_file` and +`remove` run inside it, so ordinary absolute symlinks resolve relative to the image. The `image.layer()` +declaration macro recursively flattens operation lists, allowing reusable helpers to return ordered groups of +operations. Every rule that accepts operations is wrapped in a declaration macro that flattens them the same +way, so a helper returning an ordered group works identically in `image.layer()` and in a composition; the +underlying attribute stays a flat, typed list so Buck can track every embedded source dependency. An +`image.install()` target carries `packages` beside its operations, so a reusable target declares what it +needs and `image.ops.install_from()` folds that into the installing layer's own request. `install_langs` narrows the install to the translated files of the named languages, and `install_docs` drops documentation while keeping licenses. Neither ever enters the tree, so the package database records @@ -1230,7 +1271,7 @@ sequence, not the logical image's lifetime: an initrd needs neither, while the r want both. Each package system implements them in its own driver. A sequence that installs nothing ignores both, so a composition can forward them to a layer whose operations happen to skip installation. -An `image.layer` exposes its newly persisted delta, the full stack and canonical metadata through +An `image.layer` exposes its snapshot, parent stores and canonical metadata through `image.ImageInfo`, and the same metadata through lazy `[manifest]`, `[pkgdb]` and `[sbom]` subtargets. `image.ImageSbomInfo` gives typed consumers only the two SBOM formats. Declaring a logical image declares its metadata with it, because they are intrinsic metadata facets of every logical image: an @@ -1292,23 +1333,25 @@ Every terminal driver receives the ordered layer stack and the deferred tmpfiles extension receives only the snippets that it adds to its base, because a snippet of the base would create a file of the base in the extension. The driver applies the snippets with `systemd-tmpfiles --root` before reading or emitting image content; a missing tool is an error whenever finalization is needed. The directives -run against a disposable overlay upper and can create paths or restore modes and xattrs. Image-shipped +run against a temporary root and can create paths or restore modes and xattrs. Image-shipped tmpfiles configuration is not applied implicitly; enabling it will be an explicit output option once its single-UID/GID behavior is defined. Ownership and named ACL entries are deliberately unsupported: all archive entries use uid/gid zero. -Tar uses deterministic PAX archives and stores Linux xattrs using `SCHILY.xattr.*` headers. The newc cpio -format has no general xattr representation. Tar/cpio entries are ordered and mtimes are clamped to the fixed -assembly epoch. The cpio reader/writer aligns regular-file payloads and uses `copy_file_range` when possible +Tar uses deterministic PAX archives and stores user xattrs using `SCHILY.xattr.user.*` headers. The newc cpio +format has no general xattr representation. Tar/cpio entries are ordered and encode explicit hardlinks +independently of content identity. Linux's initramfs unpacker recreates hardlinked symlinks as separate +symlinks. Mtimes are clamped to the fixed assembly epoch. Tar retains nanoseconds; newc cpio uses whole +seconds. The cpio reader/writer aligns regular-file payloads and uses `copy_file_range` when possible so large archives can share extents on reflink-capable filesystems. `compression = "zstd"` compresses the finished archive in the same action, so the uncompressed form never becomes a Buck artifact; zstd's multi-threaded output is byte-identical to its single-threaded output, so this stays reproducible. The initrd -uses it, while the kernel-modules cpio that `uki.py` appends stays raw because the distribution already -ships each module compressed; the kernel unpacks the concatenation as independently compressed segments. +uses it, while the kernel-modules cpio that `uki.py` appends stays raw because the distribution already ships +each module compressed; the kernel unpacks the concatenation as independently compressed segments. `declare_ddi()` in `ddi.bzl` deliberately distinguishes `definitions` from `partitions`. Definitions describe -new partitions to populate directly from `image.ImageInfo`: repart mounts the delta stack with a disposable -overlay upper instead of first copying a directory artifact. Partition inputs are `image.DdiInfo` outputs +new partitions to populate directly from `image.ImageInfo`: repart reads a temporary root reconstructed +from the snapshot and blobs. Partition inputs are `image.DdiInfo` outputs from an earlier split call; their blocks are copied into the new disk with their resolved type and UUID preserved. Calls emit a disk by default. `split = True` additionally exposes each newly defined partition with normalized metadata alongside its block artifact; `disk = False` makes such a call partition-only. No @@ -1371,18 +1414,16 @@ never means restating what it already does, and `install_docs` and `install_lang image wants, since documentation and translations only cost an initrd boot memory. It generates before it prunes, so the hardware database it ships is compiled from the sources it then drops. The rule archives the image into the cpio itself, at the `compression` it declares, and returns both as `image.InitrdInfo` -alongside the image's own providers. That cpio omits the package database from the paths the image's -package system declares: nothing in an initrd resolves a dependency or verifies a package, and the kernel -unpacks the whole cpio into tmpfs, so shipping it would only cost boot memory. The `[manifest]`, `[pkgdb]` -and `[sbom]` views -read the tree rather than the archive, so they still describe the complete installed set. -A bootable `image.disk` consumes `image.InitrdInfo` rather than a bare `image.ImageInfo`, so the archive is -the initrd target's own output and its compression is declared where the initrd is. A composition given -no `initrd` declares `.initrd` for itself through the same macro, inheriting its package manager, -version, and distribution; the default is therefore an ordinary target that can be built and inspected on -its own rather than an anonymous step inside the disk image. The composition publishes the provider it was -given from `[initrd]` and returns the same -instance directly, so both interfaces describe exactly the same initrd. +alongside the image's own providers. That cpio omits the package database from the paths the image's package +system declares: nothing in an initrd resolves a dependency or verifies a package, and the kernel unpacks the +whole cpio into tmpfs, so shipping it would only cost boot memory. The `[manifest]` view describes the cpio +after package-database removal. The `[pkgdb]` and `[sbom]` views describe the complete installed set. A +bootable `image.disk` consumes `image.InitrdInfo` rather than a bare `image.ImageInfo`, so the archive is the +initrd target's own output and its compression is declared where the initrd is. A composition given no +`initrd` declares `.initrd` for itself through the same macro, inheriting its package manager, version, +and distribution; the default is therefore an ordinary target that can be built and inspected on its own +rather than an anonymous step inside the disk image. The composition publishes the provider it was given from +`[initrd]` and returns the same instance directly, so both interfaces describe exactly the same initrd. `uki.py` appends the kernel-modules cpio and runs `ukify`. On x86 it also packs the microcode the image ships under `/usr/lib/firmware/{amd,intel}-ucode` into a `.ucode` section: one concatenated file per vendor @@ -1518,10 +1559,10 @@ A package installs configuration that describes state rather than carrying it: m index, `hwdb.d` with no compiled database, a locale list with no archive. On a running system scriptlets and boot-time units produce it; an image being assembled is neither, so tine builds it explicitly. Each generator is an operation of its own rather than a step of one pass, so it is positioned where the image wants it, -configured on its own terms, and adopted one at a time; what it writes persists into the delta that captures -it, so it is built once and cached with the layer rather than repeated by every terminal output. An `image` -gets the content its operations ask for and nothing else, which is the same reason terminal rules leave -package state alone. +configured on its own terms, and adopted one at a time; what it writes persists into the snapshot that +captures it, so it is built once and cached with the layer rather than repeated by every terminal output. An +`image` gets the content its operations ask for and nothing else, which is the same reason terminal rules +leave package state alone. Each generator is a no-op for an image that carries none of what it acts on, so none of them needs per-distribution knowledge: `image.ops.depmod()` is silent without kernels, `image.ops.hwdb()` without @@ -1673,11 +1714,13 @@ spec schema. Both it and the snapshot driver keep `--out` there too: the box loc targets let a caller name the file to write, or `-` for stdout when it has nowhere to write one, and one calling convention per driver beats splitting the destination by verb. -### Store image layers as deltas +### Store image snapshots with shared blobs -Copying a complete root for every image step scales with total image size rather than change size. Ordered -overlay deltas let child layers and terminal outputs reuse their ancestors. Encoding whiteouts and opaque -directories as regular files keeps those deltas compatible with Buck's artifact/CAS model. +Complete manifests describe visible filesystem state independently of Buck's metadata and filename limits. +Each action stores only newly introduced content blobs and retains its parent stores as Buck inputs. +Temporary reconstruction restores inode identity and metadata before commands run. Archive writers read +manifests and blobs directly when no authored tmpfiles rules require a filesystem. Directory exports restore +their authoritative snapshot after final Buck materialization. ### Prefer exact transactions over package-manager network access @@ -1778,10 +1821,9 @@ belong to one package system are listed in its own section instead: - A root box verifies its packages with the `rpmkeys` of its own unverified stage1, so a tampered rpm package in the seed could defeat the check for that box; every other box and every image verifies with a predecessor. Arch packages are not verified at all. -- Archive ownership is intentionally normalized to uid/gid zero. Capabilities, xattrs, and SELinux labels do - not survive as Buck directory metadata; deferred tmpfiles can restore xattrs at terminal assembly, and tar - preserves them in PAX headers, but newc cpio cannot represent general xattrs. -- Directory image output cannot represent backslashes in names; archive outputs should be used instead. +- Archive ownership is intentionally normalized to uid/gid zero. Snapshots keep user xattrs but no + capabilities, other xattr namespaces or SELinux labels. Deferred tmpfiles can restore xattrs at terminal + assembly, tar preserves user xattrs in PAX headers, and newc cpio cannot represent xattrs. - The default `/usr`-only disk has a volatile root, so the `/etc` a build writes is not what such an image boots with; first-boot defaults come from credentials instead. Package and authored state outside `/usr` is not yet translated into factory defaults or another persistent partition. How much `/etc` such an @@ -1852,8 +1894,8 @@ Near-term image gaps are: - richer ordered operations for setting file metadata directly; - deciding whether package installation and image tooling eventually need distinct compatible boxes. -The layer model should remain ordered operations captured as deltas. A provides/requires feature solver is -unnecessary unless real composition requirements appear. +The layer model should remain ordered operations captured as complete snapshots. A provides/requires feature +solver is unnecessary unless real composition requirements appear. ### Scale, configuration, and testing diff --git a/docs/user/images.md b/docs/user/images.md index 574318c7..5369d94e 100644 --- a/docs/user/images.md +++ b/docs/user/images.md @@ -6,7 +6,7 @@ SPDX-License-Identifier: MPL-2.0 # Building images How to declare, build, and run OS images. This is the user guide. The architecture behind it (the -provider vocabulary, the delta layer model, and disk composition internals) is documented in +provider vocabulary, image snapshots, and disk composition internals) is documented in [architecture.md](../design/architecture.md). ## Restrictions @@ -21,6 +21,16 @@ writable directories such as `/etc` and `/var`. If you must have files with different permissions in your image, the escape hatch is the `tmpfiles` attribute of an image or composition. +Image operations see the original supported modes, modification times, user extended attributes and explicit +hardlinks from their parent snapshot. Permission normalization applies when producing terminal outputs, +before authored `tmpfiles` rules. Snapshots keep nanosecond timestamps, and capture clamps them to +`SOURCE_DATE_EPOCH`. Cpio stores whole seconds and has no extended attributes. Linux's +initramfs unpacker recreates hardlinked symlinks as separate symlinks. + +Image paths must satisfy [UAPI.16](https://uapi-group.org/specifications/specs/file_manifest/): normalized +relative UTF-8 paths without control characters. The top-level names `Uapi16Manifest` and +`Uapi16Manifest.*` are reserved. Backslashes and literal `.wh.*` and `.esc.*` names work without escaping. + A layer that adds a program or shared library which would run on an executable stack fails to build: one whose `PT_GNU_STACK` program header is executable, or which has none on x86, where glibc then falls back to an executable stack. @@ -182,7 +192,7 @@ be trusted because procfs magic links can access paths outside the image. `image.layer` has two construction modes. An initial image takes `package_manager`, which defaults to the package manager of the distribution that the build chooses. A derived image takes `parent` and inherits the package manager of that image. A call with operations applies one ordered operation sequence in one action -and persists exactly one delta: +and persists exactly one snapshot: - `packages` and `package_sets` are rule attributes rather than operations: a layer installs them as one request before any of its operations run, so every operation sees what the layer adds. `package_sets` @@ -203,7 +213,7 @@ and persists exactly one delta: project like any other artifact. It takes the same `env` and `chroot` arguments, so the script either sees the image at `/buildroot` or has it as its own root; in the chrooted case nothing of the interpreter reaches the - delta, because the project bind carrying it lives under `/run`. + snapshot, because the project bind carrying it lives under `/run`. - `image.ops.copy` introduces a declared Buck artifact at an absolute image path; `image.ops.mkdir` and `image.ops.symlink` mutate the same root. `image.ops.remove` treats its absolute path as a glob pattern, including recursive `**`, and removes every matching file, symlink, or directory tree. A pattern that @@ -377,7 +387,7 @@ database whenever the image has a package manager. The same artifacts are expose ``` The `version` attribute sets the SBOM source version and defaults to `"0"`. Merely building the image still -produces only its latest delta. Selecting `[pkgdb]` runs the image's package system's capture driver; +produces only its latest snapshot. Selecting `[pkgdb]` runs the image's package system's capture driver; selecting either nested SBOM format runs one shared scan of the completed stack. `[manifest]` is a [UAPI.16 File Manifest](https://uapi-group.org/specifications/): an RFC7464 JSON-SEQ @@ -398,7 +408,8 @@ at all. `install_docs = False` likewise installs no documentation, keeping the l Logical images and terminal outputs are separate rule families. Terminal rules merge the layer stack only when needed: -- `image.directory` materializes a Buck directory artifact and provides `image.ImageDirectoryInfo`; +- `image.directory` builds an authoritative manifest and content blobs and provides + `image.ImageDirectoryInfo`; - `image.uki` builds the unified kernel image for the image's single installed kernel from one or more cpio `image.ImageArchiveInfo` dependencies and provides `image.UkiInfo`, named `__.efi` (defaults: target name and `0`; systemd architecture spelling, e.g. `x86-64`), the shape @@ -852,8 +863,26 @@ helper label. The same `image.InitrdInfo`, containing its logical `image.ImageIn `image.ImageArchiveInfo`, is the sole typed provider published by `[initrd]`. Merely carrying an artifact in a provider does not build it; optional actions run only when a consumer uses the artifact or a user selects its subtarget. -`[directory]` has the same representability limit as `image.directory`: it fails when the completed tree -contains a path, such as a systemd-escaped unit name, that Buck directory artifacts cannot store. +`[directory]` contains a UAPI.16 manifest and content blobs. Export it after Buck has materialized the +artifact, so the destination retains modes, times, user extended attributes, symlinks and hardlinks. + +Directory export requires read access to `/proc/sys/kernel/overflowuid`. Keep the stored manifest and blobs +immutable while exporting. The destination must not exist. Its parent must be owned by you or root. A parent +with group or other write permissions must have the sticky bit, as `/var/tmp` does. Use a parent owned by you +if Buck runs in a user namespace that does not map its owner's uid. The exporter rejects a parent owned by +the overflow uid, which is how a user namespace shows an owner that it does not map. The exporter creates a +private user namespace when needed; uid/gid zero then maps to your host uid/gid. Device nodes require +permission to create them on the destination filesystem. + +```sh +tine buck run 'tine//examples/image:boot-demo.fedora[directory]' -- /var/tmp/tine-image-export/root +``` + +The stored manifest remains authoritative if you copy the Buck artifact elsewhere; copying an exported +directory with a tool that drops metadata requires exporting it again from the stored snapshot. + +The `[manifest]` of an archive or disk describes its terminal contents, including authored tmpfiles and +package-database removal. `[pkgdb]` and `[sbom]` describe the installed package set. `[boot]` is what the disk boots, as files: the `bootable` rule run against the composition's own ESP, so a direct kernel boot or a publisher needs no second target pointing back at it. Its `[kernel]` and `[initrd]` @@ -1212,7 +1241,7 @@ tine buck build //examples/image:demo-ext.fedora tine buck build //examples/image:demo-portable.fedora ``` -These validate package installation and commands sharing one delta, incremental layering, archive packing, +These validate package installation and commands sharing one layer, incremental layering, archive packing, versioned UKI creation, semantic boot-artifact extraction, ESP-layer assembly, and disk composition. `demo-ext` is a system extension and `demo-confext` a configuration extension on top of `boot-demo`. `demo-portable` is a portable service image, and `demo-portable-ext` an extension for it. Running diff --git a/examples/image-secureboot/BUCK b/examples/image-secureboot/BUCK index 17266ad2..a36ea053 100644 --- a/examples/image-secureboot/BUCK +++ b/examples/image-secureboot/BUCK @@ -314,13 +314,15 @@ box.sh_test( name = "disk-manifest-test", labels = ["image"], box = "@tine//tools:dev.box", - test = "@tine//tests:contains.sh", - args = [ - "$(location :image[disk-manifest])", - '"name":"usr/bin/bash"', - '"name":"boot/tine-tmpfiles","type":"dir","mode":448', - '"name":"boot/EFI/Linux', - ], + args = ["$(location :image[disk-manifest])"], + script = """ +# Authored tmpfiles must set the boot directory mode in the disk's filesystem listing. +jq --seq -es ' + any(.[]; .name == "usr/bin/bash") and + any(.[]; .name == "boot/tine-tmpfiles" and .type == "dir" and .mode == 448) and + any(.[]; .name | strings | startswith("boot/EFI/Linux")) +' "$1" +""", ) # The kernel install hook is bypassed, it didn't generate its own initrd. diff --git a/examples/image/BUCK b/examples/image/BUCK index 71d8a8c0..c01a3bad 100644 --- a/examples/image/BUCK +++ b/examples/image/BUCK @@ -90,14 +90,14 @@ image.archive( format = "tar", ) -# Base-only image exercising a single-component delta stack. +# An image with a single layer. image.archive( name = "base", packages = ["bash"], format = "tar", ) -# Chained images: the child adds one file and whiteouts another. +# The child image adds one file and removes a file of its parent. image.layer( name = "chained-base", packages = ["bash", "coreutils"], @@ -122,7 +122,7 @@ image.archive( format = "tar", ) -# Package installation and mutations sharing one physical delta. +# One layer installs packages and runs commands. image.layer( name = "layered-install-base", packages = ["bash"], @@ -146,35 +146,6 @@ image.archive( format = "tar", ) -# Round-trip an escaped name and an escaped whiteout through stored deltas. -image.layer( - name = "escaped-base", - packages = ["bash", "coreutils"], - ops = [ - image.ops.mkdir("/etc/tine"), - image.ops.run( - [ - "/usr/bin/bash", - "-c", - "echo kept > '/etc/tine/esc\\x2dkept'; echo gone > '/etc/tine/esc\\x2dgone'", - ], - chroot = True, - ), - ], -) - -image.layer( - name = "escaped-child.layer", - parent = ":escaped-base", - ops = [image.ops.remove("/etc/tine/esc\\x2dgone")], -) - -image.archive( - name = "escaped", - parent = ":escaped-child.layer", - format = "tar", -) - # Bootable GPT image; `buck run :boot-demo-vm` starts it with the box's vmspawn stack. # Supply-chain and format subtargets such as `:boot-demo[sbom]` and `:boot-demo[qcow2]` build on demand. # The version stamps os-release, partition labels, and the UKI and disk file names. A product names @@ -532,30 +503,6 @@ box.sh_test( args = ["$(location :tree)"] + distribution.select({d["release"]: [name if name == "arch" else "none"] for name, d in DISTRIBUTIONS.items()}), ) -# Replacing a populated directory exercises the opaque-dir marker. -image.layer( - name = "opaque-base", - packages = ["bash", "coreutils"], - ops = [ - image.ops.run(["/usr/bin/bash", "-c", "mkdir -p /etc/tine/sub; echo old > /etc/tine/sub/old"], chroot = True), - ], -) - -image.layer( - name = "opaque-child.layer", - parent = ":opaque-base", - ops = [ - image.ops.remove("/etc/tine/sub"), - image.ops.run(["/usr/bin/bash", "-c", "mkdir -p /etc/tine/sub; echo new > /etc/tine/sub/fresh"], chroot = True), - ], -) - -image.archive( - name = "opaque", - parent = ":opaque-child.layer", - format = "tar", -) - # The UKI's module selection still carries what the demo image boots through. The smoke proves the # same thing by booting; this pins it against the real kernel without a VM, so a change to the # default list that drops one fails here with the module named. Only modules Fedora builds as diff --git a/examples/image/expected.json b/examples/image/expected.json index 92e5a631..ca9aaa2d 100644 --- a/examples/image/expected.json +++ b/examples/image/expected.json @@ -39,28 +39,28 @@ }, "x86_64": { "boot-demo-preloaded.fedora": { - "sha256": "5639ec39438ad8de3facbc193b55163785b36956abec603061c95dce4229a8d2" + "sha256": "ab9d19efdbc3fe08b8a2271fa42a492a0cf1ba80a04ecb6dcd21c31f97e4ce9c" }, "boot-demo.fedora": { - "sha256": "810f4d8cd0ae75dd5d58c09784615409a4c0c5983f44b9a7e68b3e2b758aaab0" + "sha256": "0756e175d735d6b5f28d27b0f7422661c088fbfe7fad7def9b90b2c0750eaa2c" }, "boot-demo.fedora[boot][initrd]": { - "sha256": "92c4e74ec361a81e7dcb0484dfb97ba060f2c3199f790c0524c2811f21306254" + "sha256": "13d9081dec73347d54c27ba45af327616284cedb16781282a8075c8c24d86b87" }, "boot-demo.fedora[boot][kernel]": { "sha256": "d505498dab710cf2acb7c757d7d50592f147fd418d7464bca95415b190d43c87" }, "boot-demo.fedora[boot][uki]": { - "sha256": "1dbd49e1af726cb9023fd1da842d5441736563c5dea426901a98e2b5e22c14cb" + "sha256": "5f2da6847834d25631d1d06e85e2103b70c7a99631450673efd765c4fcde64e6" }, "boot-demo.fedora[disk-manifest]": { - "sha256": "8c434b3a0a28f6c6eb6e1609bb766258dbf51c5e0b7e2c79b7161df918561713" + "sha256": "bcd43826e4c56c7888f01d2add6e47defaae0c233b36e7455d0fd8bc805b7820" }, "boot-demo.fedora[manifests][esp]": { - "sha256": "a856a70ea0b09588e28798cf88250f550ae2475f705ed6d84cea88c076b79946" + "sha256": "fa57c5b36cb2268d4ebffc2c4be284bc978f0f4cf680538ba75e34ab4e3491b2" }, "boot-demo.fedora[manifests][usr]": { - "sha256": "0451b4a974d9376d975cf92902fc3a287e1a94679947cbddaebc540d838ba861" + "sha256": "456389b2e66d0a9dd1a60af74d1496cafffd89cd0141d0d166b1cd8a6135d949" }, "boot-demo.fedora[partitions][usr-verity]": { "sha256": "8baac782ec9d31c05161bbf86cec6afab8b55d41faad269d932fd67c8680338f" @@ -72,7 +72,7 @@ "sha256": "f3dbb593040d1eed578b65d23532032df07162777707ffcf0a2a6f70d0e501f5" }, "demo-ext.fedora": { - "sha256": "fe8308bb979aae3004152ed7acc1190b343c46cc822a54c5c65e7ae2e89934ab" + "sha256": "70c293ffdf46ee6c97c4544cfaafbb96601cbcb83bf336e849f895451b7ef3d5" } } } diff --git a/examples/image/manifest_check.py b/examples/image/manifest_check.py index 4cd53714..367a0dde 100644 --- a/examples/image/manifest_check.py +++ b/examples/image/manifest_check.py @@ -27,8 +27,9 @@ def decode(data: bytes) -> list[dict[str, Any]]: objects = decode(Path(sys.argv[1]).read_bytes()) -root, files = objects[0], objects[1:] -assert root == {"mediaType": "application/vnd.uapi.16.manifest"}, f"bad root object: {root}" +assert objects[-1] == {"mediaType": "application/vnd.uapi.16.trailer"}, "missing trailer" +root, files = objects[0], objects[1:-1] +assert root.get("mediaType") == "application/vnd.uapi.16.manifest" and "name" not in root, root assert files, "the manifest lists no files at all" names = [obj["name"] for obj in files] diff --git a/image/BUCK b/image/BUCK index 9e6a2c58..9c1eb067 100644 --- a/image/BUCK +++ b/image/BUCK @@ -35,6 +35,7 @@ python.binary( "//box:isolation", "//package:installer", "//rootfs:rootfs", + "//uapi16:storage", ], visibility = ["PUBLIC"], ) @@ -48,6 +49,9 @@ python.binary( "//:util", "//archive:cpio", "//archive:tar", + "//box:isolation", + "//uapi16:storage", + "//uapi16:uapi16", ], visibility = ["PUBLIC"], ) @@ -57,10 +61,11 @@ python.binary( main = "ddi.py", deps = [ ":finalize", - ":manifest-lib", "//:specs", "//:util", "//rootfs:rootfs", + "//uapi16:storage", + "//uapi16:uapi16", ], visibility = ["PUBLIC"], ) @@ -95,6 +100,7 @@ python.binary( "//:specs", "//:util", "//archive:cpio", + "//archive:tar", "//kmod:kmod", ], visibility = ["PUBLIC"], @@ -119,13 +125,13 @@ python.test( srcs = [ "ami_import.py", "ami_import_test.py", + "archive.py", + "archive_test.py", "ddi.py", "ddi_test.py", "finalize_test.py", "layer.py", "layer_test.py", - "manifest.py", - "manifest_test.py", "preload.py", "preload_test.py", "substitute.py", @@ -141,10 +147,13 @@ python.test( "//:specs", "//:util", "//archive:cpio", + "//archive:tar", "//box:isolation", "//kmod:kmod", "//package:installer", "//rootfs:rootfs", + "//uapi16:storage", + "//uapi16:uapi16", ], ) @@ -160,30 +169,22 @@ python.binary( python.binary( name = "manifest", - main = "manifest.py", + main = "manifest_main.py", deps = [ ":finalize", "//:specs", "//:util", + "//uapi16:uapi16", ], visibility = ["PUBLIC"], ) # A driver that packages a tree lists it from inside its own action, where the tree is final. -python.library( - name = "manifest-lib", - srcs = ["manifest.py"], - deps = [ - ":finalize", - "//:specs", - "//:util", - ], -) python.library( name = "finalize", srcs = ["finalize.py"], - deps = ["//:util", "//rootfs:rootfs"], + deps = ["//:util", "//rootfs:rootfs", "//uapi16:storage", "//uapi16:uapi16"], visibility = ["PUBLIC"], ) @@ -220,6 +221,7 @@ python.binary( "//:specs", "//:util", "//kmod:kmod", + "//uapi16:uapi16", ], visibility = ["PUBLIC"], ) diff --git a/image/archive.bzl b/image/archive.bzl index 2433d063..6523fe87 100644 --- a/image/archive.bzl +++ b/image/archive.bzl @@ -22,13 +22,16 @@ ImageArchiveInfo = provider( fields = { "archive": provider_field(Artifact), "format": provider_field(str), + "manifest": provider_field(Artifact), }, ) ImageDirectoryInfo = provider( - doc = "A materialized logical image directory.", + doc = "A stored snapshot and its command for faithful directory export.", fields = { "directory": provider_field(Artifact), + "export": provider_field(RunInfo), + "manifest": provider_field(Artifact), }, ) @@ -49,6 +52,7 @@ def declare_image_archive( captures the database from the tree itself. """ out = declare_out(ctx, identifier, "image." + format + _COMPRESSION_EXT[compression]) + listing = declare_out(ctx, identifier, "image." + format + ".Uapi16Manifest") cmd = terminal_image_command( ctx, # One image can be archived in several formats, so the spec is named like the output. @@ -59,12 +63,13 @@ def declare_image_archive( spec = { "compression": compression, "format": format, + "manifest": listing.as_output(), "out": out.as_output(), "pkgdb_paths": pkgdb_paths(image) if strip_pkgdb else [], }, ) ctx.actions.run(cmd, category = "image_" + format, identifier = identifier or format) - return ImageArchiveInfo(archive = out, format = format) + return ImageArchiveInfo(archive = out, format = format, manifest = listing) ARCHIVE_ATTRS = { "compression": attrs.enum(COMPRESSIONS, default = "none"), @@ -89,16 +94,22 @@ def declare_image_directory( spec = { "compression": "none", "format": "directory", + "manifest": None, "out": out.as_output(), "pkgdb_paths": [], }, ) ctx.actions.run(cmd, category = "image_directory", identifier = identifier or "directory") - return ImageDirectoryInfo(directory = out) + export = RunInfo(args = cmd_args(tools.archive[RunInfo], "export", "--store", out)) + return ImageDirectoryInfo(directory = out, manifest = out.project("Uapi16Manifest"), export = export) + +def image_directory_providers(info: ImageDirectoryInfo) -> list[Provider]: + """Return the providers of a `[directory]` target or sub-target for `info`.""" + return [DefaultInfo(default_output = info.directory, sub_targets = {"manifest": [DefaultInfo(default_output = info.manifest)]}), info.export, info] def _image_directory_impl(ctx: AnalysisContext) -> list[Provider]: info = declare_image_directory(ctx, tools = ctx.attrs._tools[ImageToolsInfo], image = ctx.attrs.image[ImageInfo]) - return [DefaultInfo(default_output = info.directory), info] + return image_directory_providers(info) _image_directory = rule( impl = _image_directory_impl, diff --git a/image/archive.py b/image/archive.py index 925ed2ff..6b4573c0 100755 --- a/image/archive.py +++ b/image/archive.py @@ -3,16 +3,13 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""Merge a logical image into a deterministic tar, cpio, or directory. - -Archive ownership is normalized to uid/gid 0. The newc cpio format has no general -extended-attribute representation. -""" +"""Export image snapshots as reproducible archives or directory stores.""" +import argparse import os -import subprocess import sys import tempfile +from contextlib import chdir from pathlib import Path import specs @@ -20,73 +17,98 @@ import cpio import finalize +import storage import tar +import uapi16 +from isolation import AT_FDCWD, Close, open_directory, unshare_sandbox class Spec(finalize.ImageSpec): out: str format: str compression: str - # Image paths holding the package database, stripped from the archive. pkgdb_paths: list[str] + manifest: str | None -def _clamp_mtimes(tree: Path, epoch: int) -> None: - """Clamp directory-format mtimes without an archive filter.""" - for path in [tree, *tree.rglob("*")]: - if path.lstat().st_mtime > epoch: - os.utime(path, (epoch, epoch), follow_symlinks=False) - - -def _pack(tree: Path, out: Path, fmt: str, epoch: int) -> None: +def _pack(snapshot: storage.Snapshot, out: Path, fmt: str, epoch: int) -> None: if fmt == "tar": - tar.pack_tree(tree, out, epoch) + tar.pack_manifest(snapshot.objects, snapshot.content, out) elif fmt == "cpio": - cpio.pack_tree(tree, out, epoch) + cpio.pack_manifest(snapshot.objects, snapshot.content, out, epoch) else: util.fail(f"unknown archive format {fmt!r}") -def _archive(tree: Path, out: Path, fmt: str, epoch: int, compression: str) -> None: +def _archive(snapshot: storage.Snapshot, out: Path, fmt: str, epoch: int, compression: str) -> None: if fmt == "directory": if compression != "none": util.fail("archive: the directory format cannot be compressed") - - # Reject names that would wedge Buck while storing the thawed tree. - for path in tree.rglob("*"): - if "\\" in path.name: - util.fail( - f"archive: {path.relative_to(tree)} contains a backslash, which buck cannot " - "store; the directory format cannot represent this image — use tar" - ) - out.mkdir(parents=True, exist_ok=True) - subprocess.run(["cp", "-a", "--reflink=auto", f"{tree}/.", str(out)], check=True) - _clamp_mtimes(out, epoch) + storage.save(snapshot, out) elif compression == "none": - _pack(tree, out, fmt, epoch) + _pack(snapshot, out, fmt, epoch) elif compression != "zstd": util.fail(f"archive: unknown compression {compression!r}") else: - # zstd needs the finished archive, so pack it beside the output rather than in TMPDIR: the - # shared filesystem keeps the packer's reflink cloning working, and Buck only ever sees the - # compressed result. - with tempfile.TemporaryDirectory(dir=out.parent) as scratch: + with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: raw = Path(scratch) / out.name - _pack(tree, raw, fmt, epoch) + _pack(snapshot, raw, fmt, epoch) util.compress_zstd(raw, out) -def main(argv: list[str] | None = None) -> None: - spec = specs.parse(Spec, "archive", argv) +def export(argv: list[str]) -> None: + parser = argparse.ArgumentParser(prog="image-directory") + parser.add_argument("--store", required=True, type=Path) + parser.add_argument("destination", type=Path) + args = parser.parse_args(argv) + # Restoration changes the working directory, so blob references need an absolute store. + snapshot = storage.Snapshot([args.store.absolute()]) + with Close(open_directory(AT_FDCWD, str(args.destination.parent))) as parent: + parent_attributes = os.fstat(parent) + # A user namespace shows an owner that it doesn't map as the overflow uid. The caller's own uid in + # the namespace can be the overflow uid as well. A parent owned by the overflow uid is never trusted. + overflow_uid = int(Path("/proc/sys/kernel/overflowuid").read_text()) + if ( + parent_attributes.st_uid not in (0, os.geteuid()) + or parent_attributes.st_uid == overflow_uid + or (parent_attributes.st_mode & 0o022 and not parent_attributes.st_mode & 0o1000) + ): + util.fail(f"image-directory: export needs a protected parent: {args.destination.parent}") + unshare_sandbox(become_root=True) + name = args.destination.name or "." + try: + os.mkdir(name, mode=0o700, dir_fd=parent) + except FileExistsError: + util.fail(f"image-directory: destination already exists: {args.destination}") + with ( + Close(os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent)) as directory, + chdir("."), + ): + attributes = os.fstat(directory) + if attributes.st_uid != os.geteuid() or attributes.st_mode & 0o077 or os.listdir(directory): + util.fail(f"image-directory: destination changed before export: {args.destination}") + os.fchdir(directory) + storage.restore(snapshot, Path(".")) + +def main(argv: list[str] | None = None) -> None: + arguments = sys.argv[1:] if argv is None else argv + if arguments[:1] == ["export"]: + export(arguments[1:]) + return + spec = specs.parse(Spec, "archive", arguments) epoch = int(os.environ["SOURCE_DATE_EPOCH"]) out = Path(spec["out"]) - with finalize.image(spec, program="archive") as tree: - # The package database is a supply-chain artifact that the image's `[pkgdb]` subtarget - # captures separately, so an archive nothing resolves packages in can drop it. - for relative in spec["pkgdb_paths"]: - util.remove_path(tree / relative, with_parents=True) - _archive(tree, out, spec["format"], epoch, spec["compression"]) + with finalize.snapshot( + spec, program="archive", epoch=epoch, pkgdb_paths=spec["pkgdb_paths"] + ) as snapshot: + if spec["format"] == "cpio": + for obj in snapshot.objects: + obj["mTimeNSec"] = obj["mTimeNSec"] // 10**9 * 10**9 + obj.pop("extendedAttributes", None) + _archive(snapshot, out, spec["format"], epoch, spec["compression"]) + if spec["manifest"]: + uapi16.write_listing(snapshot.objects, Path(spec["manifest"])) print(f"archive: wrote {spec['format']} (epoch={epoch}) -> {out}", file=sys.stderr) diff --git a/image/archive_test.py b/image/archive_test.py new file mode 100644 index 00000000..879849bc --- /dev/null +++ b/image/archive_test.py @@ -0,0 +1,430 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Terminal metadata and archive identity regressions.""" + +import os +import shutil +import stat +import tarfile +import tempfile +import unittest +from contextlib import chdir +from decimal import Decimal +from pathlib import Path +from typing import override +from unittest import mock + +import specs + +import archive +import cpio +import finalize +import storage +import uapi16 + +EPOCH = 1000000000 + + +# Captures in these tests keep every timestamp, so they clamp to an epoch after all of them. +CAPTURE_EPOCH = 2**31 + + +class TestArchive(unittest.TestCase): + @override + def setUp(self) -> None: + self.scratch = Path(self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp"))) + source = self.scratch / "source" + (source / "empty-dir").mkdir(parents=True) + for name, mode, ns in (("a", 0o640, 123), ("b", 0o755, 456), ("empty", 0o600, 789)): + path = source / name + path.write_text("contents" if name != "empty" else "") + path.chmod(mode) + os.utime(path, ns=(ns, ns)) + os.link(source / "a", source / "linked") + os.link(source / "empty", source / "empty-linked") + (source / "link").symlink_to("/a") + for name in ("back\\slash", ".wh.literal", ".esc.literal", "TRAILER!!!"): + (source / name).write_text(name) + os.mkfifo(source / "fifo", 0o600) + os.setxattr(source / "a", "user.example", b"value") + (source / "var/lib/database").mkdir(parents=True) + (source / "var/lib/database/contents").write_text("database") + self.store = self.scratch / "store" + storage.capture(storage.Root(source), self.store, epoch=CAPTURE_EPOCH) + self.spec = finalize.ImageSpec(lower=[str(self.store)], tmpfiles=[]) + + def test_archives_keep_hardlinks_created_by_commands(self) -> None: + # A command creates a hardlink to an inherited independent file. Upper-aware capture + # must preserve the group in tar and cpio without linking the equal-content file a. + child = self.scratch / "child" + with storage.root(storage.Snapshot([self.store])) as restored: + tree = restored.path + os.link(tree / "b", tree / "new-link") + storage.capture(restored, child, [self.store], epoch=CAPTURE_EPOCH) + spec = finalize.ImageSpec(lower=[str(self.store), str(child)], tmpfiles=[]) + + # The archive writers must group paths by the captured inodeToken rather than blob digest. + with finalize.snapshot(spec, program="test", epoch=EPOCH) as snapshot: + archive._archive(snapshot, self.scratch / "image.tar", "tar", EPOCH, "none") + archive._archive(snapshot, self.scratch / "image.cpio", "cpio", EPOCH, "none") + with tarfile.open(self.scratch / "image.tar") as packed: + self.assertTrue(packed.getmember("./new-link").islnk()) + self.assertEqual(packed.getmember("./new-link").linkname, "./b") + self.assertTrue(packed.getmember("./b").isreg()) + entries = {entry.name: entry for entry in cpio.read((self.scratch / "image.cpio").read_bytes())} + self.assertEqual(entries["b"].ino, entries["new-link"].ino) + self.assertNotEqual(entries["a"].ino, entries["b"].ino) + + def test_terminal_readers_normalize_metadata_without_copying_contents(self) -> None: + # A terminal reader needs normalized modes and explicit hardlinks before it reads files. + # Preparing metadata before the read-only mount must avoid every blob copy. + (self.scratch / "source/tmp").mkdir(mode=0o700) + (self.scratch / "source/var/tmp").symlink_to("../empty-dir") + storage.capture(storage.Root(self.scratch / "source"), self.store, epoch=CAPTURE_EPOCH) + with mock.patch.object(storage, "clone_file", side_effect=AssertionError("unexpected copy")): + with finalize.image(self.spec, program="test", strategy="overlay") as tree: + self.assertEqual(stat.S_IMODE((tree / "a").stat().st_mode), 0o644) + self.assertEqual(stat.S_IMODE((tree / "b").stat().st_mode), 0o755) + self.assertEqual(stat.S_IMODE((tree / "tmp").stat().st_mode), 0o1777) + self.assertTrue((tree / "var/tmp").is_symlink()) + self.assertEqual(stat.S_IMODE((tree / "empty-dir").stat().st_mode), 0o755) + self.assertEqual(stat.S_IMODE((tree / "fifo").stat().st_mode), 0o600) + self.assertTrue((tree / "a").samefile(tree / "linked")) + self.assertFalse((tree / "a").samefile(tree / "b")) + self.assertEqual((tree / "a").read_text(), "contents") + self.assertEqual((tree / "a").stat().st_mtime_ns, 123) + + def test_terminal_readers_apply_tmpfiles_before_reading(self) -> None: + # Authored tmpfiles rules need a writable root even for a terminal reader. Their modes + # must override normalization and their new files must be present when the reader runs. + spec = finalize.ImageSpec( + lower=self.spec["lower"], tmpfiles=["z /a 0600 - - -", "f /created 0640 - - - authored"] + ) + with finalize.image(spec, program="test") as tree: + self.assertEqual(stat.S_IMODE((tree / "a").stat().st_mode), 0o600) + self.assertEqual((tree / "created").read_text(), "authored") + self.assertTrue((tree / "a").samefile(tree / "linked")) + + def test_archive_writers_read_blobs_without_reconstruction(self) -> None: + # A terminal without tmpfiles can package its snapshot directly. Tar must preserve + # declared links, independent equal files, original nanoseconds and user attributes. + out = self.scratch / "image.tar" + with mock.patch.object(storage, "root", side_effect=AssertionError("unexpected reconstruction")): + with finalize.snapshot(self.spec, program="test", epoch=EPOCH) as snapshot: + archive._archive(snapshot, out, "tar", EPOCH, "none") + with tarfile.open(out) as packed: + objects = {entry.name: entry for entry in packed.getmembers()} + self.assertTrue(objects["./linked"].islnk()) + self.assertEqual(objects["./linked"].linkname, "./a") + self.assertTrue(objects["./b"].isreg()) + self.assertEqual(objects["./a"].mode, 0o644) + self.assertEqual(objects["./a"].pax_headers["mtime"], "0.000000123") + self.assertEqual(objects["./a"].pax_headers["SCHILY.xattr.user.example"], "value") + self.assertEqual(objects["./link"].linkname, "/a") + self.assertTrue(objects["./fifo"].isfifo()) + self.assertIn("./back\\slash", objects) + + def test_cpio_preserves_nonempty_and_empty_hardlinks(self) -> None: + # Newc needs one payload and a shared inode per declared group, including empty files. + # Equal contents alone must not give unrelated regular files the same inode. + out = self.scratch / "image.cpio" + with finalize.snapshot(self.spec, program="test", epoch=EPOCH) as snapshot: + archive._archive(snapshot, out, "cpio", EPOCH, "none") + objects = {obj.name: obj for obj in cpio.read(out.read_bytes())} + self.assertGreater(objects["./TRAILER!!!"].size, 0) + self.assertEqual(objects["a"].ino, objects["linked"].ino) + self.assertEqual(objects["a"].nlink, 2) + self.assertEqual(objects["empty"].ino, objects["empty-linked"].ino) + self.assertNotEqual(objects["a"].ino, objects["b"].ino) + dest = self.scratch / "unpacked" + with out.open("rb") as stream: + cpio.unpack(stream.fileno(), dest) + self.assertTrue((dest / "a").samefile(dest / "linked")) + self.assertTrue((dest / "empty").samefile(dest / "empty-linked")) + self.assertFalse((dest / "a").samefile(dest / "b")) + + def test_tmpfiles_and_database_removal_match_exported_manifest(self) -> None: + # Apply terminal tmpfiles and database removal before writing both the archive and its + # listing. The manifest must describe the modes and timestamps that tar actually encodes. + out, listing = self.scratch / "image.tar", self.scratch / "listing" + spec = specs.write( + self.scratch / "spec.json", + dict(self.spec) + | { + "out": str(out), + "format": "tar", + "compression": "none", + "manifest": str(listing), + "pkgdb_paths": ["var/lib/database"], + "tmpfiles": ["z /a 0600 - - -", "f /created 0640 - - - authored"], + }, + ) + with mock.patch.dict(os.environ, {"SOURCE_DATE_EPOCH": str(EPOCH)}): + archive.main(["--spec", str(spec)]) + objects = {obj.get("name", "."): obj for obj in uapi16.read(listing)} + with tarfile.open(out) as packed: + for entry in packed.getmembers(): + name = entry.name.removeprefix("./") + obj = objects[name] + if not entry.issym(): + self.assertEqual(entry.mode, obj["mode"]) + ns = int(Decimal(entry.pax_headers.get("mtime", str(entry.mtime))) * 10**9) + self.assertEqual(ns, obj["mTimeNSec"]) + stream = packed.extractfile("./created") + assert stream is not None + with stream: + self.assertEqual(stream.read(), b"authored") + self.assertEqual(objects["a"]["mode"], 0o600) + self.assertEqual(objects["linked"]["mode"], 0o600) + self.assertNotIn("var/lib/database", objects) + self.assertNotIn("var", objects) + + def test_directory_export_restores_after_materialization(self) -> None: + # The directory artifact contains only safe blob names and a uapi16. Copying that + # artifact before export must not lose modes, times, hardlinks or logical filenames. + out, copied = self.scratch / "directory", self.scratch / "copied" + with finalize.snapshot(self.spec, program="test", epoch=EPOCH) as snapshot: + archive._archive(snapshot, out, "directory", EPOCH, "none") + shutil.copytree(out, copied, copy_function=shutil.copyfile) + with chdir(self.scratch): + archive.export(["--store", "copied", "exported"]) + exported = self.scratch / "exported" + self.assertEqual((exported / "a").stat().st_mtime_ns, 123) + self.assertEqual(stat.S_IMODE((exported / "a").stat().st_mode), 0o644) + self.assertTrue((exported / "a").samefile(exported / "linked")) + self.assertFalse((exported / "a").samefile(exported / "b")) + self.assertTrue((exported / "back\\slash").exists()) + self.assertEqual({p.name for p in out.iterdir()}, {"blobs", uapi16.MANIFEST}) + + def test_export_refuses_a_destination_created_during_validation(self) -> None: + # Another process inserts a destination and a symlink while the store is validated. + # Export must fail without following that symlink or changing the unrelated file. + destination, victim = self.scratch / "exported", self.scratch / "victim" + victim.write_text("unrelated") + read = storage.Snapshot + + def raced(stores: list[Path]) -> storage.Snapshot: + snapshot = read(stores) + destination.mkdir() + (destination / "a").symlink_to(victim) + return snapshot + + with mock.patch.object(storage, "Snapshot", side_effect=raced), self.assertRaises(SystemExit): + archive.export(["--store", str(self.store), str(destination)]) + self.assertEqual(victim.read_text(), "unrelated") + + def test_export_keeps_the_open_directory_after_rename(self) -> None: + # Another process renames the private destination after it is opened and inserts a symlink. + # Restoration must stay in the opened directory and leave the symlink's target unchanged. + destination, saved = self.scratch / "exported", self.scratch / "renamed" + victim = self.scratch / "victim" + victim.mkdir() + (victim / "a").write_text("unrelated") + restore = storage.restore + + def raced(snapshot: storage.Snapshot, tree: Path) -> None: + destination.rename(saved) + destination.symlink_to(victim, target_is_directory=True) + restore(snapshot, tree) + + with mock.patch.object(storage, "restore", side_effect=raced): + archive.export(["--store", str(self.store), str(destination)]) + self.assertEqual((victim / "a").read_text(), "unrelated") + self.assertEqual((saved / "a").read_text(), "contents") + + def test_export_rejects_a_replacement_directory(self) -> None: + # Another process replaces the destination before open with a private directory containing + # a symlink. Export must reject the nonempty replacement without following the symlink. + destination, saved = self.scratch / "exported", self.scratch / "renamed" + replacement, victim = self.scratch / "replacement", self.scratch / "victim" + replacement.mkdir(mode=0o700) + victim.write_text("unrelated") + (replacement / "a").symlink_to(victim) + open_directory = os.open + + def raced(path: str | Path, flags: int, *, dir_fd: int | None = None) -> int: + if path == destination.name: + destination.rename(saved) + replacement.rename(destination) + return open_directory(path, flags, dir_fd=dir_fd) + + with mock.patch.object(archive.os, "open", side_effect=raced), self.assertRaises(SystemExit): + archive.export(["--store", str(self.store), str(destination)]) + self.assertEqual(victim.read_text(), "unrelated") + + def test_export_keeps_the_parent_directory_after_rename(self) -> None: + # Another user replaces an ancestor after mkdir. The destination then resolves to an existing + # private empty directory. Opening through the original parent must retain the created root. + parent, saved = self.scratch / "parent", self.scratch / "renamed" + parent.mkdir() + destination = parent / "out" + victim = self.scratch / "victim" + (victim / "out").mkdir(parents=True, mode=0o700) + os.utime(victim / "out", ns=(111, 111)) + open_directory = os.open + + def raced(path: str | Path, flags: int, *, dir_fd: int | None = None) -> int: + if path == destination.name: + parent.rename(saved) + parent.symlink_to(victim, target_is_directory=True) + return open_directory(path, flags, dir_fd=dir_fd) + + with mock.patch.object(archive.os, "open", side_effect=raced): + archive.export(["--store", str(self.store), str(destination)]) + self.assertEqual(list((victim / "out").iterdir()), []) + self.assertEqual((victim / "out").stat().st_mtime_ns, 111) + self.assertEqual((saved / "out/a").read_text(), "contents") + + def test_export_requires_a_protected_parent(self) -> None: + # A writable nonsticky parent permits another user to replace the new root with an existing + # private empty directory. Reject that parent before creating or changing either directory. + parent = self.scratch / "shared" + parent.mkdir() + parent.chmod(0o777) + private = parent / "private" + private.mkdir(mode=0o700) + destination = parent / "out" + with self.assertRaises(SystemExit): + archive.export(["--store", str(self.store), str(destination)]) + self.assertFalse(destination.exists()) + self.assertEqual(stat.S_IMODE(private.stat().st_mode), 0o700) + + # A trusted sticky parent prevents other users from renaming the exporter's private root. + parent.chmod(0o1777) + archive.export(["--store", str(self.store), str(destination)]) + self.assertEqual((destination / "a").read_text(), "contents") + + def test_export_creates_a_protected_parent_with_a_shared_umask(self) -> None: + # Direct execution can inherit a shared umask. A missing parent must be created with + # protected permissions so the exporter can restore the image beneath it. + destination = self.scratch / "missing" / "out" + previous = os.umask(0o002) + try: + archive.export(["--store", str(self.store), str(destination)]) + finally: + os.umask(previous) + self.assertEqual((destination / "a").read_text(), "contents") + + def test_export_rejects_an_unmapped_owner_matching_the_caller(self) -> None: + # A user namespace shows an unmapped parent owner as the overflow uid, which can equal the + # caller's uid. Report the overflow uid as both the caller's uid and the owner of the parent, and + # check that the export fails without creating the destination. + destination = self.scratch / "out" + parent = self.scratch.stat() + overflow = int(Path("/proc/sys/kernel/overflowuid").read_text()) + fstat = os.fstat + + def attributes(fd: int) -> os.stat_result: + result = fstat(fd) + if result.st_ino == parent.st_ino and result.st_dev == parent.st_dev: + values = list(result) + values[4] = overflow + return os.stat_result(values) + return result + + with ( + mock.patch.object(archive.os, "geteuid", return_value=overflow), + mock.patch.object(archive.os, "fstat", side_effect=attributes), + self.assertRaises(SystemExit), + ): + archive.export(["--store", str(self.store), str(destination)]) + self.assertFalse(destination.exists()) + + def test_database_removal_resolves_intermediate_symlinks(self) -> None: + # A database can be reached through a relative or absolute directory symlink. Direct + # terminal export and tmpfiles finalization must remove it inside the image in either case. + source = self.scratch / "source" + (source / "var/lib").rename(source / "relocated") + for index, target in enumerate(("../relocated", "/relocated")): + (source / "var/lib").unlink(missing_ok=True) + (source / "var/lib").symlink_to(target) + store = self.scratch / f"store-{index}" + storage.capture(storage.Root(source), store, epoch=CAPTURE_EPOCH) + for rules in ([], ["d /empty-dir - - -"]): + with self.subTest(target=target, rules=rules): + spec = finalize.ImageSpec(lower=[str(store)], tmpfiles=rules) + with finalize.snapshot( + spec, program="test", epoch=EPOCH, pkgdb_paths=["var/lib/database"] + ) as snapshot: + names = {obj["name"] for obj in snapshot.objects[1:]} + self.assertNotIn("relocated/database", names) + self.assertNotIn("relocated/database/contents", names) + self.assertIn("var/lib", names) + + def test_database_removal_updates_the_root_timestamp(self) -> None: + # Removing the last var subtree changes the root mtime. The direct snapshot path must + # match filesystem removal when an existing directory has a no-op tmpfiles rule. + source = self.scratch / "source" + for present in (True, False): + if not present: + shutil.rmtree(source / "var/lib/database") + os.utime(source, ns=(111, 111)) + storage.capture(storage.Root(source), self.store, epoch=CAPTURE_EPOCH) + for rules in ([], ["d /empty-dir - - -"]): + with self.subTest(present=present, rules=rules): + spec = finalize.ImageSpec(lower=self.spec["lower"], tmpfiles=rules) + with finalize.snapshot( + spec, program="test", epoch=EPOCH, pkgdb_paths=["var/lib/database"] + ) as snapshot: + self.assertEqual(snapshot.objects[0]["mTimeNSec"], EPOCH * 10**9) + self.assertNotIn("var", {obj["name"] for obj in snapshot.objects[1:]}) + + def test_missing_database_does_not_require_reconstruction(self) -> None: + # A cached snapshot can contain a device even when the current worker cannot create it. + # An absent database without symlink parents must still be archivable directly from blobs. + objects = uapi16.read(self.store / uapi16.MANIFEST) + objects.append( + {"name": "device", "type": "chr", "mode": 0o600, "mTimeNSec": 111, "major": 1, "minor": 3} + ) + uapi16.write_objects(objects, self.store / uapi16.MANIFEST) + out = self.scratch / "image.tar" + with mock.patch.object(storage, "root", side_effect=AssertionError("unexpected reconstruction")): + with finalize.snapshot( + self.spec, program="test", epoch=EPOCH, pkgdb_paths=["absent/database"] + ) as snapshot: + archive._archive(snapshot, out, "tar", EPOCH, "none") + with tarfile.open(out) as packed: + self.assertTrue(packed.getmember("./device").ischr()) + + def test_missing_database_leaves_nonempty_parent_times_unchanged(self) -> None: + # A missing database under a nonempty directory removes nothing. Its parent timestamps + # must remain unchanged rather than being set to the terminal epoch. + source = self.scratch / "source" + shutil.rmtree(source / "var/lib/database") + (source / "var/lib/keep").touch() + os.utime(source / "var/lib", ns=(333, 333)) + os.utime(source, ns=(111, 111)) + storage.capture(storage.Root(source), self.store, epoch=CAPTURE_EPOCH) + with finalize.snapshot( + self.spec, program="test", epoch=EPOCH, pkgdb_paths=["var/lib/database"] + ) as snapshot: + objects = {obj.get("name", "."): obj for obj in snapshot.objects} + self.assertEqual(objects["."]["mTimeNSec"], 111) + self.assertEqual(objects["var/lib"]["mTimeNSec"], 333) + + def test_cpio_listing_uses_terminal_timestamp_precision(self) -> None: + # Cpio finalization must round the listing to whole seconds and omit user attributes, + # while keeping the explicit hardlink group used by the archive. + out, listing = self.scratch / "image.cpio", self.scratch / "listing" + spec = specs.write( + self.scratch / "spec.json", + dict(self.spec) + | { + "out": str(out), + "manifest": str(listing), + "format": "cpio", + "compression": "none", + "pkgdb_paths": [], + }, + ) + with mock.patch.dict(os.environ, {"SOURCE_DATE_EPOCH": str(EPOCH)}): + archive.main(["--spec", str(spec)]) + objects = {obj.get("name", "."): obj for obj in uapi16.read(listing)} + self.assertEqual(objects["a"]["mTimeNSec"], 0) + self.assertNotIn("extendedAttributes", objects["a"]) + self.assertEqual(objects["a"]["inodeToken"], objects["linked"]["inodeToken"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/image/compose.bzl b/image/compose.bzl index 4713ed92..64d7923c 100644 --- a/image/compose.bzl +++ b/image/compose.bzl @@ -15,6 +15,7 @@ load( "ImageArchiveInfo", "declare_image_archive", "declare_image_directory", + "image_directory_providers", ) load( ":boot.bzl", @@ -152,6 +153,7 @@ def _initrd_image_impl(ctx: AnalysisContext) -> list[Provider]: return image_providers( default_outputs = [cpio.archive], extra = [InitrdInfo(cpio = cpio, image = image), cpio], + sub_targets = {"manifest": [DefaultInfo(default_output = cpio.manifest)]}, image = image, ) @@ -236,6 +238,7 @@ def _archive_impl(ctx: AnalysisContext) -> list[Provider]: return image_providers( default_outputs = [archive.archive], extra = [archive], + sub_targets = {"manifest": [DefaultInfo(default_output = archive.manifest)]}, image = image, ) @@ -484,7 +487,8 @@ def _disk_impl(ctx: AnalysisContext) -> list[Provider]: "initrd": [ DefaultInfo( default_output = initrd_info.cpio.archive, - sub_targets = published_metadata_subtargets(initrd_info.image, "{}.initrd".format(basename), version), + sub_targets = published_metadata_subtargets(initrd_info.image, "{}.initrd".format(basename), version) + | {"manifest": [DefaultInfo(default_output = initrd_info.cpio.manifest)]}, ), initrd_info, ], @@ -539,7 +543,7 @@ def _disk_impl(ctx: AnalysisContext) -> list[Provider]: sub_targets = dict(composed.sub_targets) sub_targets.update(manifest_sub_targets(manifests)) sub_targets.update(boot_sub_targets) - sub_targets["directory"] = [DefaultInfo(default_output = directory.directory), directory] + sub_targets["directory"] = image_directory_providers(directory) if bootable: sub_targets["ami"] = declare_ami( ctx.actions, @@ -594,7 +598,7 @@ def _disk_impl(ctx: AnalysisContext) -> list[Provider]: default_outputs = [raw], extra = [directory, composed.info, published] + boot_providers, image = top, - sub_targets = sub_targets | published_metadata_subtargets(top, basename, version), + sub_targets = published_metadata_subtargets(top, basename, version) | sub_targets, ) _DISK_ATTRS = ( diff --git a/image/ddi.bzl b/image/ddi.bzl index e56bbb49..17b4a5c0 100644 --- a/image/ddi.bzl +++ b/image/ddi.bzl @@ -439,10 +439,8 @@ def declare_ddi( ) sub_targets = manifest_sub_targets(manifests) if manifest != None: - # Qualified, unlike the provider field: a logical image publishes a `manifest` subtarget of - # its own (image.bzl:image_metadata_subtargets) listing its whole tree, and a composition - # merges both sets of subtargets into one namespace. sub_targets["disk-manifest"] = [DefaultInfo(default_output = manifest)] + sub_targets["manifest"] = [DefaultInfo(default_output = manifest)] if root_hash != None: sub_targets["roothash"] = [DefaultInfo(default_output = root_hash.hash), info] diff --git a/image/ddi.py b/image/ddi.py index 244ff3ba..15da8294 100755 --- a/image/ddi.py +++ b/image/ddi.py @@ -21,8 +21,8 @@ import util import finalize -import manifest -import rootfs +import storage +import uapi16 _SEED_NAMESPACE = uuid.UUID("5af2de99-4f9f-4e0b-a04b-bde36b068c4f") @@ -260,14 +260,15 @@ def _extension_seed(identity: str, name: str, releases: list[Release]) -> uuid.U return uuid.uuid5(_EXTENSION_SEED_NAMESPACE, digest.hexdigest()) -def _os_release(tree: Path) -> dict[str, str]: +def _os_release(snapshot: storage.Snapshot) -> dict[str, str]: """Parse the parent image's os-release for strict extension matching.""" + objects = storage.by_name(snapshot.objects) for candidate in ("usr/lib/os-release", "etc/os-release"): - path = tree / candidate - if not path.is_file(): + obj = objects.get(storage.resolve(objects, candidate)) + if obj is None or obj["type"] != "reg": continue fields = {} - for line in path.read_text().splitlines(): + for line in snapshot.content(obj).read_text().splitlines(): key, sep, value = line.strip().partition("=") if key and sep and not key.startswith("#"): fields[key] = value.strip("\"'") @@ -290,25 +291,18 @@ def pin_release(parent: dict[str, str], release: dict[str, str]) -> dict[str, st return strict | release -def _extension_layers(spec: Spec, extension: ExtensionSpec) -> tuple[list[Release], list[str | Path]]: - """Return the extension-release files and the layers the extension packages. - - With a parent, the fields are pinned to the parent's os-release and the layers are the delta - above it. Without a parent, the function returns the releases and layers of the spec unchanged. - """ +def _extension_releases(spec: Spec, extension: ExtensionSpec) -> list[Release]: releases = extension["releases"] parent = extension["parent"] - lowers: list[str | Path] = list(spec["lower"]) if not parent: - return releases, lowers - with rootfs.rootfs("/buildroot", lowers=lowers) as tree: - fields = _os_release(tree) + return releases + fields = _os_release(storage.Snapshot(spec["lower"][:parent])) if "ID" not in fields: util.fail("ddi: the parent os-release lacks ID") pinned = [ Release(path=release["path"], fields=pin_release(fields, release["fields"])) for release in releases ] - return pinned, lowers[parent:] + return pinned def empty_directories(tree: Path, directories: list[str]) -> None: @@ -472,12 +466,16 @@ def main(argv: list[str] | None = None) -> None: for index, partition in enumerate(partitions, start=len(written)) ] extension = spec["extension"] - releases: list[Release] = [] - lowers = None - if extension: - releases, lowers = _extension_layers(spec, extension) + releases = _extension_releases(spec, extension) if extension else [] with ( - finalize.image(spec, program="ddi", lowers=lowers, binds=binds) as tree, + finalize.image( + spec, + program="ddi", + binds=binds, + base=extension["parent"] if extension else 0, + readonly=False, + strategy="copy", + ) as tree, tempfile.TemporaryDirectory(prefix="repart.") as scratch_dir, ): if extension: @@ -511,7 +509,7 @@ def main(argv: list[str] | None = None) -> None: listings = [] for partition in written: if partition.manifest is not None: - manifest.write(tree, partition.manifest, epoch, roots=partition.definition.sources) + uapi16.write(tree, partition.manifest, epoch, roots=partition.definition.sources) listings.append(partition.manifest) # An imported partition was listed by the invocation that filled it, and this one holds @@ -519,7 +517,7 @@ def main(argv: list[str] | None = None) -> None: # partition this invocation mounts over a directory another one holds is the one on top. if spec["manifest"]: imported = [it.manifest for it in partitions if it.manifest is not None] - manifest.merge(imported + listings, Path(spec["manifest"])) + uapi16.merge(imported + listings, Path(spec["manifest"])) scratch = Path(scratch_dir) repart_definitions = scratch / "repart.d" diff --git a/image/extension.bzl b/image/extension.bzl index 146995c5..db3415ab 100644 --- a/image/extension.bzl +++ b/image/extension.bzl @@ -165,7 +165,7 @@ def _declare_preset( default_outputs = [info.disk], extra = [info, published] + extra, image = image, - sub_targets = output.sub_targets | published_metadata_subtargets(image, basename, version, filename), + sub_targets = published_metadata_subtargets(image, basename, version, filename) | output.sub_targets, ) def declare_extension( diff --git a/image/finalize.py b/image/finalize.py index a27e6015..168104c1 100644 --- a/image/finalize.py +++ b/image/finalize.py @@ -25,6 +25,8 @@ from util import fail import rootfs +import storage +import uapi16 class ImageSpec(TypedDict): @@ -34,40 +36,51 @@ class ImageSpec(TypedDict): tmpfiles: list[str] -# The directories whose whole point is a mode, so a layer cannot carry them: sticky and -# world-writable, which is exactly what Buck cannot store and the tar filter strips. _WORLD_WRITABLE = ("tmp", "var/tmp") -def world_writable(tree: Path) -> None: - """Restore the mode on the temporary directories, which nothing upstream of here can keep. - - Buck stores no mode beyond the executable bit, so a package's `0o1777` is gone by the time a - layer is a build artifact however faithfully it was extracted. Here is the last point before - the image is written, and the only one where the mode survives into it. - """ - for name in _WORLD_WRITABLE: - path = tree / name - if path.is_dir() and not path.is_symlink(): - path.chmod(0o1777) - - @contextlib.contextmanager def image( spec: ImageSpec, *, program: str, - lowers: Sequence[str | Path] | None = None, binds: Sequence[tuple[str | Path, str | Path]] | None = None, + base: int = 0, + readonly: bool = True, + strategy: storage.Strategy = "auto", ) -> Generator[Path]: """Mount the image stack the spec names and yield the finalized tree.""" - with rootfs.rootfs( - "/buildroot", lowers=spec["lower"] if lowers is None else lowers, binds=binds - ) as tree: - # Before the snippets, so an image that states a mode of its own still wins. - world_writable(tree) + with _root( + spec, program=program, binds=binds, base=base, readonly=readonly, strategy=strategy + ) as restored: + yield restored.path + + +@contextlib.contextmanager +def _root( + spec: ImageSpec, + *, + program: str, + binds: Sequence[tuple[str | Path, str | Path]] | None = None, + base: int = 0, + readonly: bool = True, + strategy: storage.Strategy = "auto", +) -> Generator[storage.Root]: + snapshot = storage.Snapshot(spec["lower"]) + if base: + storage.delta(snapshot, base) + normalize_snapshot(snapshot) + with ( + storage.root( + snapshot, + # Authored tmpfiles rules write into the root. They fail on a read-only root. + readonly=readonly and not spec["tmpfiles"], + strategy=strategy, + ) as restored, + rootfs.rootfs("/buildroot", bind=restored.path, binds=binds) as tree, + ): apply_tmpfiles(tree, spec["tmpfiles"], program=program) - yield tree + yield storage.Root(tree, restored.upper) def apply_tmpfiles(tree: Path, snippets: list[str], *, program: str) -> None: @@ -355,3 +368,65 @@ def locale_gen(tree: Path) -> None: # it wrote nothing. Only 0 means that the archive contains the locale. if rc != 0: fail(f"locale_gen: `{' '.join(cmd)}` failed (rc={rc})") + + +def normalize_snapshot(snapshot: storage.Snapshot) -> None: + for obj in snapshot.objects: + if obj["type"] == "dir": + obj["mode"] = 0o1777 if obj.get("name") in _WORLD_WRITABLE else 0o755 + elif obj["type"] == "reg": + obj["mode"] = 0o755 if obj["mode"] & 0o111 else 0o644 + + +def _remove(objects: dict[str, uapi16.Object], relative: str, epoch: int) -> None: + path = Path(relative) + name = storage.resolve(objects, relative, follow=False) + removed = False + for child in list(objects): + if child == name or child.startswith(name + "/"): + del objects[child] + removed = True + while True: + path = path.parent + name = storage.resolve(objects, str(path)) + directory = objects.get(name) + if directory is None or directory["type"] != "dir": + return + if removed: + # Removing a file from a real tree sets the mtime of its directory to the current time. + # snapshot() clamps that time to the epoch. + directory["mTimeNSec"] = epoch * 10**9 + # remove_path() stops at a parent that is a symlink. + if name == "." or objects[storage.resolve(objects, str(path), follow=False)]["type"] == "lnk": + return + if any(child.startswith(name + "/") for child in objects): + return + del objects[name] + removed = True + + +@contextlib.contextmanager +def snapshot( + spec: ImageSpec, + *, + program: str, + epoch: int, + pkgdb_paths: Sequence[str] = (), +) -> Generator[storage.Snapshot]: + with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: + if spec["tmpfiles"]: + out = Path(scratch) / "snapshot" + with _root(spec, program=program, readonly=False) as restored: + storage.capture(restored, out, spec["lower"], epoch=epoch) + result = storage.Snapshot([*spec["lower"], out]) + else: + result = storage.Snapshot(spec["lower"]) + normalize_snapshot(result) + objects = storage.by_name(result.objects) + for relative in pkgdb_paths: + _remove(objects, relative, epoch) + result.objects = list(objects.values()) + uapi16.renumber(result.objects) + for obj in result.objects: + obj["mTimeNSec"] = min(obj.get("mTimeNSec", epoch * 10**9), epoch * 10**9) + yield result diff --git a/image/finalize_test.py b/image/finalize_test.py index e24d7271..18740743 100644 --- a/image/finalize_test.py +++ b/image/finalize_test.py @@ -224,27 +224,3 @@ def test_writes_into_the_archive_of_the_image(self) -> None: "en_US.UTF-8", ], ) - - -class TestWorldWritable(unittest.TestCase): - @override - def setUp(self) -> None: - self.tree = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="finalize."))) - - def test_restores_the_mode_a_layer_could_not_carry(self) -> None: - for name in ("tmp", "var/tmp"): - (self.tree / name).mkdir(parents=True, mode=0o755) - finalize.world_writable(self.tree) - for name in ("tmp", "var/tmp"): - self.assertEqual((self.tree / name).stat().st_mode & 0o7777, 0o1777) - - def test_leaves_an_image_that_ships_neither_alone(self) -> None: - finalize.world_writable(self.tree) - self.assertFalse((self.tree / "tmp").exists()) - - def test_does_not_chmod_through_a_symlink(self) -> None: - (self.tree / "var").mkdir() - (self.tree / "elsewhere").mkdir(mode=0o755) - (self.tree / "var/tmp").symlink_to("../elsewhere") - finalize.world_writable(self.tree) - self.assertEqual((self.tree / "elsewhere").stat().st_mode & 0o7777, 0o755) diff --git a/image/image.bzl b/image/image.bzl index d625d4e5..cfe57182 100644 --- a/image/image.bzl +++ b/image/image.bzl @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""Logical filesystem images built as ordered overlay deltas.""" +"""Logical filesystem images stored as complete snapshots and immutable blobs.""" load("//:specs.bzl", "specs") load("//box:defs.bzl", "box") @@ -120,7 +120,7 @@ ImageSbomInfo = provider( ) ImageInfo = provider( - doc = "A logical filesystem image represented by an ordered delta stack and lazy metadata.", + doc = "A logical filesystem image represented by snapshot stores and lazy metadata.", fields = { "box": provider_field(Dependency), # Accumulated install specs; they seed the local-packages closure of every derived layer @@ -645,14 +645,12 @@ def declare_image( ) package_manager_info = package_manager[PackageManagerInfo] - delta = declare_out(ctx, identifier, "delta", dir = True) - work = declare_out(ctx, identifier, "overlay.work", dir = True) if layers else None + delta = declare_out(ctx, identifier, "snapshot", dir = True) spec = { "install": None, "lower": layers, "operations": [_encode_operation(operation, tools.python) for operation in operations], "out": delta.as_output(), - "work": work.as_output() if work != None else None, } if package_manager_info != None: system = package_manager_info.package_system[PackageSystemInfo] diff --git a/image/layer.py b/image/layer.py index f9408036..512ed515 100755 --- a/image/layer.py +++ b/image/layer.py @@ -3,7 +3,7 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""Apply ordered operations against one mounted root and capture its overlay delta.""" +"""Apply ordered operations against one mounted root and capture its complete snapshot.""" import glob # noqa: F401 # Preload for Path.glob before entering the image chroot. import os @@ -25,6 +25,7 @@ import finalize import installer import rootfs +import storage from isolation import AT_FDCWD, Bind, Close, open_directory @@ -41,7 +42,6 @@ class LayerInstall(TypedDict): class Spec(TypedDict): lower: list[str] out: str - work: str | None install: LayerInstall | None operations: list[object] @@ -149,25 +149,6 @@ def _remove_glob(tree: Path, value: str) -> None: util.remove_path(path) -def _normalize_mode(path: Path) -> None: - """Normalize file permissions for path. - - See docs/user/images.md. - """ - mode = path.lstat().st_mode - if stat.S_ISDIR(mode): - path.chmod(0o755) - elif stat.S_ISREG(mode): - path.chmod(0o755 if mode & 0o111 else 0o644) - # ignore symlinks (meaningless mode), whiteout nodes, and non-files - - -def _normalize_modes(tree: Path) -> None: - _normalize_mode(tree) - for path in tree.rglob("*"): - _normalize_mode(path) - - # The ELF fields the stack check reads; see elf(5). Per class: the header after e_ident up to # e_phnum, and the offset of p_flags in a program header. Per data encoding: the byte order. _ELF_CLASSES = {1: ("HHIIIIIHHH", 24), 2: ("HHIQQQIHHH", 4)} @@ -257,10 +238,10 @@ def _install(install: LayerInstall, target: Path, scratch: Path) -> None: request = installer.InstallSpec( arch=install["arch"], packages_dir=str(Path(install["packages_dir"]).absolute()), - target=None, + snapshot=None, installroot=str(target), lower=[], - work=None, + target=None, box_config=False, langs=install["langs"], docs=install["docs"], @@ -311,37 +292,25 @@ def main(argv: list[str] | None = None) -> None: install = spec["install"] lower = spec["lower"] - if lower: - if spec["work"] is None: - util.fail("image lower stack needs a work overlay directory") - mounted = rootfs.rootfs( - "/buildroot", - lowers=lower, - upperdir=out, - workdir=spec["work"], - apivfs=True, - ) - else: - if spec["work"] is not None: - util.fail("image work overlay directory requires a lower stack") - mounted = rootfs.rootfs( - "/buildroot", - bind=out, - capture_bind=True, - apivfs=True, - ) - - with mounted as target, tempfile.TemporaryDirectory(prefix="layer.") as scratch: - # Packages land before the operations so every one of them sees what this layer installs. - if install is not None: - _install(install, target, Path(scratch)) - for operation in operations: - _apply(operation, target) - _normalize_modes(out) - stacks = _executable_stacks(out) - if stacks: - util.fail("image: executable stacks are not allowed, see docs/user/images.md:\n" + "\n".join(stacks)) - print(f"image: applied {len(operations)} ops over {len(lower)} lower(s) -> {out}", file=sys.stderr) + with ( + storage.root(storage.Snapshot(lower)) as restored, + tempfile.TemporaryDirectory(prefix="layer.", dir="/var/tmp") as scratch, + ): + tree = restored.path + with rootfs.rootfs("/buildroot", bind=tree, apivfs=True, strict_unmount=True) as target: + if install is not None: + _install(install, target, Path(scratch)) + for operation in operations: + _apply(operation, target) + stacks = _executable_stacks(tree) + if stacks: + util.fail( + "image: executable stacks are not allowed, see docs/user/images.md:\n" + "\n".join(stacks) + ) + storage.capture(restored, out, lower, epoch=int(os.environ["SOURCE_DATE_EPOCH"])) + print( + f"image: applied {len(operations)} ops over {len(lower)} parent store(s) -> {out}", file=sys.stderr + ) if __name__ == "__main__": diff --git a/image/layer_test.py b/image/layer_test.py index 8cae59e8..3dd7598b 100644 --- a/image/layer_test.py +++ b/image/layer_test.py @@ -8,6 +8,8 @@ import errno import os +import shutil +import stat import struct import tempfile import unittest @@ -18,8 +20,15 @@ import specs +import finalize import layer import rootfs +import storage +import uapi16 +from isolation import umount2 + +# Captures in these tests keep every timestamp, so they clamp to an epoch after all of them. +CAPTURE_EPOCH = 2**31 class TestImageWrites(unittest.TestCase): @@ -33,6 +42,8 @@ def setUp(self) -> None: self.out = self.scratch / "out" self.out.mkdir() (self.project / "source").write_text("copied", encoding="utf-8") + # The sandbox sets SOURCE_DATE_EPOCH for every action, and layer.main() clamps times to it. + self.enterContext(mock.patch.dict(os.environ, {"SOURCE_DATE_EPOCH": str(2**31)})) def _build(self, operations: list[object], lower: Path | None = None) -> None: spec = specs.write( @@ -40,7 +51,6 @@ def _build(self, operations: list[object], lower: Path | None = None) -> None: { "out": str(self.out), "lower": [str(lower)] if lower is not None else [], - "work": str(self.scratch / "work") if lower is not None else None, "install": None, "operations": operations, }, @@ -59,7 +69,7 @@ def test_absolute_symlinks_resolve_inside_the_image(self) -> None: with ( chdir(self.project), - rootfs.rootfs(self.scratch / "target", bind=self.out, capture_bind=True) as target, + rootfs.rootfs(self.scratch / "target", bind=self.out) as target, ): layer._apply(["copy", "source", "/alias/file"], target) layer._apply(["os_release", {"ID": "test"}], target) @@ -74,10 +84,13 @@ def test_a_copy_source_must_stay_relative_to_the_project(self) -> None: layer._apply(["copy", source, "/destination"], self.out) def test_a_derived_layer_exposes_a_readonly_project_only_to_copy_and_run(self) -> None: + # Copy and run must see a read-only project, while removals affect only the image snapshot. lower = self.scratch / "lower" lower.mkdir() (lower / "cache.pyc").write_text("image cache", encoding="utf-8") (self.project / "cache.pyc").write_text("cache", encoding="utf-8") + stored = self.scratch / "stored" + storage.capture(storage.Root(lower), stored, epoch=CAPTURE_EPOCH) def run(tag: str, cmd: object, env: object, cwd: str | None = None) -> None: self.assertEqual(cwd, layer.PROJECT) @@ -94,14 +107,88 @@ def run(tag: str, cmd: object, env: object, cwd: str | None = None) -> None: ["run", ["command"], {}, True], ["remove", "/**/*.pyc"], ], - lower, + stored, ) command.assert_called_once() - self.assertEqual((self.out / "copied").read_text(encoding="utf-8"), "copied") - self.assertTrue((self.out / ".wh.cache.pyc").is_file()) + with storage.root(storage.Snapshot([stored, self.out])) as restored: + tree = restored.path + self.assertEqual((tree / "copied").read_text(encoding="utf-8"), "copied") + self.assertFalse((tree / "cache.pyc").exists()) self.assertTrue((self.project / "cache.pyc").exists()) + def test_commands_see_inherited_metadata_and_can_override_it(self) -> None: + # Capture before artifact copying, then check a command sees the original metadata and + # explicit hardlinks. Its metadata changes must become authoritative in the child snapshot. + source = self.scratch / "source-root" + source.mkdir() + (source / "file").write_text("contents") + (source / "file").chmod(0o640) + os.utime(source / "file", ns=(123, 123)) + os.link(source / "file", source / "linked") + parent = self.scratch / "parent" + storage.capture(storage.Root(source), parent, epoch=CAPTURE_EPOCH) + copied = self.scratch / "copied-parent" + shutil.copytree(parent, copied, copy_function=shutil.copyfile) + + def command(*args: object) -> None: + path = Path("/buildroot/file") + self.assertEqual(path.stat().st_mtime_ns, 123) + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o640) + self.assertTrue(path.samefile("/buildroot/linked")) + path.chmod(0o600) + os.utime(path, ns=(456, 456)) + + with mock.patch.object(layer, "_run", side_effect=command): + self._build([["run", ["command"], {}, False]], copied) + with storage.root(storage.Snapshot([copied, self.out])) as restored: + tree = restored.path + self.assertEqual(stat.S_IMODE((tree / "file").stat().st_mode), 0o600) + self.assertEqual((tree / "linked").stat().st_mtime_ns, 456) + self.assertEqual(list((self.out / "blobs").iterdir()), []) + + def test_a_live_writer_prevents_capture(self) -> None: + # A command retains an open image file after it returns. Strict unmount must fail before + # capture can publish contents that the retained descriptor could still change. + descriptor = -1 + + def command(*args: object) -> None: + nonlocal descriptor + descriptor = os.open("/buildroot/file", os.O_WRONLY | os.O_CREAT, 0o644) + + try: + with mock.patch.object(layer, "_run", side_effect=command), self.assertRaises(OSError) as error: + self._build([["run", ["command"], {}, False]]) + self.assertEqual(error.exception.errno, errno.EBUSY) + self.assertFalse((self.out / uapi16.MANIFEST).exists()) + finally: + if descriptor >= 0: + os.close(descriptor) + if Path("/buildroot").is_mount(): + umount2("/buildroot", 0) + + def test_commands_create_hardlinks_to_inherited_files(self) -> None: + # A command links an inherited independent file and changes its data and mode. Capture + # must use upper inode identity so the next image keeps the new aliases hardlinked. + source = self.scratch / "source-root" + source.mkdir() + (source / "file").write_text("contents") + parent = self.scratch / "parent" + storage.capture(storage.Root(source), parent, epoch=CAPTURE_EPOCH) + + def command(*args: object) -> None: + os.link("/buildroot/file", "/buildroot/linked") + Path("/buildroot/file").write_text("changed") + Path("/buildroot/linked").chmod(0o600) + + with mock.patch.object(layer, "_run", side_effect=command): + self._build([["run", ["command"], {}, False]], parent) + with storage.root(storage.Snapshot([parent, self.out])) as restored: + tree = restored.path + self.assertTrue((tree / "file").samefile(tree / "linked")) + self.assertEqual((tree / "linked").read_text(), "changed") + self.assertEqual(stat.S_IMODE((tree / "file").stat().st_mode), 0o600) + class TestRemove(unittest.TestCase): def test_plain_path_is_also_a_glob(self) -> None: @@ -159,9 +246,11 @@ def test_pattern_must_be_an_absolute_image_path(self) -> None: class TestNormalization(unittest.TestCase): def test_modes(self) -> None: + # Terminal policy normalizes regular files and directories without following links or + # changing special-file modes. Authored tmpfiles rules run after this normalization. with tempfile.TemporaryDirectory(prefix="layer-test.", dir="/var/tmp") as scratch: - tree = Path(scratch) - tree.chmod(0o700) + tree = Path(scratch) / "source" + tree.mkdir(mode=0o700) (tree / "sub").mkdir() (tree / "sub").chmod(0o1777) (tree / "sub/file").write_text("file") @@ -175,7 +264,12 @@ def test_modes(self) -> None: (tree / "link").symlink_to("sub/secret") os.mkfifo(tree / "fifo", 0o600) - layer._normalize_modes(tree) + store = Path(scratch) / "store" + storage.capture(storage.Root(tree), store, epoch=CAPTURE_EPOCH) + snapshot = storage.Snapshot([store]) + finalize.normalize_snapshot(snapshot) + tree = Path(scratch) / "normalized" + storage.restore(snapshot, tree) self.assertEqual(tree.stat().st_mode & 0o7777, 0o755) self.assertEqual((tree / "sub").stat().st_mode & 0o7777, 0o755) @@ -184,7 +278,6 @@ def test_modes(self) -> None: self.assertEqual((tree / "sub/script").stat().st_mode & 0o7777, 0o755) self.assertEqual((tree / "sub/mount").stat().st_mode & 0o7777, 0o755) self.assertTrue((tree / "link").is_symlink()) - # not content: whatever it is, it is not ours to rewrite self.assertEqual((tree / "fifo").lstat().st_mode & 0o7777, 0o600) diff --git a/image/manifest_main.py b/image/manifest_main.py new file mode 100755 index 00000000..6057e7f3 --- /dev/null +++ b/image/manifest_main.py @@ -0,0 +1,33 @@ +#!/usr/bin/python3 + +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Export the finalized image's UAPI.16 listing.""" + +import os +import sys +from pathlib import Path + +import specs + +import finalize +import uapi16 + + +class Spec(finalize.ImageSpec): + out: str + + +def main(argv: list[str] | None = None) -> None: + spec = specs.parse(Spec, "manifest", argv) + + out = Path(spec["out"]) + epoch = int(os.environ["SOURCE_DATE_EPOCH"]) + with finalize.snapshot(spec, program="manifest", epoch=epoch) as snapshot: + written = uapi16.write_listing(snapshot.objects, out) + print(f"manifest: wrote {written} objects -> {out}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/image/uki.py b/image/uki.py index 4bbb1550..0eceecb6 100755 --- a/image/uki.py +++ b/image/uki.py @@ -199,7 +199,7 @@ def _microcode(tree: Path, scratch: Path, epoch: int, efi_arch: str) -> Path | N for path in root.rglob("*"): path.chmod(0o755 if path.is_dir() else 0o644) archive = scratch / "microcode.cpio" - cpio.pack_tree(root, archive, epoch, block_align=False) + cpio.pack_tree(root, archive, epoch) print(f"uki: microcode for {', '.join(packed)} in {archive.stat().st_size // 1024} KiB", file=sys.stderr) return archive diff --git a/package/BUCK b/package/BUCK index b936ac1e..8bd1f280 100644 --- a/package/BUCK +++ b/package/BUCK @@ -16,7 +16,7 @@ python.library( name = "builder", srcs = ["builder.py"], visibility = ["PUBLIC"], - deps = ["//:util", "//rootfs:rootfs"], + deps = ["//:util", "//rootfs:rootfs", "//uapi16:storage"], ) python.library( @@ -29,7 +29,7 @@ python.library( python.library( name = "installer", srcs = ["installer.py"], - deps = ["//:specs", "//:util", "//rootfs:rootfs"], + deps = ["//:specs", "//:util", "//rootfs:rootfs", "//uapi16:storage"], visibility = ["PUBLIC"], ) diff --git a/package/build.bzl b/package/build.bzl index 8408778c..15e93af9 100644 --- a/package/build.bzl +++ b/package/build.bzl @@ -74,7 +74,7 @@ def package_build( category: str, spec: dict[str, typing.Any], ) -> list[Provider]: - """Run the `build` driver of `package_system` on the layer stack `stack` that `base` starts. + """Run the `build` driver of `package_system` on the snapshot stores `stack` that `base` starts. `spec` contains the fields of the driver spec that the calling rule owns. `packages` names the output directory of the packages. A dev build keeps its build directory between runs if `keep_build_dir` is diff --git a/package/builder.py b/package/builder.py index b16320b9..4467600d 100644 --- a/package/builder.py +++ b/package/builder.py @@ -9,14 +9,15 @@ import shutil import tempfile -from collections.abc import Callable -from contextlib import AbstractContextManager, nullcontext +from collections.abc import Callable, Generator +from contextlib import contextmanager, nullcontext from pathlib import Path from typing import TypedDict import util import rootfs +import storage # The source tree is mounted at topdir/CHECKOUT, and the persistent build directory at topdir/BUILD. CHECKOUT = "CHECKOUT" @@ -27,7 +28,7 @@ class Spec(TypedDict): # `build_dir` is a build directory that Buck keeps between runs, or None for a clean build. build_dir: str | None - # `lower` lists the layers of the buildroot, lowest first. + # `lower` lists the snapshot stores of the buildroot, lowest first. lower: list[str] out: str # `source_tree` is a prepared source tree to build in place, or None to build from the declared sources. @@ -76,8 +77,13 @@ def run(spec: Spec, build: Callable[[Path, Path, Path], int]) -> int: return rc -def buildroot(lower: list[str], topdir: Path) -> AbstractContextManager[Path]: - """Chroot into the buildroot with the layers `lower`, with topdir at `CHROOT_TOPDIR`.""" - return rootfs.rootfs( - "/buildroot", lowers=lower, binds=[(topdir, CHROOT_TOPDIR)], apivfs=True, chroot=True - ) +@contextmanager +def buildroot(lower: list[str], topdir: Path) -> Generator[Path]: + """Chroot into the buildroot that the snapshot stores `lower` restore, with topdir at `CHROOT_TOPDIR`.""" + with ( + storage.root(storage.Snapshot(lower)) as restored, + rootfs.rootfs( + "/buildroot", bind=restored.path, binds=[(topdir, CHROOT_TOPDIR)], apivfs=True, chroot=True + ) as root, + ): + yield root diff --git a/package/buildroot.bzl b/package/buildroot.bzl index c7132e6a..6a0ecfeb 100644 --- a/package/buildroot.bzl +++ b/package/buildroot.bzl @@ -86,7 +86,7 @@ def package_buildroot( build_requires: list[str], buildroot_packages: list[Dependency], ) -> list[Artifact]: - """Return the layer stack, bottom to top, that one package builds on. + """Return the snapshot stores, bottom to top, that one package builds on. The stack is the shared base root, plus a delta that installs `build_requires`. The solve of the delta prefers the packages of `buildroot_packages` over the repositories of the package manager. diff --git a/package/install.bzl b/package/install.bzl index 1dd655f4..b8089e39 100644 --- a/package/install.bzl +++ b/package/install.bzl @@ -21,7 +21,7 @@ load(":system.bzl", "PackageSystemInfo") _EXTRA_REPO_PRIORITY = 50 _RootInfo = provider( - doc = "The installed root tree carried out of an anonymous target.", + doc = "The snapshot store of an installed root, carried out of an anonymous target.", fields = {"root": provider_field(Artifact)}, ) @@ -117,8 +117,7 @@ def _install_actions( system = package_manager.package_system[PackageSystemInfo] closure = resolve_packages(ctx, package_manager_dep, box_dep, install, stack, extra_packages, identifier = identifier, install_files = install_files) prefix = identifier + "/" if identifier != None else "" - out = ctx.actions.declare_output(prefix + ("install.delta" if stack else "root"), dir = True) - work = ctx.actions.declare_output(prefix + "install.work", dir = True) if stack else None + out = ctx.actions.declare_output(prefix + ("install.snapshot" if stack else "root.snapshot"), dir = True) cmd = cmd_args( box.run(box = box_dep[box.Info], exe = system.install), specs.args( @@ -132,8 +131,8 @@ def _install_actions( "langs": [], "lower": stack, "packages_dir": closure, - "target": out.as_output(), - "work": work.as_output() if work != None else None, + "snapshot": out.as_output(), + "target": None, }, ), ) @@ -179,5 +178,5 @@ def install_packages( }, ).artifact("root") # A promise is resolved before the anon target is analysed, so its path kind is asserted here. - return ctx.actions.assert_short_path(ctx.actions.assert_has_content_based_path(root), short_path = "root") + return ctx.actions.assert_short_path(ctx.actions.assert_has_content_based_path(root), short_path = "root.snapshot") return _install_actions(ctx, package_manager, box_dep, install, stack, extra_packages, install_files, identifier) diff --git a/package/installer.py b/package/installer.py index 7e02df5a..bbcc4d98 100644 --- a/package/installer.py +++ b/package/installer.py @@ -4,10 +4,9 @@ """Run one native package installation against the root its spec names. Every package system's installer answers the same request: a directory holding the exact package -closure, and either a fresh root to fill, a delta to persist over a lower stack, or a root the -caller has already mounted. Which of those it is, and how the root is mounted and captured, is -the same work whichever system fills it, so it happens here and the driver is left with the -transaction itself. +closure, and either a snapshot to capture over lower snapshots, or a root the caller has already +mounted. Every package system restores and captures the root the same way, so this module does it and +the driver only runs the transaction. """ import os @@ -21,6 +20,8 @@ from util import fail import rootfs +import storage +import uapi16 # A fixed install path avoids embedding Buck hashes and supports scriptlet chroots. BUILDROOT = "/buildroot" @@ -61,11 +62,13 @@ class InstallSpec(TypedDict): arch: str packages_dir: str - # Either an output root, bound at /buildroot to install into, or a root the caller mounted. - target: str | None + # The install captures the snapshot store `snapshot` over the snapshot stores in `lower`. An image layer + # sets `installroot` to the root that it mounted instead. + snapshot: str | None installroot: str | None lower: list[str] - work: str | None + # Only a box install sets `target`, to the directory that the sandbox mounts. + target: str | None box_config: bool langs: list[str] docs: bool @@ -137,13 +140,12 @@ def _normalize(installroot: Path, spec: InstallSpec) -> None: def run( prog: str, - install: Callable[[Path, Path, InstallSpec, bool], None], + install: Callable[[Path, Path, InstallSpec], None], argv: list[str] | None = None, ) -> None: """Mount the root this invocation names and hand it to `install`. - The callback receives the closure directory, the root to install into, the spec, and whether - it is layering over packages a lower stack already carries. + The callback receives the closure directory, the root to install into and the spec. """ spec = specs.parse(InstallSpec, prog, argv) os.environ.update( @@ -162,49 +164,38 @@ def run( # An installer resolves its own paths against the root, so give it no relative ones. packages_dir = Path(spec["packages_dir"]).absolute() - layered = bool(spec["lower"]) + lowers = spec["lower"] if spec["installroot"] is not None: - if spec["target"] is not None or layered or spec["work"] is not None: - fail(f"{prog}: installroot excludes target, lower, and work") + if spec["snapshot"] is not None or spec["target"] is not None or lowers: + fail(f"{prog}: installroot excludes snapshot, target and lower") installroot = Path(spec["installroot"]).absolute() - install(packages_dir, installroot, spec, layered) + install(packages_dir, installroot, spec) _normalize(installroot, spec) return - if spec["target"] is None: - fail(f"{prog}: one of target and installroot is required") - - # Bind sources must exist. - target = Path(spec["target"]) - target.mkdir(parents=True, exist_ok=True) - - if layered: - if spec["work"] is None: - fail(f"{prog}: lower needs a work overlay directory") - # The closure is empty when the lower layers contain every package of the layer, for example in - # the buildroot of a package whose spec needs nothing beyond the base root, or in a box layer - # that adds nothing to its parent. The layer then stays empty, and the package manager doesn't - # run. Buck still expects the work directory as an output. - if not any(packages_dir.iterdir()): - Path(spec["work"]).mkdir(parents=True, exist_ok=True) - return - root = rootfs.rootfs( - BUILDROOT, - lowers=spec["lower"], - upperdir=target, - workdir=spec["work"], - apivfs=True, - ) - else: - root = rootfs.rootfs(BUILDROOT, bind=target, capture_bind=True, apivfs=True) + if spec["snapshot"] is None: + fail(f"{prog}: one of snapshot and installroot is required") + out = Path(spec["snapshot"]) - with root: - install(packages_dir, Path(BUILDROOT), spec, layered) - _normalize(Path(BUILDROOT), spec) + # The closure is empty when the lower snapshots contain every package of the layer, for example in the + # buildroot of a package whose spec needs nothing beyond the base root. The package manager then doesn't + # run, and the snapshot only repeats the manifest of the snapshot below it. + if lowers and not any(packages_dir.iterdir()): + (out / "blobs").mkdir(parents=True) + shutil.copyfile(Path(lowers[-1]) / uapi16.MANIFEST, out / uapi16.MANIFEST) + else: + with storage.root(storage.Snapshot(lowers)) as restored: + with rootfs.rootfs(BUILDROOT, bind=restored.path, apivfs=True, strict_unmount=True): + install(packages_dir, Path(BUILDROOT), spec) + _normalize(Path(BUILDROOT), spec) + storage.capture(restored, out, lowers, epoch=int(os.environ["SOURCE_DATE_EPOCH"])) - # The sandbox stacks a box's layers as Buck stores them, without turning markers back into - # overlayfs whiteouts. Deletions are not supported at box runtime right now. - if layered and spec["box_config"] and (deleted := rootfs.deletions(target)): - markers = ", ".join(map(str, deleted)) - fail(f"{prog}: a box layer cannot delete from the layers below it, but has {markers}") + if spec["target"] is None: + return + snapshot = storage.Snapshot([*lowers, out]) + # The sandbox stacks the directories of a box's layers and decodes no whiteouts. A file that a + # layer deletes would still show up from the layer below. + if deleted := storage.delta(snapshot, len(lowers)): + fail(f"{prog}: a box layer cannot delete from the layers below it, but deletes {', '.join(deleted)}") + storage.write_buck_tree(snapshot, Path(spec["target"])) diff --git a/package/installer_test.py b/package/installer_test.py index 0d9556f6..c92b1f9f 100644 --- a/package/installer_test.py +++ b/package/installer_test.py @@ -7,11 +7,11 @@ import subprocess import tempfile import unittest -from contextlib import nullcontext from pathlib import Path from unittest import mock import installer +import storage class TestPathRules(unittest.TestCase): @@ -40,6 +40,20 @@ def test_keeps_authselect_nsswitch_link(self) -> None: self.assertEqual(nsswitch.readlink(), Path("/etc/authselect/nsswitch.conf")) +def _snapshot(root: Path, name: str, files: dict[str, str]) -> Path: + """Capture a tree with `files` as the snapshot store `name` under `root`, with the epoch of the tests.""" + tree = root / f"{name}.tree" + # A box root contains the mount points of the API filesystems. Without them, the install would create + # them and a box layer would export them. + for path in ("etc", "dev", "proc", "run", "tmp", "var/tmp"): + (tree / path).mkdir(parents=True) + for path, content in files.items(): + (tree / path).write_text(content) + store = root / name + storage.capture(storage.Root(tree), store, epoch=0) + return store + + class TestRun(unittest.TestCase): def test_package_scripts_tolerate_unmapped_acl_groups(self) -> None: # The test box mounts /var/tmp as tmpfs, so a host mounted with noacl cannot hide this failure. @@ -47,21 +61,21 @@ def test_package_scripts_tolerate_unmapped_acl_groups(self) -> None: with ( self.subTest(mode=mode), tempfile.TemporaryDirectory(dir="/var/tmp") as scratch, - mock.patch.dict(os.environ, {"SYSTEMD_IN_CHROOT": "0"}), + mock.patch.dict(os.environ, {"SYSTEMD_IN_CHROOT": "0", "SOURCE_DATE_EPOCH": "0"}), ): root = Path(scratch) - target, lower = root / "target", root / "lower" + target = root / "target" target.mkdir() - lower.mkdir() + lower = _snapshot(root, "lower", {}) if mode == "layered" else None (root / "packages").mkdir() (root / "packages" / "systemd.rpm").write_text("") spec = installer.InstallSpec( arch="x86_64", packages_dir=str(root / "packages"), - target=None if mode == "mounted" else str(target), + snapshot=None if mode == "mounted" else str(root / "snapshot"), installroot=str(target) if mode == "mounted" else None, - lower=[str(lower)] if mode == "layered" else [], - work=str(root / "work") if mode == "layered" else None, + lower=[str(lower)] if lower is not None else [], + target=None, box_config=False, langs=[], docs=True, @@ -71,7 +85,6 @@ def install( packages_dir: Path, installroot: Path, request: installer.InstallSpec, - layered: bool, ) -> None: (installroot / "journal").mkdir() # Debian's systemd postinst applies an ACL for adm (GID 4), which a namespace @@ -87,26 +100,27 @@ def install( installer.run("install", install, []) def test_a_box_layer_may_add_but_not_delete(self) -> None: + # The sandbox mounts the directories of a box's layers without decoding deletions. Install a box + # layer over a lower snapshot that adds a file, and once more so that it also deletes a file of the + # lower snapshot. The first install must export only the added file, the second must fail. for deletes in (False, True): with ( self.subTest(deletes=deletes), tempfile.TemporaryDirectory(dir="/var/tmp") as scratch, - mock.patch.dict(os.environ), + mock.patch.dict(os.environ, {"SOURCE_DATE_EPOCH": "0"}), ): root = Path(scratch) - lower = root / "lower" - (lower / "etc").mkdir(parents=True) - (lower / "doomed").write_text("") + lower = _snapshot(root, "lower", {"doomed": ""}) (root / "packages").mkdir() (root / "packages" / "systemd.rpm").write_text("") spec = installer.InstallSpec( arch="x86_64", packages_dir=str(root / "packages"), - target=str(root / "target"), + snapshot=str(root / "snapshot"), installroot=None, lower=[str(lower)], - work=str(root / "work"), - box_config=True, + target=str(root / "target"), + box_config=False, langs=[], docs=True, ) @@ -115,7 +129,6 @@ def install( packages_dir: Path, installroot: Path, request: installer.InstallSpec, - layered: bool, deletes: bool = deletes, ) -> None: (installroot / "added").write_text("") @@ -124,26 +137,27 @@ def install( with mock.patch.object(installer.specs, "parse", return_value=spec): if deletes: - with self.assertRaisesRegex(SystemExit, r"cannot delete .*\.wh\.doomed"): + with self.assertRaisesRegex(SystemExit, "cannot delete .*doomed"): installer.run("install", install, []) else: installer.run("install", install, []) + self.assertEqual([path.name for path in (root / "target").iterdir()], ["added"]) - def test_an_empty_closure_leaves_the_layer_empty(self) -> None: - # A layered install with no packages to install must not run the package manager. Buck still - # expects the work directory as an output. Install an empty package directory over one lower - # layer, and check that the layer is empty and that the work directory exists. + def test_an_empty_closure_repeats_the_lower_snapshot(self) -> None: + # A layered install with no packages to install must not run the package manager. Install an empty + # package directory over one lower snapshot, and check that the package manager doesn't run and + # that the new snapshot restores the same tree as the lower snapshot. with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: root = Path(scratch) - (root / "lower").mkdir() + lower = _snapshot(root, "lower", {"etc/os-release": "ID=test\n"}) (root / "packages").mkdir() spec = installer.InstallSpec( arch="x86_64", packages_dir=str(root / "packages"), - target=str(root / "target"), + snapshot=str(root / "snapshot"), installroot=None, - lower=[str(root / "lower")], - work=str(root / "work"), + lower=[str(lower)], + target=None, box_config=False, langs=[], docs=True, @@ -152,51 +166,10 @@ def test_an_empty_closure_leaves_the_layer_empty(self) -> None: with mock.patch.object(installer.specs, "parse", return_value=spec): installer.run("install", install, []) install.assert_not_called() - self.assertEqual(list((root / "target").iterdir()), []) - self.assertTrue((root / "work").is_dir()) - - def test_fresh_root_mount_owns_capture_after_an_install_failure(self) -> None: - with tempfile.TemporaryDirectory(dir="/var/tmp") as scratch: - root = Path(scratch) - target = root / "target" - spec = installer.InstallSpec( - arch="x86_64", - packages_dir=str(root / "packages"), - target=str(target), - installroot=None, - lower=[], - work=None, - box_config=False, - langs=[], - docs=True, - ) - - def fail( - packages_dir: Path, - installroot: Path, - request: installer.InstallSpec, - layered: bool, - ) -> None: - raise RuntimeError("install failed") - - with ( - mock.patch.dict(os.environ), - mock.patch.object(installer.specs, "parse", return_value=spec), - mock.patch.object( - installer.rootfs, - "rootfs", - return_value=nullcontext(), - ) as mounted, - self.assertRaisesRegex(RuntimeError, "install failed"), - ): - installer.run("install", fail, []) - - mounted.assert_called_once_with( - installer.BUILDROOT, - bind=target, - capture_bind=True, - apivfs=True, + self.assertEqual( + storage.Snapshot([lower, root / "snapshot"]).objects, storage.Snapshot([lower]).objects ) + self.assertEqual(list((root / "snapshot/blobs").iterdir()), []) if __name__ == "__main__": diff --git a/package_system/deb/BUCK b/package_system/deb/BUCK index 9b3ab39e..cc4b686c 100644 --- a/package_system/deb/BUCK +++ b/package_system/deb/BUCK @@ -46,7 +46,7 @@ python.binary( "//:util", "//package:href", "//package:transaction", - "//rootfs:rootfs", + "//uapi16:storage", ], ) @@ -134,6 +134,6 @@ python.test( "//package:installer", "//package:snapshotter", "//package:transaction", - "//rootfs:rootfs", + "//uapi16:storage", ], ) diff --git a/package_system/deb/install.py b/package_system/deb/install.py index 46263e58..f3e4646b 100755 --- a/package_system/deb/install.py +++ b/package_system/deb/install.py @@ -202,8 +202,8 @@ def install( # `plan.py` pins repositories above 1000, so it can select a version that is older # than the installed version. "--allow-downgrades", - # APT resolves the transaction again here. Refuse a removal as `plan.py` does: removing - # a package of a lower layer would write whiteouts into this layer. + # APT resolves the transaction again here. Refuse a removal as `plan.py` does, because a + # transaction cannot express a removal. "--no-remove", "install", *(str(package) for package in packages), @@ -237,7 +237,7 @@ def scrub(installroot: Path) -> None: update.unlink() -def _install(packages_dir: Path, installroot: Path, spec: installer.InstallSpec, layered: bool) -> None: +def _install(packages_dir: Path, installroot: Path, spec: installer.InstallSpec) -> None: with installer.fresh_machine_id(installroot): install( packages_dir, @@ -246,10 +246,7 @@ def _install(packages_dir: Path, installroot: Path, spec: installer.InstallSpec, langs=spec["langs"], docs=spec["docs"], ) - # Scrub only after a transaction that installed a package. On an overlay, `unlink()` of a - # file of a lower layer writes a whiteout into this layer. - if any(path.suffix == ".deb" for path in packages_dir.iterdir()): - scrub(installroot) + scrub(installroot) def main(argv: list[str] | None = None) -> None: diff --git a/package_system/deb/plan.py b/package_system/deb/plan.py index 3f692fee..9a1894b9 100755 --- a/package_system/deb/plan.py +++ b/package_system/deb/plan.py @@ -25,7 +25,7 @@ import util from util import MAGIC, decompressor -import rootfs +import storage import transaction from href import relative_href @@ -195,7 +195,7 @@ def solve( scratch = Path(stack.enter_context(TemporaryDirectory(prefix="deb-plan.", dir="/var/tmp"))) root = scratch / "root" if lower: - root = stack.enter_context(rootfs.rootfs("/installroot", lowers=lower)) + root = stack.enter_context(storage.root(storage.Snapshot(lower))).path status = root / "var/lib/dpkg/status" status.parent.mkdir(parents=True, exist_ok=True) status.touch(exist_ok=True) diff --git a/package_system/pacman/BUCK b/package_system/pacman/BUCK index 2ffed2c5..0933d49b 100644 --- a/package_system/pacman/BUCK +++ b/package_system/pacman/BUCK @@ -46,7 +46,7 @@ python.binary( "//:util", "//package:href", "//package:transaction", - "//rootfs:rootfs", + "//uapi16:storage", ], ) @@ -100,7 +100,7 @@ python.test( name = "plan-test", box = "//tools:dev.box", srcs = ["alpm.py", "plan.py", "plan_test.py"], - deps = ["//:specs", "//:util", "//package:href", "//package:transaction", "//rootfs:rootfs"], + deps = ["//:specs", "//:util", "//package:href", "//package:transaction", "//uapi16:storage"], ) python.test( diff --git a/package_system/pacman/install.py b/package_system/pacman/install.py index ca2c6957..34b33f31 100755 --- a/package_system/pacman/install.py +++ b/package_system/pacman/install.py @@ -135,10 +135,7 @@ def parkdb(installroot: Path, epoch: int) -> None: f"{epoch}\n" if index and lines[index - 1].strip() == "%INSTALLDATE%" else line for index, line in enumerate(lines) ) - # Entries a lower layer already parked are unchanged, and writing one anyway would copy - # the whole inherited database up into this layer's delta. - if stamped != text: - desc.write_text(stamped, encoding="utf-8") + desc.write_text(stamped, encoding="utf-8") def scrub(installroot: Path) -> None: @@ -147,7 +144,7 @@ def scrub(installroot: Path) -> None: (installroot / "var/log/pacman.log").unlink(missing_ok=True) -def _install(packages_dir: Path, installroot: Path, spec: installer.InstallSpec, layered: bool) -> None: +def _install(packages_dir: Path, installroot: Path, spec: installer.InstallSpec) -> None: with installer.fresh_machine_id(installroot), tempfile.TemporaryDirectory(prefix="pacman.") as scratch: install( packages_dir, diff --git a/package_system/pacman/plan.py b/package_system/pacman/plan.py index a2f9d271..10fbc6d9 100755 --- a/package_system/pacman/plan.py +++ b/package_system/pacman/plan.py @@ -27,7 +27,7 @@ from util import fail import alpm -import rootfs +import storage import transaction from href import relative_href @@ -110,8 +110,7 @@ def main(argv: list[str] | None = None) -> None: with ExitStack() as stack: scratch = Path(stack.enter_context(TemporaryDirectory(prefix="plan."))) if spec["lower"]: - # An ephemeral upper keeps the lower stack unchanged while alpm writes into the root. - root = stack.enter_context(rootfs.rootfs("/installroot", lowers=spec["lower"])) + root = stack.enter_context(storage.root(storage.Snapshot(spec["lower"]))).path else: root = scratch / "root" root.mkdir() diff --git a/package_system/rpm/BUCK b/package_system/rpm/BUCK index 2b9c8ab2..1abf20a1 100644 --- a/package_system/rpm/BUCK +++ b/package_system/rpm/BUCK @@ -35,7 +35,7 @@ python.binary( name = "plan", main = "plan.py", boxes = [_BOX], - deps = [":macros", "//:specs", "//:util", "//package:transaction", "//rootfs:rootfs"], + deps = [":macros", "//:specs", "//:util", "//package:transaction", "//uapi16:storage"], ) python.library(name = "rpmbuild", srcs = ["rpmbuild.py"], deps = ["//:util", "//package:builder"]) @@ -96,7 +96,7 @@ python.test( name = "plan-test", box = _BOX, srcs = ["createrepo.py", "plan.py", "plan_test.py"], - deps = [":macros", "//:specs", "//:util", "//package:transaction", "//rootfs:rootfs"], + deps = [":macros", "//:specs", "//:util", "//package:transaction", "//uapi16:storage"], ) python.test( diff --git a/package_system/rpm/build_test.py b/package_system/rpm/build_test.py index beb2bb5f..8e702152 100644 --- a/package_system/rpm/build_test.py +++ b/package_system/rpm/build_test.py @@ -33,6 +33,15 @@ def setUp(self) -> None: self.project.mkdir() self.auxiliary = self.scratch / "auxiliary" self.auxiliary.write_text("packaging input\n") + # The tests replace the buildroot mount, so the buildroot doesn't need a snapshot to restore. + self.enterContext(mock.patch.object(build.builder.storage, "Snapshot")) + self.enterContext( + mock.patch.object( + build.builder.storage, + "root", + return_value=contextlib.nullcontext(build.builder.storage.Root(Path("buildroot"))), + ) + ) def specification( self, diff --git a/package_system/rpm/buildrequires.py b/package_system/rpm/buildrequires.py index 45f67bd1..8c5a939a 100755 --- a/package_system/rpm/buildrequires.py +++ b/package_system/rpm/buildrequires.py @@ -28,7 +28,7 @@ class Spec(rpmbuild.Parse): - # `lower` lists the layers of the buildroot with declared BuildRequires installed, bottom first. + # `lower` lists the snapshot stores of the buildroot with declared BuildRequires installed, bottom first. lower: list[str] # `source_tree` is the source tree of an in-place build, or None. source_tree: str | None diff --git a/package_system/rpm/install.py b/package_system/rpm/install.py index 112e016e..8e6ae691 100755 --- a/package_system/rpm/install.py +++ b/package_system/rpm/install.py @@ -166,14 +166,13 @@ def install_into_root( scrub(installroot) -def _install(packages_dir: Path, installroot: Path, spec: installer.InstallSpec, layered: bool) -> None: +def _install(packages_dir: Path, installroot: Path, spec: installer.InstallSpec) -> None: install_into_root( packages_dir, installroot, arch=spec["arch"], - # Installed packages satisfy dependencies for an incremental install; a root the caller - # mounted says so by already carrying an rpmdb. - system=layered or (installroot / DBPATH / "rpmdb.sqlite").exists(), + # Installed packages satisfy dependencies for an install over a root that contains an rpmdb. + system=(installroot / DBPATH / "rpmdb.sqlite").exists(), langs=spec["langs"], docs=spec["docs"], ) diff --git a/package_system/rpm/plan.py b/package_system/rpm/plan.py index 9045461d..bbb80cd5 100755 --- a/package_system/rpm/plan.py +++ b/package_system/rpm/plan.py @@ -23,7 +23,7 @@ from util import fail import macros -import rootfs +import storage import transaction # A multilib package in a pinned-arch transaction indicates a bad solve. @@ -249,8 +249,7 @@ def main(argv: list[str] | None = None) -> None: with ExitStack() as stack: installroot = None if solve_spec["lower"]: - # An ephemeral upper keeps the lower stack unchanged. - installroot = stack.enter_context(rootfs.rootfs("/installroot", lowers=solve_spec["lower"])) + installroot = stack.enter_context(storage.root(storage.Snapshot(solve_spec["lower"]))).path tx = plan( repos, install, diff --git a/rootfs/rootfs.py b/rootfs/rootfs.py index 45f8479a..d3e11ebb 100644 --- a/rootfs/rootfs.py +++ b/rootfs/rootfs.py @@ -1,11 +1,9 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""Mount target roots and translate stored OCI markers to native overlayfs state. +"""Mount target roots and capture output trees in a form Buck can store. -Buck cannot store device nodes, xattrs, or backslashes in paths, so deltas encode them as -regular marker files. Sidecar layers reconstruct markers on entry; persisted uppers are -captured back to the storage form after unmounting. +Buck cannot store backslashes in paths, so `capture()` escapes names that contain them. `readonly_project()` makes the project read-only while a build driver runs. `source_overlay()` gives a tool that writes into its sources, such as rpmbuild, a view of them that discards the writes. @@ -13,12 +11,10 @@ import errno import os -import shutil import stat import tempfile from collections.abc import Generator, Mapping, Sequence from contextlib import ExitStack, contextmanager, suppress -from enum import Enum, auto from pathlib import Path from isolation import ( @@ -35,184 +31,45 @@ chroot as _chroot, ) -_WH = ".wh." # `.wh.`: a whiteout hiding from lower layers -_OPAQUE = ".wh..wh..opq" # marks its parent dir opaque (hide everything below it) _ESC = ".esc." # `.esc.`: a name buck can't store, percent-escaped -_OPAQUE_XATTR = b"user.overlay.opaque" # an opaque dir's native form (userxattr namespace) -# OCI delta to native overlay translation. - - -def _escape(name: str) -> str: +def escape(name: str) -> str: """Escape a path component Buck cannot store, preserving injectivity.""" if "\\" not in name and not name.startswith(_ESC): return name return _ESC + name.replace("%", "%25").replace("\\", "%5C") -def _unescape(name: str) -> str: - return name.removeprefix(_ESC).replace("%5C", "\\").replace("%25", "%") - - -class _Kind(Enum): - """What a stored delta entry means once decoded.""" - - NORMAL = auto() # a plain entry, carried through under its true name - WHITEOUT = auto() # deletes the payload name from lower layers - OPAQUE = auto() # marks its parent dir opaque - - -def _classify(name: str) -> tuple[_Kind, str]: - """Decode a stored name into its marker kind and payload.""" - true = _unescape(name) if name.startswith(_ESC) else name - if true == _OPAQUE: - return _Kind.OPAQUE, "" - if true.startswith(_WH): - return _Kind.WHITEOUT, true[len(_WH) :] - return _Kind.NORMAL, true - - -def deletions(delta: Path) -> list[Path]: - """The marker files of a stored delta that delete from the layers below it.""" - markers = (path for path in delta.rglob("*") if _classify(path.name)[0] is not _Kind.NORMAL) - return sorted(path.relative_to(delta) for path in markers) - - -def _whiteout_marker(name: str) -> str: - """Encode a possibly unstorable whiteout target.""" - return _escape(_WH + name) - - -def _whiteout(directory: Path, name: str) -> None: - directory.mkdir(parents=True, exist_ok=True) - os.mknod(str(directory / name), stat.S_IFCHR | 0o600, os.makedev(0, 0)) - - -def _thaw(src: Path, dst: Path) -> None: - """Copy an escaped entry under its true name, recursively decoding markers.""" - if src.is_symlink() or not src.is_dir(): - shutil.copy2(src, dst, follow_symlinks=False) - return - dst.mkdir(mode=stat.S_IMODE(src.lstat().st_mode)) - for child in src.iterdir(): - kind, payload = _classify(child.name) - if kind is _Kind.OPAQUE: - os.setxattr(str(dst), _OPAQUE_XATTR, b"y") - elif kind is _Kind.WHITEOUT: - _whiteout(dst, payload) - else: - _thaw(child, dst / payload) - shutil.copystat(src, dst, follow_symlinks=False) - - -def _markers(delta: Path, dest: Path) -> tuple[Path | None, Path | None]: - """Express stored markers as sparse sidecar layers above and below `delta`.""" - above, below = dest / "above", dest / "below" - used_above = used_below = False - for p in delta.rglob("*"): - rel = p.parent.relative_to(delta) - if any(part.startswith(_ESC) for part in rel.parts): - continue # inside an escaped dir: thawed wholesale when its ancestor was hit - kind, payload = _classify(p.name) - if kind is _Kind.OPAQUE: - opaque = below / rel - opaque.mkdir(parents=True, exist_ok=True) - os.setxattr(str(opaque), _OPAQUE_XATTR, b"y") - _whiteout(above / rel, p.name) # hide the marker file itself - used_above = used_below = True - elif kind is _Kind.WHITEOUT: - _whiteout(above / rel, payload) # whiteout the deleted target - _whiteout(above / rel, p.name) # hide the literal marker file (`.wh.` or `.esc.`) - used_above = True - elif p.name.startswith(_ESC): # an escaped normal entry - (above / rel).mkdir(parents=True, exist_ok=True) - _thaw(p, above / rel / payload) # materialize it under its true name - _whiteout(above / rel, p.name) # hide the `.esc.` entry itself - used_above = True - return (above if used_above else None, below if used_below else None) - - -def _reconstruct(lowers: list[Path], scratch: Path) -> list[Path]: - """Frame stored deltas as an overlayfs lowerdir list in top-to-bottom order.""" - components: list[Path] = [] - for i, lo in enumerate(reversed(lowers)): - above, below = _markers(lo, scratch / f"m{i}") - if above is not None: - components.append(above) - components.append(lo) - if below is not None: - components.append(below) - return components - - -def _is_opaque(d: Path) -> bool: - try: - return os.getxattr(str(d), _OPAQUE_XATTR) == b"y" - except OSError: - return False - - -def _in_lower(lower: Path, rel: Path) -> bool: - """Return whether the stored layer `lower` has an entry at `rel`, without following its symlinks.""" - path = lower - for part in rel.parts: - if path.is_symlink() or not path.is_dir(): - return False - path = path / _escape(part) - if not os.path.lexists(path): - return False - return True - - -def capture(tree: Path, lowers: Sequence[Path] | None = None) -> None: - """Rewrite native overlay state and unstorable names into regular marker files. - - `lowers` are the stored layers below `tree`, if `tree` is an overlay upper directory. - """ +def capture(tree: Path) -> None: + """Make every directory of `tree` writable for its owner and escape the names Buck cannot store.""" # Buck must be able to delete and rename through package-supplied read-only directories. for p in tree.rglob("*"): st = p.lstat() if stat.S_ISDIR(st.st_mode) and stat.S_IMODE(st.st_mode) & 0o700 != 0o700: p.chmod(stat.S_IMODE(st.st_mode) | 0o700) for p in sorted(tree.rglob("*"), key=lambda q: len(q.parts), reverse=True): - st = p.lstat() - if stat.S_ISCHR(st.st_mode) and st.st_rdev == 0: - p.unlink() - (p.parent / _whiteout_marker(p.name)).write_bytes(b"") - continue - if stat.S_ISDIR(st.st_mode) and _is_opaque(p): - os.removexattr(str(p), _OPAQUE_XATTR) - # overlayfs marks a directory of the upper layer opaque when the directory is renamed - # into a directory that a lower layer also contains. dpkg unpacks every new directory - # under a temporary name and renames it into place, so a Debian layer gets an opaque mark - # on every new directory. - # The mark hides nothing when no lower layer has an entry at the path, so capture() writes - # no marker for it. - rel = p.relative_to(tree) - if lowers is None or any(_in_lower(lower, rel) for lower in lowers): - (p / _OPAQUE).write_bytes(b"") - escaped = _escape(p.name) + escaped = escape(p.name) if escaped != p.name: p.rename(p.parent / escaped) @contextmanager -def capture_on_exit(tree: str | Path, lowers: Sequence[Path] | None = None) -> Generator[Path]: +def capture_on_exit(tree: str | Path) -> Generator[Path]: """Capture a tree after use without hiding an error from the body.""" tree = Path(tree) try: yield tree except BaseException as error: try: - capture(tree, lowers) + capture(tree) except BaseException as capture_error: error.add_note( f"rootfs capture of {tree} also failed: {capture_error.__class__.__name__}: {capture_error}" ) raise else: - capture(tree, lowers) + capture(tree) # Root setup. @@ -220,11 +77,7 @@ def capture_on_exit(tree: str | Path, lowers: Sequence[Path] | None = None) -> G @contextmanager def source_overlay(source: Path, target: Path) -> Generator[Path]: - """Mount `source` at `target` as an overlay, and discard every write when the context exits. - - `rootfs()` decodes the `.wh.` and `.esc.` marker files of a stored image layer. This function - does not, because a checkout is not an image layer and can contain such names. - """ + """Mount `source` at `target` as an overlay, and discard every write when the context exits.""" # An overlay shows the directory that a mount in its lower directory covers, not the mounted tree. # A `tine mount` covers a directory of the project, so directory submounts need overlays of their own. # statmount() returns mount points with their symlinks resolved, so `source` needs them resolved too. @@ -245,7 +98,7 @@ def source_overlay(source: Path, target: Path) -> Generator[Path]: upper.chmod(stat.S_IMODE(attributes.st_mode)) os.utime(upper, ns=(attributes.st_atime_ns, attributes.st_mtime_ns)) try: - stack.enter_context(Overlay((lower,), upper, work, merged)).mount() + stack.enter_context(Overlay(lower, upper, work, merged)).mount() except OSError as error: if error.errno == errno.EINVAL: error.add_note( @@ -341,60 +194,16 @@ def chroot(target: str | Path) -> Generator[Path]: def rootfs( target: str | Path, *, - bind: str | Path | None = None, - capture_bind: bool = False, - lowers: Sequence[str | Path] | None = None, - upperdir: str | Path | None = None, - workdir: str | Path | None = None, + bind: str | Path, apivfs: bool = False, binds: Sequence[tuple[str | Path, str | Path]] | None = None, chroot: bool = False, + strict_unmount: bool = False, ) -> Generator[Path]: - """Mount a bind or overlay root, optionally chrooting and persisting its output. - - Persistent overlay uppers and bind sources with `capture_bind=True` are captured after all - mounts have been torn down. - """ - if capture_bind and bind is None: - raise ValueError("rootfs(capture_bind=True) needs bind=") + """Bind `bind` at `target` with optional API filesystems, extra binds and a chroot.""" target = Path(target) with ExitStack() as stack: - if lowers is not None: - # _in_lower() treats a symlink as a missing directory. Without resolve(), capture() would find - # no entry in a layer that the caller passes as a symlink. - resolved = [Path(lo).resolve() for lo in lowers] - scratch = Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="rootfs."))) - components = _reconstruct(resolved, scratch) - if not components: - empty = scratch / "empty" - empty.mkdir() - components.append(empty) - if upperdir is not None: - if workdir is None: - raise ValueError("rootfs(upperdir=...) needs a matching workdir=") - upper, work = Path(upperdir), Path(workdir) - else: - # A writable ephemeral upper also permits a single-component stack. - upper, work = scratch / "upper", scratch / "work" - upper.mkdir(parents=True, exist_ok=True) - work.mkdir(parents=True, exist_ok=True) - if upperdir is not None: - # Enter before registering the unmount so capture runs after it. - stack.enter_context(capture_on_exit(upper, resolved)) - # capture() reads and rewrites the upperdir after the unmount, so a captured upper needs a - # strict unmount. Nobody reads an ephemeral upper, so it is unmounted lazily like every - # other mount. - mounted: Overlay | Bind = stack.enter_context( - Overlay(tuple(components), upper, work, target, strict_unmount=upperdir is not None) - ) - elif bind is not None: - if capture_bind: - # Enter before registering the unmount so capture runs after it. - stack.enter_context(capture_on_exit(bind)) - # capture() reads the bind source after the unmount, like the upper of an overlay. - mounted = stack.enter_context(Bind(Path(bind), target, strict_unmount=capture_bind)) - else: - raise ValueError("rootfs needs bind= or lowers=") + mounted = stack.enter_context(Bind(Path(bind), target, strict_unmount=strict_unmount)) mounted.mount() # An image layer can contain an absolute symlink at a mount point. The mount points resolve inside # the root, so that the symlink doesn't place a mount outside of the root. diff --git a/rootfs/rootfs_test.py b/rootfs/rootfs_test.py index 651a23ed..474d61c3 100644 --- a/rootfs/rootfs_test.py +++ b/rootfs/rootfs_test.py @@ -41,8 +41,6 @@ def setUp(self) -> None: (self.source / "escaping-dangling-link").symlink_to(self.scratch / "absent") (self.source / "empty").mkdir() (self.source / "readonly").mkdir(mode=0o555) - for name in (".wh.original", ".wh..wh..opq", ".esc.literal"): - (self.source / name).write_text(name, encoding="utf-8") # overlayfs takes the mode and the times of the mount root from the upper directory. The # setgid bit and the old timestamp make the test fail if the root has the mode and the # times of a fresh upper directory. @@ -59,7 +57,7 @@ def test_an_invalid_overlay_has_a_possible_locked_mount_hint(self) -> None: pass self.assertIn("Locked mounts below the source can cause EINVAL", "".join(raised.exception.__notes__)) - def test_preserves_the_tree_without_decoding_image_markers(self) -> None: + def test_preserves_the_tree(self) -> None: with chdir(self.project), rootfs.source_overlay(Path("input"), self.scratch / "build/src") as tree: original = {path.relative_to(self.source) for path in self.source.rglob("*")} self.assertEqual({path.relative_to(tree) for path in tree.rglob("*")}, original) @@ -185,7 +183,7 @@ def test_is_a_tmpfs_outside_a_run_action(self) -> None: mock.patch.dict(os.environ), ): os.environ.pop("BUCK_SCRATCH_PATH", None) - with rootfs.rootfs(scratch / "target", lowers=[lower], apivfs=True) as tree: + with rootfs.rootfs(scratch / "target", bind=lower, apivfs=True) as tree: (tree / "var/tmp/staged").write_text("staged", encoding="utf-8") self.assertTrue((tree / "var/tmp").is_mount()) self.assertEqual(list(scratch.glob("var-tmp.*")), []) @@ -204,7 +202,7 @@ def test_is_backed_where_tmpdir_points(self) -> None: with ( mock.patch.dict(os.environ, environment), mock.patch.object(tempfile, "tempdir", None), - rootfs.rootfs(scratch / "target", lowers=[lower], apivfs=True) as tree, + rootfs.rootfs(scratch / "target", bind=lower, apivfs=True) as tree, ): (tree / "var/tmp/staged").write_text("staged", encoding="utf-8") backing = next(scratch.glob("var-tmp.*")) @@ -227,13 +225,15 @@ def test_an_absolute_symlink_in_a_layer_resolves_inside_the_root(self) -> None: self.subTest(apivfs=apivfs), rootfs.rootfs( scratch / "target", - lowers=[lower], + bind=lower, binds=[] if apivfs else [(scratch, "/run/project")], apivfs=apivfs, ) as tree, ): + # The bind of `scratch` is on the same filesystem as the root, so is_mount() can't detect + # it. `scratch` contains `lower`, and the root doesn't. if not apivfs: - self.assertTrue((tree / outside.relative_to("/") / "project").is_mount()) + self.assertTrue((tree / outside.relative_to("/") / "project/lower").is_dir()) self.assertFalse(outside.is_mount()) self.assertFalse((outside / "project").exists()) @@ -350,7 +350,7 @@ def test_captures_after_success(self) -> None: with rootfs.capture_on_exit(output): pass - capture.assert_called_once_with(output, None) + capture.assert_called_once_with(output) def test_capture_failure_does_not_hide_body_failure(self) -> None: output = Path("output") @@ -376,46 +376,20 @@ def test_capture_failure_after_success_is_reported(self) -> None: class TestCapture(unittest.TestCase): - def test_keeps_an_opaque_mark_only_where_a_lower_layer_has_the_directory(self) -> None: - with tempfile.TemporaryDirectory(dir="/var/tmp") as directory: - lower, upper = Path(directory) / "lower", Path(directory) / "upper" - (lower / "usr/share/doc/old").mkdir(parents=True) - (lower / "usr/share/doc/old/file").write_text("old") - for name in ("old", "new"): - (upper / "usr/share/doc" / name).mkdir(parents=True, exist_ok=True) - os.setxattr(str(upper / "usr/share/doc" / name), b"user.overlay.opaque", b"y") - - rootfs.capture(upper, [lower]) - - self.assertEqual(rootfs.deletions(upper), [Path("usr/share/doc/old/.wh..wh..opq")]) - self.assertRaises(OSError, os.getxattr, str(upper / "usr/share/doc/new"), b"user.overlay.opaque") - - def test_keeps_every_opaque_mark_without_lower_layers(self) -> None: + def test_makes_directories_writable_and_escapes_backslashes(self) -> None: + # Buck can't remove a tree through a directory without owner write access, and it can't store a + # name with a backslash. Capture a tree with a read-only directory that contains such a name, and + # check that the directory becomes writable and the name is escaped. with tempfile.TemporaryDirectory(dir="/var/tmp") as directory: tree = Path(directory) - (tree / "new").mkdir() - os.setxattr(str(tree / "new"), b"user.overlay.opaque", b"y") + (tree / "readonly").mkdir() + (tree / "readonly/back\\slash").write_text("kept") + (tree / "readonly").chmod(0o555) rootfs.capture(tree) - self.assertEqual(rootfs.deletions(tree), [Path("new/.wh..wh..opq")]) - - -class TestDeletions(unittest.TestCase): - def test_lists_whiteouts_and_opaque_markers_only(self) -> None: - with tempfile.TemporaryDirectory(dir="/var/tmp") as directory: - delta = Path(directory) - (delta / "etc/opaque").mkdir(parents=True) - (delta / "etc/added").write_text("added") - (delta / "etc/.wh.removed").write_bytes(b"") - (delta / "etc/opaque/.wh..wh..opq").write_bytes(b"") - (delta / ".esc..wh.esc%5Caped").write_bytes(b"") - (delta / ".esc.back%5Cslash").write_text("kept") - - self.assertEqual( - rootfs.deletions(delta), - [Path(".esc..wh.esc%5Caped"), Path("etc/.wh.removed"), Path("etc/opaque/.wh..wh..opq")], - ) + self.assertEqual((tree / "readonly").stat().st_mode & 0o777, 0o755) + self.assertEqual([path.name for path in (tree / "readonly").iterdir()], [".esc.back%5Cslash"]) if __name__ == "__main__": diff --git a/ruff.toml b/ruff.toml index 5ddc2873..5677281c 100644 --- a/ruff.toml +++ b/ruff.toml @@ -21,6 +21,7 @@ src = [ "package_system/rpm", "remote_cache", "rootfs", + "uapi16", ] lint.select = [ "E", # pycodestyle diff --git a/uapi16/BUCK b/uapi16/BUCK new file mode 100644 index 00000000..ef9b1d60 --- /dev/null +++ b/uapi16/BUCK @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""UAPI.16 manifests and the snapshot stores built on them.""" + +load("//python:defs.bzl", "python") + +python.library( + name = "uapi16", + srcs = ["uapi16.py"], + deps = ["//:util"], + visibility = ["tine//..."], +) + +python.library( + name = "storage", + srcs = ["storage.py"], + deps = [":uapi16", "//:util", "//box:isolation", "//rootfs:rootfs"], + visibility = ["tine//..."], +) + +python.test( + name = "test", + box = "//tools:dev.box", + srcs = ["storage_test.py", "uapi16_test.py"], + deps = [":storage", ":uapi16", "//:util", "//box:isolation"], +) diff --git a/uapi16/pyproject.toml b/uapi16/pyproject.toml new file mode 100644 index 00000000..a0e58bb2 --- /dev/null +++ b/uapi16/pyproject.toml @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +# Zed needs a Python manifest here to route these files to their own ty instance. diff --git a/uapi16/storage.py b/uapi16/storage.py new file mode 100644 index 00000000..fa1a14d0 --- /dev/null +++ b/uapi16/storage.py @@ -0,0 +1,334 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Store complete filesystem snapshots independently of artifact filesystem metadata.""" + +import os +import re +import stat +import tempfile +from collections.abc import Generator, Sequence +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass, replace +from pathlib import Path, PurePosixPath +from typing import Literal + +from util import ReflinkUnavailable, clone_file, fail, reflink + +import rootfs +import uapi16 +from isolation import Bind, Overlay + +type Strategy = Literal["auto", "copy", "overlay"] + + +class Snapshot: + def __init__(self, stores: Sequence[str | Path]) -> None: + self.stores = [Path(store) for store in stores] + self.objects: list[uapi16.Object] = ( + uapi16.read(self.stores[-1] / uapi16.MANIFEST) + if self.stores + else [{"mediaType": uapi16.MEDIA_TYPE, "type": "dir", "mode": 0o755}] + ) + self.blobs: dict[str, Path] = {} + for store in self.stores: + for blob in (store / "blobs").iterdir(): + self.blobs.setdefault(blob.name, blob) + self._validate() + + def _validate(self) -> None: + directories = {""} + groups: dict[int, uapi16.Object] = {} + for obj in self.objects[1:]: + name = obj["name"] + if name.rpartition("/")[0] not in directories: + fail(f"storage: {name!r} does not follow its parent directory") + if obj["type"] == "dir": + directories.add(name) + if token := obj.get("inodeToken"): + if obj["type"] == "dir": + fail(f"storage: directory {name!r} has an inodeToken") + inode = obj.copy() + del inode["name"] + if groups.setdefault(token, inode) != inode: + fail(f"storage: hardlink group {token} has conflicting metadata") + if obj["type"] == "reg": + digest = obj["sha256"] + if obj["contents"] != [{"file": f"blobs/{digest}"}]: + fail(f"storage: unsupported content reference for {name!r}") + self.content(obj) + + def content(self, obj: uapi16.Object) -> Path: + digest = obj["sha256"] + if not isinstance(digest, str) or not re.fullmatch("[0-9a-f]{64}", digest): + fail(f"storage: invalid SHA-256 digest {digest!r}") + if digest not in self.blobs: + fail(f"storage: missing blob {digest} for {obj['name']!r}") + path = self.blobs[digest] + st = path.lstat() + if not stat.S_ISREG(st.st_mode) or st.st_size != obj["size"]: + fail(f"storage: invalid blob {digest} for {obj['name']!r}") + return path + + +@dataclass(frozen=True) +class Root: + path: Path + upper: Path | None = None + + def entries(self) -> Generator[uapi16.Entry]: + for entry in uapi16.entries(self.path): + if self.upper is not None and entry.st.st_nlink > 1 and not stat.S_ISDIR(entry.st.st_mode): + # OverlayFS can report different origin inodes for paths linked after copy-up. + # The upper inode identifies the hardlink group that commands actually modified. + try: + st = (self.upper / entry.name).lstat() + except FileNotFoundError: + pass + else: + entry = replace(entry, st=st) + yield entry + + +def add_blob(source: Path, destination: Path) -> None: + if not destination.exists(): + clone_file(source, destination) + destination.chmod(0o444) + + +def capture(tree: Root, out: Path, parents: Sequence[str | Path] = (), *, epoch: int) -> None: + """Write the snapshot of `tree` to `out`, with modification times clamped to `epoch`.""" + parent = Snapshot(parents) + blobs = out / "blobs" + blobs.mkdir(parents=True, exist_ok=True) + found = list(tree.entries()) + tokens = uapi16.hardlink_tokens(found) + root = uapi16.file_object(uapi16.Entry("", tree.path, tree.path.lstat()), epoch=epoch) + root.pop("name") + root["mediaType"] = uapi16.MEDIA_TYPE + objects = [root] + inodes: dict[tuple[int, int], uapi16.Object] = {} + for entry in found: + if entry.inode in inodes: + obj = inodes[entry.inode].copy() + obj["name"] = entry.name + else: + obj = uapi16.file_object(entry, epoch=epoch, token=tokens.get(entry.inode)) + if obj["type"] == "reg": + digest = obj["sha256"] + obj["contents"] = [{"file": f"blobs/{digest}"}] + if digest not in parent.blobs: + add_blob(entry.path, blobs / digest) + inodes[entry.inode] = obj + objects.append(obj) + for obj in objects: + # Snapshot ownership is root even when a package manager installs another owner. + obj["uid"] = obj["gid"] = 0 + uapi16.write_objects(objects, out / uapi16.MANIFEST) + + +def save(snapshot: Snapshot, out: Path) -> None: + blobs = out / "blobs" + blobs.mkdir(parents=True, exist_ok=True) + for obj in snapshot.objects[1:]: + if obj["type"] == "reg": + add_blob(snapshot.content(obj), blobs / obj["sha256"]) + uapi16.write_objects(snapshot.objects, out / uapi16.MANIFEST) + + +def by_name(objects: Sequence[uapi16.Object]) -> dict[str, uapi16.Object]: + # The root object has no name. resolve() returns "." for it. + return {obj.get("name", "."): obj for obj in objects} + + +def resolve(objects: dict[str, uapi16.Object], name: str, *, follow: bool = True) -> str: + resolved = PurePosixPath() + parts = list(PurePosixPath(name).parts) + hops = 0 + while parts: + part = parts.pop(0) + if part == "..": + resolved = resolved.parent + continue + candidate = resolved / part + obj = objects.get(str(candidate)) + if obj is None or obj["type"] != "lnk" or (not parts and not follow): + resolved = candidate + continue + hops += 1 + if hops > 40: + fail(f"storage: too many levels of symbolic links in {name!r}") + target = PurePosixPath(uapi16.symlink_target(obj)) + if target.is_absolute(): + resolved = PurePosixPath() + target = target.relative_to("/") + parts = [*target.parts, *parts] + return str(resolved) + + +def metadata(path: Path, obj: uapi16.Object) -> None: + if "uid" in obj or "gid" in obj: + os.chown(path, obj.get("uid", 0), obj.get("gid", 0), follow_symlinks=False) + if obj["type"] != "lnk": + path.chmod(obj["mode"]) + for name, value in obj.get("extendedAttributes", {}).get("user", {}).items(): + os.setxattr(path, "user." + name, uapi16.xattr_value(value), follow_symlinks=False) + if "mTimeNSec" in obj: + os.utime(path, ns=(obj["mTimeNSec"], obj["mTimeNSec"]), follow_symlinks=False) + + +def restore( + snapshot: Snapshot, + tree: Path, + *, + metacopy: bool = False, + upper: Path | None = None, +) -> None: + """Restore private files or prepare metadata and hardlinks for OverlayFS.""" + tree.mkdir(parents=True, exist_ok=True) + groups: dict[int, Path] = {} + directories = [(tree, snapshot.objects[0])] + linked_parents = set() + if upper is not None: + upper.mkdir() + directories.append((upper, snapshot.objects[0])) + linked_parents = { + str(parent) + for obj in snapshot.objects[1:] + if obj.get("inodeToken") + for parent in Path(obj["name"]).parents + } + for obj in snapshot.objects[1:]: + kind = obj["type"] + token = obj.get("inodeToken") + # index=off breaks lower hardlinks on copy-up, so writable groups start in the upper. + private = upper is not None and token + path = (upper if private else tree) / obj["name"] + if kind == "dir": + path.mkdir() + directories.append((path, obj)) + if upper is not None and obj["name"] in linked_parents: + (upper / obj["name"]).mkdir() + directories.append((upper / obj["name"], obj)) + continue + if token and token in groups: + os.link(groups[token], path, follow_symlinks=False) + continue + if kind == "reg": + if metacopy and not private: + with path.open("wb") as stream: + stream.truncate(obj["size"]) + os.setxattr(path, "user.overlay.metacopy", b"") + os.setxattr(path, "user.overlay.redirect", ("/" + obj["sha256"]).encode()) + else: + # Metadata restoration and writes must never change a cached blob's inode. + clone_file(snapshot.content(obj), path) + elif kind == "lnk": + path.symlink_to(uapi16.symlink_target(obj)) + elif kind == "fifo": + os.mkfifo(path) + elif kind in ("chr", "blk"): + mode = stat.S_IFCHR if kind == "chr" else stat.S_IFBLK + os.mknod(path, mode | obj["mode"], os.makedev(obj["major"], obj["minor"])) + elif kind == "sock": + os.mknod(path, stat.S_IFSOCK | obj["mode"]) + else: + fail(f"storage: unsupported inode type {kind!r}") + metadata(path, obj) + if token: + groups[token] = path + # Creating children changes directory mtimes, so restore directories last. + for path, obj in reversed(directories): + metadata(path, obj) + + +def _identities(objects: Sequence[uapi16.Object]) -> dict[str, tuple[uapi16.Object, frozenset[str]]]: + """Map each name to its object without the inodeToken and to the names of its hardlink group.""" + groups: dict[int, set[str]] = {} + for obj in objects: + if token := obj.get("inodeToken"): + groups.setdefault(token, set()).add(obj["name"]) + identities: dict[str, tuple[uapi16.Object, frozenset[str]]] = {} + for obj in objects: + rest = obj.copy() + token = rest.pop("inodeToken", None) + identities[obj["name"]] = (rest, frozenset(groups[token]) if token else frozenset()) + return identities + + +def _reflinks(snapshot: Snapshot, scratch: Path) -> bool: + # clone_file() reflinks through copy_file_range() where the filesystem supports reflinks. On other + # filesystems it copies every byte, and copying every blob is slower than an overlay mount. + blob = next(iter(snapshot.blobs.values()), None) + if blob is None: + return True + try: + reflink(blob, scratch / "reflink") + except ReflinkUnavailable: + return False + return True + + +def delta(snapshot: Snapshot, base: int) -> list[str]: + """Keep the objects that differ from the first `base` stores, and their parent directories. + + Return the names that the first `base` stores contain and the snapshot does not. + """ + inherited = _identities(Snapshot(snapshot.stores[:base]).objects[1:]) + current = _identities(snapshot.objects[1:]) + changed = {name for name, obj in current.items() if inherited.get(name) != obj} + needed = {str(parent) for name in changed for parent in Path(name).parents} + snapshot.objects = [snapshot.objects[0]] + [ + obj for obj in snapshot.objects[1:] if obj["name"] in changed or obj["name"] in needed + ] + return sorted(inherited.keys() - current.keys()) + + +def write_buck_tree(snapshot: Snapshot, out: Path) -> None: + """Write the snapshot as a directory that contains only what Buck stores.""" + out.mkdir(parents=True, exist_ok=True) + out.chmod(0o755) + for obj in snapshot.objects[1:]: + path = out.joinpath(*map(rootfs.escape, obj["name"].split("/"))) + kind = obj["type"] + if kind == "dir": + path.mkdir() + path.chmod(0o755) + elif kind == "reg": + clone_file(snapshot.content(obj), path) + path.chmod(0o755 if obj["mode"] & 0o111 else 0o644) + elif kind == "lnk": + path.symlink_to(uapi16.symlink_target(obj)) + else: + fail(f"storage: Buck cannot store the {kind} inode {obj['name']!r}") + + +@contextmanager +def root(snapshot: Snapshot, *, readonly: bool = False, strategy: Strategy = "auto") -> Generator[Root]: + # Overlay refuses a read-only mount with one lower layer and no data layer. An empty snapshot + # has no blob directories to use as data layers. + if not snapshot.stores: + strategy = "copy" + with tempfile.TemporaryDirectory(prefix="snapshot-root.", dir="/var/tmp") as temporary: + scratch = Path(temporary) + tree = scratch / "root" + if strategy == "auto": + strategy = "copy" if _reflinks(snapshot, scratch) else "overlay" + if strategy == "copy": + restore(snapshot, tree) + with ExitStack() as stack: + if readonly: + bound = stack.enter_context(Bind(tree, tree, readonly=True)) + bound.mount() + yield Root(tree) + return + lower = scratch / "lower" + upper = None if readonly else scratch / "upper" + work = None if readonly else scratch / "work" + restore(snapshot, lower, metacopy=True, upper=upper) + if work is not None: + work.mkdir() + datadirs = tuple(store / "blobs" for store in snapshot.stores) + with Overlay(lower, upper, work, tree, datadirs=datadirs) as mounted: + mounted.mount() + yield Root(tree, upper) diff --git a/uapi16/storage_test.py b/uapi16/storage_test.py new file mode 100644 index 00000000..b15fe36c --- /dev/null +++ b/uapi16/storage_test.py @@ -0,0 +1,514 @@ +# SPDX-FileCopyrightText: Amutable GmbH +# SPDX-License-Identifier: MPL-2.0 + +"""Snapshot storage and reconstruction regressions.""" + +import errno +import os +import shutil +import socket +import stat +import tempfile +import unittest +from pathlib import Path +from typing import override +from unittest import mock + +from util import ReflinkUnavailable, clone_file + +import storage +import uapi16 +from isolation import Overlay + +# Captures in these tests keep every timestamp, so they clamp to an epoch after all of them. +CAPTURE_EPOCH = 2**31 + + +class TestStorage(unittest.TestCase): + @override + def setUp(self) -> None: + self.scratch = Path(self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp"))) + self.tree = self.scratch / "source" + self.tree.mkdir() + self.store = self.scratch / "store" + + def files(self) -> None: + for name, mode, time in (("a", 0o640, 111), ("b", 0o755, 1800000000000000222)): + path = self.tree / name + path.write_text("identical") + path.chmod(mode) + os.utime(path, ns=(time, time)) + os.link(self.tree / "a", self.tree / "linked") + + def test_roundtrip_after_artifact_metadata_is_lost(self) -> None: + # Copy the store without timestamps or hardlinks. The manifest must restore inode identity, + # original modes and times, literal marker names, symlinks, empty directories and user xattrs. + self.files() + for name in ("back\\slash", ".wh.literal", ".wh..wh..opq", ".esc.literal"): + (self.tree / name).write_text(name) + (self.tree / "empty").mkdir() + (self.tree / "link").symlink_to("/a") + os.utime(self.tree / "link", ns=(333, 333), follow_symlinks=False) + os.utime(self.tree / "empty", ns=(444, 444)) + os.setxattr(self.tree / "a", "user.example", b"\x00\xff") + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + copied = self.scratch / "copied" + shutil.copytree(self.store, copied, copy_function=shutil.copyfile) + materialized = self.scratch / "restored" + storage.restore(storage.Snapshot([copied]), materialized) + with ( + storage.root(storage.Snapshot([copied]), readonly=True) as readonly, + storage.root(storage.Snapshot([copied])) as restored, + ): + for tree in (materialized, readonly.path, restored.path): + with self.subTest(tree=tree): + self.assertEqual((tree / "a").stat().st_mtime_ns, 111) + self.assertEqual(stat.S_IMODE((tree / "a").stat().st_mode), 0o640) + self.assertEqual((tree / "b").stat().st_mtime_ns, 1800000000000000222) + self.assertEqual(stat.S_IMODE((tree / "b").stat().st_mode), 0o755) + self.assertTrue((tree / "a").samefile(tree / "linked")) + self.assertFalse((tree / "a").samefile(tree / "b")) + self.assertEqual(os.getxattr(tree / "a", "user.example"), b"\x00\xff") + self.assertEqual(os.readlink(tree / "link"), "/a") + self.assertEqual((tree / "link").lstat().st_mtime_ns, 333) + self.assertEqual((tree / "empty").stat().st_mtime_ns, 444) + self.assertEqual({p.name for p in tree.iterdir()}, {p.name for p in self.tree.iterdir()}) + child = self.scratch / "child" + storage.capture(restored, child, [copied], epoch=CAPTURE_EPOCH) + self.assertEqual((child / uapi16.MANIFEST).read_bytes(), (copied / uapi16.MANIFEST).read_bytes()) + + def test_capture_clamps_times_to_the_epoch(self) -> None: + # A package manager writes the time of the install into every file and directory that it + # changes. Capture must store the epoch for a time after it and keep an earlier time, so two + # installs at different times write the same manifest. Capture a tree with one file before and + # one file after the epoch, and check the times in the manifest. + self.files() + storage.capture(storage.Root(self.tree), self.store, epoch=1700000000) + objects = {obj.get("name"): obj for obj in storage.Snapshot([self.store]).objects} + self.assertEqual(objects["a"]["mTimeNSec"], 111) + self.assertEqual(objects["b"]["mTimeNSec"], 1700000000 * 10**9) + + def test_writes_and_metadata_leave_blobs_unchanged(self) -> None: + # Writes and metadata changes in a restored root must preserve declared hardlinks, keep + # equal-content independent files separate and leave the cached blob unchanged. + self.files() + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + blob = next((self.store / "blobs").iterdir()) + before = blob.read_bytes(), blob.stat() + with storage.root(storage.Snapshot([self.store])) as restored: + root = restored.path + self.assertTrue((root / "a").samefile(root / "linked")) + self.assertFalse((root / "a").samefile(root / "b")) + self.assertEqual((root / "a").stat().st_mtime_ns, 111) + self.assertEqual((root / "b").stat().st_mtime_ns, 1800000000000000222) + (root / "b").chmod(0o604) + os.chown(root / "b", 0, 0, follow_symlinks=False) + os.utime(root / "b", ns=(444, 444)) + self.assertEqual((root / "a").stat().st_mtime_ns, 111) + self.assertEqual(stat.S_IMODE((root / "a").stat().st_mode), 0o640) + (root / "a").write_text("linked write") + (root / "linked").chmod(0o600) + os.utime(root / "a", ns=(555, 555)) + self.assertEqual((root / "linked").read_text(), "linked write") + self.assertEqual(stat.S_IMODE((root / "a").stat().st_mode), 0o600) + self.assertEqual((root / "linked").stat().st_mtime_ns, 555) + self.assertEqual((root / "b").read_text(), "identical") + (root / "b").write_text("independent write") + self.assertEqual((root / "a").read_text(), "linked write") + self.assertEqual(blob.read_bytes(), before[0]) + self.assertEqual(blob.stat().st_mtime_ns, before[1].st_mtime_ns) + self.assertEqual(blob.stat().st_mode, before[1].st_mode) + self.assertEqual(blob.stat().st_uid, before[1].st_uid) + + def test_readonly_reconstruction_never_copies_blob_data(self) -> None: + # Sparse lower metadata files restore independent files and declared hardlinks without + # copying any data. Reads preserve metadata, while writes and chmod fail with EROFS. + self.files() + (self.tree / "nested").mkdir() + (self.tree / "independent").write_text("identical") + (self.tree / "empty").touch() + os.link(self.tree / "a", self.tree / "nested/linked") + os.utime(self.tree / "nested", ns=(999, 999)) + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + with mock.patch.object(storage, "clone_file", wraps=storage.clone_file) as copy: + with storage.root(storage.Snapshot([self.store]), readonly=True, strategy="overlay") as restored: + root = restored.path + self.assertEqual(copy.call_count, 0) + self.assertTrue((root / "a").samefile(root / "nested/linked")) + self.assertFalse((root / "b").samefile(root / "independent")) + self.assertEqual((root / "independent").read_text(), "identical") + self.assertEqual((root / "empty").read_bytes(), b"") + self.assertEqual((root / "nested").stat().st_mtime_ns, 999) + self.assertEqual(copy.call_count, 0) + for operation in ( + lambda: (root / "a").write_text("changed"), + lambda: (root / "independent").chmod(0o600), + ): + with self.assertRaises(OSError) as error: + operation() + self.assertEqual(error.exception.errno, errno.EROFS) + + def test_commands_can_create_new_hardlinks(self) -> None: + # A command creates a hardlink to an inherited independent file. Writes must update the + # alias, and capturing the result must record a hardlink group without changing its blob. + self.files() + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + child = self.scratch / "child" + with storage.root(storage.Snapshot([self.store]), strategy="overlay") as restored: + root = restored.path + os.link(root / "b", root / "new-link") + (root / "b").write_text("new group") + (root / "new-link").chmod(0o600) + os.utime(root / "new-link", ns=(777, 777)) + self.assertEqual((root / "new-link").read_text(), "new group") + self.assertEqual(stat.S_IMODE((root / "b").stat().st_mode), 0o600) + entries = {entry.name: entry for entry in restored.entries()} + self.assertEqual(entries["b"].inode, entries["new-link"].inode) + storage.capture(restored, child, [self.store], epoch=CAPTURE_EPOCH) + with storage.root(storage.Snapshot([self.store, child])) as restored: + root = restored.path + self.assertTrue((root / "b").samefile(root / "new-link")) + self.assertEqual((root / "new-link").read_text(), "new group") + self.assertEqual((root / "b").stat().st_mtime_ns, 777) + self.assertEqual((root / "a").read_text(), "identical") + + def test_extension_compares_hardlink_membership(self) -> None: + # An earlier new group changes token numbers without changing inherited files. Extension + # selection must omit those files and include every member when a group gains an alias. + (self.tree / "z").write_text("inherited") + os.link(self.tree / "z", self.tree / "z-linked") + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + (self.tree / "a").write_text("new group") + os.link(self.tree / "a", self.tree / "a-linked") + child = self.scratch / "child" + storage.capture(storage.Root(self.tree), child, [self.store], epoch=CAPTURE_EPOCH) + snapshot = storage.Snapshot([self.store, child]) + storage.delta(snapshot, 1) + with storage.root(snapshot, strategy="copy") as restored: + self.assertEqual({p.name for p in restored.path.iterdir()}, {"a", "a-linked"}) + self.assertTrue((restored.path / "a").samefile(restored.path / "a-linked")) + + # A new alias changes group membership even when the original members keep the same token. + (self.tree / "a").unlink() + (self.tree / "a-linked").unlink() + os.link(self.tree / "z", self.tree / "new-alias") + alias = self.scratch / "alias" + storage.capture(storage.Root(self.tree), alias, [self.store], epoch=CAPTURE_EPOCH) + snapshot = storage.Snapshot([self.store, alias]) + storage.delta(snapshot, 1) + with storage.root(snapshot, strategy="copy") as restored: + self.assertEqual({p.name for p in restored.path.iterdir()}, {"z", "z-linked", "new-alias"}) + self.assertTrue((restored.path / "z").samefile(restored.path / "new-alias")) + self.assertTrue((restored.path / "z-linked").samefile(restored.path / "new-alias")) + + def test_writable_reconstruction_copies_only_declared_groups(self) -> None: + # A writable overlay needs private data only for declared hardlink groups. Independent + # files must read from blobs until chmod or a write copies the affected path into the upper. + self.files() + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + with mock.patch.object(storage, "clone_file", wraps=storage.clone_file) as copy: + with storage.root(storage.Snapshot([self.store]), strategy="overlay") as restored: + root = restored.path + assert restored.upper is not None + self.assertEqual(copy.call_count, 1) + self.assertTrue((restored.upper / "a").samefile(restored.upper / "linked")) + self.assertFalse((restored.upper / "b").exists()) + self.assertEqual((root / "b").read_text(), "identical") + self.assertFalse((restored.upper / "b").exists()) + (root / "b").chmod(0o600) + self.assertTrue((restored.upper / "b").exists()) + self.assertEqual(copy.call_count, 1) + + def test_available_reflinks_use_private_inodes_without_overlay(self) -> None: + # Simulate successful reflinks with private files on the test filesystem. Automatic + # reconstruction must preserve inode independence, hardlinks and metadata without OverlayFS. + self.files() + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + blob = next((self.store / "blobs").iterdir()) + before = blob.read_bytes(), blob.stat() + + for readonly in (False, True): + with ( + self.subTest(readonly=readonly), + mock.patch.object(storage, "reflink", side_effect=clone_file), + mock.patch.object(storage, "Overlay", side_effect=AssertionError("unexpected overlay")), + storage.root(storage.Snapshot([self.store]), readonly=readonly) as restored, + ): + root = restored.path + self.assertIsNone(restored.upper) + self.assertTrue((root / "a").samefile(root / "linked")) + self.assertFalse((root / "a").samefile(root / "b")) + self.assertFalse((root / "a").samefile(blob)) + self.assertEqual((root / "a").stat().st_mtime_ns, 111) + self.assertEqual(stat.S_IMODE((root / "b").stat().st_mode), 0o755) + if readonly: + with self.assertRaises(OSError) as error: + (root / "a").write_text("changed") + self.assertEqual(error.exception.errno, errno.EROFS) + else: + os.link(root / "b", root / "new-link") + self.assertTrue((root / "b").samefile(root / "new-link")) + (root / "a").chmod(0o600) + (root / "linked").write_text("changed") + self.assertEqual((root / "a").read_text(), "changed") + self.assertEqual((root / "b").read_text(), "identical") + self.assertEqual(blob.read_bytes(), before[0]) + self.assertEqual(blob.stat().st_mode, before[1].st_mode) + self.assertEqual(blob.stat().st_mtime_ns, before[1].st_mtime_ns) + + def test_unavailable_reflinks_fall_back_to_overlay(self) -> None: + # Automatic reconstruction reflinks one blob to test the filesystem. When that fails, it + # must reconstruct an overlay with the original contents and metadata. + self.files() + (self.tree / "b").write_text("different contents") + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + + with mock.patch.object( + storage, "reflink", side_effect=ReflinkUnavailable(errno.EXDEV, "different filesystems") + ) as reflink: + with storage.root(storage.Snapshot([self.store])) as restored: + root = restored.path + self.assertIsNotNone(restored.upper) + self.assertEqual(reflink.call_count, 1) + self.assertEqual((root / "a").read_text(), "identical") + self.assertEqual((root / "b").read_text(), "different contents") + self.assertTrue((root / "a").samefile(root / "linked")) + self.assertEqual((root / "a").stat().st_mtime_ns, 111) + + def test_metadata_failure_does_not_select_another_backend(self) -> None: + # An unsupported metadata operation is a reconstruction error rather than evidence that + # reflinks are unavailable. Report the metadata failure without attempting an overlay. + self.files() + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + + with ( + mock.patch.object(storage, "reflink", side_effect=clone_file), + mock.patch.object(storage, "metadata", side_effect=OSError(errno.EOPNOTSUPP, "metadata")), + mock.patch.object(storage, "Overlay", side_effect=AssertionError("unexpected overlay")), + self.assertRaises(OSError) as error, + ): + with storage.root(storage.Snapshot([self.store])): + pass + self.assertEqual(error.exception.errno, errno.EOPNOTSUPP) + + def test_independent_hardlink_groups_can_share_one_blob(self) -> None: + # Two declared groups have equal contents but different metadata. Each group must keep + # its own inode when chmod and writes modify the other group in the temporary root. + self.files() + os.link(self.tree / "b", self.tree / "second-link") + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + self.assertEqual(len(list((self.store / "blobs").iterdir())), 1) + with storage.root(storage.Snapshot([self.store])) as restored: + root = restored.path + self.assertTrue((root / "a").samefile(root / "linked")) + self.assertTrue((root / "b").samefile(root / "second-link")) + self.assertFalse((root / "a").samefile(root / "b")) + (root / "linked").chmod(0o600) + (root / "a").write_text("first group") + self.assertEqual((root / "second-link").read_text(), "identical") + self.assertEqual(stat.S_IMODE((root / "b").stat().st_mode), 0o755) + (root / "second-link").write_text("second group") + self.assertEqual((root / "b").read_text(), "second group") + self.assertEqual((root / "linked").read_text(), "first group") + + def test_reuse_and_metadata_only_changes(self) -> None: + # Changing only metadata must alter the complete manifest while reusing the parent's blob. + self.files() + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + child = self.scratch / "child" + with storage.root(storage.Snapshot([self.store])) as restored: + root = restored.path + (root / "b").chmod(0o600) + os.utime(root / "b", ns=(999, 999)) + storage.capture(restored, child, [self.store], epoch=CAPTURE_EPOCH) + self.assertEqual(list((child / "blobs").iterdir()), []) + self.assertNotEqual( + (child / uapi16.MANIFEST).read_bytes(), (self.store / uapi16.MANIFEST).read_bytes() + ) + with storage.root(storage.Snapshot([self.store, child]), readonly=True) as restored: + root = restored.path + self.assertEqual(stat.S_IMODE((root / "b").stat().st_mode), 0o600) + self.assertEqual((root / "b").stat().st_mtime_ns, 999) + + def test_deletions_and_replacements(self) -> None: + # Delete inherited paths, replace files and directories, and rename a fresh directory over + # an inherited directory. Each child snapshot must describe the complete resulting tree. + (self.tree / "dir").mkdir() + (self.tree / "dir/old").write_text("old") + (self.tree / "deleted").write_text("deleted") + (self.tree / "file").write_text("file") + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + child = self.scratch / "child" + with storage.root(storage.Snapshot([self.store])) as restored: + root = restored.path + (root / "deleted").unlink() + shutil.rmtree(root / "dir") + (root / "fresh").mkdir() + (root / "fresh/new").write_text("new") + (root / "fresh").rename(root / "dir") + (root / "file").unlink() + (root / "file").mkdir() + (root / "file/child").write_text("child") + storage.capture(restored, child, [self.store], epoch=CAPTURE_EPOCH) + with storage.root(storage.Snapshot([self.store, child])) as restored: + root = restored.path + self.assertFalse((root / "deleted").exists()) + self.assertEqual([p.name for p in (root / "dir").iterdir()], ["new"]) + self.assertEqual((root / "file/child").read_text(), "child") + shutil.rmtree(root / "file") + (root / "file").write_text("replacement") + grandchild = self.scratch / "grandchild" + storage.capture(restored, grandchild, [self.store, child], epoch=CAPTURE_EPOCH) + with storage.root(storage.Snapshot([self.store, child, grandchild])) as restored: + root = restored.path + self.assertEqual((root / "file").read_text(), "replacement") + self.assertEqual({p.name for p in child.iterdir()}, {"blobs", uapi16.MANIFEST}) + self.assertNotIn(b"overlay.opaque", (child / uapi16.MANIFEST).read_bytes()) + + def test_capture_native_overlay_changes(self) -> None: + # An external overlay can introduce native whiteouts and opaque directories. Capturing + # its visible filesystem must omit deleted paths and overlay markers from the snapshot. + (self.tree / "opaque").mkdir() + (self.tree / "opaque/old").write_text("old") + (self.tree / "deleted").write_text("deleted") + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + lower, upper, work, merged = (self.scratch / name for name in ("lower", "upper", "work", "merged")) + storage.restore(storage.Snapshot([self.store]), lower) + upper.mkdir() + work.mkdir() + child = self.scratch / "child" + with Overlay(lower, upper, work, merged) as overlay: + overlay.mount() + (merged / "deleted").unlink() + shutil.rmtree(merged / "opaque") + (merged / "opaque").mkdir() + (merged / "opaque/new").write_text("new") + self.assertEqual(os.getxattr(upper / "opaque", "user.overlay.opaque"), b"y") + storage.capture(storage.Root(merged, upper), child, [self.store], epoch=CAPTURE_EPOCH) + with storage.root(storage.Snapshot([self.store, child])) as restored: + root = restored.path + self.assertFalse((root / "deleted").exists()) + self.assertEqual([path.name for path in (root / "opaque").iterdir()], ["new"]) + self.assertNotIn(b"overlay.opaque", (child / uapi16.MANIFEST).read_bytes()) + self.assertEqual({path.name for path in child.iterdir()}, {"blobs", uapi16.MANIFEST}) + + def test_failed_commands_remove_the_temporary_root(self) -> None: + # A command failure must remove the temporary root while retaining the reusable snapshot. + self.files() + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + root: Path | None = None + with self.assertRaisesRegex(RuntimeError, "command failed"): + with storage.root(storage.Snapshot([self.store])) as restored: + root = restored.path + (root / "a").write_text("changed") + raise RuntimeError("command failed") + assert root is not None + self.assertFalse(root.exists()) + with storage.root(storage.Snapshot([self.store])) as restored: + root = restored.path + self.assertEqual((root / "a").read_text(), "identical") + + def test_child_stores_only_new_contents(self) -> None: + # A child reuses every inherited content digest and writes one blob for two new equal files. + self.files() + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + child = self.scratch / "child" + with storage.root(storage.Snapshot([self.store])) as restored: + root = restored.path + (root / "new").write_text("new contents") + (root / "new-independent").write_text("new contents") + storage.capture(restored, child, [self.store], epoch=CAPTURE_EPOCH) + self.assertEqual(len(list((child / "blobs").iterdir())), 1) + with storage.root(storage.Snapshot([self.store, child]), readonly=True) as restored: + root = restored.path + self.assertEqual((root / "a").read_text(), "identical") + self.assertFalse((root / "new").samefile(root / "new-independent")) + + def test_readonly_empty_snapshot(self) -> None: + # An image without parent stores still needs an empty, read-only root with its default mode. + with storage.root(storage.Snapshot([]), readonly=True) as restored: + root = restored.path + self.assertEqual(list(root.iterdir()), []) + self.assertEqual(stat.S_IMODE(root.stat().st_mode), 0o755) + with self.assertRaises(OSError) as error: + (root / "new").touch() + self.assertEqual(error.exception.errno, errno.EROFS) + + def test_device_nodes(self) -> None: + # Preserve device numbers when the namespace permits mknod; unprivileged hosts may deny it. + try: + os.mknod(self.tree / "device", stat.S_IFCHR | 0o600, os.makedev(1, 3)) + except PermissionError: + self.skipTest("the sandbox does not permit device nodes") + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + with storage.root(storage.Snapshot([self.store])) as restored: + root = restored.path + attributes = (root / "device").lstat() + self.assertTrue(stat.S_ISCHR(attributes.st_mode)) + self.assertEqual(attributes.st_rdev, os.makedev(1, 3)) + + def test_special_files(self) -> None: + # FIFOs and sockets need manifest inode types because Buck only stores regular blobs. + os.mkfifo(self.tree / "fifo", 0o600) + with socket.socket(socket.AF_UNIX) as sock: + sock.bind(str(self.tree / "sock")) + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + for readonly in (False, True): + with ( + self.subTest(readonly=readonly), + storage.root(storage.Snapshot([self.store]), readonly=readonly) as restored, + ): + root = restored.path + self.assertTrue(stat.S_ISFIFO((root / "fifo").lstat().st_mode)) + self.assertTrue(stat.S_ISSOCK((root / "sock").lstat().st_mode)) + self.assertEqual(stat.S_IMODE((root / "fifo").stat().st_mode), 0o600) + + def test_buck_tree_contains_only_what_buck_stores(self) -> None: + # A box layer directory must contain only regular files, directories and symlinks with modes + # that Buck stores. Capture a tree with a hardlink, an extended attribute, a private directory + # and a name containing a backslash. The Buck tree must contain separate files with normalized + # modes, no extended attributes and the escaped name. + self.files() + os.setxattr(self.tree / "a", "user.example", b"value") + (self.tree / "private").mkdir(mode=0o700) + (self.tree / "back\\slash").write_text("escaped") + (self.tree / "link").symlink_to("a") + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + out = self.scratch / "out" + storage.write_buck_tree(storage.Snapshot([self.store]), out) + self.assertEqual(stat.S_IMODE((out / "a").stat().st_mode), 0o644) + self.assertEqual(stat.S_IMODE((out / "b").stat().st_mode), 0o755) + self.assertEqual(stat.S_IMODE((out / "private").stat().st_mode), 0o755) + self.assertFalse((out / "a").samefile(out / "linked")) + self.assertEqual((out / "linked").read_text(), "identical") + self.assertEqual(os.listxattr(out / "a"), []) + self.assertEqual((out / ".esc.back%5Cslash").read_text(), "escaped") + self.assertEqual(os.readlink(out / "link"), "a") + + def test_buck_tree_rejects_special_files(self) -> None: + # Buck cannot store a FIFO. Writing the Buck tree of a snapshot that contains one must fail. + os.mkfifo(self.tree / "fifo") + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + with self.assertRaises(SystemExit): + storage.write_buck_tree(storage.Snapshot([self.store]), self.scratch / "out") + + def test_delta_keeps_changes_and_returns_deletions(self) -> None: + # Capture a parent with three files, then change one, add one and delete one in a child. + # delta() must keep the changed and added files with their directory and return the deleted + # name. + (self.tree / "dir").mkdir() + for name in ("dir/kept", "dir/changed", "gone"): + (self.tree / name).write_text(name) + storage.capture(storage.Root(self.tree), self.store, epoch=CAPTURE_EPOCH) + (self.tree / "dir/changed").write_text("different") + (self.tree / "dir/new").write_text("new") + (self.tree / "gone").unlink() + child = self.scratch / "child" + storage.capture(storage.Root(self.tree), child, [self.store], epoch=CAPTURE_EPOCH) + snapshot = storage.Snapshot([self.store, child]) + self.assertEqual(storage.delta(snapshot, 1), ["gone"]) + self.assertEqual({obj["name"] for obj in snapshot.objects[1:]}, {"dir", "dir/changed", "dir/new"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/image/manifest.py b/uapi16/uapi16.py old mode 100755 new mode 100644 similarity index 60% rename from image/manifest.py rename to uapi16/uapi16.py index 5dfd8189..9db6c073 --- a/image/manifest.py +++ b/uapi16/uapi16.py @@ -1,45 +1,23 @@ -#!/usr/bin/python3 - # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""List everything a logical image ships, as a UAPI.16 File Manifest. - -The format is an RFC7464 JSON-SEQ sequence: a root object naming the media type, then one object -per inode, each directory immediately followed by its own contents. Two consumers want this: a -comparison of the paths two images ship (a path in both a system extension and the /usr it merges -onto silently shadows an OS file), and per-image size accounting. Both need every inode type an OS -tree holds, so a type that cannot be listed is an error rather than an omission. - -A driver writing a listing of what it is about to package imports this rather than running it: only -the driver holds the finished tree, and only it knows which paths of that tree its artifact carries. -`roots` is how it says so, and names stay relative to the image, so a listing reads as where each -path lands on a machine rather than where it sits inside one partition. -""" +"""Read, write and combine UAPI.16 filesystem listings.""" import base64 import hashlib import json import os import stat -import sys from collections import Counter from collections.abc import Iterator, Sequence from dataclasses import dataclass from pathlib import Path -from typing import cast +from typing import TypedDict, cast -import specs from util import fail -import finalize - - -class Spec(finalize.ImageSpec): - out: str - - MEDIA_TYPE = "application/vnd.uapi.16.manifest" +TRAILER_TYPE = "application/vnd.uapi.16.trailer" # RFC7464 frames every JSON text with a leading record separator and a trailing line feed. _RS = b"\x1e" @@ -47,7 +25,7 @@ class Spec(finalize.ImageSpec): # A directory holding a manifest cannot also list it, so the format reserves the name at the top # level; deeper in the tree it is an ordinary file name. -_RESERVED = "Uapi16Manifest" +MANIFEST = "Uapi16Manifest" _INODE_TYPES = ( (stat.S_ISREG, "reg"), @@ -62,6 +40,35 @@ class Spec(finalize.ImageSpec): _NANOSECONDS = 10**9 +class Content(TypedDict, total=False): + file: str + literal: str + + +class Xattr(TypedDict, total=False): + text: str + base64: str + + +class Object(TypedDict, total=False): + """One record of a UAPI.16 manifest.""" + + mediaType: str + name: str + type: str + mode: int + uid: int + gid: int + mTimeNSec: int + size: int + major: int + minor: int + sha256: str + contents: list[Content] + inodeToken: int + extendedAttributes: dict[str, dict[str, Xattr]] + + @dataclass(frozen=True) class Entry: """One inode below the image root, under the name the manifest gives it.""" @@ -83,19 +90,17 @@ def inode_type(mode: int) -> str: def check_name(name: str) -> str: - """Reject a path the format cannot encode, rather than listing the image incompletely. - - Nothing normalizes here: scandir yields neither "." nor ".." nor an empty component, so what - is left is the format's own restrictions on the bytes and on the reserved file name. - """ + """Reject a path the format cannot encode, rather than listing the image incompletely.""" try: name.encode("utf-8") except UnicodeEncodeError: fail(f"manifest: {name!r} is not valid UTF-8, which a manifest name must be") if any(ord(character) < 32 or ord(character) == 127 for character in name): fail(f"manifest: {name!r} contains a control character") + if any(part in ("", ".", "..") for part in name.split("/")): + fail(f"manifest: {name!r} must be a normalized relative path") top = name.split("/", 1)[0] - if top == _RESERVED or top.startswith(_RESERVED + "."): + if top == MANIFEST or top.startswith(MANIFEST + "."): fail(f"manifest: {name!r} is the reserved manifest file name") return name @@ -172,10 +177,10 @@ def _digest(path: Path) -> str: return hashlib.file_digest(stream, "sha256").hexdigest() -def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> dict[str, object]: - """The record for one inode, with its fields in the order the specification lists them.""" +def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> Object: + """Describe one inode using UAPI.16 metadata and content fields.""" kind = inode_type(entry.st.st_mode) - obj: dict[str, object] = {"name": entry.name, "type": kind} + obj: Object = {"name": entry.name, "type": kind} target = os.fsencode(os.readlink(entry.path)) if kind == "lnk" else b"" if kind == "reg": @@ -189,9 +194,31 @@ def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> dict[s obj["mode"] = stat.S_IMODE(entry.st.st_mode) obj["uid"] = entry.st.st_uid obj["gid"] = entry.st.st_gid - # Clamped exactly as every other reproducible output here clamps: what a package stamped - # survives, what this build stamped collapses onto the epoch. - obj["mTime"] = min(entry.st.st_mtime_ns, epoch * _NANOSECONDS) + obj["mTimeNSec"] = min(entry.st.st_mtime_ns, epoch * _NANOSECONDS) + if entry.st.st_mtime_ns < 0: + fail(f"manifest: {entry.name!r} has a timestamp before the UNIX epoch") + attributes: dict[str, Xattr] = {} + for name in sorted(os.listxattr(entry.path, follow_symlinks=False)): + if not name.startswith("user.") or name.startswith("user.overlay."): + continue + short = name.removeprefix("user.") + if not short or any(ord(c) < 32 or ord(c) == 127 for c in short): + fail(f"manifest: unsupported extended attribute name {name!r}") + try: + short.encode("utf-8") + except UnicodeEncodeError: + fail(f"manifest: unsupported extended attribute name {name!r}") + value = os.getxattr(entry.path, name, follow_symlinks=False) + try: + text = value.decode("utf-8") + except UnicodeDecodeError: + text = None + if text is None or "\0" in text: + attributes[short] = {"base64": base64.b64encode(value).decode("ascii")} + else: + attributes[short] = {"text": text} + if attributes: + obj["extendedAttributes"] = {"user": attributes} if token is not None: obj["inodeToken"] = token if kind == "lnk": @@ -205,20 +232,44 @@ def file_object(entry: Entry, *, epoch: int, token: int | None = None) -> dict[s return obj -def _record(obj: dict[str, object]) -> bytes: - return _RS + json.dumps(obj, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + _LF +def symlink_target(obj: Object) -> str: + return os.fsdecode(base64.b64decode(obj["contents"][0]["literal"])) + + +def xattr_value(value: Xattr) -> bytes: + return value["text"].encode() if "text" in value else base64.b64decode(value["base64"]) + + +def write_listing(objects: Sequence[Object], out: Path) -> int: + # The `contents` of a regular file names a blob, and the blob only exists inside a store. + listed: list[Object] = [] + for obj in objects: + copy = obj.copy() + if copy.get("type") != "lnk": + copy.pop("contents", None) + listed.append(copy) + return write_objects(listed, out) + + +def _record(obj: Object) -> bytes: + return ( + _RS + + json.dumps(obj, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + + _LF + ) def records(tree: Path, epoch: int, roots: Sequence[str] = ()) -> Iterator[bytes]: """The whole sequence for `tree`: the root object, then one object per inode below it.""" # The root object carries the media type alone, which is what marks the sequence a manifest. - # Its mode and mtime would be the ephemeral overlay upper's rather than the image's, and the - # format defaults a nameless first object to a directory anyway. + # A partition manifest lists a part of the image, and the image root is not part of it. The format + # defaults a nameless first object to a directory. yield _record({"mediaType": MEDIA_TYPE}) found = list(entries(tree, roots)) tokens = hardlink_tokens(found) for entry in found: yield _record(file_object(entry, epoch=epoch, token=tokens.get(entry.inode))) + yield _record({"mediaType": TRAILER_TYPE}) def write(tree: Path, out: Path, epoch: int, roots: Sequence[str] = ()) -> int: @@ -231,21 +282,55 @@ def write(tree: Path, out: Path, epoch: int, roots: Sequence[str] = ()) -> int: return written -def _objects(source: Path) -> Iterator[dict[str, object]]: - """The named objects of a manifest, the root object it opens with aside.""" +def read(source: Path) -> list[Object]: + """Read a complete manifest, rejecting truncated sequences and trailing data.""" data = source.read_bytes() - head, separator, rest = data.partition(_RS) - if not separator or head: + if not data.startswith(_RS): fail(f"manifest: {source} does not open with a record") - for index, record in enumerate(rest.split(_RS)): + objects: list[Object] = [] + names: set[str] = set() + trailer = False + for record in data[1:].split(_RS): + if trailer: + fail(f"manifest: {source} has data after its trailer") if not record.endswith(_LF): fail(f"manifest: {source} holds a record that no line feed ends") - obj = cast(dict[str, object], json.loads(record)) - if index == 0: - if obj != {"mediaType": MEDIA_TYPE}: - fail(f"manifest: {source} does not open with a {MEDIA_TYPE} object") + try: + obj = cast(Object, json.loads(record)) + except (ValueError, UnicodeDecodeError) as error: + fail(f"manifest: {source} has invalid JSON: {error}") + if not isinstance(obj, dict): + fail(f"manifest: {source} holds a non-object record") + if not objects: + if obj.get("mediaType") != MEDIA_TYPE or obj.get("name") is not None: + fail(f"manifest: {source} does not open with a {MEDIA_TYPE} root object") + elif obj.get("mediaType") == TRAILER_TYPE: + text = record.decode("utf-8") + _, end = json.JSONDecoder().raw_decode(text, len(text) - len(text.lstrip())) + if obj != {"mediaType": TRAILER_TYPE} or text[end:].lstrip(" \t\r") != "\n": + fail(f"manifest: {source} has an invalid trailer or data after its trailer") + trailer = True continue - yield obj + elif obj.get("mediaType") not in (None, MEDIA_TYPE): + continue + else: + name = obj.get("name") + if not isinstance(name, str) or name in names: + fail(f"manifest: {source} has a missing or duplicate name: {name!r}") + names.add(check_name(name)) + objects.append(obj) + if not trailer: + fail(f"manifest: {source} is incomplete: missing trailer") + return objects + + +def write_objects(objects: Sequence[Object], out: Path) -> int: + out.parent.mkdir(parents=True, exist_ok=True) + with out.open("wb") as stream: + for obj in objects: + stream.write(_record(obj)) + stream.write(_record({"mediaType": TRAILER_TYPE})) + return len(objects) + 1 def order(name: str) -> list[bytes]: @@ -264,11 +349,11 @@ def merge(sources: Sequence[Path], out: Path) -> int: between them. A name more than one of them lists is taken from the last, which is the partition mounted over a directory the others merely hold. """ - objects: dict[str, dict[str, object]] = {} + objects: dict[str, Object] = {} offset = 0 for source in sources: highest = 0 - for obj in _objects(source): + for obj in read(source)[1:]: token = obj.get("inodeToken") if isinstance(token, int): # A token numbers an inode within one manifest, so each part keeps its own grouping @@ -278,25 +363,15 @@ def merge(sources: Sequence[Path], out: Path) -> int: objects[str(obj["name"])] = obj offset += highest - written = 1 - out.parent.mkdir(parents=True, exist_ok=True) - with out.open("wb") as stream: - stream.write(_record({"mediaType": MEDIA_TYPE})) - for name in sorted(objects, key=order): - stream.write(_record(objects[name])) - written += 1 - return written - - -def main(argv: list[str] | None = None) -> None: - spec = specs.parse(Spec, "manifest", argv) - - out = Path(spec["out"]) - epoch = int(os.environ["SOURCE_DATE_EPOCH"]) - with finalize.image(spec, program="manifest") as tree: - written = write(tree, out, epoch) - print(f"manifest: wrote {written} objects -> {out}", file=sys.stderr) + ordered = [objects[name] for name in sorted(objects, key=order)] + renumber(ordered) + return write_objects([{"mediaType": MEDIA_TYPE}, *ordered], out) -if __name__ == "__main__": - main() +def renumber(objects: Sequence[Object]) -> None: + tokens: dict[int, int] = {} + counts = Counter(obj.get("inodeToken") for obj in objects if obj.get("inodeToken")) + for obj in objects: + token = obj.pop("inodeToken", None) + if isinstance(token, int) and counts[token] > 1: + obj["inodeToken"] = tokens.setdefault(token, len(tokens) + 1) diff --git a/image/manifest_test.py b/uapi16/uapi16_test.py similarity index 78% rename from image/manifest_test.py rename to uapi16/uapi16_test.py index 5375e9ae..3dabfab9 100644 --- a/image/manifest_test.py +++ b/uapi16/uapi16_test.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Amutable GmbH # SPDX-License-Identifier: MPL-2.0 -"""Tests for the UAPI.16 file manifest. +"""Tests for the UAPI.16 file uapi16. buck test tine//image:test """ @@ -16,8 +16,9 @@ from collections.abc import Sequence from pathlib import Path from typing import override +from unittest import mock -import manifest +import uapi16 # The epoch every image action runs under (box/runtime.bzl); any value works, but using the real # one keeps the clamping assertions honest about what a build actually produces. @@ -33,16 +34,17 @@ def parse(data: bytes) -> list[dict[str, object]]: for record in rest.split(b"\x1e"): assert record.endswith(b"\n"), "a record is not terminated by a line feed" objects.append(json.loads(record.decode("utf-8"))) - return objects + assert objects[-1] == {"mediaType": uapi16.TRAILER_TYPE} + return objects[:-1] class TreeTest(unittest.TestCase): @override def setUp(self) -> None: - self.tree = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="manifest."))) + self.tree = Path(self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest."))) def manifest(self, tree: Path | None = None, roots: Sequence[str] = ()) -> list[dict[str, object]]: - return parse(b"".join(manifest.records(tree or self.tree, EPOCH, roots))) + return parse(b"".join(uapi16.records(tree or self.tree, EPOCH, roots))) def named(self, tree: Path | None = None) -> dict[str, dict[str, object]]: return {str(obj["name"]): obj for obj in self.manifest(tree)[1:]} @@ -54,18 +56,20 @@ def test_the_first_object_is_the_root_and_names_the_media_type(self) -> None: root = self.manifest()[0] self.assertEqual(root, {"mediaType": "application/vnd.uapi.16.manifest"}) - def test_an_empty_tree_is_the_root_object_alone(self) -> None: + def test_an_empty_tree_has_no_named_objects(self) -> None: + # An empty filesystem still has its root and trailer, with no named file objects. self.assertEqual(len(self.manifest()), 1) def test_each_record_is_framed_by_a_separator_and_a_line_feed(self) -> None: + # A tree with one file needs three JSON-SEQ records: the root, file and trailer. (self.tree / "a").write_text("a") - data = b"".join(manifest.records(self.tree, EPOCH)) - self.assertEqual(data.count(b"\x1e"), 2) + data = b"".join(uapi16.records(self.tree, EPOCH)) + self.assertEqual(data.count(b"\x1e"), 3) self.assertTrue(data.endswith(b"\n")) def test_the_json_carries_no_gratuitous_whitespace(self) -> None: (self.tree / "a").write_text("a") - self.assertNotIn(b" ", b"".join(manifest.records(self.tree, EPOCH))) + self.assertNotIn(b" ", b"".join(uapi16.records(self.tree, EPOCH))) class TestOrder(TreeTest): @@ -92,14 +96,14 @@ def test_the_bytes_do_not_depend_on_the_order_the_tree_was_created_in(self) -> N names = ["gamma", "alpha", "beta", "delta"] outputs = [] for order in (names, list(reversed(names))): - with tempfile.TemporaryDirectory(prefix="manifest.") as scratch: + with tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.") as scratch: tree = Path(scratch) for name in order: (tree / name).mkdir() (tree / name / "file").write_text(name) os.utime(tree / name / "file", (EPOCH - 1, EPOCH - 1)) os.utime(tree / name, (EPOCH - 1, EPOCH - 1)) - outputs.append(b"".join(manifest.records(tree, EPOCH))) + outputs.append(b"".join(uapi16.records(tree, EPOCH))) self.assertEqual(outputs[0], outputs[1]) @@ -193,7 +197,9 @@ def test_a_file_reached_by_one_name_alone_carries_no_token(self) -> None: def test_a_link_count_above_one_is_not_enough_when_the_other_name_is_outside_the_tree( self, ) -> None: - outside = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="manifest.outside."))) + outside = Path( + self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.outside.")) + ) (outside / "one").write_text("shared") os.link(outside / "one", self.tree / "inside") self.assertEqual((self.tree / "inside").stat().st_nlink, 2) @@ -229,24 +235,25 @@ def test_a_unix_socket_is_listed_as_one(self) -> None: def test_the_inode_type_names_follow_the_format(self) -> None: self.assertEqual( - [manifest.inode_type(mode) for mode in (stat.S_IFBLK, stat.S_IFSOCK, stat.S_IFLNK)], + [uapi16.inode_type(mode) for mode in (stat.S_IFBLK, stat.S_IFSOCK, stat.S_IFLNK)], ["blk", "sock", "lnk"], ) def test_an_inode_type_with_no_representation_fails_rather_than_being_guessed(self) -> None: with self.assertRaises(SystemExit): - manifest.inode_type(0) + uapi16.inode_type(0) class TestDeviceNodes(unittest.TestCase): """Device nodes, from a fabricated inode: this sandbox refuses mknod.""" - def device(self, mode: int, rdev: int) -> dict[str, object]: + def device(self, mode: int, rdev: int) -> uapi16.Object: st = os.stat_result( (mode, 0, 0, 1, 0, 0, 0, 0, EPOCH - 1, 0), {"st_rdev": rdev, "st_mtime_ns": (EPOCH - 1) * 10**9}, ) - return manifest.file_object(manifest.Entry(name="dev/x", path=Path("dev/x"), st=st), epoch=EPOCH) + with mock.patch.object(uapi16.os, "listxattr", return_value=[]): + return uapi16.file_object(uapi16.Entry(name="dev/x", path=Path("dev/x"), st=st), epoch=EPOCH) def test_a_character_device_carries_its_numbers_and_nothing_a_file_would(self) -> None: self.assertEqual( @@ -259,7 +266,7 @@ def test_a_character_device_carries_its_numbers_and_nothing_a_file_would(self) - "mode": 0o666, "uid": 0, "gid": 0, - "mTime": (EPOCH - 1) * 10**9, + "mTimeNSec": (EPOCH - 1) * 10**9, }, ) @@ -273,12 +280,12 @@ class TestTimestamps(TreeTest): def test_an_mtime_past_the_epoch_is_clamped_onto_it(self) -> None: (self.tree / "file").write_text("x") os.utime(self.tree / "file", (EPOCH + 10, EPOCH + 10)) - self.assertEqual(self.named()["file"]["mTime"], EPOCH * 10**9) + self.assertEqual(self.named()["file"]["mTimeNSec"], EPOCH * 10**9) def test_an_mtime_before_the_epoch_survives_in_nanoseconds(self) -> None: (self.tree / "file").write_text("x") os.utime(self.tree / "file", ns=(0, (EPOCH - 5) * 10**9 + 123)) - self.assertEqual(self.named()["file"]["mTime"], (EPOCH - 5) * 10**9 + 123) + self.assertEqual(self.named()["file"]["mTimeNSec"], (EPOCH - 5) * 10**9 + 123) class TestUnencodableNames(TreeTest): @@ -304,28 +311,32 @@ def test_a_reserved_suffix_at_the_top_level_is_refused_too(self) -> None: class TestWrite(TreeTest): - def test_the_written_file_is_the_sequence_and_the_count_includes_the_root(self) -> None: + def test_the_count_includes_the_root_and_trailer(self) -> None: + # Writing one file must report three records and finish with the mandatory trailer. (self.tree / "file").write_text("x") - out = Path(self.enterContext(tempfile.TemporaryDirectory(prefix="manifest.out."))) / "m" - self.assertEqual(manifest.write(self.tree, out, EPOCH), 2) + out = ( + Path(self.enterContext(tempfile.TemporaryDirectory(dir="/var/tmp", prefix="manifest.out."))) + / "m" + ) + self.assertEqual(uapi16.write(self.tree, out, EPOCH), 3) self.assertEqual([obj.get("name") for obj in parse(out.read_bytes())], [None, "file"]) class TestScope(unittest.TestCase): def test_a_leading_and_trailing_slash_is_not_part_of_the_name(self) -> None: - self.assertEqual(manifest.scope(["/usr:", "/boot/"]), ["boot", "usr:"]) + self.assertEqual(uapi16.scope(["/usr:", "/boot/"]), ["boot", "usr:"]) def test_the_same_root_named_twice_is_listed_once(self) -> None: - self.assertEqual(manifest.scope(["/usr", "usr/"]), ["usr"]) + self.assertEqual(uapi16.scope(["/usr", "usr/"]), ["usr"]) def test_a_root_below_another_is_dropped_rather_than_listed_twice(self) -> None: - self.assertEqual(manifest.scope(["/usr", "/usr/lib", "/boot"]), ["boot", "usr"]) + self.assertEqual(uapi16.scope(["/usr", "/usr/lib", "/boot"]), ["boot", "usr"]) def test_a_name_that_merely_shares_a_prefix_is_kept(self) -> None: - self.assertEqual(manifest.scope(["/usr", "/usrfoo"]), ["usr", "usrfoo"]) + self.assertEqual(uapi16.scope(["/usr", "/usrfoo"]), ["usr", "usrfoo"]) def test_the_whole_tree_covers_every_other_root(self) -> None: - self.assertEqual(manifest.scope(["/", "/usr"]), [""]) + self.assertEqual(uapi16.scope(["/", "/usr"]), [""]) class TestScopedTree(TreeTest): @@ -361,7 +372,7 @@ def test_no_roots_is_the_whole_tree(self) -> None: class TestMerge(TreeTest): def part(self, name: str, roots: Sequence[str]) -> Path: out = self.tree / name - manifest.write(self.tree / "parts", out, EPOCH, roots) + uapi16.write(self.tree / "parts", out, EPOCH, roots) return out @override @@ -374,7 +385,7 @@ def setUp(self) -> None: def merged(self, *sources: Path) -> list[dict[str, object]]: out = self.tree / "merged" - manifest.merge(sources, out) + uapi16.merge(sources, out) return parse(out.read_bytes()) def test_the_whole_is_every_part_under_one_root_object(self) -> None: @@ -418,5 +429,44 @@ def test_something_that_is_not_a_manifest_is_refused(self) -> None: self.merged(self.tree / "junk") +class TestReader(TreeTest): + def test_requires_trailer_and_rejects_trailing_data(self) -> None: + # A partial write and data after a trailer must fail instead of becoming a snapshot. + out = self.tree / "manifest" + root = b'\x1e{"mediaType":"application/vnd.uapi.16.manifest"}\n' + trailer = b'\x1e{"mediaType":"application/vnd.uapi.16.trailer"}\n' + for data in (root, root + trailer + trailer, root + trailer + b" ", root + trailer + b"\n"): + out.write_bytes(data) + with self.subTest(data=data), self.assertRaises(SystemExit): + uapi16.read(out) + out.write_bytes(root + trailer) + self.assertEqual(uapi16.read(out), [{"mediaType": uapi16.MEDIA_TYPE}]) + + def test_accepts_whitespace_before_the_trailers_line_feed(self) -> None: + # JSON whitespace may precede the trailer's final LF, but any bytes after that LF + # are trailing data and must still fail. + out = self.tree / "manifest" + root = b'\x1e{"mediaType":"application/vnd.uapi.16.manifest"}\n' + trailer = b'\x1e { "mediaType": "application/vnd.uapi.16.trailer" } \t\r\n' + out.write_bytes(root + trailer) + self.assertEqual(uapi16.read(out), [{"mediaType": uapi16.MEDIA_TYPE}]) + out.write_bytes(root + trailer + b" ") + with self.assertRaises(SystemExit): + uapi16.read(out) + + def test_accepts_root_metadata(self) -> None: + # A storage manifest includes root metadata, which must survive parsing unchanged. + out = self.tree / "manifest" + objects: list[uapi16.Object] = [{"mediaType": uapi16.MEDIA_TYPE, "mode": 0o750}] + uapi16.write_objects(objects, out) + self.assertEqual(uapi16.read(out), objects) + + def test_rejects_non_relative_names(self) -> None: + # UAPI.16 forbids path traversal and unnormalized names even in an external uapi16. + for name in ("", "/a", "a/", "a//b", "a/../b", "./a", "a/.", "a/..", "a\n"): + with self.subTest(name=name), self.assertRaises(SystemExit): + uapi16.check_name(name) + + if __name__ == "__main__": unittest.main() diff --git a/util.py b/util.py index 835be99d..1a8276cb 100644 --- a/util.py +++ b/util.py @@ -5,6 +5,7 @@ import bz2 import errno +import fcntl import gzip import http.client import io @@ -32,6 +33,10 @@ _FETCH_ATTEMPTS = 4 +class ReflinkUnavailable(OSError): + pass + + def fail(message: str) -> NoReturn: sys.exit(f"tine: {message}") @@ -208,6 +213,18 @@ def clone_file(src: Path, dst: Path, allow_link: bool = False) -> None: shutil.copymode(src, dst) +def reflink(src: Path, dst: Path) -> None: + """Raise ReflinkUnavailable instead of copying bytes when the filesystem cannot clone src.""" + with open(src, "rb") as source, open(dst, "wb") as destination: + try: + fcntl.ioctl(destination.fileno(), fcntl.FICLONE, source.fileno()) + except OSError as error: + if error.errno in _COPY_FALLBACK_ERRNOS | {errno.EACCES, errno.EBADF, errno.ENOTTY, errno.EPERM}: + raise ReflinkUnavailable(error.errno, error.strerror) from error + raise + shutil.copymode(src, dst) + + def named_files(source: Path, name: str) -> list[Path]: """Find files called `name` under `source`, and return their paths relative to `source`.""" found: list[Path] = [] @@ -241,7 +258,7 @@ def remove_path(path: Path, with_parents: bool = False) -> None: return for parent in path.parents: # `parent.parent` stops the walk at the filesystem root. - if parent == parent.parent or not parent.is_dir() or any(parent.iterdir()): + if parent == parent.parent or parent.is_symlink() or not parent.is_dir() or any(parent.iterdir()): break parent.rmdir() diff --git a/util_test.py b/util_test.py index 188e4f8c..b6396346 100644 --- a/util_test.py +++ b/util_test.py @@ -6,10 +6,12 @@ buck test tine//:test """ +import errno import stat import tempfile import unittest from pathlib import Path +from unittest import mock import util @@ -82,3 +84,38 @@ def test_finds_nested_files_without_following_links(self) -> None: (source / "dangling").symlink_to("missing") self.assertEqual(util.named_files(source, "wanted"), [Path("nested/wanted"), Path("wanted")]) + + +class TestReflink(unittest.TestCase): + def test_reflink_mode_errors_are_not_unavailability(self) -> None: + # A successful reflink followed by an unsupported mode change must report the metadata + # error. A caller must not mistake that failure for missing reflink support. + root = scratch(self) + source, destination = root / "source", root / "destination" + source.write_bytes(b"cached contents") + with ( + mock.patch.object(util.fcntl, "ioctl", return_value=0), + mock.patch.object(util.shutil, "copymode", side_effect=OSError(errno.EOPNOTSUPP, "mode")), + self.assertRaises(OSError) as error, + ): + util.reflink(source, destination) + self.assertNotIsInstance(error.exception, util.ReflinkUnavailable) + self.assertEqual(error.exception.errno, errno.EOPNOTSUPP) + + def test_reflink_never_falls_back_to_a_byte_copy(self) -> None: + # A caller choosing a storage strategy must distinguish an unsupported reflink from + # a successful clone. Reject unsupported or denied clones without copying any bytes. + root = scratch(self) + source, destination = root / "source", root / "destination" + source.write_bytes(b"cached contents") + for number in (errno.EXDEV, errno.EOPNOTSUPP, errno.EPERM): + with ( + self.subTest(errno=number), + mock.patch.object(util.fcntl, "ioctl", side_effect=OSError(number, "unavailable")), + mock.patch.object(util, "copy_range") as copy, + self.assertRaises(OSError) as error, + ): + util.reflink(source, destination) + self.assertEqual(error.exception.errno, number) + copy.assert_not_called() + self.assertEqual(source.read_bytes(), b"cached contents")