Fix package discovery and build failures in NixCI

README.md

@@ -59,7 +59,15 @@ runtime and tools with this bundle and may need code or dependency changes.
 
 ## Building
 
-For each of the following examples, the build result will be found in a file or directory named `result`.
+NixCI builds all Linux package outputs and keeps running after failures so that
+one broken dependency does not hide failures in other modules. Every
+`packages.x86_64-linux` entry is a derivation, including the literal dotted keys
+`"deploymentModules.<id>"` and `"phony-oci-bundles.<id>"`; these are not nested
+package sets. The top-level `deploymentModules` and `phony-oci-bundles` maps
+preserve shorthand access such as `nix build .#deploymentModules.nodejs-22`.
+
+The `nix build` commands below create a `result` symlink. Image builders instead
+take an explicit local output directory.
 
 To build the full bundle, including compatibility modules (this will take a while, and a lot of disk space):
 
@@ -79,32 +87,48 @@ To build a specific module, for example `python-3.10`, you can do:
 nix build .#'"python-3.10"'
 ```
 
+Image outputs are executable builders, not image derivations. NixCI builds the
+scripts and their pinned runtime closures; it never stores full images in the
+Nix store or uploads them to the binary cache. Run a builder with a new output
+directory on the local machine. Existing directories and paths under
+`/nix/store` are rejected. Temporary files and partial output are cleaned up on
+failure. The parent directory must exist and have room for both the working
+tree and the finished artifact.
+
 To build a disk with only a few modules for development purposes:
 
 ```
-nix build .#bundle-squashfs
+nix run .#bundle-squashfs -- ./development-disk
 ```
 
 or
 
 ```
-nix build .#custom-bundle-squashfs
+nix run .#custom-bundle-squashfs -- ./custom-disk
+```
+
+These directories contain `disk.sqsh`. To build the full disk or its compressed
+tarball (containing `disk.raw`):
+
+```
+nix run .#bundle-image -- ./full-disk
+nix run .#bundle-image-tarball -- ./full-disk-tarball
 ```
 
 To build the full bundle as a gzip-compressed tar OCI image:
 
 ```
-nix build .#bundle-oci
+nix run .#bundle-oci -- ./bundle-oci
 ```
 
 The result is an OCI image layout directory with the reference `bundle`, suitable
 for copying with an OCI-layout-aware tool such as
-`skopeo copy oci:./result:bundle docker://REGISTRY/REPOSITORY:TAG`.
-The output also provides a copy helper that keeps the exact image in its Nix
-closure and preserves the manifest digest:
+`skopeo copy oci:./bundle-oci:bundle docker://REGISTRY/REPOSITORY:TAG`.
+The copy helper takes the local image directory explicitly and preserves the
+manifest digest:
 
 ```
-nix run .#bundle-oci.copyTo -- docker://REGISTRY/REPOSITORY:TAG --authfile ./auth.json --digestfile ./digest
+nix run .#bundle-oci.copyTo -- ./bundle-oci docker://REGISTRY/REPOSITORY:TAG --authfile ./auth.json --digestfile ./digest
 ```
 
 The destination can use any Skopeo-supported transport, including `oci:`.
@@ -127,9 +151,11 @@ timestamps, ownership, extended headers and gzip headers are deterministic.
 The filesystem is generated directly from the same pinned full bundle used by
 `disk-script`: its complete store closure and `/etc/nixmodules` metadata, with
 unchanged store paths, file contents, modes and symlink targets. All guest
-UIDs/GIDs are 11000, matching the legacy disk builders. Existing disk and
-per-module OCI builders are unchanged. There is no disk conversion or dependency
-update. The tar archive preserves hardlinks and extended attributes. A consumer
+UIDs/GIDs are 11000, matching the disk builders. Per-module OCI outputs are also
+scripts: for example, `nix run .#phony-oci-bundles.nodejs-22 -- ./node-oci`.
+Automation that previously copied images from `nix build` results must invoke
+these scripts instead. There is no disk conversion or dependency update.
+The tar archive preserves hardlinks and extended attributes. A consumer
 that converts the tar to EROFS must leave filesystem compression disabled for
 files to remain DAX-eligible. This uses more unpacked disk space than compressed
 EROFS; it does not automatically share all guest memory.

flake.nix

@@ -170,8 +170,12 @@
       packages.x86_64-linux = import ./pkgs {
         inherit pkgs self;
       };
+      phony-oci-bundles = builtins.mapAttrs
+        (id: _: self.packages.x86_64-linux."phony-oci-bundles.${id}")
+        self.modules;
       checks.x86_64-linux.module-registry =
         assert import ./scripts/module-registry-test.nix;
+        assert builtins.all pkgs.lib.isDerivation (builtins.attrValues self.packages.x86_64-linux);
         assert builtins.attrNames self.modules == builtins.attrNames self.deploymentModules;
         assert builtins.attrNames self.activeModules == builtins.attrNames self.activeDeploymentModules;
         assert builtins.all

nix-ci.nix

@@ -0,0 +1,4 @@
+{
+  systems = [ "x86_64-linux" ];
+  fail-fast = false;
+}

pkgs/bundle-image-tarball/builder.sh

@@ -2,8 +2,9 @@
 
 PATH="${env[PATH]}"
 out="${outputs[out]}"
-bundleImage="${env[bundle-image]}"
+bundleImage="$PWD/disk"
+"${env[imageBuilder]}" "$bundleImage"
 
 echo "making tarball..."
 mkdir -p "$out"
-tar --use-compress-program=pigz -Scf $out/disk.raw.tar.gz -C ${bundleImage} disk.raw
+tar --use-compress-program=pigz -Scf "$out/disk.raw.tar.gz" -C "$bundleImage" disk.raw

pkgs/bundle-image-tarball/default.nix

@@ -1,18 +1,13 @@
-{ system, bash, lib, bundle-image, revstring, coreutils, gnutar, pigz }:
-
-derivation {
-  name = "nixmodules-${revstring}";
-  builder = "${bash}/bin/bash";
-  args = [ ./builder.sh ];
-  inherit system;
-  __structuredAttrs = true;
-  unsafeDiscardReferences.out = true;
+{ pkgs, lib, bundle-image, revstring, coreutils, gnutar, pigz }:
+let
+  tools = [ coreutils gnutar pigz ];
+in
+pkgs.callPackage ../image-script.nix { } {
+  name = "build-nixmodules-image-tarball";
+  runtimeInputs = tools;
+  builder = ./builder.sh;
   env = {
-    PATH = lib.makeBinPath [
-      coreutils
-      gnutar
-      pigz
-    ];
-    inherit bundle-image;
+    PATH = lib.makeBinPath tools;
+    imageBuilder = lib.getExe bundle-image;
   };
 }

pkgs/bundle-image/default.nix

@@ -1,35 +1,15 @@
-{ system
-, bash
-, lib
-, bundle
-, revstring
-, coreutils
-, findutils
-, closureInfo
-, squashfsTools
-, fetchFromGitHub
-, pkgs
-, diskName
-}:
-
+{ pkgs, lib, bundle, revstring, coreutils, findutils, closureInfo, squashfsTools, diskName }:
 let
-  label = "nixmodules-${revstring}";
+  tools = [ coreutils findutils squashfsTools ];
 in
-
-derivation {
-  name = label;
-  builder = "${bash}/bin/bash";
-  args = [ ./builder.sh ];
-  inherit system;
-  __structuredAttrs = true;
-  unsafeDiscardReferences.out = true;
+pkgs.callPackage ../image-script.nix { } {
+  name = "build-nixmodules-image";
+  runtimeInputs = tools;
+  builder = ./builder.sh;
   env = {
-    inherit label bundle diskName;
-    PATH = lib.makeBinPath [
-      coreutils
-      findutils
-      squashfsTools
-    ];
+    label = "nixmodules-${revstring}";
+    inherit bundle diskName;
+    PATH = lib.makeBinPath tools;
     diskClosureInfo = closureInfo { rootPaths = [ bundle ]; };
   };
 }

pkgs/bundle-oci/builder.sh

@@ -0,0 +1,11 @@
+. .attrs.sh
+export PATH="${env[PATH]}"
+out="${outputs[out]}"
+mkdir -p root/nix/store root/etc/nixmodules "$out/blobs/sha256"
+cp -a --reflink=auto "${env[bundle]}/etc/nixmodules/." root/etc/nixmodules/
+while IFS= read -r path; do
+  cp -a --reflink=auto "$path" root/nix/store/
+done < "${env[bundleClosure]}/store-paths"
+chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules
+bash "${env[layerScript]}" root layer.tar.gz
+python3 "${env[layoutScript]}" layer.tar.gz "$out" "${env[architecture]}" "${env[metadata]}"

pkgs/bundle-oci/check.nix

@@ -1,4 +1,4 @@
-{ pkgs }:
+{ pkgs, ztoc-rs }:
 
 let
   dependency = pkgs.runCommand "bundle-oci-dependency" { } ''
@@ -25,17 +25,73 @@ let
     sourceTimestamp = 1700000000;
   };
   closure = pkgs.closureInfo { rootPaths = [ bundle ]; };
+  disk = pkgs.callPackage ../bundle-image {
+    inherit bundle;
+    revstring = "fixture";
+    diskName = "disk.raw";
+  };
+  tarball = pkgs.callPackage ../bundle-image-tarball {
+    bundle-image = disk;
+    revstring = "fixture";
+  };
+  moduleImage = pkgs.callPackage ../mk-phony-oci { inherit ztoc-rs; } {
+    inherit module;
+    moduleId = "fixture";
+  };
+  failingBuilder = pkgs.callPackage ../image-script.nix { } {
+    name = "failing-image";
+    runtimeInputs = [ ];
+    env = { };
+    builder = pkgs.writeText "fail-image.sh" "exit 1";
+  };
 in
 pkgs.runCommand "bundle-oci-check"
 {
   nativeBuildInputs = [ pkgs.gnutar pkgs.gzip pkgs.python3 ];
 }
   ''
-    python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} ${./layout.py} ${./layer.sh}
-    ${image.copyTo}/bin/copy-nixmodules-bundle-oci "oci:$PWD/copied:bundle" \
+    mkdir "local images"
+    cd "local images"
+    ${pkgs.lib.getExe image} "$PWD/image"
+    if ${pkgs.lib.getExe image} "$PWD/image"; then
+      echo "Builder overwrote an existing directory" >&2
+      exit 1
+    fi
+    if ${pkgs.lib.getExe image} /nix/store/forbidden-image; then
+      echo "Builder accepted an output in the Nix store" >&2
+      exit 1
+    fi
+    ln -s /nix/store store-link
+    if ${pkgs.lib.getExe image} "$PWD/store-link/forbidden-image"; then
+      echo "Builder accepted a symlink into the Nix store" >&2
+      exit 1
+    fi
+    ${pkgs.lib.getExe disk} "$PWD/disk"
+    metadata=$(readlink -f ${bundle}/etc/nixmodules/modules.json)
+    ${pkgs.squashfsTools}/bin/unsquashfs -cat "$PWD/disk/disk.raw" "''${metadata#/}" > modules.json
+    cmp modules.json ${bundle}/etc/nixmodules/modules.json
+    ${pkgs.lib.getExe tarball} "$PWD/tarball"
+    tar -tzf "$PWD/tarball/disk.raw.tar.gz" | grep -x disk.raw
+    ${pkgs.lib.getExe moduleImage} "$PWD/module-image"
+    python3 - "$PWD/module-image" <<'PY'
+    import json, pathlib, sys
+    root = pathlib.Path(sys.argv[1])
+    index = json.loads((root / "index.json").read_text())
+    manifest = index["manifests"][0]
+    assert manifest["annotations"]["org.opencontainers.image.ref.name"] == "fixture"
+    body = json.loads((root / "blobs/sha256" / manifest["digest"].split(":")[1]).read_text())
+    assert len(body["layers"]) == 2
+    for layer in body["layers"]:
+        assert (root / "blobs/sha256" / layer["digest"].split(":")[1]).stat().st_size == layer["size"]
+    PY
+    if ${pkgs.lib.getExe failingBuilder} "$PWD/failed-image"; then exit 1; fi
+    test ! -e "$PWD/failed-image"
+    test -z "$(find . -maxdepth 1 -name '*.work.*' -print)"
+    python3 ${./check.py} "$PWD/image" ${bundle} ${closure}/store-paths ${dependency} ${./layout.py} ${./layer.sh}
+    ${image.copyTo}/bin/copy-nixmodules-bundle-oci "$PWD/image" "oci:$PWD/copied:bundle" \
       --authfile ${pkgs.writeText "empty-auth.json" ''{"auths":{}}''} \
       --digestfile "$PWD/copied.digest"
-    python3 - ${image} "$PWD/copied" "$PWD/copied.digest" <<'PY'
+    python3 - "$PWD/image" "$PWD/copied" "$PWD/copied.digest" <<'PY'
     import json, pathlib, sys
     original, copied, digest_file = map(pathlib.Path, sys.argv[1:])
     descriptor = json.loads((original / "index.json").read_text())["manifests"][0]

pkgs/bundle-oci/check.py

@@ -120,15 +120,18 @@ def compare(source, target):
 for metadata in (bundle / "etc/nixmodules").iterdir():
     compare(metadata, root / "etc/nixmodules" / metadata.name)
 packed_dependency = root / str(dependency).lstrip("/")
-assert (packed_dependency / "data").stat().st_ino == (
-    packed_dependency / "hardlink"
-).stat().st_ino
+if (dependency / "data").stat().st_ino == (dependency / "hardlink").stat().st_ino:
+    assert (packed_dependency / "data").stat().st_ino == (
+        packed_dependency / "hardlink"
+    ).stat().st_ino
 print("OCI descriptors, complete closure, metadata, content, modes, symlinks, ownership and hardlinks verified")
 
 xattr_root = pathlib.Path("xattrs")
 xattr_root.mkdir()
 xattr_file = xattr_root / ("long-name-" * 20)
 xattr_file.write_bytes(b"xattrs\n")
+# NAR substitution does not retain hardlink identity, so create this fixture at runtime.
+os.link(xattr_file, xattr_root / "zz-hardlink")
 try:
     os.setxattr(xattr_file, "user.fixture", b"preserved")
     xattrs_supported = True
@@ -143,14 +146,16 @@ def compare(source, target):
     assert member.isfile() and source.extractfile(member).read() == b"xattrs\n"
     if xattrs_supported:
         assert member.pax_headers["SCHILY.xattr.user.fixture"] == "preserved"
+restored = pathlib.Path("xattrs-restored")
+restored.mkdir()
+subprocess.run([
+    "tar", "--extract", "--gzip", "--file=xattrs.tar.gz",
+    "--directory", str(restored), "--no-same-owner",
+    "--xattrs", "--xattrs-include=*",
+], check=True)
+assert (restored / xattr_file.name).stat().st_ino == (restored / "zz-hardlink").stat().st_ino
+print("Runtime hardlink fixture preserved in archive and extraction")
 if xattrs_supported:
-    restored = pathlib.Path("xattrs-restored")
-    restored.mkdir()
-    subprocess.run([
-        "tar", "--extract", "--gzip", "--file=xattrs.tar.gz",
-        "--directory", str(restored), "--no-same-owner",
-        "--xattrs", "--xattrs-include=*",
-    ], check=True)
     assert os.getxattr(restored / xattr_file.name, "user.fixture") == b"preserved"
     print("Extended attributes preserved in archive and extraction")
 os.utime(xattr_file, (12345, 67890))