From 2760a5cc6919a9bea7cd9b30a153cd09d5138d33 Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Thu, 22 Oct 2026 20:51:13 +0000 Subject: [PATCH 1/4] Expose buildable package targets for NixCI Flatten deployment and OCI package exports into derivations, preserve top-level module access, and collect all CI failures instead of cancelling at the first error. Assisted-by: Replit --- README.md | 7 +++++++ flake.nix | 4 ++++ nix-ci.nix | 4 ++++ pkgs/default.nix | 25 ++++++++++++------------- 4 files changed, 27 insertions(+), 13 deletions(-) create mode 100644 nix-ci.nix diff --git a/README.md b/README.md index 1732385f..cb4c4782 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,13 @@ runtime and tools with this bundle and may need code or dependency changes. ## Building +NixCI builds all Linux package outputs and keeps running after failures so that +one broken dependency does not hide failures in other modules. Every +`packages.x86_64-linux` entry is a derivation, including the literal dotted keys +`"deploymentModules."` and `"phony-oci-bundles."`; these are not nested +package sets. The top-level `deploymentModules` and `phony-oci-bundles` maps +preserve shorthand access such as `nix build .#deploymentModules.nodejs-22`. + For each of the following examples, the build result will be found in a file or directory named `result`. To build the full bundle, including compatibility modules (this will take a while, and a lot of disk space): diff --git a/flake.nix b/flake.nix index d38084cd..003159fd 100644 --- a/flake.nix +++ b/flake.nix @@ -170,8 +170,12 @@ packages.x86_64-linux = import ./pkgs { inherit pkgs self; }; + phony-oci-bundles = builtins.mapAttrs + (id: _: self.packages.x86_64-linux."phony-oci-bundles.${id}") + self.modules; checks.x86_64-linux.module-registry = assert import ./scripts/module-registry-test.nix; + assert builtins.all pkgs.lib.isDerivation (builtins.attrValues self.packages.x86_64-linux); assert builtins.attrNames self.modules == builtins.attrNames self.deploymentModules; assert builtins.attrNames self.activeModules == builtins.attrNames self.activeDeploymentModules; assert builtins.all diff --git a/nix-ci.nix b/nix-ci.nix new file mode 100644 index 00000000..d4950b3a --- /dev/null +++ b/nix-ci.nix @@ -0,0 +1,4 @@ +{ + systems = [ "x86_64-linux" ]; + fail-fast = false; +} diff --git a/pkgs/default.nix b/pkgs/default.nix index 95f7adc6..edf8970a 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -25,6 +25,15 @@ let ztoc-rs = self.inputs.ztoc-rs.packages.x86_64-linux.default; }; + phony-oci-bundles = mapAttrs + (moduleId: _: mkPhonyOCI { + inherit moduleId; + module = self.deploymentModules.${moduleId}; + }) + modules; + + prefixedPackages = prefix: mapAttrs' (name: value: nameValuePair "${prefix}.${name}" value); + bundle-fn = pkgs.callPackage ./bundle { inherit self; }; bundle-squashfs-fn = @@ -87,17 +96,7 @@ rec { diskName = "disk.sqsh"; }; - phony-oci-bundles = mapAttrs - ( - moduleId: _: - mkPhonyOCI { - inherit moduleId; - module = self.deploymentModules.${moduleId}; - } - ) - modules; - - deploymentModules = self.deploymentModules; - } - // modules +// modules +// prefixedPackages "deploymentModules" self.deploymentModules + // prefixedPackages "phony-oci-bundles" phony-oci-bundles From d84f8cd8687d206d53767ce7585093f8bb2f203f Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Thu, 22 Oct 2026 23:48:29 +0000 Subject: [PATCH 2/4] Build images locally through nix run scripts Keep image artifacts out of Nix outputs and binary caches. Convert disk, tarball, and OCI outputs into runtime builders, and exercise them with temporary fixtures including destination safety and cleanup checks. Assisted-by: Replit --- README.md | 41 +++++++++++----- pkgs/bundle-image-tarball/builder.sh | 5 +- pkgs/bundle-image-tarball/default.nix | 25 ++++------ pkgs/bundle-image/default.nix | 38 ++++---------- pkgs/bundle-oci/builder.sh | 11 +++++ pkgs/bundle-oci/check.nix | 64 ++++++++++++++++++++++-- pkgs/bundle-oci/default.nix | 71 +++++++++++++-------------- pkgs/default.nix | 4 +- pkgs/image-script.nix | 48 ++++++++++++++++++ pkgs/mk-phony-oci/builder.sh | 40 +++++++-------- pkgs/mk-phony-oci/default.nix | 32 +++--------- 11 files changed, 234 insertions(+), 145 deletions(-) create mode 100644 pkgs/bundle-oci/builder.sh create mode 100644 pkgs/image-script.nix diff --git a/README.md b/README.md index cb4c4782..2dbb3e35 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,8 @@ one broken dependency does not hide failures in other modules. Every package sets. The top-level `deploymentModules` and `phony-oci-bundles` maps preserve shorthand access such as `nix build .#deploymentModules.nodejs-22`. -For each of the following examples, the build result will be found in a file or directory named `result`. +The `nix build` commands below create a `result` symlink. Image builders instead +take an explicit local output directory. To build the full bundle, including compatibility modules (this will take a while, and a lot of disk space): @@ -86,32 +87,48 @@ To build a specific module, for example `python-3.10`, you can do: nix build .#'"python-3.10"' ``` +Image outputs are executable builders, not image derivations. NixCI builds the +scripts and their pinned runtime closures; it never stores full images in the +Nix store or uploads them to the binary cache. Run a builder with a new output +directory on the local machine. Existing directories and paths under +`/nix/store` are rejected. Temporary files and partial output are cleaned up on +failure. The parent directory must exist and have room for both the working +tree and the finished artifact. + To build a disk with only a few modules for development purposes: ``` -nix build .#bundle-squashfs +nix run .#bundle-squashfs -- ./development-disk ``` or ``` -nix build .#custom-bundle-squashfs +nix run .#custom-bundle-squashfs -- ./custom-disk +``` + +These directories contain `disk.sqsh`. To build the full disk or its compressed +tarball (containing `disk.raw`): + +``` +nix run .#bundle-image -- ./full-disk +nix run .#bundle-image-tarball -- ./full-disk-tarball ``` To build the full bundle as a gzip-compressed tar OCI image: ``` -nix build .#bundle-oci +nix run .#bundle-oci -- ./bundle-oci ``` The result is an OCI image layout directory with the reference `bundle`, suitable for copying with an OCI-layout-aware tool such as -`skopeo copy oci:./result:bundle docker://REGISTRY/REPOSITORY:TAG`. -The output also provides a copy helper that keeps the exact image in its Nix -closure and preserves the manifest digest: +`skopeo copy oci:./bundle-oci:bundle docker://REGISTRY/REPOSITORY:TAG`. +The copy helper takes the local image directory explicitly and preserves the +manifest digest: ``` -nix run .#bundle-oci.copyTo -- docker://REGISTRY/REPOSITORY:TAG --authfile ./auth.json --digestfile ./digest +nix run .#bundle-oci.copyTo -- ./bundle-oci docker://REGISTRY/REPOSITORY:TAG --authfile ./auth.json --digestfile ./digest ``` The destination can use any Skopeo-supported transport, including `oci:`. @@ -134,9 +151,11 @@ timestamps, ownership, extended headers and gzip headers are deterministic. The filesystem is generated directly from the same pinned full bundle used by `disk-script`: its complete store closure and `/etc/nixmodules` metadata, with unchanged store paths, file contents, modes and symlink targets. All guest -UIDs/GIDs are 11000, matching the legacy disk builders. Existing disk and -per-module OCI builders are unchanged. There is no disk conversion or dependency -update. The tar archive preserves hardlinks and extended attributes. A consumer +UIDs/GIDs are 11000, matching the disk builders. Per-module OCI outputs are also +scripts: for example, `nix run .#phony-oci-bundles.nodejs-22 -- ./node-oci`. +Automation that previously copied images from `nix build` results must invoke +these scripts instead. There is no disk conversion or dependency update. +The tar archive preserves hardlinks and extended attributes. A consumer that converts the tar to EROFS must leave filesystem compression disabled for files to remain DAX-eligible. This uses more unpacked disk space than compressed EROFS; it does not automatically share all guest memory. diff --git a/pkgs/bundle-image-tarball/builder.sh b/pkgs/bundle-image-tarball/builder.sh index 926bbf4e..b437d6fc 100644 --- a/pkgs/bundle-image-tarball/builder.sh +++ b/pkgs/bundle-image-tarball/builder.sh @@ -2,8 +2,9 @@ PATH="${env[PATH]}" out="${outputs[out]}" -bundleImage="${env[bundle-image]}" +bundleImage="$PWD/disk" +"${env[imageBuilder]}" "$bundleImage" echo "making tarball..." mkdir -p "$out" -tar --use-compress-program=pigz -Scf $out/disk.raw.tar.gz -C ${bundleImage} disk.raw +tar --use-compress-program=pigz -Scf "$out/disk.raw.tar.gz" -C "$bundleImage" disk.raw diff --git a/pkgs/bundle-image-tarball/default.nix b/pkgs/bundle-image-tarball/default.nix index 5fa484b1..bdde44c4 100644 --- a/pkgs/bundle-image-tarball/default.nix +++ b/pkgs/bundle-image-tarball/default.nix @@ -1,18 +1,13 @@ -{ system, bash, lib, bundle-image, revstring, coreutils, gnutar, pigz }: - -derivation { - name = "nixmodules-${revstring}"; - builder = "${bash}/bin/bash"; - args = [ ./builder.sh ]; - inherit system; - __structuredAttrs = true; - unsafeDiscardReferences.out = true; +{ pkgs, lib, bundle-image, revstring, coreutils, gnutar, pigz }: +let + tools = [ coreutils gnutar pigz ]; +in +pkgs.callPackage ../image-script.nix { } { + name = "build-nixmodules-image-tarball"; + runtimeInputs = tools; + builder = ./builder.sh; env = { - PATH = lib.makeBinPath [ - coreutils - gnutar - pigz - ]; - inherit bundle-image; + PATH = lib.makeBinPath tools; + imageBuilder = lib.getExe bundle-image; }; } diff --git a/pkgs/bundle-image/default.nix b/pkgs/bundle-image/default.nix index 5a43bf69..513bafd2 100644 --- a/pkgs/bundle-image/default.nix +++ b/pkgs/bundle-image/default.nix @@ -1,35 +1,15 @@ -{ system -, bash -, lib -, bundle -, revstring -, coreutils -, findutils -, closureInfo -, squashfsTools -, fetchFromGitHub -, pkgs -, diskName -}: - +{ pkgs, lib, bundle, revstring, coreutils, findutils, closureInfo, squashfsTools, diskName }: let - label = "nixmodules-${revstring}"; + tools = [ coreutils findutils squashfsTools ]; in - -derivation { - name = label; - builder = "${bash}/bin/bash"; - args = [ ./builder.sh ]; - inherit system; - __structuredAttrs = true; - unsafeDiscardReferences.out = true; +pkgs.callPackage ../image-script.nix { } { + name = "build-nixmodules-image"; + runtimeInputs = tools; + builder = ./builder.sh; env = { - inherit label bundle diskName; - PATH = lib.makeBinPath [ - coreutils - findutils - squashfsTools - ]; + label = "nixmodules-${revstring}"; + inherit bundle diskName; + PATH = lib.makeBinPath tools; diskClosureInfo = closureInfo { rootPaths = [ bundle ]; }; }; } diff --git a/pkgs/bundle-oci/builder.sh b/pkgs/bundle-oci/builder.sh new file mode 100644 index 00000000..1861a4fb --- /dev/null +++ b/pkgs/bundle-oci/builder.sh @@ -0,0 +1,11 @@ +. .attrs.sh +export PATH="${env[PATH]}" +out="${outputs[out]}" +mkdir -p root/nix/store root/etc/nixmodules "$out/blobs/sha256" +cp -a --reflink=auto "${env[bundle]}/etc/nixmodules/." root/etc/nixmodules/ +while IFS= read -r path; do + cp -a --reflink=auto "$path" root/nix/store/ +done < "${env[bundleClosure]}/store-paths" +chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules +bash "${env[layerScript]}" root layer.tar.gz +python3 "${env[layoutScript]}" layer.tar.gz "$out" "${env[architecture]}" "${env[metadata]}" diff --git a/pkgs/bundle-oci/check.nix b/pkgs/bundle-oci/check.nix index f9720788..e3a4d296 100644 --- a/pkgs/bundle-oci/check.nix +++ b/pkgs/bundle-oci/check.nix @@ -1,4 +1,4 @@ -{ pkgs }: +{ pkgs, ztoc-rs }: let dependency = pkgs.runCommand "bundle-oci-dependency" { } '' @@ -25,17 +25,73 @@ let sourceTimestamp = 1700000000; }; closure = pkgs.closureInfo { rootPaths = [ bundle ]; }; + disk = pkgs.callPackage ../bundle-image { + inherit bundle; + revstring = "fixture"; + diskName = "disk.raw"; + }; + tarball = pkgs.callPackage ../bundle-image-tarball { + bundle-image = disk; + revstring = "fixture"; + }; + moduleImage = pkgs.callPackage ../mk-phony-oci { inherit ztoc-rs; } { + inherit module; + moduleId = "fixture"; + }; + failingBuilder = pkgs.callPackage ../image-script.nix { } { + name = "failing-image"; + runtimeInputs = [ ]; + env = { }; + builder = pkgs.writeText "fail-image.sh" "exit 1"; + }; in pkgs.runCommand "bundle-oci-check" { nativeBuildInputs = [ pkgs.gnutar pkgs.gzip pkgs.python3 ]; } '' - python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} ${./layout.py} ${./layer.sh} - ${image.copyTo}/bin/copy-nixmodules-bundle-oci "oci:$PWD/copied:bundle" \ + mkdir "local images" + cd "local images" + ${pkgs.lib.getExe image} "$PWD/image" + if ${pkgs.lib.getExe image} "$PWD/image"; then + echo "Builder overwrote an existing directory" >&2 + exit 1 + fi + if ${pkgs.lib.getExe image} /nix/store/forbidden-image; then + echo "Builder accepted an output in the Nix store" >&2 + exit 1 + fi + ln -s /nix/store store-link + if ${pkgs.lib.getExe image} "$PWD/store-link/forbidden-image"; then + echo "Builder accepted a symlink into the Nix store" >&2 + exit 1 + fi + ${pkgs.lib.getExe disk} "$PWD/disk" + metadata=$(readlink -f ${bundle}/etc/nixmodules/modules.json) + ${pkgs.squashfsTools}/bin/unsquashfs -cat "$PWD/disk/disk.raw" "''${metadata#/}" > modules.json + cmp modules.json ${bundle}/etc/nixmodules/modules.json + ${pkgs.lib.getExe tarball} "$PWD/tarball" + tar -tzf "$PWD/tarball/disk.raw.tar.gz" | grep -x disk.raw + ${pkgs.lib.getExe moduleImage} "$PWD/module-image" + python3 - "$PWD/module-image" <<'PY' + import json, pathlib, sys + root = pathlib.Path(sys.argv[1]) + index = json.loads((root / "index.json").read_text()) + manifest = index["manifests"][0] + assert manifest["annotations"]["org.opencontainers.image.ref.name"] == "fixture" + body = json.loads((root / "blobs/sha256" / manifest["digest"].split(":")[1]).read_text()) + assert len(body["layers"]) == 2 + for layer in body["layers"]: + assert (root / "blobs/sha256" / layer["digest"].split(":")[1]).stat().st_size == layer["size"] + PY + if ${pkgs.lib.getExe failingBuilder} "$PWD/failed-image"; then exit 1; fi + test ! -e "$PWD/failed-image" + test -z "$(find . -maxdepth 1 -name '*.work.*' -print)" + python3 ${./check.py} "$PWD/image" ${bundle} ${closure}/store-paths ${dependency} ${./layout.py} ${./layer.sh} + ${image.copyTo}/bin/copy-nixmodules-bundle-oci "$PWD/image" "oci:$PWD/copied:bundle" \ --authfile ${pkgs.writeText "empty-auth.json" ''{"auths":{}}''} \ --digestfile "$PWD/copied.digest" - python3 - ${image} "$PWD/copied" "$PWD/copied.digest" <<'PY' + python3 - "$PWD/image" "$PWD/copied" "$PWD/copied.digest" <<'PY' import json, pathlib, sys original, copied, digest_file = map(pathlib.Path, sys.argv[1:]) descriptor = json.loads((original / "index.json").read_text())["manifests"][0] diff --git a/pkgs/bundle-oci/default.nix b/pkgs/bundle-oci/default.nix index ae357cff..e3c9353b 100644 --- a/pkgs/bundle-oci/default.nix +++ b/pkgs/bundle-oci/default.nix @@ -1,8 +1,9 @@ -{ runCommand +{ pkgs , lib , stdenv , bundle , closureInfo +, coreutils , gnutar , gzip , python3 @@ -11,43 +12,37 @@ , revision ? null , sourceTimestamp ? null }: - let - bundleClosure = closureInfo { rootPaths = [ bundle ]; }; - metadata = builtins.toJSON { inherit revision sourceTimestamp; }; - image = runCommand "nixmodules-bundle-oci" - { - nativeBuildInputs = [ gnutar gzip python3 ]; - inherit bundle bundleClosure; + tools = [ coreutils gnutar gzip python3 pkgs.bash ]; + script = pkgs.callPackage ../image-script.nix { } { + name = "build-nixmodules-bundle-oci"; + runtimeInputs = tools; + builder = ./builder.sh; + env = { + inherit bundle; + bundleClosure = closureInfo { rootPaths = [ bundle ]; }; + PATH = lib.makeBinPath tools; architecture = lib.toLower stdenv.hostPlatform.parsed.cpu.name; - __structuredAttrs = true; - # The embedded closure must not retain references to the builder's store. - unsafeDiscardReferences.out = true; - passthru.copyTo = writeShellApplication { - name = "copy-nixmodules-bundle-oci"; - runtimeInputs = [ skopeo ]; - text = '' - if [ "$#" -lt 1 ]; then - echo "usage: copy-nixmodules-bundle-oci [skopeo copy options]" >&2 - exit 1 - fi - destination="$1" - shift - exec skopeo --insecure-policy copy --preserve-digests \ - "oci:${image}:bundle" "$destination" "$@" - ''; - }; - } - '' - mkdir -p root/nix/store root/etc/nixmodules "$out/blobs/sha256" - cp -a --reflink=auto "$bundle/etc/nixmodules/." root/etc/nixmodules/ - while IFS= read -r path; do - cp -a --reflink=auto "$path" root/nix/store/ - done < "$bundleClosure/store-paths" - chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules - - bash ${./layer.sh} root layer.tar.gz - python3 ${./layout.py} layer.tar.gz "$out" "$architecture" ${lib.escapeShellArg metadata} - ''; + metadata = builtins.toJSON { inherit revision sourceTimestamp; }; + layerScript = ./layer.sh; + layoutScript = ./layout.py; + }; + }; in -image +script.overrideAttrs (_: { + passthru.copyTo = writeShellApplication { + name = "copy-nixmodules-bundle-oci"; + runtimeInputs = [ skopeo coreutils ]; + text = '' + if [ "$#" -lt 2 ]; then + echo "usage: copy-nixmodules-bundle-oci [skopeo copy options]" >&2 + exit 2 + fi + image=$(realpath -- "$1") + destination="$2" + shift 2 + exec skopeo --insecure-policy copy --preserve-digests \ + "oci:$image:bundle" "$destination" "$@" + ''; + }; +}) diff --git a/pkgs/default.nix b/pkgs/default.nix index edf8970a..ad204ed1 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -59,7 +59,9 @@ rec { sourceTimestamp = self.lastModified or null; }; - bundle-oci-check = pkgs.callPackage ./bundle-oci/check.nix { }; + bundle-oci-check = pkgs.callPackage ./bundle-oci/check.nix { + ztoc-rs = self.inputs.ztoc-rs.packages.x86_64-linux.default; + }; custom-bundle = bundle-fn { moduleIds = dev-module-ids; diff --git a/pkgs/image-script.nix b/pkgs/image-script.nix new file mode 100644 index 00000000..78c624ab --- /dev/null +++ b/pkgs/image-script.nix @@ -0,0 +1,48 @@ +{ lib, writeShellApplication, coreutils, bash, writeText }: +{ name, runtimeInputs, env, builder }: +let + attributes = writeText "${name}-attributes" '' + declare -A env=( + ${lib.concatStringsSep "\n" (lib.mapAttrsToList (key: value: "[${lib.escapeShellArg key}]=${lib.escapeShellArg (toString value)}") env)} + ) + declare -A outputs=([out]="$IMAGE_WORK/output") + ''; +in +writeShellApplication { + inherit name; + runtimeInputs = [ coreutils bash ] ++ runtimeInputs; + text = '' + if [ "$#" -ne 1 ]; then + echo "usage: ${name} " >&2 + exit 2 + fi + destination=$(realpath -m -- "$1") + case "$destination" in + /nix/store|/nix/store/*) + echo "Image output must be outside /nix/store" >&2 + exit 2 + ;; + esac + mkdir -- "$destination" + work="" + complete=false + cleanup() { + if [ -n "$work" ]; then + chmod -R u+w -- "$work" + rm -rf -- "$work" + fi + if [ "$complete" = false ]; then rm -rf -- "$destination"; fi + } + trap cleanup EXIT + work=$(mktemp -d "''${destination}.work.XXXXXX") + export IMAGE_WORK="$work" + cp ${attributes} "$work/.attrs.sh" + ( + cd "$work" + ${bash}/bin/bash -euo pipefail ${builder} + ) >&2 + cp -a --reflink=auto "$work/output/." "$destination/" + complete=true + printf '%s\n' "$destination" + ''; +} diff --git a/pkgs/mk-phony-oci/builder.sh b/pkgs/mk-phony-oci/builder.sh index e99f3554..cfa0d192 100644 --- a/pkgs/mk-phony-oci/builder.sh +++ b/pkgs/mk-phony-oci/builder.sh @@ -3,35 +3,35 @@ set -eou pipefail PATH="${env[PATH]}" out="${outputs[out]}" -mkdir $out +mkdir "$out" mkdir image -echo '{"imageLayoutVersion": "1.0.0"}' > ${out}/oci-layout +echo '{"imageLayoutVersion": "1.0.0"}' > "${out}/oci-layout" -tar -Pcpf ${out}/layer.tar --hard-dereference --sort=name --mtime="1970-01-01T00:00:01Z" \ +tar -Pcpf "${out}/layer.tar" --hard-dereference --sort=name --mtime="1970-01-01T00:00:01Z" \ --owner=0 --group=0 --verbatim-files-from --files-from "${env[diskClosureInfo]}"/store-paths -DIFFID=$(sha256sum ${out}/layer.tar | cut -f -1 -d ' ') -pigz ${out}/layer.tar -LAYER_DIGEST=$(sha256sum ${out}/layer.tar.gz | cut -f -1 -d ' ') -cat ${out}/layer.tar.gz | ztoc > ${out}/ztoc -ZTOC_DIGEST=$(sha256sum ${out}/ztoc | cut -f -1 -d ' ') +DIFFID=$(sha256sum "${out}/layer.tar" | cut -f -1 -d ' ') +pigz "${out}/layer.tar" +LAYER_DIGEST=$(sha256sum "${out}/layer.tar.gz" | cut -f -1 -d ' ') +ztoc < "${out}/layer.tar.gz" > "${out}/ztoc" +ZTOC_DIGEST=$(sha256sum "${out}/ztoc" | cut -f -1 -d ' ') -LAYER_SIZE=$(stat --printf='%s' ${out}/layer.tar.gz) -ZTOC_SIZE=$(stat --printf='%s' ${out}/ztoc) +LAYER_SIZE=$(stat --printf='%s' "${out}/layer.tar.gz") +ZTOC_SIZE=$(stat --printf='%s' "${out}/ztoc") -mkdir -p ${out}/blobs/sha256/ -mv ${out}/layer.tar.gz "${out}/blobs/sha256/${LAYER_DIGEST}" -mv ${out}/ztoc "${out}/blobs/sha256/${ZTOC_DIGEST}" +mkdir -p "${out}/blobs/sha256/" +mv "${out}/layer.tar.gz" "${out}/blobs/sha256/${LAYER_DIGEST}" +mv "${out}/ztoc" "${out}/blobs/sha256/${ZTOC_DIGEST}" ## Phony Image -echo '{}' > ${out}/config.json -CONFIG_DIGEST=$(sha256sum ${out}/config.json | cut -f -1 -d ' ') -CONFIG_SIZE=$(stat --printf='%s' ${out}/config.json) +echo '{}' > "${out}/config.json" +CONFIG_DIGEST=$(sha256sum "${out}/config.json" | cut -f -1 -d ' ') +CONFIG_SIZE=$(stat --printf='%s' "${out}/config.json") mv "${out}/config.json" "${out}/blobs/sha256/${CONFIG_DIGEST}" -cat < ${out}/manifest.json +cat < "${out}/manifest.json" { "schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json", @@ -58,11 +58,11 @@ cat < ${out}/manifest.json ] } EOF -MANIFEST_DIGEST=$(sha256sum ${out}/manifest.json | cut -f -1 -d ' ') -MANIFEST_SIZE=$(stat --printf='%s' ${out}/manifest.json) +MANIFEST_DIGEST=$(sha256sum "${out}/manifest.json" | cut -f -1 -d ' ') +MANIFEST_SIZE=$(stat --printf='%s' "${out}/manifest.json") mv "${out}/manifest.json" "${out}/blobs/sha256/${MANIFEST_DIGEST}" -cat < ${out}/index.json +cat < "${out}/index.json" { "schemaVersion": 2, "mediaType": "application/vnd.oci.image.index.v1+json", diff --git a/pkgs/mk-phony-oci/default.nix b/pkgs/mk-phony-oci/default.nix index 73f6f4c9..234956ca 100644 --- a/pkgs/mk-phony-oci/default.nix +++ b/pkgs/mk-phony-oci/default.nix @@ -1,34 +1,16 @@ -{ system -, bash -, lib -, coreutils -, closureInfo -, gnutar -, pigz -, ztoc-rs -}: - +{ pkgs, lib, coreutils, closureInfo, gnutar, pigz, ztoc-rs }: { module, moduleId }: let sanitizedModuleId = builtins.replaceStrings [ ":" ] [ "_" ] moduleId; + tools = [ coreutils gnutar pigz ztoc-rs ]; in -derivation { - name = "oci-image-${sanitizedModuleId}"; - builder = "${bash}/bin/bash"; - args = [ ./builder.sh ]; - inherit system; - __structuredAttrs = true; - unsafeDiscardReferences.out = true; - outputs = [ "out" ]; +pkgs.callPackage ../image-script.nix { } { + name = "build-oci-image-${sanitizedModuleId}"; + runtimeInputs = tools; + builder = ./builder.sh; env = { MODULE_ID = moduleId; - PATH = lib.makeBinPath [ - coreutils - coreutils - gnutar - pigz - ztoc-rs - ]; + PATH = lib.makeBinPath tools; diskClosureInfo = closureInfo { rootPaths = [ module ]; }; }; } From f55d55f44e464674888b55c92dfbcba18306c7a3 Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Thu, 22 Oct 2026 23:53:54 +0000 Subject: [PATCH 3/4] Retain image helper paths in runtime closures Coerce path-valued builder environment inputs into store files instead of discarding their dependency context with toString. Assisted-by: Replit --- pkgs/image-script.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/image-script.nix b/pkgs/image-script.nix index 78c624ab..8fe0a609 100644 --- a/pkgs/image-script.nix +++ b/pkgs/image-script.nix @@ -1,9 +1,10 @@ { lib, writeShellApplication, coreutils, bash, writeText }: { name, runtimeInputs, env, builder }: let + runtimeValue = value: if builtins.isPath value then "${value}" else toString value; attributes = writeText "${name}-attributes" '' declare -A env=( - ${lib.concatStringsSep "\n" (lib.mapAttrsToList (key: value: "[${lib.escapeShellArg key}]=${lib.escapeShellArg (toString value)}") env)} + ${lib.concatStringsSep "\n" (lib.mapAttrsToList (key: value: "[${lib.escapeShellArg key}]=${lib.escapeShellArg (runtimeValue value)}") env)} ) declare -A outputs=([out]="$IMAGE_WORK/output") ''; From d5dc78c6b8440146156d85bb128aae614756c82f Mon Sep 17 00:00:00 2001 From: Ryan Mulligan Date: Fri, 23 Oct 2026 00:06:38 +0000 Subject: [PATCH 4/4] Make hardlink fixture independent of binary cache substitution Create and verify a hardlink at test runtime, and only require cached closure inputs to retain links that still exist in the source. Assisted-by: Replit --- pkgs/bundle-oci/check.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/pkgs/bundle-oci/check.py b/pkgs/bundle-oci/check.py index f101a060..e6f1b0d0 100644 --- a/pkgs/bundle-oci/check.py +++ b/pkgs/bundle-oci/check.py @@ -120,15 +120,18 @@ def compare(source, target): for metadata in (bundle / "etc/nixmodules").iterdir(): compare(metadata, root / "etc/nixmodules" / metadata.name) packed_dependency = root / str(dependency).lstrip("/") -assert (packed_dependency / "data").stat().st_ino == ( - packed_dependency / "hardlink" -).stat().st_ino +if (dependency / "data").stat().st_ino == (dependency / "hardlink").stat().st_ino: + assert (packed_dependency / "data").stat().st_ino == ( + packed_dependency / "hardlink" + ).stat().st_ino print("OCI descriptors, complete closure, metadata, content, modes, symlinks, ownership and hardlinks verified") xattr_root = pathlib.Path("xattrs") xattr_root.mkdir() xattr_file = xattr_root / ("long-name-" * 20) xattr_file.write_bytes(b"xattrs\n") +# NAR substitution does not retain hardlink identity, so create this fixture at runtime. +os.link(xattr_file, xattr_root / "zz-hardlink") try: os.setxattr(xattr_file, "user.fixture", b"preserved") xattrs_supported = True @@ -143,14 +146,16 @@ def compare(source, target): assert member.isfile() and source.extractfile(member).read() == b"xattrs\n" if xattrs_supported: assert member.pax_headers["SCHILY.xattr.user.fixture"] == "preserved" +restored = pathlib.Path("xattrs-restored") +restored.mkdir() +subprocess.run([ + "tar", "--extract", "--gzip", "--file=xattrs.tar.gz", + "--directory", str(restored), "--no-same-owner", + "--xattrs", "--xattrs-include=*", +], check=True) +assert (restored / xattr_file.name).stat().st_ino == (restored / "zz-hardlink").stat().st_ino +print("Runtime hardlink fixture preserved in archive and extraction") if xattrs_supported: - restored = pathlib.Path("xattrs-restored") - restored.mkdir() - subprocess.run([ - "tar", "--extract", "--gzip", "--file=xattrs.tar.gz", - "--directory", str(restored), "--no-same-owner", - "--xattrs", "--xattrs-include=*", - ], check=True) assert os.getxattr(restored / xattr_file.name, "user.fixture") == b"preserved" print("Extended attributes preserved in archive and extraction") os.utime(xattr_file, (12345, 67890))