review: 7 fixes, cuts, docs against the code

.github/workflows/build-boot.yml

@@ -116,7 +116,7 @@ jobs:
         id: ref
         # Same 7-char short SHA docker/metadata-action's type=sha emits, so
         # the ref matches the tag actually pushed below.
-        run: echo "image=ghcr.io/${{ github.repository }}/boot:$(cat boot/kernel/VERSION)-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
+        run: echo "image=ghcr.io/${{ github.repository }}/boot:${{ steps.ver.outputs.kernel }}-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
 
       - name: Download digests
         uses: actions/download-artifact@v8.0.2
@@ -149,7 +149,7 @@ jobs:
         working-directory: /tmp/digests
         run: |
           docker buildx imagetools create \
-            $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '${{ steps.meta.outputs.json }}') \
+            $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
             $(printf 'ghcr.io/${{ github.repository }}/boot@sha256:%s ' *)
 
       - name: Inspect

.github/workflows/build-os-images.yml

@@ -1,7 +1,7 @@
 # Reusable: the images gating workflow calls this after the boot/silkd
 # carriers build, passing which changed and their sha-pinned refs. It detects
 # changed image dirs for the matrix, builds base then flavors, and pushes to
-# ghcr; base gets its same-dir *.sh files as buildx secrets.
+# ghcr; base gets install-agent.sh as a buildx secret.
 #   - a changed carrier (boot or silkd) refolds into every image; otherwise
 #     only the git-changed image dirs rebuild;
 #   - a base change rebuilds every flavor (they are FROM base);
@@ -87,7 +87,6 @@ jobs:
           if [[ "$CARRIER_CHANGED" == "true" || "$MANUAL" == "true" || -z "$BASE" ]] ||
              git diff --name-only "$BASE" "$SHA" -- "$wf/build-os-images.yml" "$wf/images.yml" | grep -q .; then
             all_names
-            all_changed=""
           else
             all_changed=$(git diff --name-only "$BASE" "$SHA" -- 'os-image/' ':!*.md' | sort -u)
             declare -A names
@@ -103,29 +102,13 @@ jobs:
             fi
           fi
 
-          # Build matrix entries with a per-entry no-cache flag. Cache must be
-          # dropped when any non-Dockerfile input changed: shared files
-          # directly under os-image/{name}/ (depth 3) or tag-specific files
-          # next to the Dockerfile (depth 4) — both are injected as buildx
-          # secrets, which never bust docker layer cache by content.
+          no_cache=false
+          if [[ "$MANUAL_NO_CACHE" == "true" ]]; then
+            no_cache=true
+          fi
           base_entries=()
           flavor_entries=()
           for name in "${!names[@]}"; do
-            needs_no_cache="false"
-            if [[ "$MANUAL_NO_CACHE" == "true" ]]; then
-              needs_no_cache="true"
-            elif [[ -n "$all_changed" ]]; then
-              while IFS= read -r f; do
-                [[ -z "$f" ]] && continue
-                depth=$(echo "$f" | awk -F'/' '{print NF}')
-                base_name=$(basename "$f")
-                if [[ "$depth" -eq 3 ]] || { [[ "$depth" -eq 4 ]] && [[ "$base_name" != "Dockerfile" ]]; }; then
-                  needs_no_cache="true"
-                  break
-                fi
-              done <<< "$(echo "$all_changed" | grep "^os-image/$name/")"
-            fi
-
             platforms="linux/amd64,linux/arm64"
             if [[ -f "os-image/$name/platforms" ]]; then
               platforms=$(cat "os-image/$name/platforms")
@@ -135,7 +118,7 @@ jobs:
               [[ -z "$dockerfile" ]] && continue
               tag=$(echo "$dockerfile" | cut -d'/' -f3)
               if [[ -n "$tag" ]]; then
-                entry="{\"name\":\"$name\",\"tag\":\"$tag\",\"no_cache\":\"$needs_no_cache\",\"platforms\":\"$platforms\"}"
+                entry="{\"name\":\"$name\",\"tag\":\"$tag\",\"no_cache\":\"$no_cache\",\"platforms\":\"$platforms\"}"
                 if [[ "$name" == "base" ]]; then
                   base_entries+=("$entry")
                 else
@@ -205,29 +188,6 @@ jobs:
           hash=$(find "os-image/${{ matrix.name }}" -type f ! -name Dockerfile ! -name '*.md' -print0 | sort -z | xargs -0 sha256sum | sha256sum | cut -d' ' -f1)
           echo "hash=$hash" >> "$GITHUB_OUTPUT"
 
-      - name: Discover secret files
-        id: secrets
-        run: |
-          base="os-image/${{ matrix.name }}"
-          tag="${{ matrix.tag }}"
-          {
-            echo "files<<EOF"
-            for f in "$base"/*.sh; do
-              [ -f "$f" ] || continue
-              name="$(basename "$f")"
-              id="sandbox_$(echo "${name%.sh}" | tr '.-' '__')"
-              echo "${id}=${f}"
-            done
-            for f in "$base/$tag"/*; do
-              [ -f "$f" ] || continue
-              name="$(basename "$f")"
-              [ "$name" = "Dockerfile" ] && continue
-              id="$(echo "$name" | tr '.-' '__')"
-              echo "${id}=${f}"
-            done
-            echo "EOF"
-          } >> "$GITHUB_OUTPUT"
-
       - name: Build and push
         uses: docker/build-push-action@v7.4.0
         with:
@@ -245,7 +205,7 @@ jobs:
             INPUTS_HASH=${{ steps.inputs.outputs.hash }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          secret-files: ${{ steps.secrets.outputs.files }}
+          secret-files: sandbox_install_agent=os-image/base/install-agent.sh
           no-cache: ${{ matrix.no_cache == 'true' }}
           cache-from: type=gha,scope=${{ matrix.name }}-${{ matrix.tag }}
           cache-to: type=gha,mode=max,scope=${{ matrix.name }}-${{ matrix.tag }}

.github/workflows/build-silkd.yml

@@ -142,7 +142,7 @@ jobs:
         working-directory: /tmp/digests
         run: |
           docker buildx imagetools create \
-            $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< '${{ steps.meta.outputs.json }}') \
+            $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
             $(printf 'ghcr.io/${{ github.repository }}/silkd@sha256:%s ' *)
 
       - name: Inspect

.github/workflows/silkd.yml

@@ -36,4 +36,4 @@ jobs:
 
       - name: root-only tests
         working-directory: silkd
-        run: sudo -E env "PATH=$PATH" "HOME=$HOME" cargo test --locked --test exec_e2e --test git_e2e --test fs_e2e --test find_e2e --test tun_e2e -- --ignored --skip throughput
+        run: sudo -E env "PATH=$PATH" "HOME=$HOME" cargo test --locked --test '*' -- --ignored --skip throughput

README.md

@@ -99,15 +99,14 @@ sandboxd](docs/deploy.md) covers release binaries, the full config and systemd.
   `boot/**`, `silkd/**`, or `os-image/**` it builds the
   changed carriers (via `build-boot.yml` / `build-silkd.yml`,
   `workflow_call`) then os-images, in order, so the chain is deterministic
-- `build-os-images.yml` — bakes base + flavors FROM the sha-pinned carriers
+- `build-os-images.yml` — bakes base + flavors FROM the carriers, sha-pinned
+  when rebuilt in the same run
 - `release.yml` — on a version tag, checks the tag against silkd's version,
   then builds the release binaries and archives
 - `publish-pypi.yml` — on an `sdk-*-v*` tag, builds and publishes the
   matching package via PyPI Trusted Publishing (OIDC, per-package
   environment)
 
-On a fresh repo run build-boot first — images build `FROM` the boot artifact.
-
 ## Boot chain
 
 ```

boot/Dockerfile

@@ -10,17 +10,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     xz-utils curl ca-certificates python3 \
     && rm -rf /var/lib/apt/lists/*
 ARG KERNEL_VERSION
-# Mirror chain: KERNEL_MIRROR (cdn.kernel.org by default), then kernel.org's
-# edge mirror and an rsync mirror. Download to a file first — a mid-stream
-# failure must not leave a half-extracted tree — and verify against the pinned SHA256
-# (kernel/SHA256, updated together with VERSION): the fallback mirrors are
-# third parties and this kernel boots untrusted-code sandboxes.
 ARG KERNEL_MIRROR=https://cdn.kernel.org/pub/linux/kernel
 WORKDIR /linux
 COPY kernel/SHA256 /tmp/SHA256
-# The hash is taken from the exact line naming this kernel version, so a
-# multi-line SHA256 file (e.g. keeping the previous kernel's entry during a
-# bump) can never pair the tarball with the wrong hash.
+# Mirrors are third parties, so the tarball is checked against this version's kernel/SHA256 line before extraction.
 RUN set -e; \
     hash=$(awk -v f="linux-${KERNEL_VERSION}.tar.xz" '$2==f{print $1; exit}' /tmp/SHA256); \
     [ -n "$hash" ] || { echo "kernel/SHA256 has no entry for linux-${KERNEL_VERSION}.tar.xz"; exit 1; }; \
@@ -36,10 +29,7 @@ RUN set -e; \
 ARG TARGETARCH
 COPY kernel/sandbox.config /tmp/sandbox.config
 COPY kernel/sandbox-arm64.config /tmp/sandbox-arm64.config
-# merge_config only warns and olddefconfig silently drops renamed or
-# dependency-orphaned symbols — assert the boot- and container-critical set
-# so a kernel bump cannot lose one with a green build. Boot entry and console
-# are per-arch: amd64 = PVH, arm64 = direct Image boot + PL031 RTC.
+# olddefconfig silently drops renamed or orphaned symbols, so assert the boot- and container-critical set.
 RUN set -e; \
     case "$TARGETARCH" in \
       amd64) make x86_64_defconfig kvm_guest.config; \

boot/kernel/sandbox.config

@@ -1,5 +1,5 @@
-# sandbox guest kernel fragment, applied over `x86_64_defconfig` +
-# `kvm_guest.config` (see boot/Dockerfile). Everything the boot path touches
+# sandbox guest kernel fragment, applied over the arch defconfig (plus
+# `kvm_guest.config` on x86_64; see boot/Dockerfile). Everything the boot path touches
 # is built in (=y): no modules to load, no initramfs modprobe, no udev settle.
 
 ## No module support at all — the strongest "nothing to wait for" guarantee.
@@ -196,6 +196,5 @@ CONFIG_TRANSPARENT_HUGEPAGE_MADVISE=y
 # CONFIG_KEXEC is not set
 # CONFIG_FTRACE is not set
 CONFIG_DEBUG_INFO_NONE=y
-# Boot-time W+X page-table scan: debug-only cost on every boot (it also
-# flagged 6 W+X pages on the PVH path — tracked separately).
+# Boot-time W+X page-table scan: debug-only cost on every boot.
 # CONFIG_DEBUG_WX is not set

docs/cluster.md

@@ -312,7 +312,9 @@ release and adoption an evacuation uses:
   `disk_reserve`, under `memory_limit` and under `max_running`.
 - When a peer adopts, the owner forgets the claim and the wake answers 404, as
   after an evacuation. The client's `Lookup` finds the peer, which wakes the
-  claim there. The hand-off counts in `sandboxd_wake_handoffs_total`.
+  claim there. The hand-off counts in `sandboxd_wake_handoffs_total`. A preview
+  URL minted before the hand-off stops working until it is minted again, as
+  after an evacuation.
 - When no peer adopts (no peer, or every peer full or closed), the owner takes
   the row back and the wake answers 429 as before. A node runs one hand-off at
   a time, and after one fails it asks no peer for 30 seconds, so clients

docs/deploy.md

@@ -656,7 +656,7 @@ golden fail the same way, it is retired again at most once every 30 s.
 
 The systemd unit is
 [`packaging/sandboxd.service`](https://github.com/cocoonstack/sandbox/blob/main/packaging/sandboxd.service);
-install it as shipped. Its `KillMode` and `TimeoutStopSec` are explained below.
+install it as shipped. The unit runs `/usr/local/bin/sandboxd -config /etc/sandboxd/config.json`, and its `KillMode` and `TimeoutStopSec` are explained below.
 `LimitCORE=0` keeps a crash from writing sandboxd's memory, which holds every
 running claim's host-only env values, to a core file. For the same reason,
 run sandboxd hosts with swap off or encrypted.
@@ -1148,7 +1148,7 @@ all at once or not at all. Every field falls into one of three classes:
   - per pool: `egress`, `warmup`, `capture_trim`, `storage`, `reader`, `egress_upstream_env`, `tun`, for existing keys and for keys new in the file;
   - `tun`: `exits`, `mtu`, `port` and `limits`; its `subnet`, `listen`, `vni`, `senders` and `reuse_after_seconds`, and adding or removing the block, need a restart;
   - `egress_classes`: each class's `egress` and `egress_upstream_env`;
-  - node-wide: `egress_internal_allow`, `secrets`, `egress_upstream`, `egress_usage_bytes`, `egress_usage_interval_seconds`, `egress_conn_rate`, `disk_reserve`, `memory_limit`, `max_claims`, `max_running`, `move_concurrency`, `max_fork_count`, `release_delay_seconds`, `vmm_restart`, `sync_claims`, `checkpoint_ttl_hours`, and the unpooled defaults `idle_hibernate_seconds`, `archive_after_seconds` and `archive_delete_after_seconds`.
+  - node-wide: `egress_internal_allow`, `secrets`, `egress_upstream`, `egress_usage_bytes`, `egress_usage_interval_seconds`, `egress_conn_rate`, `disk_reserve`, `memory_limit`, `balloon`, `max_claims`, `max_running`, `move_concurrency`, `max_fork_count`, `release_delay_seconds`, `vmm_restart`, `sync_claims`, `checkpoint_ttl_hours`, and the unpooled defaults `idle_hibernate_seconds`, `archive_after_seconds` and `archive_delete_after_seconds`.
 - **Left to their API**: a pool's targets (`warm`, `warm_max`, idle and archive
   durations) belong to `PUT /v1/pools`. A reload never changes them and lists
   them as `ignored`. A pool key new in the file gets its settings at once and stays

docs/performance.md

@@ -68,7 +68,7 @@ keep an idle relay for 30 seconds and send later RPCs on it; use mode C of
 ## Relay connection reuse
 
 Every data-plane RPC used to pay a TCP dial, an HTTP Upgrade and, behind a
-TLS edge, a handshake. From proto 2 silkd serves RPCs back to back, and both
+TLS edge, a handshake. silkd now serves RPCs back to back on one connection, and both
 SDKs park up to 8 idle relay connections per handle for a 30s idle window.
 
 `e2e/cmd/rpcbench`, warm `rt:24.04` sandbox, `net=none`, n=200 `fs_stat`

docs/sandboxd-api.md

@@ -120,8 +120,8 @@ or the operator token, which claims as root but never with `volumes` (403).
   0, and the next call that reaches the guest restores it with a fresh lease
   of the claim's length, so its next expiry archives it again; a call that
   wakes it between the hibernate and the export grants that fresh lease too.
-  A failed hibernate or export at expiry keeps the claim and retries on every
-  reap tick (5 s); it never falls back to destroy. A failed claim (`failed` set, see
+  A failed hibernate or export at expiry keeps the claim and retries it a
+  minute later; it never falls back to destroy. A failed claim (`failed` set, see
   [dead VMMs](deploy.md#dead-vmms-and-host-restarts)) is destroyed at expiry even
   under `archive`. A volume claim or an egress-lane claim
   cannot hibernate, so `archive` on one answers 409 with the same error as a
@@ -962,7 +962,7 @@ has no `preview_listen`. The signed token embeds the sandbox id, port, and
 owner `advertise_addr`, so any node's preview listener can serve it (forwarding
 to the owner's main listener) and a released sandbox's URL answers 404 —
 no revocation list. A browser request to the minted URL answers 503
-`lease lost; retry` for a replicated claim while its node's lease is overdue, 503 `sandbox moving to another node; retry` while an [evacuation](#post-v1drain) has yet to move it, and 429 when it has to wake a sandbox while the disk reserve is low or the node is at `max_running`. See [deploy](deploy.md#preview-urls).
+`lease lost; retry` for a replicated claim while its node's lease is overdue, 503 `sandbox moving to another node; retry` while an [evacuation](#post-v1drain) has yet to move it, and 429 when it has to wake a sandbox while the disk reserve is low or the node is at `max_running`; a wake that [hands an archived claim to a peer](cluster.md#waking-an-archived-claim-on-a-full-node) answers 404, as after a move. See [deploy](deploy.md#preview-urls).
 
 ## POST /v1/sandboxes/{id}/checkpoint
 
@@ -1273,8 +1273,8 @@ archive/unarchive/archive-delete counters, `sandboxd_dns_malformed_total` (guest
 for average latency, `sandboxd_claim_duration_seconds{tier}`, a histogram of pool
 claims by tier, `sandboxd_claim_redirects_total`, claims this node handed to a
 peer, `sandboxd_wakes_refused_total{reason}`, wakes `max_running` (`max_running`),
-`disk_reserve` (`disk`) refused with 429, and wakes `memory_limit` (`memory`)
-refused here whether or not a hand-off ran, `sandboxd_wake_handoffs_total`,
+`disk_reserve` (`disk`) and `memory_limit` (`memory`)
+refused here, whether or not a hand-off ran, `sandboxd_wake_handoffs_total`,
 archived claims this node [handed to a peer with room](cluster.md#waking-an-archived-claim-on-a-full-node)
 instead, and `sandboxd_wake_memory_fallbacks_total`, archived wakes over
 `memory_limit` this node woke itself because no hand-off ran or no peer adopted. `sandboxd_balloon_inflations_total`, `sandboxd_balloon_deflations_total` and `sandboxd_balloon_failures_total` count the balloon sweep's changes and the ones that failed, which retry a minute later. On a node with a `tun` block it adds the packet path's counters:

docs/sdk.md

@@ -347,7 +347,7 @@ out, err := sb.Exec(ctx, "cat", "/tmp/state")   // sessions & memory intact
 guest does can fall between the snapshot point and the stop. The handle
 stays valid: the first call that reaches the guest restores the VM (adding
 roughly a restore's latency, tens of milliseconds on bare metal). On a node at
-[`max_running`](deploy.md#configuration) that call returns a 429 `*APIError`,
+[`max_running`](deploy.md#configuration), or below `disk_reserve`, that call returns a 429 `*APIError`,
 which the SDK does not retry, and the sandbox stays hibernated. The TTL
 keeps running — a hibernated sandbox is still reaped at its deadline, so
 claim with a `WithTimeout` that covers the idle period or `Renew` before it
@@ -705,7 +705,7 @@ sb = client.Attach(ownerAddr, id, token)  // bind a known handle, no lookup roun
 
 err = client.PutTenant(ctx, sandbox.TenantSpec{Name: "u:42", Token: tok, MaxClaims: 10, EgressClass: "desk"})
 err = client.DeleteTenant(ctx, "u:42")
-res, _ = client.PutTenantCluster(ctx, sandbox.TenantSpec{Name: "u:42", MaxClaims: 20}) // token kept
+res, _ = client.PutTenantCluster(ctx, sandbox.TenantSpec{Name: "u:42", MaxClaims: 20, EgressClass: "desk"}) // token kept
 res, _ = client.DeleteTenantCluster(ctx, "u:42")
 page, _ := client.Tenants(ctx, "", 0)     // page.Tenants: names, caps, live claims, removed tenants; never tokens; page.Next pages
 page, _ = client.SetTenants(ctx, specs)   // replace the whole set