Whole-repo static audit: fence egress and admission, snapshot leak, archive retry spacing, 30 more fixes

.github/workflows/sandboxd.yml

@@ -15,6 +15,7 @@ on:
       - "protocol/**"
       - "Makefile"
       - ".golangci.yml"
+      - ".goreleaser.yml"
       - ".github/workflows/sandboxd.yml"
   pull_request:
     paths:
@@ -26,6 +27,7 @@ on:
       - "protocol/**"
       - "Makefile"
       - ".golangci.yml"
+      - ".goreleaser.yml"
       - ".github/workflows/sandboxd.yml"
   workflow_dispatch:
 
@@ -57,9 +59,11 @@ jobs:
           go-version-file: sandboxd/go.mod
           cache-dependency-path: "**/go.sum"
 
-      - name: Build sandboxd
-        working-directory: sandboxd
-        run: go build ./...
+      - name: Check the release config
+        uses: goreleaser/goreleaser-action@v7.3.0
+        with:
+          version: v2.18.2
+          args: check
 
       - name: Lint
         run: make go-lint

Makefile

@@ -15,6 +15,8 @@ EXTRACT_IMAGE ?= $(BOOT_IMAGE)
 GO_MODULES := protocol/wire sandboxd sdk/go e2e mcp
 GO_OSES := linux darwin
 
+.DEFAULT_GOAL := help
+
 ## Location to install dependencies to
 LOCALBIN ?= $(shell pwd)/bin
 $(LOCALBIN):

docs/advanced-network.md

@@ -69,8 +69,7 @@ answers.
 | 10 | 2 | flags | 0 | bit 0 `ON`: packet mode is on for this box |
 
 Flag bits 1 to 15 are reserved and sent as 0; a peer ignores a bit it does
-not know. A later revision uses one to open several streams per guest, one
-per `tun0` queue.
+not know.
 
 A host that answers without `ON` closes the stream after the hello; the guest
 removes `tun0`. A host that refuses a guest answers without `ON` rather than
@@ -129,7 +128,7 @@ guest -> tun0 -> silkd --vsock 2051--> sandboxd --VXLAN--> exit (vxlan0, forward
   disk write on the claim path; each release is appended to
   `<data_dir>/tun-freed.log` so its reuse window outlives a restart. A node
   with a `mesh` must have `meta_store.cell` and allocates it in the cell's
-  [table](deploy.md#shared-tun-address-table), one round trip per claim, and stays off (proxy mode only) while the
+  [table](deploy.md#shared-tun-address-table), one round trip per candidate address, and stays off (proxy mode only) while the
   meta store is unreachable. A freed address is reused after
   `tun.reuse_after_seconds`, so the subnet's size over that window bounds the
   sustained claim rate: a `/8` with the 5-day default carries about 38 new tun
@@ -139,8 +138,8 @@ guest -> tun0 -> silkd --vsock 2051--> sandboxd --VXLAN--> exit (vxlan0, forward
   whatever the guest wrote; a GSO frame is cut into wire-sized packets with
   full checksums; each gets an Ethernet header (claim MAC to the exit's MAC)
   and a VXLAN header and leaves for the exit picked by the claim's profile
-  label (`default` when it has none). The outer source port follows a hash of
-  the inner 5-tuple.
+  label (`default` when it has none). The outer source port is one of `senders`
+  sockets, picked by a hash of the protocol, the ports and the inner destination.
 - **Inbound**: a VXLAN packet from an exit's VTEP whose inner destination is
   a claim on this node goes back to `100.64.0.2` and onto that claim's stream;
   anything else is dropped. Inbound passes only for a flow the claim opened,
@@ -152,7 +151,7 @@ guest -> tun0 -> silkd --vsock 2051--> sandboxd --VXLAN--> exit (vxlan0, forward
   and port-unreachable reach the guest's socket.
 - **ARP**: sandboxd answers ARP from the exits for its own claims only,
   announces a claim with a gratuitous ARP when it is armed (claim, wake,
-  move-in), after a reload of `tun.exits` and every 45 seconds, and never
+  move-in), after every config reload and every 45 seconds, and never
   forwards a frame the guest built.
 - **Policy**: a flow table keyed by 5-tuple; the first packet of a flow is
   judged by `EvalFlow` (bare `host: "*"` rules only) and the internal-address

docs/benchmarks.md

@@ -74,9 +74,9 @@ On exit it removes every `sbx-*` VM on the cocoon root, a serving
 sandboxd's claims included, so run it only on a node that serves no claims.
 The daemon also reclaims every `sbx-` VM at start.
 
-Knobs (environment variables): `WARM`/`WARM_N` (warm-pool depth and burst
-size — the burst must stay within the depth, or refill loses the race and
-the tail measures clones), `CLONE_N`, `COLD_N`, `RPC_N`, `PULL_MB`,
+Knobs (environment variables): `WARM`/`WARM_N` (warm-pool depth and the
+number of back-to-back warm-tier claims, which must stay within the depth, or
+refill loses the race and the tail measures clones), `CLONE_N`, `COLD_N`, `RPC_N`, `PULL_MB`,
 `PULL_N`, `BURST_N` (concurrent clone-claim burst; 0 skips the stage),
 `ADDR`/`TOKEN` (the throwaway daemon's listen address and token).
 

docs/cluster.md

@@ -3,7 +3,7 @@
 A cluster is a set of sandboxd nodes joined through a
 [hashicorp/memberlist](https://github.com/hashicorp/memberlist) SWIM mesh.
 Gossip carries only placement hints — per-pool warm counts, promoted-template
-hashes, available volume names, each node's data-plane address, and a digest
+hashes, available volume names, each node's data-plane and client addresses, and a digest
 of the cluster-invariant config.
 Gossip carries no per-sandbox state, so a stale view costs at most
 one extra redirect, never correctness. A single node with no seeds is a valid
@@ -444,7 +444,7 @@ guest resumes on the source unless the target committed.
 
 **Counting.** The target admits the move like a new claim: drain,
 `disk_reserve`, `memory_limit`, node `max_claims` and `max_running`, the
-tenant's `max_claims` and `move_concurrency`, and holds a reservation from admission to its commit. A
+tenant's `max_claims`, and the node's `move_concurrency`, and holds a reservation from admission to its commit. A
 [cordon](sandboxd-api.md#post-v1cordon) refuses only new claims, so a cordoned target admits the move. The
 reservation counts in every cap but is no claim: listings, reaps and gossip
 never see it. At the commit the reservation becomes the claim under one lock,
@@ -596,7 +596,9 @@ checkpoint store and keeps each claim's registry row on the latest snapshot:
   ports, preview, fork, checkpoint and promote answer `503 lease lost; retry` (a read through a
   [reader](deploy.md#reading-sleeping-claims-without-waking-them) after the `101`
   gets an `error` frame instead), and `/owner` answers 404 so
-  a `Lookup` moves on to the other nodes. A claim is replicated when its policy
+  a `Lookup` moves on to the other nodes. Its proxy, SOCKS5 and intercept doors deny those claims and the trip
+  ends their open tunnels, and a new claim of a replicating pool meets an
+  [admission refusal](sandboxd-api.md#admission-refusals), which redirects it to a peer. A claim is replicated when its policy
   pool has `replicate_seconds`, from the moment it is claimed, adopted or
   loaded, whether or not a round has run, so the fence covers it before its
   first replica; the setting is restart-only so that decision cannot go stale.

docs/deploy.md

@@ -104,7 +104,7 @@ the intersection of the two.
 | `cocoon_bin` | `cocoon` | cocoon CLI binary |
 | `restore_mode` | unset | clone and wake-restore memory mode: `copy`, `ondemand`, or `mmap`; use `mmap` for dense pools |
 | `no_direct_io` | false | use buffered writable disks for Cloud Hypervisor cold boots and clones; recommended for dense ephemeral pools to avoid direct-I/O CoW journal contention |
-| `sync_claims` | false | fsync `claims.json` and its directory on every commit (claim, release, renew, hibernate, wake, fork, archive, reap), about 0.3 ms each on NVMe; without it a host power loss can leave an empty journal, which fails the next start's reconcile until the file is removed, after which hibernate snapshots are reaped as orphans and archive checkpoints stay stranded. Off by default; turn it on when running claims must come back from a power loss (see `vmm_restart`) |
+| `sync_claims` | false | fsync `claims.json` and its directory on every commit (claim, release, renew, hibernate, wake, fork, archive, reap), about 0.3 ms each on NVMe; without it a host power loss can leave an empty journal, which fails the next start's reconcile until the file is removed, after which hibernate snapshots are reaped as orphans and archive checkpoints stay stranded. Off by default; turn it on when running claims must come back from a power loss (see `vmm_restart`). It does not cover `usage.jsonl`, whose last lines a power loss can still drop |
 | `cocoond_socket` | unset | `cocoon daemon`'s read-only API socket (by default `<cocoon run_dir>/cocoond.sock`); set, the node detects VMM exits from its event stream and polls only as a 60 s backstop. See [dead VMMs](#dead-vmms-and-host-restarts) |
 | `vmm_restart` | `cold` | what the node does with a running claim whose VMM exited outside a transition (a VMM crash, or a host restart that took it): `cold` cold-boots the claim's own VM from its disk under the same sandbox id and token (guest memory, processes, and the instance-metadata document are lost), `none` keeps the claim listed as `failed` for the caller to release or wake. Egress-lane and volume claims, a VM record gone from cocoon, and a VMM that exits more than 3 times in 10 minutes are marked `failed` under either value. See [dead VMMs](#dead-vmms-and-host-restarts) |
 | `no_balloon` | false | boot pool and template VMs without the virtio-balloon (cocoon otherwise returns 25% of guest memory to the host); clones inherit it from the golden. A guest that thrashes before deflate-on-OOM fires — a 16G build tier running a large typecheck — needs its whole memory |
@@ -429,7 +429,7 @@ generation sweep.
 | `<prefix><id>/meta.json` | The live record, uploaded last as the commit marker. Absent once the record is deleted. Its current generation is `export-` plus the first 16 hex characters of the SHA-256 of the object body. |
 | `<prefix><id>/labels.json` | Optional labels, removed with the record. |
 | `<prefix>manifests/<id>/<gen>.json` | One JSON manifest per published generation: `lineage` and, per file, `path`, `size` and the present `chunks` (`index`, `sha256`). `<gen>` is `export-<16 hex>`. |
-| `<prefix>chunks/<lineage>/<sha256>` | A 16 MiB chunk. A lineage is the dedup scope: it defaults to the record id, and every checkpoint and archive of one claim shares the claim's sandbox id. A chunk is shared by every record of its lineage that names the same digest, and only by those. |
+| `<prefix>chunks/<lineage>/<sha256>` | A chunk of up to 16 MiB; a file's last chunk is shorter. A lineage is the dedup scope: it defaults to the record id, and every checkpoint and archive of one claim shares the claim's sandbox id. A chunk is shared by every record of its lineage that names the same digest, and only by those. |
 | `<prefix>lineages/<lineage>/<id>` | Zero-byte marker: record `<id>` published into another record's lineage `<lineage>`, written after the manifest and before `meta.json`; a record in its own lineage writes none. A publish into a shared lineage reads only the records these list, so a crash between manifest and marker only costs a missed dedup. It only drives dedup, never chunk liveness. |
 
 **Offline reclamation only.** Stop every sandboxd node using the prefix and
@@ -1202,7 +1202,7 @@ some of them on a node:
 | `portsmoke`, `guestserver` | the guest-port relay; `guestserver` is its guest listener | `port-e2e.sh` |
 | `ringsmoke` | the output ring cap and exec as a user | |
 | `androidsmoke`, `browsersmoke` | the android and browser flavors | |
-| `pullbench`, `pushbench`, `rpcbench`, `qaab` | performance measurements | |
+| `pullbench`, `pushbench`, `rpcbench` | performance measurements | |
 
 `scripts/bench.sh` is the published [benchmark](benchmarks.md#reproduce)
 procedure and `scripts/boot-bench.sh` times the boot phases.

docs/egress.md

@@ -158,7 +158,7 @@ proxy over the same vsock path. To stop it bypassing the proxy, sandboxd locks
 the NIC at claim — until then a warm VM of the pool, and the pool's golden
 build, sit on the bridge unlocked, running only the image's own services, never
 tenant code: an nftables netdev table (`sandbox_egress_<tap>`) with an
-ingress hook on the guest's tap drops every guest-initiated packet, so the guest's
+ingress hook on the guest's tap drops every packet the guest sends, so the guest's
 only egress is the audited vsock proxy. A locked guest therefore cannot renew a
 DHCP lease and loses its NIC address when the lease ends, which nothing on this
 lane needs. The

docs/sdk.md

@@ -219,7 +219,7 @@ without `Stdin`, `WriteFile`, `OpenPty`, `Watch`, `DialPort`, `Lsp.Request`,
 `Logs`, `Attach`, and the file, git, process and session calls. `Run` with
 `Stdin` and `Push` stream the caller's reader, so they fail once and the next
 call reaches the new owner. An
-error from the caller's own writer is returned as it is and never retried, and a
+error from the caller's own writer is returned wrapped (match it with `errors.Is`) and never retried, and a
 released sandbox still answers 404.
 Volume sandboxes and egress-lane sandboxes cannot hibernate, fork, checkpoint,
 or promote, nor take `WithArchiveOnExpire`; each answers 409, as does a promote

docs/security.md

@@ -34,7 +34,7 @@ exposing any part of a deployment beyond a single trusted host.
   `host: "*"` rules and the internal-address guard allow, under per-claim
   rate, flow and quota limits.
   On the egress lane: the same vsock paths plus a NIC whose every
-  guest-initiated packet is dropped by an nftables lock in the host
+  outgoing packet, replies and ARP included, is dropped by an nftables lock in the host
   root netns ([egress](egress.md)); the lock is fail-closed and applied
   before the claim is handed out. On both lanes the proxy refuses
   loopback, private, link-local (cloud metadata), CGN, the

docs/silkd.md

@@ -28,7 +28,7 @@ RPC still runs ends that RPC's input and is served after it, except in the
 upload of `fs_write` or `fs_push`: there a request frame before `data_end`
 fails the upload with `bad_request` and is dropped. A client may close
 after any terminal frame. A frame that does not parse is answered with
-`bad_request` and closes the connection. Binary payloads ride base64 in `data`
+`bad_request` and, outside an upload, closes the connection. Binary payloads ride base64 in `data`
 fields. Frames are capped at 8 MiB; requests carry `"v": 1` and unknown
 fields are ignored (the forward-compatibility story).
 
@@ -42,7 +42,7 @@ connection-bound verbs are `fs_watch`, `pty_open`, `lsp_request`, and
 `port_forward`.
 
 The authoritative wire contract is the shared fixture corpus in
-`protocol/wire/fixtures/v1`, round-tripped by the Rust and Go test suites; the
+`protocol/wire/fixtures/v1`: the Go suite round-trips every fixture, Rust parses the requests and round-trips the responses, and the
 Python suite round-trips the request frames and checks that each response
 decodes to the fixture's `type` tag and `data` — a frame the Rust or Go side
 cannot parse fails CI.

e2e/cmd/androidsmoke/main.go

@@ -80,7 +80,7 @@ func run(addr, token, template string) error {
 }
 
 func initTree(ctx context.Context, sb *sandbox.Sandbox) error {
-	// The framework (zygote, system_server) starts minutes after init on a nested first boot; silkd answers long before
+	// The framework starts minutes after init on a nested first boot; silkd answers earlier.
 	deadline := time.Now().Add(5 * time.Minute)
 	for {
 		out, err := sb.Exec(ctx, "/system/bin/sh", "-c", shellPath+"; ps -A")

e2e/cmd/browsersmoke/main.go

@@ -104,7 +104,7 @@ func openTarget(ctx context.Context, sb *sandbox.Sandbox) error {
 }
 
 func cdpJSON(ctx context.Context, sb *sandbox.Sandbox, method, path string) (map[string]any, error) {
-	body, err := harness.HTTPOverPort(ctx, sb, cdpPort, method, path, nil)
+	body, err := harness.HTTPOverPort(ctx, sb, cdpPort, method, path)
 	if err != nil {
 		return nil, err
 	}

e2e/cmd/coldproof/main.go

@@ -16,7 +16,7 @@ func main() {
 	var (
 		addr     = flag.String("addr", "127.0.0.1:7777", "sandboxd address")
 		token    = flag.String("token", "", "node api token")
-		template = flag.String("template", "rt:24.04", "template ref (must be unpooled for a true cold boot)")
+		template = flag.String("template", "rt:24.04", "template ref; only an unpooled one measures a cold boot")
 		n        = flag.Int("n", 1, "iterations")
 	)
 	flag.Parse()

e2e/cmd/crossnode/main.go

@@ -3,6 +3,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"os"
@@ -29,6 +30,9 @@ func main() {
 }
 
 func run(addrA, addrB, token, template string) error {
+	if addrB == "" {
+		return errors.New("-b is required")
+	}
 	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
 	defer cancel()
 

e2e/cmd/egresssmoke/main.go

@@ -1,15 +1,5 @@
-// egresssmoke is the guarded-egress acceptance: a sandbox reaches an allowed
-// origin through the host egress proxy with a host-injected credential it never
-// holds, direct egress is blocked (no NIC on the none lane, an nft lock on the
-// egress lane), and a disallowed host is denied.
-//
-// Injection is checked against a public echo (-echo) reached through the proxy,
-// which the pool policy must allow; a host-local origin bound at -reach is the
-// negative control the guest must fail to reach directly. Example config:
-//
-//	"secrets":[{"name":"probe","header":"X-Egress-Token","value_env":"EGRESS_PROBE_TOKEN"}],
-//	"pools":[{"template":"…","net":"none","size":"small","warm":1,
-//	  "egress":{"allow":[{"host":"postman-echo.com","secret":"probe"}]}}]
+// egresssmoke checks guarded egress: an allowed origin gets the host-injected
+// credential, direct egress is blocked, and a disallowed host is denied.
 package main
 
 import (
@@ -36,19 +26,18 @@ func main() {
 	netShape := flag.String("net", "none", "claim lane: none|egress")
 	reach := flag.String("reach", "127.0.0.1", "host the guest reaches the origin at")
 	nicAddr := flag.String("nicaddr", "", "static CIDR to bring the egress-lane NIC up with (default route via -reach)")
-	guarded := flag.Bool("guarded", true, "pool has an egress policy; false asserts the unlocked NIC reaches directly (negative control)")
 	echo := flag.String("echo", "postman-echo.com", "public HTTP host echoing request headers at /get, for the injection check")
 	flag.Parse()
 
-	if err := run(*addr, *token, *template, *wantToken, *netShape, *reach, *nicAddr, *echo, *guarded); err != nil {
+	if err := run(*addr, *token, *template, *wantToken, *netShape, *reach, *nicAddr, *echo); err != nil {
 		fmt.Fprintln(os.Stderr, "egresssmoke:", err)
 		os.Exit(1)
 	}
 	fmt.Println("EGRESSSMOKE PASS")
 }
 
-func run(addr, token, template, wantToken, netShape, reach, nicAddr, echo string, guarded bool) error {
-	if guarded && wantToken == "" {
+func run(addr, token, template, wantToken, netShape, reach, nicAddr, echo string) error {
+	if wantToken == "" {
 		return errors.New("-secret is required for the injection check")
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
@@ -79,16 +68,9 @@ func run(addr, token, template, wantToken, netShape, reach, nicAddr, echo string
 	}
 
 	out, _ := sb.Exec(ctx, "sh", "-c", fmt.Sprintf("curl -s -m 3 --noproxy '*' %s || echo BLOCKED", target))
-	if guarded && !strings.Contains(out, "BLOCKED") {
+	if !strings.Contains(out, "BLOCKED") {
 		return fmt.Errorf("guest reached the origin without the proxy: %q", out)
 	}
-	if !guarded {
-		if !strings.Contains(out, "REACHED") {
-			return fmt.Errorf("unlocked NIC did not reach the origin: %q", out)
-		}
-		fmt.Println("  unlocked NIC reaches the origin directly (negative control)")
-		return nil
-	}
 	fmt.Println("  direct egress blocked")
 
 	// The SSRF guard refuses private and loopback destinations even when allow-listed, so injection needs a public echo.
@@ -107,8 +89,12 @@ func run(addr, token, template, wantToken, netShape, reach, nicAddr, echo string
 	}
 	fmt.Println("  allowed origin reached with no -x: proxy variables present in the exec")
 
-	if out, _ := sb.Exec(ctx, "sh", "-c", "env; cat /proc/1/environ 2>/dev/null | tr '\\0' '\\n'"); strings.Contains(out, wantToken) {
-		return fmt.Errorf("secret value leaked into the guest")
+	env, err := sb.Exec(ctx, "sh", "-c", "env; cat /proc/1/environ 2>/dev/null | tr '\\0' '\\n'")
+	if err != nil {
+		return fmt.Errorf("leak probe: %w", err)
+	}
+	if strings.Contains(env, wantToken) {
+		return errors.New("secret value leaked into the guest")
 	}
 	fmt.Println("  secret absent from guest env")
 

e2e/cmd/guestserver/main.go

@@ -12,7 +12,6 @@ import (
 	"sync"
 )
 
-// halfCloseSink greets, shuts its write side, and keeps reading what the client sends after.
 type halfCloseSink struct {
 	mu  sync.Mutex
 	got string