follow-up: off-loop reader teardown, one upload rebind, Python dial-failure parity, reap log out of the lock

e2e/cmd/androidsmoke/main.go

@@ -1,7 +1,4 @@
-// androidsmoke is the android-flavor acceptance: claim (none/xlarge) → Android
-// init tree over the relay → adb CNXN handshake on the adb port →
-// checkpoint/branch of the booted guest. No session step: the guest
-// ships no bash.
+// androidsmoke checks the android flavor: claim, the init tree over the relay, the adb handshake, and checkpoint/branch.
 package main
 
 import (

e2e/cmd/browsersmoke/main.go

@@ -1,6 +1,4 @@
-// browsersmoke is the browser-flavor acceptance: claim the browser flavor → CDP
-// /json/version over the relay → open a target → checkpoint/branch of the
-// warmed browser.
+// browsersmoke checks the browser flavor: claim, CDP /json/version over the relay, a new target, and checkpoint/branch.
 package main
 
 import (

e2e/cmd/coldproof/main.go

@@ -1,7 +1,4 @@
-// coldproof measures a cold claim end-to-end and proves the guest is live:
-// it claims an unpooled template (RunCold path), then execs inside the guest
-// and prints the guest's own /proc/uptime and kernel — the guest-side clock
-// is the evidence that claim-ready means a booted, exec-serving system.
+// coldproof times a cold claim and prints the guest uptime and kernel as proof it booted.
 package main
 
 import (

e2e/cmd/credsmoke/main.go

@@ -1,6 +1,4 @@
-// credsmoke is the claim-credential acceptance: claims each send their own env-injected header,
-// query or body credential through the proxy (-inject-size: on an "inject" pool), the guest never
-// holds the value, and rotation, refusal, clearing and forks behave as the API says.
+// credsmoke checks claim credentials end to end through the proxy; -inject-size names an "inject" pool.
 package main
 
 import (

e2e/cmd/crossnode/main.go

@@ -1,8 +1,4 @@
-// crossnode is the two-node store acceptance: a checkpoint created on node
-// A is claimed on node B through the shared checkpoint store; run it with a
-// template absent from B's image store so the claim exercises the engine's
-// clone --pull path (base blobs resolved by digest). The marker written on
-// A proves the memory state crossed nodes.
+// crossnode claims on node B a checkpoint made on node A through the shared store; use a template absent from B's image store.
 package main
 
 import (

e2e/cmd/demo/main.go

@@ -1,5 +1,4 @@
-// demo exercises a live sandboxd through the SDK and prints per-iteration
-// latencies — the bare-metal e2e script drives it on a real node.
+// demo claims, execs and releases through the SDK and prints per-iteration latencies.
 package main
 
 import (

e2e/cmd/guestserver/main.go

@@ -1,10 +1,4 @@
-// guestserver is the in-guest listener the port-relay e2e drives: a loopback
-// HTTP server that reports the protocol it was reached over. It stands in for
-// any guest daemon during hardware tests — the relay carries bytes, so proving
-// HTTP/1.1 and h2c both survive it needs a server that speaks both.
-//
-// It is uploaded into a claimed sandbox by portsmoke; nothing in the stock
-// image provides an HTTP listener.
+// guestserver is the in-guest HTTP/1.1 and h2c listener portsmoke uploads to test the port relay.
 package main
 
 import (

e2e/cmd/interceptsmoke/main.go

@@ -1,8 +1,4 @@
-// interceptsmoke is the HTTPS-interception acceptance: a none-lane sandbox whose
-// golden had the cluster root baked reaches an HTTPS echo through the proxy; the
-// leaf the node's intermediate signed validates against that baked root, the TLS
-// is terminated (so the guest sees our issuer, not the origin's), and the secret
-// is injected into the HTTPS request — the guarantees plaintext already had.
+// interceptsmoke checks HTTPS interception: a none-lane guest with the baked cluster root reaches an HTTPS echo with the secret injected.
 package main
 
 import (

e2e/cmd/lifecycle/main.go

@@ -1,7 +1,4 @@
-// lifecycle exercises sandboxd's auto-archive: a claim idles into hibernation,
-// then to a store checkpoint with its local VM dropped, and the next access
-// wakes it from the store with the same id and guest state intact. The node
-// must run a pool with idle_hibernate_seconds + archive_after_seconds set.
+// lifecycle checks that an idle claim archives and wakes with its id and guest state; the pool needs idle_hibernate_seconds and archive_after_seconds.
 package main
 
 import (
@@ -82,7 +79,6 @@ func run(addr, token, template, netShape string, wait time.Duration) error {
 	return nil
 }
 
-// waitArchived returns how long the reaper's idle→hibernate→archive ladder took to archive the claim.
 func waitArchived(ctx context.Context, client *sandbox.Client, timeout time.Duration) (time.Duration, error) {
 	start := time.Now()
 	deadline := start.Add(timeout)

e2e/cmd/meshsmoke/main.go

@@ -1,8 +1,4 @@
-// meshsmoke proves cluster template routing on a real two-node mesh:
-// a claim at the entry node redirects to the peer that pools the key, a
-// promote there is reachable by name from the entry node, and a name-based
-// delete follows gossip to the owner. Run against node A of a two-node
-// cluster where only node B pools the template.
+// meshsmoke checks template routing on a two-node mesh; run it against node A where only node B pools the template.
 package main
 
 import (

e2e/cmd/portsmoke/main.go

@@ -1,7 +1,4 @@
-// portsmoke proves sandboxd's guest-port relay on real hardware: it starts a
-// listener inside a claimed microVM and drives
-// GET /v1/sandboxes/{id}/ports/{port} end to end — the path an edge proxy
-// takes to reach a sandbox that has no NIC. Run by scripts/port-e2e.sh.
+// portsmoke checks the guest-port relay end to end against a claimed microVM; scripts/port-e2e.sh runs it.
 package main
 
 import (

e2e/cmd/pullbench/main.go

@@ -1,6 +1,4 @@
-// pullbench measures fs_pull throughput end to end — the hardware A/B for
-// silkd's chunk-frame render path: writes size MiB of random bytes in the
-// guest, then times n sequential Pulls.
+// pullbench times n sequential fs_pull transfers of size MiB from the guest.
 package main
 
 import (

e2e/cmd/pushbench/main.go

@@ -1,5 +1,4 @@
-// pushbench measures fs_write throughput end to end — the hardware A/B for
-// the SDK's data-frame encode path: writes size MiB to the guest n times.
+// pushbench times n fs_write transfers of size MiB to the guest.
 package main
 
 import (

e2e/cmd/qaab/main.go

@@ -1,5 +1,4 @@
-// qaab measures one A/B arm against a live sandboxd: a concurrent cold-claim
-// burst, then a concurrent in-guest write burst, reported as one JSON row.
+// qaab measures one A/B arm: a concurrent cold-claim burst, then a concurrent in-guest write burst, as one JSON row.
 package main
 
 import (

e2e/cmd/ringsmoke/main.go

@@ -1,6 +1,4 @@
-// ringsmoke drives two silkd paths the other smokes leave alone: a detached
-// process that overflows its output ring one byte at a time, whose logs must
-// replay exactly the newest 256 KiB, and exec as a named user.
+// ringsmoke checks the replay of a detached process's 256 KiB output ring and exec as a named user.
 package main
 
 import (

e2e/cmd/smoke/main.go

@@ -1,7 +1,4 @@
-// smoke drives the full SDK v2 surface against one live sandbox: files,
-// sessions, find/replace, watch, git, and pty — the bare-metal proof that
-// silkd v2 works end to end through the relay. Run by scripts/sandboxd-e2e.sh
-// on a real node.
+// smoke drives the SDK surface against one live sandbox; scripts/sandboxd-e2e.sh runs it.
 package main
 
 import (

e2e/cmd/sockssmoke/main.go

@@ -1,10 +1,4 @@
-// Command sockssmoke proves the SOCKS5 egress channel from inside a claimed
-// sandbox: an allowed host tunnels over 127.0.0.1:1080, a GET-only host is
-// reachable on the HTTP proxy but refused on SOCKS5, an unlisted host and an
-// unlisted port are refused, IMAPS rides the tunnel on its listed port, the
-// HTTP door refuses unlisted ports on CONNECT and forward requests, and no
-// tunnel opens from a pool without a policy or from one whose policy did not
-// opt in.
+// sockssmoke checks the SOCKS5 egress door from a claimed sandbox against the allow, refuse and opt-in rules of its pool policy.
 package main
 
 import (

e2e/cmd/volumesmoke/main.go

@@ -1,5 +1,4 @@
-// volumesmoke validates shared read-only dataset mounts, and optionally one
-// writable dataset mount and live volume changes, on a live Cloud Hypervisor node.
+// volumesmoke checks read-only dataset mounts and, optionally, a writable mount and live volume changes.
 package main
 
 import (

mcp/main.go

@@ -1,7 +1,4 @@
-// sandbox-mcp is a Model Context Protocol server over stdio: it exposes the
-// sandbox surface (claim, exec, files, fork, checkpoint, promote, hibernate)
-// as MCP tools. One process serves one sandboxd endpoint, configured by
-// flags or environment.
+// sandbox-mcp serves the sandbox surface as Model Context Protocol tools over stdio, one sandboxd endpoint per process.
 package main
 
 import (

mcp/tools.go

@@ -13,6 +13,8 @@ import (
 	sandbox "github.com/cocoonstack/sandbox/sdk/go"
 )
 
+const sandboxIDDesc = "id returned by create_sandbox, fork, or branch_checkpoint"
+
 var (
 	errOutputCap = errors.New("output past the cap")
 
@@ -23,43 +25,43 @@ var (
 		},
 		{
 			"exec", "Run a shell command in a sandbox, wait for it to exit, and return stdout, stderr, and the exit code as JSON. The call is cut off after 5 minutes; for servers or long jobs use spawn instead. A failed or cut-off call still returns the output collected so far next to an error field. Each stream keeps its first 1 MiB and sets truncated beyond that. A hibernated sandbox wakes transparently on this call.",
-			schema(props{"sandbox_id": str("id returned by create_sandbox, fork, or branch_checkpoint"), "command": str("shell command, run via sh -c"), "cwd": str("working directory; empty runs in the guest's default")}, "sandbox_id", "command"), toolExec,
+			schema(props{"sandbox_id": str(sandboxIDDesc), "command": str("shell command, run via sh -c"), "cwd": str("working directory; empty runs in the guest's default")}, "sandbox_id", "command"), toolExec,
 		},
 		{
 			"spawn", "Start a shell command detached in a sandbox and return its guest pid immediately. The process keeps running across later tool calls; its output goes to a per-process ring buffer that keeps the newest 256 KiB, which logs replays. Use exec instead when you need the result now.",
-			schema(props{"sandbox_id": str("id returned by create_sandbox, fork, or branch_checkpoint"), "command": str("shell command, run via sh -c"), "cwd": str("working directory; empty runs in the guest's default")}, "sandbox_id", "command"), toolSpawn,
+			schema(props{"sandbox_id": str(sandboxIDDesc), "command": str("shell command, run via sh -c"), "cwd": str("working directory; empty runs in the guest's default")}, "sandbox_id", "command"), toolSpawn,
 		},
 		{
 			"ps", "List the sandbox's tracked processes (exec, spawn, and pty commands) as JSON: pid, argv, detached, state (running or exited), exit_code once exited, and start time. Processes started inside the guest by other means are not listed.",
-			schema(props{"sandbox_id": str("id returned by create_sandbox, fork, or branch_checkpoint")}, "sandbox_id"), toolPs,
+			schema(props{"sandbox_id": str(sandboxIDDesc)}, "sandbox_id"), toolPs,
 		},
 		{
 			"kill", "Send a signal to a tracked process. Killing a process that already exited is a no-op success; the guest never re-signals a reaped pid.",
-			schema(props{"sandbox_id": str("id returned by create_sandbox, fork, or branch_checkpoint"), "pid": integer("guest pid from spawn or ps"), "signal": integer("signal number, e.g. 15 for SIGTERM; 0 sends SIGKILL")}, "sandbox_id", "pid"), toolKill,
+			schema(props{"sandbox_id": str(sandboxIDDesc), "pid": integer("guest pid from spawn or ps"), "signal": integer("signal number, e.g. 15 for SIGTERM; 0 sends SIGKILL")}, "sandbox_id", "pid"), toolKill,
 		},
 		{
 			"logs", "Return a tracked process's buffered stdout and stderr, plus exit_code once it has ended. The buffer keeps the newest 256 KiB per process, so redirect long output to a file when it must be complete.",
-			schema(props{"sandbox_id": str("id returned by create_sandbox, fork, or branch_checkpoint"), "pid": integer("guest pid from spawn or ps")}, "sandbox_id", "pid"), toolLogs,
+			schema(props{"sandbox_id": str(sandboxIDDesc), "pid": integer("guest pid from spawn or ps")}, "sandbox_id", "pid"), toolLogs,
 		},
 		{
 			"write_file", "Write text content to a file in a sandbox, replacing any existing file. The write is atomic (temp file plus rename); an existing file keeps its permission bits. The parent directory must already exist.",
-			schema(props{"sandbox_id": str("id returned by create_sandbox, fork, or branch_checkpoint"), "path": str("absolute path of the file"), "content": str("full file content; written verbatim")}, "sandbox_id", "path", "content"), toolWriteFile,
+			schema(props{"sandbox_id": str(sandboxIDDesc), "path": str("absolute path of the file"), "content": str("full file content; written verbatim")}, "sandbox_id", "path", "content"), toolWriteFile,
 		},
 		{
 			"read_file", "Return the whole content of a regular file up to 1 MiB in a sandbox as text; bytes that are not valid UTF-8 are replaced, so binary content is lossy. A larger file, a directory, or a device is an error: use exec with head, tail, or a checksum instead. A missing path is an error.",
-			schema(props{"sandbox_id": str("id returned by create_sandbox, fork, or branch_checkpoint"), "path": str("absolute path of the file")}, "sandbox_id", "path"), toolReadFile,
+			schema(props{"sandbox_id": str(sandboxIDDesc), "path": str("absolute path of the file")}, "sandbox_id", "path"), toolReadFile,
 		},
 		{
 			"list_dir", "List one directory in a sandbox (not recursive) as JSON entries with name, kind (file, dir, symlink, or other), and size in bytes.",
-			schema(props{"sandbox_id": str("id returned by create_sandbox, fork, or branch_checkpoint"), "path": str("absolute path of the directory")}, "sandbox_id", "path"), toolListDir,
+			schema(props{"sandbox_id": str(sandboxIDDesc), "path": str("absolute path of the directory")}, "sandbox_id", "path"), toolListDir,
 		},
 		{
 			"fork", "Clone a sandbox into N independent children that start from its exact memory and disk state, including running processes; each child gets its own id and lives one hour. N is capped by the node's max_fork_count (default 16). All-or-nothing: on failure no child survives. The parent keeps running. An egress-lane sandbox cannot fork.",
-			schema(props{"sandbox_id": str("id returned by create_sandbox, fork, or branch_checkpoint"), "count": integer("number of children, 1 to the node's max_fork_count (default 16)")}, "sandbox_id", "count"), toolFork,
+			schema(props{"sandbox_id": str(sandboxIDDesc), "count": integer("number of children, 1 to the node's max_fork_count (default 16)")}, "sandbox_id", "count"), toolFork,
 		},
 		{
 			"checkpoint", "Capture a sandbox's full state (memory, disk, running processes) without stopping it and return a checkpoint id. branch_checkpoint claims new sandboxes from that moment any number of times; a checkpoint outlives this session until it is deleted or expires under the node's checkpoint TTL. An egress-lane sandbox cannot checkpoint.",
-			schema(props{"sandbox_id": str("id returned by create_sandbox, fork, or branch_checkpoint"), "name": str("optional human label")}, "sandbox_id"), toolCheckpoint,
+			schema(props{"sandbox_id": str(sandboxIDDesc), "name": str("optional human label")}, "sandbox_id"), toolCheckpoint,
 		},
 		{
 			"branch_checkpoint", "Claim a fresh sandbox that resumes from a checkpoint's exact captured moment and return its id; it lives one hour. The checkpoint is unchanged and can be branched again.",
@@ -69,13 +71,13 @@ var (
 		{"delete_checkpoint", "Delete the node's copy of a checkpoint and tell peers to drop theirs; a peer replica that misses that broadcast stays branchable until its own TTL sweep. Sandboxes already branched from it are unaffected.", schema(props{"checkpoint_id": str("id returned by checkpoint or list_checkpoints")}, "checkpoint_id"), toolDeleteCheckpoint},
 		{
 			"hibernate", "Snapshot a sandbox and stop its VM, freeing memory while keeping its id, files, processes, and shell state. The next call that reaches the guest (exec, spawn, ps, kill, logs, the file tools) wakes it transparently in tens of milliseconds; fork, checkpoint, promote and release act on the snapshot without waking it. An egress-lane sandbox cannot hibernate.",
-			schema(props{"sandbox_id": str("id returned by create_sandbox, fork, or branch_checkpoint")}, "sandbox_id"), toolHibernate,
+			schema(props{"sandbox_id": str(sandboxIDDesc)}, "sandbox_id"), toolHibernate,
 		},
 		{
 			"promote", "Publish the sandbox's current state as a named template on its node; later create_sandbox calls with that name (and the same net and size) start from it. Re-promoting to the same name replaces the template. An egress-lane sandbox cannot promote.",
-			schema(props{"sandbox_id": str("id returned by create_sandbox, fork, or branch_checkpoint"), "template_name": str("template name to publish as")}, "sandbox_id", "template_name"), toolPromote,
+			schema(props{"sandbox_id": str(sandboxIDDesc), "template_name": str("template name to publish as")}, "sandbox_id", "template_name"), toolPromote,
 		},
-		{"release", "Destroy a sandbox and free its resources; files and processes inside it are lost. This session forgets the id, so a second release of it is rejected as unknown.", schema(props{"sandbox_id": str("id returned by create_sandbox, fork, or branch_checkpoint")}, "sandbox_id"), toolRelease},
+		{"release", "Destroy a sandbox and free its resources; files and processes inside it are lost. This session forgets the id, so a second release of it is rejected as unknown.", schema(props{"sandbox_id": str(sandboxIDDesc)}, "sandbox_id"), toolRelease},
 		{"node_info", "Report the connected node's warm pools, promoted templates, live claims, drain state, capacity, and mesh peers as JSON. Needs the operator token; a tenant token is refused.", schema(props{}), toolNodeInfo},
 	}
 )

sandboxd/config/config.go

@@ -97,7 +97,7 @@ func (s *PoolSpec) UnmarshalJSONFrom(dec *jsontext.Decoder) error {
 }
 
 // ValidateLimits checks the warm/watermark/idle bounds shared by config and PUT /v1/pools.
-func (s PoolSpec) ValidateLimits() error {
+func (s *PoolSpec) ValidateLimits() error {
 	if s.Warm < 0 {
 		return fmt.Errorf("warm must not be negative")
 	}

sandboxd/pool/reader.go

@@ -138,20 +138,18 @@ func (m *Manager) dropReader(ctx context.Context, r *reader, force bool) {
 	m.destroyReader(ctx, r)
 }
 
-func (m *Manager) dropStaleReaders(ctx context.Context) {
+func (m *Manager) dropStaleReaders(ctx context.Context) *sync.WaitGroup {
 	rs := m.readers
 	rs.mu.Lock()
 	live := slices.Collect(maps.Values(rs.live))
 	rs.mu.Unlock()
-	for _, r := range live {
-		m.mu.Lock()
+	m.mu.Lock()
+	stale := slices.DeleteFunc(live, func(r *reader) bool {
 		sb := m.claimed[r.claim]
-		stale := sb == nil || sleepSource(sb) != r.source
-		m.mu.Unlock()
-		if stale {
-			m.dropReader(ctx, r, true)
-		}
-	}
+		return sb != nil && sleepSource(sb) == r.source
+	})
+	m.mu.Unlock()
+	return m.runBounded(ctx, len(stale), func(ctx context.Context, i int) { m.dropReader(ctx, stale[i], true) })
 }
 
 func (m *Manager) ensureReader(ctx context.Context, sb *types.Sandbox) error {

sandboxd/pool/reader_test.go

@@ -663,14 +663,44 @@ func TestReaderWithAConnectionOpenIsRetiredOnceItsClaimWakesOrLeaves(t *testing.
 		if err != nil {
 			t.Fatalf("leave=%v: %v", leave, err)
 		}
-		m.dropStaleReaders(t.Context())
+		m.dropStaleReaders(t.Context()).Wait()
 		if removes, _ := readerLifecycle(eng); !slices.Contains(removes, "sbx-reader-1") {
 			t.Errorf("leave=%v: removes=%v, want the reader destroyed although its connection is open", leave, removes)
 		}
 		_ = open.Close()
 	}
 }
 
+func TestStaleReaderTeardownNeverBlocksTheCaller(t *testing.T) {
+	eng := newFakeEngine()
+	m := newReaderManager(t, eng)
+	sb := mustClaim(t, m, testKey)
+	hibernateClaim(t, m, sb)
+	seedReaderVMs(m, testKey, "sbx-reader-1")
+	open, _, err := m.DialReader(t.Context(), sb.ID, sb.Token)
+	if err != nil {
+		t.Fatalf("DialReader: %v", err)
+	}
+	defer func() { _ = open.Close() }()
+	if _, _, _, err = m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil {
+		t.Fatalf("wake: %v", err)
+	}
+	stall := make(chan struct{})
+	eng.mu.Lock()
+	eng.removeStall = stall
+	eng.mu.Unlock()
+	started := make(chan *sync.WaitGroup, 1)
+	go func() { started <- m.dropStaleReaders(t.Context()) }()
+	select {
+	case wg := <-started:
+		close(stall)
+		wg.Wait()
+	case <-time.After(5 * time.Second):
+		close(stall)
+		t.Fatal("dropStaleReaders waited for the engine removal")
+	}
+}
+
 func TestFailedPreHibernateSyncSendsReadsToTheWakePath(t *testing.T) {
 	for _, tt := range []struct {
 		name    string