From 0773e3930c2fba7f74963e1cc19d991179f2adcc Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:07:56 +0800 Subject: [PATCH 001/127] pool: reuse setIfLive in archive and wake rollbacks, fold putReplica, drop restating comments --- sandboxd/pool/archive.go | 18 ++++-------------- sandboxd/pool/checkpoint.go | 7 +++---- sandboxd/pool/claim.go | 12 ++++-------- sandboxd/pool/failover.go | 13 +------------ sandboxd/pool/golden.go | 1 - sandboxd/pool/hibernate.go | 3 ++- sandboxd/pool/replica_test.go | 2 +- 7 files changed, 15 insertions(+), 41 deletions(-) diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index e973edf77..d435821a4 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -158,11 +158,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { js, archivedAt := m.store.set(sb), time.Now() m.mu.Unlock() if saveErr := m.store.commit(js); saveErr != nil { - m.mu.Lock() - var rb claimSnapshot - // A Release that landed meanwhile already deleted the claim; re-setting it would resurrect it - live := m.claimed[sb.ID] == sb - if live { + rb, live := m.setIfLive(sb, func() { sb.ArchiveCk, sb.ArchiveEpoch, sb.ArchivePending, sb.ReplicaCk = "", 0, prevPending, replica if replica != "" { sb.ArchiveEpoch = epoch @@ -170,9 +166,7 @@ func (m *Manager) archive(ctx context.Context, sb *types.Sandbox) error { sb.HibernateSnap, sb.VsockSocket = snap, sock m.setVMNameLocked(sb, vmName) sb.Deadline = prevDeadline - rb = m.store.set(sb) - } - m.mu.Unlock() + }) sb.Transition.Unlock() m.store.recommit(ctx, rb) if !live || replica == "" { @@ -301,14 +295,10 @@ func (m *Manager) commitWake(ctx context.Context, sb *types.Sandbox, vmName, soc js, lease := m.store.set(sb), sb.Deadline m.mu.Unlock() if err := m.store.commit(js); err != nil { - m.mu.Lock() - var rb claimSnapshot - if m.claimed[sb.ID] == sb { + rb, _ := m.setIfLive(sb, func() { sb.VsockSocket, sb.ArchiveCk, sb.ArchiveEpoch, sb.Deadline, sb.ReaderSafe = "", ck, epoch, deadline, safe m.setVMNameLocked(sb, "") - rb = m.store.set(sb) - } - m.mu.Unlock() + }) m.store.recommit(ctx, rb) return fmt.Errorf("wake %s: persist claims: %w", sb.ID, err) } diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index 62c44950c..83f358e9c 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -222,7 +222,7 @@ func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, o Clai } dir, _, _, err := m.ckpts.Fetch(ctx, ckpt.ID) if errors.Is(err, store.ErrNotFound) { - return nil, ErrUnknownCheckpoint // deleted between the pre-check and the lock + return nil, ErrUnknownCheckpoint } if err != nil { return nil, fmt.Errorf("fetch checkpoint: %w", err) @@ -368,9 +368,8 @@ func (h *healVetoes) clear(ckptID string) (aborted bool) { h.mu.Lock() defer h.mu.Unlock() delete(h.pending, ckptID) - if _, aborted = h.aborted[ckptID]; aborted { - delete(h.aborted, ckptID) - } + _, aborted = h.aborted[ckptID] + delete(h.aborted, ckptID) return aborted } diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index d5301b28f..7682bdd6e 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -12,6 +12,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" "github.com/cocoonstack/sandbox/sandboxd/usage" + "github.com/cocoonstack/sandbox/sandboxd/utils" ) // ClaimOptions is what a claim asks for beyond its key; the zero value takes every default. @@ -663,10 +664,8 @@ func (m *Manager) rollbackClaim(ctx context.Context, sbs []*types.Sandbox, bille m.recordUsage(ctx, usage.Event{Event: "release", ID: sb.ID, VMName: sb.VMName, Tenant: sb.Tenant}) } } - for _, sb := range sbs { - m.releaseTun(ctx, sb.ID, sb.TunAddr) - } for i, sb := range sbs { + m.releaseTun(ctx, sb.ID, sb.TunAddr) td := m.quiesceVolumes(ctx, sb) td.holds = holds[i] m.out.Disarm(sb.ID, m.removeOrRetry(ctx, sb.VMName, sb.ID, td)) @@ -737,11 +736,8 @@ func (c *heldConn) Close() error { // CloseWrite forwards the half-close; net.Conn does not carry it, so embedding cannot promote one. func (c *heldConn) CloseWrite() error { - cw, ok := c.Conn.(interface{ CloseWrite() error }) - if !ok { - return nil - } - return cw.CloseWrite() + utils.CloseWrite(c.Conn) + return nil } func clampTTL(ttl time.Duration) time.Duration { diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 6336d6243..efbe621a5 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -118,11 +118,6 @@ func (m *Manager) replaceArchive(ctx context.Context, sb *types.Sandbox, ck stri return m.putRow(ctx, sb, ck, false, true) } -// putReplica writes a live claim's row with no deadline: retention starts when the claim is archived or adopted. -func (m *Manager) putReplica(ctx context.Context, sb *types.Sandbox, ck string) (int64, error) { - return m.putRow(ctx, sb, ck, true, false) -} - // replaceReplica moves the row from the epoch this node holds; a row another node took or deleted refuses the write. func (m *Manager) replaceReplica(ctx context.Context, sb *types.Sandbox, ck string) (int64, error) { return m.putRow(ctx, sb, ck, true, true) @@ -324,13 +319,7 @@ func (m *Manager) dropTombstone(ctx context.Context, id string, epoch int64) { } func (m *Manager) reputArchive(ctx context.Context, sb *types.Sandbox) { - var epoch int64 - var err error - if sb.ArchiveCk != "" { - epoch, err = m.putArchive(ctx, sb, sb.ArchiveCk) - } else { - epoch, err = m.putReplica(ctx, sb, sb.ReplicaCk) - } + epoch, err := m.putRow(ctx, sb, cmp.Or(sb.ArchiveCk, sb.ReplicaCk), sb.ArchiveCk == "", false) if err != nil { if !errors.Is(err, ErrVolumeCapture) { log.WithFunc("pool.reputArchive").Warnf(ctx, "re-put archive row %s: %v", sb.ID, err) diff --git a/sandboxd/pool/golden.go b/sandboxd/pool/golden.go index ff4bd34e0..76a04a595 100644 --- a/sandboxd/pool/golden.go +++ b/sandboxd/pool/golden.go @@ -213,7 +213,6 @@ func (m *Manager) dumpGolden(ctx context.Context, name, snap, final string) erro return m.exportGolden(ctx, snap, final) } -// exportGolden exports snap into final through a unique sibling *.tmp dir. func (m *Manager) exportGolden(ctx context.Context, snap, final string) error { staging, err := os.MkdirTemp(filepath.Dir(final), filepath.Base(final)+"-*.tmp") if err != nil { diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index 72a7a668d..fa39558a8 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -282,9 +282,10 @@ func (m *Manager) idleOnce(ctx context.Context) { // Hibernates are seconds-long engine snapshots, so fan out off the housekeeping loop go func() { defer m.sweeps.idle.Store(false) + logger := log.WithFunc("pool.idleOnce") m.runBounded(ctx, len(victims), func(ctx context.Context, i int) { sb := victims[i] - logSweepResult(ctx, log.WithFunc("pool.idleOnce"), m.idleHibernate(ctx, sb, now), "idle-hibernated "+sb.ID, "idle-hibernate "+sb.ID) + logSweepResult(ctx, logger, m.idleHibernate(ctx, sb, now), "idle-hibernated "+sb.ID, "idle-hibernate "+sb.ID) }).Wait() }() } diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 8f19098e8..917d79349 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -828,7 +828,7 @@ func TestReplicaRestartAdoptsTheRowsCheckpointOfAnInterruptedRound(t *testing.T) if publishErr != nil { t.Fatalf("publish: %v", publishErr) } - if _, putErr := a.putReplica(t.Context(), sb, interrupted.ID); putErr != nil { + if _, putErr := a.putRow(t.Context(), sb, interrupted.ID, true, false); putErr != nil { t.Fatalf("put: %v", putErr) } second := interrupted.ID From 6d01e8f826c8ec2931e297df03ed7dbe38260533 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:12:53 +0800 Subject: [PATCH 002/127] pool: never serve a pooled key from a promoted template while its golden is rebuilt resolveGolden fell back to the template store whenever the pool golden was momentarily empty (first build, image change, reload), so claims on a pooled key cloned a tenant's promoted image. --- sandboxd/pool/promote_test.go | 20 +++++++++++++++++ sandboxd/pool/refillbackoff_test.go | 33 ----------------------------- sandboxd/pool/template.go | 4 ++-- 3 files changed, 22 insertions(+), 35 deletions(-) diff --git a/sandboxd/pool/promote_test.go b/sandboxd/pool/promote_test.go index 5f65fdc9b..ae47f8981 100644 --- a/sandboxd/pool/promote_test.go +++ b/sandboxd/pool/promote_test.go @@ -253,6 +253,26 @@ func TestReconcileSweepsGoldenTmpDirs(t *testing.T) { } } +func TestResolveGoldenNeverServesAPooledKeyFromATemplate(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + parent := mustClaim(t, m, testKey) + key, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:pooled", "") + if err != nil { + t.Fatalf("Promote: %v", err) + } + m.mu.Lock() + m.pools[key] = newPool(key) + m.mu.Unlock() + golden, err := m.resolveGolden(t.Context(), key, "") + if err != nil { + t.Fatalf("resolveGolden: %v", err) + } + golden.release() + if golden.dir != "" || golden.promoted { + t.Errorf("resolveGolden served pooled key from template %q; want cold-boot until the pool golden lands", golden.dir) + } +} + func TestResolveGoldenSkipsPromotedEgressTemplate(t *testing.T) { m := newTestManager(t, newFakeEngine()) staging := stageTemplate(t, m, store.TemplateID(egKey.Hash())) diff --git a/sandboxd/pool/refillbackoff_test.go b/sandboxd/pool/refillbackoff_test.go index 22e41b612..850ec8a6d 100644 --- a/sandboxd/pool/refillbackoff_test.go +++ b/sandboxd/pool/refillbackoff_test.go @@ -398,39 +398,6 @@ func TestALateClaimCloneResultOfARetiredGoldenDoesNotCount(t *testing.T) { } } -func TestAFailingCloneOfAPromotedTemplateDoesNotCount(t *testing.T) { - eng := newFakeEngine() - m := envManager(t, eng, t.TempDir()) - src, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}) - if err != nil { - t.Fatalf("source claim: %v", err) - } - key, _, err := m.Promote(t.Context(), src.ID, Cred{Token: src.Token}, "tpl:broken", "") - if err != nil { - t.Fatalf("promote: %v", err) - } - m.mu.Lock() - p := newPool(key) - m.pools[key] = p - m.mu.Unlock() - eng.mu.Lock() - eng.cloneErr = errors.New(`vm.restore -> 500 "Memory manager error"`) - eng.mu.Unlock() - clones := eng.cloneCount() - - for range refillFailStreak { - if _, err := m.ClaimProvision(t.Context(), key, ClaimOptions{}); err == nil { - t.Fatal("claim on a failing template clone succeeded") - } - } - if n := eng.cloneCount() - clones; n != refillFailStreak { - t.Fatalf("clones %d, want each claim to clone the template", n) - } - if fails := cloneFailsOf(m, p); fails != 0 { - t.Errorf("clone fails %d on the template's pool, want 0: no pool golden was cloned", fails) - } -} - func adoptedGoldenNoWarm(t *testing.T, eng *fakeEngine) (*Manager, *pool, string) { t.Helper() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey}) diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 4b4afc43e..4e217fcbd 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -375,8 +375,8 @@ func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey, tenant s if dir != "" { return goldenResolution{dir: dir, held: p, gen: gen}, nil } - if key.Net == types.NetEgress { - return goldenResolution{}, nil // never resume a live-captured template on the egress lane; cold-boot instead + if ok || key.Net == types.NetEgress { + return goldenResolution{}, nil // a pooled key waits for its golden; the egress lane never resumes a live capture } id := store.TemplateID(key.Hash()) cleanup := m.recs.rlockRec(id) From 1b32a82af4db85863a367f869cf4f21f2cd4e24a Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:13:49 +0800 Subject: [PATCH 003/127] pool: a clone cut off by engine shutdown never counts against the golden During shutdown eng.Close refuses new commands with ErrStopping; counting those as clone failures could reach the streak and delete a healthy golden's stamp, so the next start rebuilt it from a cold boot. --- sandboxd/pool/provision.go | 2 +- sandboxd/pool/refill.go | 4 ++-- sandboxd/pool/refillbackoff_test.go | 19 +++++++++++++++++++ sandboxd/pool/watermark.go | 8 +++++--- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/sandboxd/pool/provision.go b/sandboxd/pool/provision.go index 72cc78dfa..841545284 100644 --- a/sandboxd/pool/provision.go +++ b/sandboxd/pool/provision.go @@ -36,7 +36,7 @@ func (m *Manager) provisionResolved(ctx context.Context, key types.PoolKey, g go func (m *Manager) noteClaimClone(ctx context.Context, g goldenResolution, err error) { capReason := engine.CapacitySignature(err) m.mu.Lock() - g.held.countClone(ctx, g.gen, err == nil, capReason) + g.held.countClone(ctx, g.gen, err, capReason) retired := m.retireUnclonableLocked(g.held, g.gen, time.Now()) m.mu.Unlock() if retired { diff --git a/sandboxd/pool/refill.go b/sandboxd/pool/refill.go index 14d4bbd5f..9c86abe7f 100644 --- a/sandboxd/pool/refill.go +++ b/sandboxd/pool/refill.go @@ -84,7 +84,7 @@ func (m *Manager) refillOne(ctx context.Context, p *pool, golden string, gen uin sb, err := m.startVM(ctx, p.key, func(name string) (types.VMRecord, error) { return m.eng.Clone(ctx, golden, name, p.key) }) - cloned := err == nil + cloneErr := err <-m.refillSem if err == nil { if ctx.Err() == nil { @@ -112,7 +112,7 @@ func (m *Manager) refillOne(ctx context.Context, p *pool, golden string, gen uin } // A canceled context or a capacity failure says nothing about this pool's health backedOff := false - if p.countClone(ctx, gen, cloned, capReason) { + if p.countClone(ctx, gen, cloneErr, capReason) { backedOff = p.noteRefillResult(now, err != nil) } if backedOff { diff --git a/sandboxd/pool/refillbackoff_test.go b/sandboxd/pool/refillbackoff_test.go index 850ec8a6d..e0197104d 100644 --- a/sandboxd/pool/refillbackoff_test.go +++ b/sandboxd/pool/refillbackoff_test.go @@ -3,12 +3,14 @@ package pool import ( "context" "errors" + "fmt" "os" "testing" "testing/synctest" "time" "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" ) func TestRefillBacksOffOnlyAfterAnUnbrokenRunOfFailures(t *testing.T) { @@ -255,6 +257,23 @@ func TestRetiringAGoldenLeavesItsDirectoryForAClaimMidClone(t *testing.T) { } } +func TestClaimClonesCutOffByShutdownNeverRetireTheGolden(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + eng := newFakeEngine() + eng.cloneErr = fmt.Errorf("clone: %w", engine.ErrStopping) + m, p, g := adoptedGoldenNoWarm(t, eng) + + for i := range refillFailStreak { + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{}); err == nil { + t.Fatalf("claim %d on a stopping engine succeeded", i) + } + } + if goldenOf(m, p) != g { + t.Fatalf("golden %q after %d clones cut off by shutdown, want %q kept", goldenOf(m, p), refillFailStreak, g) + } + }) +} + func TestClaimClonesThatFailRetireTheGoldenOfAPoolWithNoRefill(t *testing.T) { synctest.Test(t, func(t *testing.T) { eng := newFakeEngine() diff --git a/sandboxd/pool/watermark.go b/sandboxd/pool/watermark.go index 3aeaf0ae9..b8f68592a 100644 --- a/sandboxd/pool/watermark.go +++ b/sandboxd/pool/watermark.go @@ -2,11 +2,13 @@ package pool import ( "context" + "errors" "math" "sync/atomic" "time" "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -86,11 +88,11 @@ func (p *pool) noteRefillResult(now time.Time, failed bool) bool { return first } -func (p *pool) countClone(ctx context.Context, gen uint64, cloned bool, capReason string) bool { - if ctx.Err() != nil || capReason != "" || gen != p.goldenGen { +func (p *pool) countClone(ctx context.Context, gen uint64, cloneErr error, capReason string) bool { + if ctx.Err() != nil || capReason != "" || gen != p.goldenGen || errors.Is(cloneErr, engine.ErrStopping) { return false } - if cloned { + if cloneErr == nil { p.cloneFails = 0 } else { p.cloneFails++ From e0abeae5943a299a0c56ae7e7fae1916d0cdf7e0 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:14:11 +0800 Subject: [PATCH 004/127] pool: state the claim event ordering guarantee the code gives --- sandboxd/pool/claim.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 7682bdd6e..5eeeef992 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -627,7 +627,7 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t m.rollbackClaim(ctx, sbs, false) return fmt.Errorf("persist claim: %w", saveErr) } - // The claim event follows the commit at once, so a crash cannot restore a claim nothing billed + // The claim event follows the commit, so a claim that never committed is never billed for _, sb := range sbs { m.recordUsage(ctx, usage.Event{ Event: "claim", From 3551eff90f01ac95eaa22b6cf5828b40f52c961f Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:18:47 +0800 Subject: [PATCH 005/127] sdk: never resend a redirected claim whose request may have reached the node A candidate that read the claim and then dropped the reply (timeout, reset, 502/504 edge) may have committed it; trying the next candidate or the origin claimed a second sandbox the caller never saw, billed until its TTL. Only a dial failure, a redirect, or a sandboxd refusal moves on. --- sdk/go/client.go | 21 +++++++++++++++--- sdk/go/client_test.go | 32 +++++++++++++++++++++++++++ sdk/python/cocoonsandbox/client.py | 13 ++++++++--- sdk/python/tests/test_fault_matrix.py | 28 +++++++++++++++++++++++ 4 files changed, 88 insertions(+), 6 deletions(-) diff --git a/sdk/go/client.go b/sdk/go/client.go index 816672f7f..4875cb04b 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "net" "net/http" "net/url" "slices" @@ -27,6 +28,8 @@ const ( noRedirectQueryParam = "no_redirect" ) +var errRedirectedAgain = errors.New("redirected again despite no_redirect") + // ClientOption configures Connect. type ClientOption func(*Client) @@ -336,8 +339,7 @@ func retryTransient(err error) bool { } switch he.Status { case http.StatusUnauthorized, http.StatusNotFound, http.StatusTooManyRequests, - http.StatusServiceUnavailable, http.StatusInternalServerError, - http.StatusBadGateway, http.StatusGatewayTimeout: + http.StatusServiceUnavailable, http.StatusInternalServerError: return true default: return false @@ -375,7 +377,7 @@ func redirectFallback(origin string, candidates []string, claimAt func(addr stri return claimResponse{}, err } if len(cr.Redirect) > 0 { - return claimResponse{}, fmt.Errorf("%s redirected again despite no_redirect", target) + return claimResponse{}, fmt.Errorf("%s %w", target, errRedirectedAgain) } return cr, nil } @@ -385,6 +387,9 @@ func redirectFallback(origin string, candidates []string, claimAt func(addr stri if err == nil { return addr, cr, nil } + if !claimRefused(err) { + return "", claimResponse{}, fmt.Errorf("redirect target %s: %w", addr, err) + } lastErr = err } if !retryTransient(lastErr) { @@ -397,6 +402,16 @@ func redirectFallback(origin string, candidates []string, claimAt func(addr stri return origin, cr, nil } +func claimRefused(err error) bool { + if he, ok := errors.AsType[*APIError](err); ok { + return he.Status != http.StatusBadGateway && he.Status != http.StatusGatewayTimeout + } + if op, ok := errors.AsType[*net.OpError](err); ok && op.Op == "dial" { + return true + } + return errors.Is(err, errRedirectedAgain) +} + func scatter[T any](ctx context.Context, addrs []string, probe func(ctx context.Context, addr string) (T, error)) (result T, ok bool) { scatterCtx, cancel := context.WithCancel(ctx) defer cancel() diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 6e14f1919..05d462eeb 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -469,6 +469,38 @@ func TestRedirectTriesAllCandidates(t *testing.T) { } } +func TestRedirectStopsWhenACandidateMayHaveClaimed(t *testing.T) { + var goodCalls, entryCalls atomic.Int32 + lost := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + conn, _, err := http.NewResponseController(w).Hijack() + if err == nil { + _ = conn.Close() + } + })) + t.Cleanup(lost.Close) + + good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + goodCalls.Add(1) + _ = json.MarshalWrite(w, claimResponse{ID: "sb_6", Token: "t"}) + })) + t.Cleanup(good.Close) + + entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + entryCalls.Add(1) + _ = json.MarshalWrite(w, claimResponse{ + Redirect: []string{strings.TrimPrefix(lost.URL, "http://"), strings.TrimPrefix(good.URL, "http://")}, + }) + })) + t.Cleanup(entry.Close) + + if _, err := testClient(t, entry).New(t.Context(), "rt:24.04"); err == nil { + t.Fatal("New succeeded after a candidate dropped the reply to a delivered claim") + } + if goodCalls.Load() != 0 || entryCalls.Load() != 1 { + t.Errorf("good=%d entry=%d calls, want 0 and 1: a delivered claim is never sent again", goodCalls.Load(), entryCalls.Load()) + } +} + func TestRedirectCandidateDefinitiveErrorStillTriesNextCandidate(t *testing.T) { forbidden := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusForbidden) diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 0dfebfb8f..bc088e425 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -5,6 +5,7 @@ import http.client import json import queue +import socket import ssl import threading import time @@ -302,7 +303,13 @@ def _try_each( def _retry_transient(exc: APIError) -> bool: - return exc.status in (0, 401, 404, 429, 503, 500, 502, 504) + return exc.status in (0, 401, 404, 429, 503, 500) + + +def _claim_refused(exc: APIError) -> bool: + if exc.status: + return exc.status not in (502, 504) + return exc.__cause__ is None or isinstance(exc.__cause__, (ConnectionRefusedError, socket.gaierror)) def _redirect_fallback( @@ -315,9 +322,9 @@ def attempt(addr: str) -> tuple[str, dict[str, Any]]: return addr, reply try: - return _try_each(candidates, attempt, retry=lambda _: True) + return _try_each(candidates, attempt, retry=_claim_refused) except APIError as exc: - if not _retry_transient(exc): + if not (_claim_refused(exc) and _retry_transient(exc)): raise try: return attempt(origin) diff --git a/sdk/python/tests/test_fault_matrix.py b/sdk/python/tests/test_fault_matrix.py index 8ae16e486..4ec5b225b 100644 --- a/sdk/python/tests/test_fault_matrix.py +++ b/sdk/python/tests/test_fault_matrix.py @@ -1,5 +1,7 @@ """Fault matrix for the cluster redirect follow and the lookup scatter.""" +import socket +import threading import time import pytest @@ -14,6 +16,32 @@ def test_claim_redirect_skips_dead_candidate(spawn_node, dead_addr): assert sb.id == "sb_ok" and sb.owner == good +def test_claim_redirect_stops_when_a_candidate_may_have_claimed(spawn_node): + server = socket.create_server(("127.0.0.1", 0)) + + def drop_reply(): + conn, _ = server.accept() + conn.recv(65536) + conn.close() + + threading.Thread(target=drop_reply, daemon=True).start() + lost = f"127.0.0.1:{server.getsockname()[1]}" + good_calls = [] + good = spawn_node( + {("POST", "/v1/claim"): lambda body, path: good_calls.append(body) or (200, {"id": "x", "token": "t"})} + ) + entry_calls = [] + entry = spawn_node( + {("POST", "/v1/claim"): lambda body, path: entry_calls.append(body) or (200, {"redirect": [lost, good]})} + ) + try: + with pytest.raises(APIError): + Client(entry).new("rt:24.04") + finally: + server.close() + assert not good_calls and len(entry_calls) == 1 + + def test_claim_redirect_skips_404_candidate(spawn_node): lost = spawn_node({("POST", "/v1/claim"): lambda body, path: (404, {"error": "gone"})}) good = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"id": "sb_2", "token": "t"})}) From bd5829aef235fd4253bc8d2dddcf1fa508912e9b Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:19:46 +0800 Subject: [PATCH 006/127] pool: retire a reader whose claim woke or left even while a connection holds it Only the idle timer and a new dial dropped a reader, so a tenant that kept one reader connection open across a wake or release kept an uncounted, unbilled warm VM running its own guest disk. The reap tick now drops every reader whose claim no longer sleeps on the source it serves. --- sandboxd/pool/pool.go | 1 + sandboxd/pool/reader.go | 18 ++++++++++++++++++ sandboxd/pool/reader_test.go | 27 +++++++++++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 625bea2c1..041b290ef 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -537,6 +537,7 @@ func (m *Manager) Run(ctx context.Context) { m.refillOnce(ctx) case <-reap: m.dropIdleReservations() + m.dropStaleReaders(ctx) m.retryRemovals(ctx) m.reapOnce(ctx) m.idleOnce(ctx) diff --git a/sandboxd/pool/reader.go b/sandboxd/pool/reader.go index c2d3df582..22f4a0887 100644 --- a/sandboxd/pool/reader.go +++ b/sandboxd/pool/reader.go @@ -6,9 +6,11 @@ import ( "encoding/json/v2" "errors" "fmt" + "maps" "net" "os" "path/filepath" + "slices" "sync" "time" @@ -136,6 +138,22 @@ func (m *Manager) dropReader(ctx context.Context, r *reader, force bool) { m.destroyReader(ctx, r) } +func (m *Manager) dropStaleReaders(ctx context.Context) { + rs := m.readers + rs.mu.Lock() + live := slices.Collect(maps.Values(rs.live)) + rs.mu.Unlock() + for _, r := range live { + m.mu.Lock() + sb := m.claimed[r.claim] + stale := sb == nil || sleepSource(sb) != r.source + m.mu.Unlock() + if stale { + m.dropReader(ctx, r, true) + } + } +} + func (m *Manager) ensureReader(ctx context.Context, sb *types.Sandbox) error { m.mu.Lock() want := sleepSource(sb) diff --git a/sandboxd/pool/reader_test.go b/sandboxd/pool/reader_test.go index 00c8e6b76..f8768f200 100644 --- a/sandboxd/pool/reader_test.go +++ b/sandboxd/pool/reader_test.go @@ -649,6 +649,33 @@ func TestReaderOfAStaleSleepIsRetiredEvenWithAConnectionOpen(t *testing.T) { } } +func TestReaderWithAConnectionOpenIsRetiredOnceItsClaimWakesOrLeaves(t *testing.T) { + for _, leave := range []bool{false, true} { + eng := newFakeEngine() + m := newReaderManager(t, eng) + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + seedReaderVMs(m, testKey, "sbx-reader-1") + open, _, err := m.DialReader(t.Context(), sb.ID, sb.Token) + if err != nil { + t.Fatalf("DialReader: %v", err) + } + if leave { + err = m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}) + } else { + _, _, _, err = m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) + } + if err != nil { + t.Fatalf("leave=%v: %v", leave, err) + } + m.dropStaleReaders(t.Context()) + if removes, _ := readerLifecycle(eng); !slices.Contains(removes, "sbx-reader-1") { + t.Errorf("leave=%v: removes=%v, want the reader destroyed although its connection is open", leave, removes) + } + _ = open.Close() + } +} + func TestFailedPreHibernateSyncSendsReadsToTheWakePath(t *testing.T) { for _, tt := range []struct { name string From 7bff68846e437716711eed759eb515bdbb688be2 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:22:05 +0800 Subject: [PATCH 007/127] pool: a tenant branches only its own tenant's checkpoints ClaimCheckpoint took the unguessable id as the capability, while list and delete already scoped to the caller's tenant; a tenant holding another tenant's checkpoint id could branch its memory and disk. --- docs/sandboxd-api.md | 3 ++- sandboxd/pool/checkpoint.go | 2 +- sandboxd/pool/checkpoint_test.go | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 9ab6adfcf..717d8ed44 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -937,7 +937,8 @@ for a replicated sandbox while its node's lease is overdue. Auth: node API token; body `{"ttl_seconds": 0, "no_redirect": false, "metadata": {}, "on_expire": "destroy", "env": {}}`. Claims a fresh sandbox branched from the checkpoint (a normal claim response, attributed to the caller); the checkpoint's recorded -key applies — the unguessable id is the capability to branch. The branch +key applies. A tenant token branches only its own tenant's checkpoints; any +other id answers 404, as for list and delete. The branch carries this request's `metadata`, `on_expire` and `env`, under the claim rules, never the source sandbox's. 400 metadata over a bound, an unknown `on_expire`, or a bad `env` entry. diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index 83f358e9c..43576b831 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -217,7 +217,7 @@ func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, o Clai return nil, err } defer m.recs.rlockRec(ckpt.ID)() - if ckpt.Archive { + if ckpt.Archive || !tenantOwns(o.Tenant, ckpt.Tenant) { return nil, ErrUnknownCheckpoint // a wake image, not a branchable checkpoint } dir, _, _, err := m.ckpts.Fetch(ctx, ckpt.ID) diff --git a/sandboxd/pool/checkpoint_test.go b/sandboxd/pool/checkpoint_test.go index 061cf344a..e85887925 100644 --- a/sandboxd/pool/checkpoint_test.go +++ b/sandboxd/pool/checkpoint_test.go @@ -277,6 +277,12 @@ func TestCheckpointTenantIsolation(t *testing.T) { t.Errorf("root listing %+v, %v; want both records", ckpts, err) } + if _, err := m.ClaimCheckpoint(t.Context(), ckA.ID, ClaimOptions{Tenant: "beta"}); !errors.Is(err, ErrUnknownCheckpoint) { + t.Errorf("cross-tenant branch: %v, want ErrUnknownCheckpoint", err) + } + if _, err := m.ClaimCheckpoint(t.Context(), ckA.ID, ClaimOptions{Tenant: "acme"}); err != nil { + t.Errorf("own branch: %v", err) + } if err := m.DeleteCheckpoint(t.Context(), ckA.ID, "beta", DeleteFleet); !errors.Is(err, ErrUnknownCheckpoint) { t.Errorf("cross-tenant delete: %v, want ErrUnknownCheckpoint", err) } From a8d23b26655a7447a1c4f92d0126445295689c65 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:23:54 +0800 Subject: [PATCH 008/127] pool: a wake hand-off whose peer adopted before its reply timed out answers 404 takeBack found the row taken and dropped the claim, but handOffForRoom treated that ErrUnknownSandbox as a failure: the wake answered 429, armed the quiet period and missed wake_handoffs_total. --- sandboxd/pool/archive.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandboxd/pool/archive.go b/sandboxd/pool/archive.go index d435821a4..8a490af17 100644 --- a/sandboxd/pool/archive.go +++ b/sandboxd/pool/archive.go @@ -268,7 +268,7 @@ func (m *Manager) handOffForRoom(ctx context.Context, sb *types.Sandbox) bool { logger := log.WithFunc("pool.handOffForRoom") err := m.handOff(ctx, sb, true) switch { - case err == nil: + case err == nil || errors.Is(err, ErrUnknownSandbox): m.counters.wakeHandoffs.Add(1) logger.Infof(ctx, "handed archived %s to a peer with room to wake it", sb.ID) return true From 137e817e4678dcf1b822bdf319fc2d030eae28c1 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:26:34 +0800 Subject: [PATCH 009/127] config: archive_failover requires mesh.cluster_key Every hand-off presents the claim's bearer token to an address learned from gossip; checkpoint_peer_heal and live moves already require an encrypted mesh for the same reason. --- docs/cluster.md | 5 +++-- docs/deploy.md | 2 +- sandboxd/config/config.go | 4 ++-- sandboxd/config/config_test.go | 7 +++++-- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 17db8d908..aa6eb6405 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -31,8 +31,9 @@ Give every node a `mesh` block: join mesh: … connection refused`), so two nodes that name only each other cannot cold-start together: list the node's own `bind` alongside its peers and every node comes up in any order, joining whoever is already there -- `cluster_key` — optional base64 key (16/24/32 bytes) enabling gossip - encryption; all members must share it +- `cluster_key` — base64 key (16/24/32 bytes) enabling gossip encryption; + all members must share it. Required by `archive_failover`, + `checkpoint_peer_heal` and live moves Two constraints, fine for a homogeneous cluster: all nodes share the same `api_token` and tenant set (the SDK replays whichever token authorized a diff --git a/docs/deploy.md b/docs/deploy.md index 842959f96..9be870b2b 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -138,7 +138,7 @@ the intersection of the two. | `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a none-lane claim is hibernated once it has had no open data-plane connection (relay, buffered exec, preview), no egress request in flight and no [advanced-network](advanced-network.md) packet carried for this long; the clock restarts when the last connection closes, so a long command is never cut short. The next call that reaches the guest wakes it transparently. Per-pool `idle_hibernate_seconds` does the same for that pool's claims; pooled keys ignore the node-wide value, and egress pools reject it because they cannot resume safely. Opt in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | | `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call that reaches the guest restores it transparently (a checkpoint restore's latency) with a fresh lease of the length the claim was granted (the server default of 5m when it asked for none); a lease moves only when a client calls [`/renew`](sandboxd-api.md#post-v1sandboxesidrenew). Keep `idle_hibernate_seconds` under the leases your claims use, or a woken sandbox that idles again is destroyed by the reap before it can hibernate, its archive already consumed by the wake; one still in use when its lease ends is destroyed the same way, unless it claimed `on_expire: archive`, which archives at the lease end whatever this setting. A call that wakes a hibernated claim past its lease, before the export, starts that fresh lease too. Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | | `archive_delete_after_seconds` | 0 (keep) | purge an archived claim's store checkpoint this long after it was archived; the claim is then gone for good. On S3, chunks and manifests remain until [offline reclamation](#reclaiming-s3-storage). On archive this retention window replaces the live claim deadline; 0 clears the deadline so the archive is kept forever. An [evacuation](cluster.md#evacuating-a-node) keeps a longer live deadline, and a claim of a pool without an archive tier, whose `on_expire` is not `archive`, keeps its deadline until its first wake. Same node-wide/per-pool split | -| `archive_failover` | false | let another node adopt this node's archived claims once its node lease expires, and the running claims of pools with `replicate_seconds`: each archive and replication round also writes a row (token SHA-256, owner, epoch, checkpoint id, retention deadline, and the claim record with its env, `guest: false` values included, in plaintext) to the shared PostgreSQL, and a `Lookup` that reaches any node takes the claim over there (see [archived claims across node loss](cluster.md#archived-claims-across-node-loss)). Requires `meta_store` with a `cell` (rows are scoped by it, so meshes sharing one database never adopt from each other), a `checkpoint_store` of kind `s3` and a `mesh`, all checked at config load. The node also renews a lease row every 5 s (see [node lease](cluster.md#node-lease)). With the meta store down, archives, wakes, releases and env writes of archived claims, and env writes and renews of hibernated replicated claims, fail and retry, and replicated claims stop being served once the lease is overdue (see [running claims](cluster.md#running-claims-across-node-loss)); other running claims never touch it. Restart only | +| `archive_failover` | false | let another node adopt this node's archived claims once its node lease expires, and the running claims of pools with `replicate_seconds`: each archive and replication round also writes a row (token SHA-256, owner, epoch, checkpoint id, retention deadline, and the claim record with its env, `guest: false` values included, in plaintext) to the shared PostgreSQL, and a `Lookup` that reaches any node takes the claim over there (see [archived claims across node loss](cluster.md#archived-claims-across-node-loss)). Requires `meta_store` with a `cell` (rows are scoped by it, so meshes sharing one database never adopt from each other), a `checkpoint_store` of kind `s3` and a `mesh` with a `cluster_key` (a hand-off presents the claim's token to an address learned from gossip), all checked at config load. The node also renews a lease row every 5 s (see [node lease](cluster.md#node-lease)). With the meta store down, archives, wakes, releases and env writes of archived claims, and env writes and renews of hibernated replicated claims, fail and retry, and replicated claims stop being served once the lease is overdue (see [running claims](cluster.md#running-claims-across-node-loss)); other running claims never touch it. Restart only | | `mesh` | unset | join a cluster ([Clusters](cluster.md)); unset = single node | | `pools[]` | — | warm pools, keyed by `(template, net, size)`. `warm` defaults to 4 only when both `warm` and `warm_max` are omitted; an explicit `warm: 0` is kept, so `warm: 0` under a `warm_max` starts the pool empty and grows it on demand, and `warm: 0` alone registers the key with no warm VMs (`PUT /v1/pools` never defaults `warm`); `net` is `none` (the default) or `egress`; `size` is a tier, below (default `small`). Retune online without a restart via [`PUT /v1/pools`](sandboxd-api.md#put-v1pools) — omitted pools drain, and their goldens are deleted once they are idle (`warm: 0` keeps the golden). This is the **first-boot seed**: once a node takes a `PUT /v1/pools`, the applied set persists to `/pools.json` and overrides this section on every later boot (a startup log notes it); delete `pools.json` to return to config-owned pools. With a `meta_store` `cell`, the cell's set overrides it instead (see [shared pool set](#shared-pool-set)). Egress stays config-owned either way. A template re-pulled or re-imported under the same ref is picked up within a minute: the pool rebuilds its golden and replaces its warm VMs. See [state ownership](cluster.md#state-ownership) | diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 36a4a78f6..188900813 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -835,8 +835,8 @@ func (c *Config) validateMemoryLimit() error { func (c *Config) validateMetaStore() error { ms := c.MetaStore switch { - case c.ArchiveFailover && (ms == nil || ms.Cell == "" || !c.S3Checkpoints() || c.Mesh == nil): - return fmt.Errorf("archive_failover requires meta_store with a cell, an s3 checkpoint_store and a mesh") + case c.ArchiveFailover && (ms == nil || ms.Cell == "" || !c.S3Checkpoints() || c.Mesh == nil || c.Mesh.ClusterKey == ""): + return fmt.Errorf("archive_failover requires meta_store with a cell, an s3 checkpoint_store and a mesh with a cluster_key") case ms == nil: return nil case ms.Kind != "pg": diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 07284879a..80ca3c552 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -214,7 +214,7 @@ func TestLoadArchiveFailover(t *testing.T) { parts := map[string]string{ "meta_store": `"meta_store":{"kind":"pg","dsn_env":"SANDBOX_TEST_DSN","cell":"c1"}`, "checkpoint_store": `"checkpoint_store":{"kind":"s3","s3":{"bucket":"b"}}`, - "mesh": `"mesh":{"bind":"node1:7946"}`, + "mesh": `"mesh":{"bind":"node1:7946","cluster_key":"MDEyMzQ1Njc4OWFiY2RlZg=="}`, } body := func(skip string) string { b := `{"api_token":"r","advertise_addr":"node1:7777","archive_failover":true,"pools":[]` @@ -240,11 +240,14 @@ func TestLoadArchiveFailover(t *testing.T) { if _, err := Load(writeConfig(t, strings.Replace(body(""), `,"cell":"c1"`, "", 1))); err == nil || !strings.Contains(err.Error(), "archive_failover requires") { t.Errorf("archive_failover without meta_store.cell: %v, want a refusal", err) } + if _, err := Load(writeConfig(t, strings.Replace(body(""), `,"cluster_key":"MDEyMzQ1Njc4OWFiY2RlZg=="`, "", 1))); err == nil || !strings.Contains(err.Error(), "archive_failover requires") { + t.Errorf("archive_failover without mesh.cluster_key: %v, want a refusal", err) + } } func TestLoadReplicateSeconds(t *testing.T) { t.Setenv("SANDBOX_TEST_DSN", "postgres://sandboxd@db/sandboxd") - failover := `"archive_failover":true,"meta_store":{"kind":"pg","dsn_env":"SANDBOX_TEST_DSN","cell":"c1"},"checkpoint_store":{"kind":"s3","s3":{"bucket":"b"}},"mesh":{"bind":"node1:7946"},` + failover := `"archive_failover":true,"meta_store":{"kind":"pg","dsn_env":"SANDBOX_TEST_DSN","cell":"c1"},"checkpoint_store":{"kind":"s3","s3":{"bucket":"b"}},"mesh":{"bind":"node1:7946","cluster_key":"MDEyMzQ1Njc4OWFiY2RlZg=="},` body := func(on, pool string) string { return `{"api_token":"r","advertise_addr":"node1:7777",` + on + `"pools":[` + pool + `]}` } From e6300df16a74ab11c53d195e4b3f0d45c785f75c Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:27:55 +0800 Subject: [PATCH 010/127] pool, docs: state why the egress lane refuses hibernate per the cocoon contract A bridge tap survives vm hibernate and restore of the same VM (contract B11); the refusal holds because restore resumes the guest inside the call, before sandboxd can re-lock (B10). --- docs/egress.md | 5 +++-- sandboxd/pool/hibernate.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/egress.md b/docs/egress.md index 8ef17af3a..96baec91b 100644 --- a/docs/egress.md +++ b/docs/egress.md @@ -178,8 +178,9 @@ presence; git keeps running as on any lane with a NIC. A golden built under another verdict is rebuilt, not adopted. Egress-lane sandboxes never archive and do not hibernate, fork, checkpoint, or -promote: cocoon resumes a guest before its fresh tap can be re-locked, so any -resume from a snapshot would open an unlocked-NIC window. Keeping the lane live +promote: cocoon resumes a restored or cloned guest inside the call, before +sandboxd can re-lock its tap, so any resume from a snapshot would open an +unlocked-NIC window. Keeping the lane live holds the lock unbroken from claim to release; the idle sweep skips the lane and those four verbs are refused (409) on it. diff --git a/sandboxd/pool/hibernate.go b/sandboxd/pool/hibernate.go index fa39558a8..e01fd42db 100644 --- a/sandboxd/pool/hibernate.go +++ b/sandboxd/pool/hibernate.go @@ -80,7 +80,7 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error return ErrVolumeCapture } if sb.Key.Net == types.NetEgress { - // cocoon resumes the guest before its fresh tap can be re-locked + // restore resumes the guest inside the call, before sandboxd could re-lock its tap return ErrNoEgressHibernate } if err := m.settleCapture(ctx, sb); err != nil { From f2cf6259a177abbb685000c4c035c241bfe74fd9 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:27:55 +0800 Subject: [PATCH 011/127] pool: record a checkpoint's tun state under the transition lock it captures under publishCheckpoint read TunOff before exportSource took sb.Transition, so a SetTun enable landing in between captured a guest with tun0 up into a checkpoint whose branches start with tun off. --- sandboxd/pool/capture.go | 17 +++++++++-------- sandboxd/pool/checkpoint.go | 7 ++----- sandboxd/pool/template.go | 2 +- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/sandboxd/pool/capture.go b/sandboxd/pool/capture.go index ad3c4a096..964016e10 100644 --- a/sandboxd/pool/capture.go +++ b/sandboxd/pool/capture.go @@ -8,6 +8,7 @@ import ( "github.com/projecteru2/core/log" + "github.com/cocoonstack/sandbox/sandboxd/outbound" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -57,37 +58,37 @@ func (m *Manager) captureAllowed(sb *types.Sandbox) error { return nil } -func (m *Manager) exportSource(ctx context.Context, sb *types.Sandbox, exportDir, from string) (snap, vmName string, guestEnv bool, err error) { +func (m *Manager) exportSource(ctx context.Context, sb *types.Sandbox, exportDir, from string) (snap, vmName string, guestEnv bool, tun outbound.TunState, err error) { if lockErr := m.lockTransition(sb); lockErr != nil { - return "", "", false, lockErr + return "", "", false, outbound.TunState{}, lockErr } if from != "" && sb.HibernateSnap != from { sb.Transition.Unlock() - return "", "", false, errWokeMeanwhile + return "", "", false, outbound.TunState{}, errWokeMeanwhile } if captureErr := m.captureAllowed(sb); captureErr != nil { sb.Transition.Unlock() - return "", "", false, captureErr + return "", "", false, outbound.TunState{}, captureErr } if settleErr := m.settleCapture(ctx, sb); settleErr != nil { sb.Transition.Unlock() - return "", "", false, settleErr + return "", "", false, outbound.TunState{}, settleErr } m.trimForCapture(ctx, sb) snap, cleanup, err := m.sourceSnap(ctx, sb) if err != nil { sb.Transition.Unlock() - return "", "", false, err + return "", "", false, outbound.TunState{}, err } defer cleanup() - vmName, guestEnv = sb.VMName, m.heldGuestEnv(sb) + vmName, guestEnv, tun = sb.VMName, m.heldGuestEnv(sb), m.tunState(sb) // A fresh snapshot is this capture's alone, so dials need not wait out its export; a wake consumes a hibernate snapshot if snap != sb.HibernateSnap { sb.Transition.Unlock() } else { defer sb.Transition.Unlock() } - return snap, vmName, guestEnv, m.eng.SnapshotExport(ctx, snap, exportDir) + return snap, vmName, guestEnv, tun, m.eng.SnapshotExport(ctx, snap, exportDir) } // A failed trim only leaves the capture larger. diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index 43576b831..8065fd89c 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -171,7 +171,6 @@ func (m *Manager) FetchCheckpoint(ctx context.Context, ckptID string) (string, [ } func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID, name, tenant, from string, archive bool) (types.Checkpoint, string, error) { - tun := m.tunState(sb) ckpt := types.Checkpoint{ ID: ckID, Name: name, @@ -181,8 +180,6 @@ func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID CreatedAt: time.Now(), PolicySource: sb.PolicySource, NoEgress: sb.NoEgress, - TunOff: !tun.On, - TunProfile: tun.Profile, Archive: archive, } if archive { @@ -193,11 +190,11 @@ func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID return types.Checkpoint{}, "", fmt.Errorf("stage checkpoint: %w", err) } defer func() { _ = os.RemoveAll(staging) }() - srcSnap, _, guestEnv, err := m.exportSource(ctx, sb, filepath.Join(staging, store.ExportDir), from) + srcSnap, _, guestEnv, tun, err := m.exportSource(ctx, sb, filepath.Join(staging, store.ExportDir), from) if err != nil { return types.Checkpoint{}, "", fmt.Errorf("checkpoint %s: %w", sb.ID, err) } - ckpt.GuestEnv = guestEnv + ckpt.GuestEnv, ckpt.TunOff, ckpt.TunProfile = guestEnv, !tun.On, tun.Profile meta, err := json.Marshal(ckpt) if err != nil { return types.Checkpoint{}, "", fmt.Errorf("encode checkpoint meta: %w", err) diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 4e217fcbd..243490655 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -73,7 +73,7 @@ func (m *Manager) Promote(ctx context.Context, id string, cred Cred, template, t return types.PoolKey{}, "", fmt.Errorf("stage template: %w", err) } defer func() { _ = os.RemoveAll(staging) }() - _, vmName, guestEnv, err := m.exportSource(ctx, sb, filepath.Join(staging, store.ExportDir), "") + _, vmName, guestEnv, _, err := m.exportSource(ctx, sb, filepath.Join(staging, store.ExportDir), "") if err != nil { return types.PoolKey{}, "", fmt.Errorf("promote %s: %w", sb.ID, err) } From 3464794595c1043ac34b91bd9edb2ef1d4251bbc Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:29:11 +0800 Subject: [PATCH 012/127] sdk/go: Run returns a stdin read failure alongside the exit code pumpStdin treated any read error from Cmd.Stdin as EOF, so a reader that failed mid-stream truncated the command's input and Run reported success, unlike os/exec. --- sdk/go/sandbox.go | 18 +++++++++++++----- sdk/go/sandbox_test.go | 16 ++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/sdk/go/sandbox.go b/sdk/go/sandbox.go index 48bae30db..945301d32 100644 --- a/sdk/go/sandbox.go +++ b/sdk/go/sandbox.go @@ -211,6 +211,7 @@ func (s *Sandbox) run(ctx context.Context, cmd Cmd) (int, error) { defer l.close() stdinDone := make(chan struct{}) + var stdinErr error if cmd.Stdin == nil { // A guest that already answered and closed fails this send; the frames it sent still come back below _ = conn.Send(wire.StdinClose{}) @@ -218,15 +219,16 @@ func (s *Sandbox) run(ctx context.Context, cmd Cmd) (int, error) { } else { go func() { defer close(stdinDone) - pumpStdin(conn, cmd.Stdin) + stdinErr = pumpStdin(conn, cmd.Stdin) }() } pid, code, exited, err := pumpStdio(ctx, conn, cmd.Stdout, cmd.Stderr) // A pump still parked in Read would write into the next RPC, so its connection is not kept + pumped := false select { case <-stdinDone: - err = l.done(err) + err, pumped = l.done(err), true default: } if err != nil { @@ -240,6 +242,9 @@ func (s *Sandbox) run(ctx context.Context, cmd Cmd) (int, error) { if !exited { return 0, unexpected(&wire.Done{}) } + if pumped && stdinErr != nil { + return int(code), &callerError{fmt.Errorf("read stdin: %w", stdinErr)} + } return int(code), nil } @@ -354,18 +359,21 @@ func markGone(err error, id, addr string) error { } // pumpStdin chunks the reader into stdin frames. -func pumpStdin(conn *silkd.Conn, r io.Reader) { +func pumpStdin(conn *silkd.Conn, r io.Reader) error { buf := make([]byte, stdinChunk) for { n, err := r.Read(buf) if n > 0 { if sendErr := conn.Send(&wire.Stdin{Data: buf[:n]}); sendErr != nil { - return + return nil } } if err != nil { _ = conn.Send(wire.StdinClose{}) - return + if errors.Is(err, io.EOF) { + return nil + } + return err } } } diff --git a/sdk/go/sandbox_test.go b/sdk/go/sandbox_test.go index f29455d82..c8f0f09ca 100644 --- a/sdk/go/sandbox_test.go +++ b/sdk/go/sandbox_test.go @@ -12,6 +12,7 @@ import ( "slices" "strings" "testing" + "testing/iotest" "time" "github.com/cocoonstack/sandbox/protocol/wire" @@ -69,6 +70,21 @@ func TestRunStreamsStdin(t *testing.T) { } } +func TestRunReportsAStdinReadFailure(t *testing.T) { + sb := testSandbox(t, newAgentServer(t, silkdtest.ServeConn)) + broken := errors.New("upstream reset") + + var out strings.Builder + code, err := sb.Run(t.Context(), Cmd{ + Argv: []string{"cat"}, + Stdin: io.MultiReader(strings.NewReader("partial"), iotest.ErrReader(broken)), + Stdout: &out, + }) + if !errors.Is(err, broken) || code != 0 || out.String() != "partial" { + t.Errorf("code=%d out=%q err=%v, want the exit code with the stdin failure", code, out.String(), err) + } +} + func TestRunReturnsAtItsContextDeadline(t *testing.T) { sb := testSandbox(t, newAgentServer(t, silkdtest.ServeConn)) From c78696f623f2312451d9db1c479787b2e6ed2bba Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:29:48 +0800 Subject: [PATCH 013/127] sdk/go: bound each Lookup owner probe like the Python SDK A tagged 404 runs Lookup, and a released sandbox answers 404 everywhere, so a blackholed peer held the caller for the transport dial timeout when its ctx had no deadline. --- sdk/go/client.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/sdk/go/client.go b/sdk/go/client.go index 4875cb04b..61a9b28a5 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -185,13 +185,16 @@ func (c *Client) Volumes(ctx context.Context) ([]VolumeInfo, error) { // Lookup finds the sandbox owner using its id and token. func (c *Client) Lookup(ctx context.Context, id, token string) (*Sandbox, error) { - if owner, err := c.ownerAt(ctx, c.addr, id, token); err == nil { + probe := func(ctx context.Context, addr string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, peersTimeout) + defer cancel() + return c.ownerAt(ctx, addr, id, token) + } + if owner, err := probe(ctx, c.addr); err == nil { return &Sandbox{ID: id, token: token, c: c, owner: owner}, nil } addrs, peersErr := c.peersOrErr(ctx) - owner, ok := scatter(ctx, addrs, func(ctx context.Context, addr string) (string, error) { - return c.ownerAt(ctx, addr, id, token) - }) + owner, ok := scatter(ctx, addrs, probe) if !ok { return nil, errors.Join(fmt.Errorf("lookup %s: no owner found", id), peersErr) } From 6b03c15d87c541af60e1d5106a3b1b72bdcf558e Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:31:40 +0800 Subject: [PATCH 014/127] silkd: kill a language server by its process group The server was spawned in silkd's group and killed by pid, so a reaped, stopped or TTL-expired server orphaned its own children (cargo check, proc-macro-srv, tsserver); exec and session already group-kill. --- silkd/src/lsp.rs | 4 ++++ silkd/tests/lsp_e2e.rs | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/silkd/src/lsp.rs b/silkd/src/lsp.rs index 73579d7d2..de90636b2 100644 --- a/silkd/src/lsp.rs +++ b/silkd/src/lsp.rs @@ -55,6 +55,7 @@ impl Broker { .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::null()) + .process_group(0) .kill_on_drop(true); if let Some(dir) = root { cmd.current_dir(dir); @@ -174,6 +175,9 @@ struct Server { } async fn kill(mut server: Server) { + if let Some(pid) = server.child.id() { + sysutil::signal_group(pid, libc::SIGKILL); + } let _ = server.child.start_kill(); let _ = server.child.wait().await; } diff --git a/silkd/tests/lsp_e2e.rs b/silkd/tests/lsp_e2e.rs index 6055cca09..4fa754105 100644 --- a/silkd/tests/lsp_e2e.rs +++ b/silkd/tests/lsp_e2e.rs @@ -129,6 +129,40 @@ async fn lsp_stop_kills_an_idle_server() { assert_stopped(&state, &server_id).await; } +#[tokio::test] +async fn lsp_stop_kills_the_servers_children_too() { + let _env_lock = ENV_LOCK.lock().await; + let (env, state, server_id) = started_server( + "grouptest", + "#!/bin/sh\nsleep 300 &\necho $! > \"$(dirname \"$0\")/child\"\nwait\n", + ) + .await; + let pid_file = env.path().join("child"); + let deadline = tokio::time::Instant::now() + DEADLINE; + let pid: libc::pid_t = loop { + if let Some(pid) = std::fs::read_to_string(&pid_file) + .ok() + .and_then(|s| s.trim().parse().ok()) + { + break pid; + } + assert!( + tokio::time::Instant::now() < deadline, + "server never forked" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + }; + lsp_stop(&state, &server_id).await; + // SAFETY: kill(2) with signal 0 only probes whether the pid exists; it takes no pointers. + while unsafe { libc::kill(pid, 0) } == 0 { + assert!( + tokio::time::Instant::now() < deadline, + "server child {pid} outlived lsp_stop" + ); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + } +} + #[tokio::test] async fn lsp_request_reaps_when_the_client_vanishes_before_ready() { let _env_lock = ENV_LOCK.lock().await; From 77d3510acc3874548170be878d34a2cf235c4582 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:32:16 +0800 Subject: [PATCH 015/127] e2e: release the claim on every driver error path lifecycle left an archived claim behind on any failure, which made the next run see Archived=1 at once; coldproof and demo leaked a claim to its TTL on an exec error. --- e2e/cmd/coldproof/main.go | 2 ++ e2e/cmd/demo/main.go | 9 ++++++--- e2e/cmd/lifecycle/main.go | 13 +++++++------ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/e2e/cmd/coldproof/main.go b/e2e/cmd/coldproof/main.go index 92f78b180..a1d4f26cd 100644 --- a/e2e/cmd/coldproof/main.go +++ b/e2e/cmd/coldproof/main.go @@ -44,11 +44,13 @@ func run(addr, token, template string, n int) error { claim := time.Since(start) uptime, err := sb.Exec(ctx, "cat", "/proc/uptime") if err != nil { + _ = sb.Close() return fmt.Errorf("exec uptime: %w", err) } exec1 := time.Since(start) kernel, err := sb.Exec(ctx, "uname", "-r") if err != nil { + _ = sb.Close() return fmt.Errorf("exec uname: %w", err) } fmt.Printf("iter=%d claim=%.1fms first_exec_done=%.1fms guest_uptime=%ss guest_kernel=%s\n", diff --git a/e2e/cmd/demo/main.go b/e2e/cmd/demo/main.go index c6321a940..b172d864e 100644 --- a/e2e/cmd/demo/main.go +++ b/e2e/cmd/demo/main.go @@ -53,12 +53,15 @@ func run(addr, token, template, netShape, size string, n, ttl int, leak bool) er } claimed := time.Now() out, err := sb.Exec(ctx, "echo", "42") + if err == nil && out != "42\n" { + err = fmt.Errorf("got %q, want 42", out) + } if err != nil { + if !leak { + _ = sb.Close() + } return fmt.Errorf("exec in %s: %w", sb.ID, err) } - if out != "42\n" { - return fmt.Errorf("exec in %s: got %q, want 42", sb.ID, out) - } execDone := time.Now() fmt.Printf("iter=%d id=%s claim=%.1fms exec=%.1fms\n", i, sb.ID, claimed.Sub(start).Seconds()*1000, execDone.Sub(claimed).Seconds()*1000) diff --git a/e2e/cmd/lifecycle/main.go b/e2e/cmd/lifecycle/main.go index a629da757..69758f817 100644 --- a/e2e/cmd/lifecycle/main.go +++ b/e2e/cmd/lifecycle/main.go @@ -40,6 +40,7 @@ func run(addr, token, template, netShape string, wait time.Duration) error { if err != nil { return err } + defer func() { _ = sb.Close() }() id := sb.ID marker := fmt.Appendf(nil, "archive-marker-%d", time.Now().UnixNano()) if err = sb.WriteFile(ctx, markerPath, marker, nil); err != nil { @@ -47,7 +48,7 @@ func run(addr, token, template, netShape string, wait time.Duration) error { } fmt.Printf("claimed id=%s, marker written\n", id) - elapsed, err := waitArchived(ctx, client, 1, wait) + elapsed, err := waitArchived(ctx, client, wait) if err != nil { return err } @@ -81,8 +82,8 @@ func run(addr, token, template, netShape string, wait time.Duration) error { return nil } -// waitArchived returns how long the reaper's idle→hibernate→archive ladder took to archive n claims. -func waitArchived(ctx context.Context, client *sandbox.Client, n int, timeout time.Duration) (time.Duration, error) { +// waitArchived returns how long the reaper's idle→hibernate→archive ladder took to archive the claim. +func waitArchived(ctx context.Context, client *sandbox.Client, timeout time.Duration) (time.Duration, error) { start := time.Now() deadline := start.Add(timeout) for { @@ -90,12 +91,12 @@ func waitArchived(ctx context.Context, client *sandbox.Client, n int, timeout ti if err != nil { return 0, err } - if info.Archived == n { + if info.Archived == 1 { return time.Since(start), nil } if time.Now().After(deadline) { - return 0, fmt.Errorf("archived never reached %d within %s (hibernated=%d archived=%d claimed=%d)", - n, timeout, info.Hibernated, info.Archived, info.Claimed) + return 0, fmt.Errorf("archived never reached 1 within %s (hibernated=%d archived=%d claimed=%d)", + timeout, info.Hibernated, info.Archived, info.Claimed) } time.Sleep(500 * time.Millisecond) } From 00d91eaf7e3da8599f72e584d265da1e785d46a1 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:35:28 +0800 Subject: [PATCH 016/127] scripts: make archive-e2e.sh executable like its siblings --- scripts/archive-e2e.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/archive-e2e.sh diff --git a/scripts/archive-e2e.sh b/scripts/archive-e2e.sh old mode 100644 new mode 100755 From 8f969574850d19392c36e88a5209643b33fa3efe Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:34:16 +0800 Subject: [PATCH 017/127] review: mcp tool arg types above the handlers, parseCommand beside parse --- mcp/tools.go | 102 +++++++++++++++++++++++++-------------------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/mcp/tools.go b/mcp/tools.go index 1b7a74cfa..c76e09cd1 100644 --- a/mcp/tools.go +++ b/mcp/tools.go @@ -96,6 +96,49 @@ func (a sandboxArg) id() string { return a.SandboxID } type sandboxIDer interface{ id() string } +type cmdArgs struct { + sandboxArg + Command string `json:"command"` + Cwd string `json:"cwd"` +} + +type killArgs struct { + sandboxArg + PID uint32 `json:"pid"` + Signal int32 `json:"signal"` +} + +type logsArgs struct { + sandboxArg + PID uint32 `json:"pid"` +} + +type writeFileArgs struct { + sandboxArg + Path string `json:"path"` + Content string `json:"content"` +} + +type pathArgs struct { + sandboxArg + Path string `json:"path"` +} + +type forkArgs struct { + sandboxArg + Count int `json:"count"` +} + +type checkpointArgs struct { + sandboxArg + Name string `json:"name"` +} + +type promoteArgs struct { + sandboxArg + TemplateName string `json:"template_name"` +} + // cappedOutput keeps the first execOutputCap bytes: an agent that tails a firehose must not grow this process. type cappedOutput struct { strings.Builder @@ -148,20 +191,6 @@ func toolCreateSandbox(ctx context.Context, s *server, raw jsontext.Value) (stri return jsonText(map[string]any{"sandbox_id": sb.ID, "deadline": sb.Deadline}), nil } -type cmdArgs struct { - sandboxArg - Command string `json:"command"` - Cwd string `json:"cwd"` -} - -func parseCommand(s *server, raw jsontext.Value) (cmdArgs, *sandbox.Sandbox, error) { - args, sb, err := s.parseAndBox[cmdArgs](raw) - if err == nil && args.Command == "" { - return args, nil, errors.New("command must not be empty") - } - return args, sb, err -} - func toolExec(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := parseCommand(s, raw) if err != nil { @@ -208,12 +237,6 @@ func toolPs(ctx context.Context, s *server, raw jsontext.Value) (string, error) return jsonText(procs), nil } -type killArgs struct { - sandboxArg - PID uint32 `json:"pid"` - Signal int32 `json:"signal"` -} - func toolKill(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := s.parseAndBox[killArgs](raw) if err != nil { @@ -225,11 +248,6 @@ func toolKill(ctx context.Context, s *server, raw jsontext.Value) (string, error return "killed", nil } -type logsArgs struct { - sandboxArg - PID uint32 `json:"pid"` -} - func toolLogs(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := s.parseAndBox[logsArgs](raw) if err != nil { @@ -247,12 +265,6 @@ func toolLogs(ctx context.Context, s *server, raw jsontext.Value) (string, error return jsonText(out), nil } -type writeFileArgs struct { - sandboxArg - Path string `json:"path"` - Content string `json:"content"` -} - func toolWriteFile(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := s.parseAndBox[writeFileArgs](raw) if err != nil { @@ -264,11 +276,6 @@ func toolWriteFile(ctx context.Context, s *server, raw jsontext.Value) (string, return fmt.Sprintf("wrote %d bytes to %s", len(args.Content), args.Path), nil } -type pathArgs struct { - sandboxArg - Path string `json:"path"` -} - func toolReadFile(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := s.parseAndBox[pathArgs](raw) if err != nil { @@ -295,11 +302,6 @@ func toolListDir(ctx context.Context, s *server, raw jsontext.Value) (string, er return jsonText(entries), nil } -type forkArgs struct { - sandboxArg - Count int `json:"count"` -} - func toolFork(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := s.parseAndBox[forkArgs](raw) if err != nil { @@ -317,11 +319,6 @@ func toolFork(ctx context.Context, s *server, raw jsontext.Value) (string, error return jsonText(map[string]any{"sandbox_ids": ids}), nil } -type checkpointArgs struct { - sandboxArg - Name string `json:"name"` -} - func toolCheckpoint(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := s.parseAndBox[checkpointArgs](raw) if err != nil { @@ -383,11 +380,6 @@ func toolHibernate(ctx context.Context, s *server, raw jsontext.Value) (string, return "hibernated (the next call that reaches the guest wakes it transparently)", nil } -type promoteArgs struct { - sandboxArg - TemplateName string `json:"template_name"` -} - func toolPromote(ctx context.Context, s *server, raw jsontext.Value) (string, error) { args, sb, err := s.parseAndBox[promoteArgs](raw) if err != nil { @@ -420,6 +412,14 @@ func toolNodeInfo(ctx context.Context, s *server, _ jsontext.Value) (string, err return jsonText(info), nil } +func parseCommand(s *server, raw jsontext.Value) (cmdArgs, *sandbox.Sandbox, error) { + args, sb, err := s.parseAndBox[cmdArgs](raw) + if err == nil && args.Command == "" { + return args, nil, errors.New("command must not be empty") + } + return args, sb, err +} + func parse(raw jsontext.Value, v any) error { if len(raw) == 0 { return nil From 154bde1c4d57dfb0fa65af9e3b50997d67ffc40a Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:34:29 +0800 Subject: [PATCH 018/127] review: sdk fork/promote/renew wire types beside Sandbox, APIError beside ClientOption --- sdk/go/client.go | 56 ++++++++++------------------------------------- sdk/go/sandbox.go | 32 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/sdk/go/client.go b/sdk/go/client.go index 61a9b28a5..05cda7ea6 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -33,50 +33,6 @@ var errRedirectedAgain = errors.New("redirected again despite no_redirect") // ClientOption configures Connect. type ClientOption func(*Client) -type claimEncoder func(noRedirect, requirePromoted bool) ([]byte, error) - -type claimPoster func(addr string, body []byte) (claimResponse, error) - -type volumeListResponse struct { - Volumes []VolumeInfo `json:"volumes"` -} - -type forkRequest struct { - Token string `json:"token"` - Count int `json:"count"` - TTLSeconds int `json:"ttl_seconds,omitzero"` -} - -type forkResponse struct { - Children []claimResponse `json:"children"` -} - -type promoteRequest struct { - Token string `json:"token"` - Template string `json:"template"` -} - -type promoteResponse struct { - Key struct { - Template string `json:"template"` - Net string `json:"net"` - Size string `json:"size"` - } `json:"key"` - ContentDigest string `json:"content_digest"` -} - -type renewRequest struct { - TTLSeconds int `json:"ttl_seconds,omitzero"` -} - -type renewResponse struct { - Deadline time.Time `json:"deadline"` -} - -type errorResponse struct { - Error string `json:"error"` -} - // APIError is a non-2xx control-plane reply. type APIError struct { Verb string @@ -91,6 +47,18 @@ func (e *APIError) Error() string { return fmt.Sprintf("%s: http %d", e.Verb, e.Status) } +type claimEncoder func(noRedirect, requirePromoted bool) ([]byte, error) + +type claimPoster func(addr string, body []byte) (claimResponse, error) + +type volumeListResponse struct { + Volumes []VolumeInfo `json:"volumes"` +} + +type errorResponse struct { + Error string `json:"error"` +} + type claimRequest struct { Template string `json:"template"` Net string `json:"net,omitempty"` diff --git a/sdk/go/sandbox.go b/sdk/go/sandbox.go index 945301d32..98b21466f 100644 --- a/sdk/go/sandbox.go +++ b/sdk/go/sandbox.go @@ -65,6 +65,38 @@ func (e *ownerGoneError) Error() string { return e.err.Error() } func (e *ownerGoneError) Unwrap() error { return e.err } +type forkRequest struct { + Token string `json:"token"` + Count int `json:"count"` + TTLSeconds int `json:"ttl_seconds,omitzero"` +} + +type forkResponse struct { + Children []claimResponse `json:"children"` +} + +type promoteRequest struct { + Token string `json:"token"` + Template string `json:"template"` +} + +type promoteResponse struct { + Key struct { + Template string `json:"template"` + Net string `json:"net"` + Size string `json:"size"` + } `json:"key"` + ContentDigest string `json:"content_digest"` +} + +type renewRequest struct { + TTLSeconds int `json:"ttl_seconds,omitzero"` +} + +type renewResponse struct { + Deadline time.Time `json:"deadline"` +} + // Sandbox is a claimed sandbox handle; all data-plane calls dial its owning // node directly. type Sandbox struct { From b6f3fac83d872a9e50c92b0cd3f07c12df98fe99 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:34:43 +0800 Subject: [PATCH 019/127] wire: inline the response decoder func type, trim the encodeTagged clause conn.go already states --- protocol/wire/frame.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/protocol/wire/frame.go b/protocol/wire/frame.go index 53fb3e561..d9c5defed 100644 --- a/protocol/wire/frame.go +++ b/protocol/wire/frame.go @@ -97,7 +97,7 @@ var ( } // responseDecoders dispatches on the type tag first: one key differs in shape across variants. - responseDecoders = map[string]respDecoder{ + responseDecoders = map[string]func([]byte) (Response, error){ "started": decodeResp[Started], "stdout": decodeResp[Stdout], "stderr": decodeResp[Stderr], @@ -628,8 +628,6 @@ type GitBranches struct { func (GitBranches) RespType() string { return "git_branches" } -type respDecoder func([]byte) (Response, error) - type bulkResponse struct { head []byte mk func([]byte) Response @@ -759,7 +757,7 @@ func decodeBulk(line, head []byte, scratch *[]byte) ([]byte, bool) { return out[:n], true } -// encodeTagged splices the tag head in front of v's fields, so wire tags never live on the structs; the spare capacity byte lets Conn.Send append the newline without a copy. +// encodeTagged splices the tag head in front of v's fields, so wire tags never live on the structs. func encodeTagged(head string, v any) ([]byte, error) { body, err := json.Marshal(v) if err != nil { From f9be6e6462daf068d0919ef008f50aa5e51d66ac Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:05 +0800 Subject: [PATCH 020/127] sdk: one newClaim for the option loop and one templateQuery for the template delete query --- sdk/go/checkpoint.go | 5 +---- sdk/go/client.go | 30 ++++++++++++++++-------------- sdk/go/template.go | 18 ++++-------------- 3 files changed, 21 insertions(+), 32 deletions(-) diff --git a/sdk/go/checkpoint.go b/sdk/go/checkpoint.go index 9c09ba01a..9c243b896 100644 --- a/sdk/go/checkpoint.go +++ b/sdk/go/checkpoint.go @@ -49,10 +49,7 @@ type Checkpoint struct { // New claims a sandbox branched from the checkpoint, following an owner redirect. func (ck *Checkpoint) New(ctx context.Context, opts ...Option) (*Sandbox, error) { - var claim claimRequest - for _, opt := range opts { - opt(&claim) - } + claim := newClaim("", opts) if len(claim.Volumes) > 0 || claim.ClaimRef != "" { return nil, errors.New("checkpoint claims do not accept WithVolumes or WithClaimRef") } diff --git a/sdk/go/client.go b/sdk/go/client.go index 05cda7ea6..22c18be73 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -20,11 +20,7 @@ import ( ) const ( - idleConnsPerHost = 100 - - templateQueryParam = "template" - netQueryParam = "net" - sizeQueryParam = "size" + idleConnsPerHost = 100 noRedirectQueryParam = "no_redirect" ) @@ -126,10 +122,7 @@ func Connect(addr string, opts ...ClientOption) (*Client, error) { // New claims a sandbox for template. func (c *Client) New(ctx context.Context, template string, opts ...Option) (*Sandbox, error) { - claim := claimRequest{Template: template} - for _, opt := range opts { - opt(&claim) - } + claim := newClaim(template, opts) addr, cr, err := claimFollow(c.addr, "claim", func(noRedirect, requirePromoted bool) ([]byte, error) { claim.NoRedirect, claim.RequirePromoted = noRedirect, requirePromoted return encodeBody("claim", claim) @@ -176,11 +169,8 @@ func (c *Client) Attach(ownerAddr, id, token string) *Sandbox { // DeleteTemplate removes a promoted template by name. func (c *Client) DeleteTemplate(ctx context.Context, template string, opts ...Option) error { - claim := claimRequest{Template: template} - for _, opt := range opts { - opt(&claim) - } - u := url.Values{templateQueryParam: {claim.Template}, netQueryParam: {claim.Net}, sizeQueryParam: {claim.Size}} + claim := newClaim(template, opts) + u := templateQuery(claim.Template, claim.Net, claim.Size) redirect, err := c.deleteTemplates(ctx, c.addr, u) if err != nil || len(redirect) == 0 { return err @@ -414,3 +404,15 @@ func apiError(verb string, resp *http.Response) error { _, _ = io.Copy(io.Discard, resp.Body) return &APIError{Verb: verb, Status: resp.StatusCode, Message: er.Error} } + +func newClaim(template string, opts []Option) claimRequest { + claim := claimRequest{Template: template} + for _, opt := range opts { + opt(&claim) + } + return claim +} + +func templateQuery(name, net, size string) url.Values { + return url.Values{"template": {name}, "net": {net}, "size": {size}} +} diff --git a/sdk/go/template.go b/sdk/go/template.go index 30fb7aafb..12d855bce 100644 --- a/sdk/go/template.go +++ b/sdk/go/template.go @@ -1,9 +1,6 @@ package sandbox -import ( - "context" - "net/url" -) +import "context" // Template is a promoted template bound to the node that holds it. type Template struct { @@ -18,10 +15,7 @@ type Template struct { // New claims a sandbox cloned from the template; its key axes cannot be overridden. func (t *Template) New(ctx context.Context, opts ...Option) (*Sandbox, error) { - claim := claimRequest{Template: t.Name} - for _, opt := range opts { - opt(&claim) - } + claim := newClaim(t.Name, opts) if err := claim.rejectPinnedAxes(); err != nil { return nil, err } @@ -41,12 +35,8 @@ func (t *Template) New(ctx context.Context, opts ...Option) (*Sandbox, error) { // Delete removes the template from its owning node, never a same-name template elsewhere. func (t *Template) Delete(ctx context.Context) error { - u := url.Values{ - templateQueryParam: {t.Name}, - netQueryParam: {t.net}, - sizeQueryParam: {t.size}, - noRedirectQueryParam: {"1"}, - } + u := templateQuery(t.Name, t.net, t.size) + u.Set(noRedirectQueryParam, "1") _, err := t.c.deleteTemplates(ctx, t.addr, u) return err } From 7bff188d9b4419610a263c539fd76b51a38fec04 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:05 +0800 Subject: [PATCH 021/127] sdk: probe peek compares with errors.Is --- sdk/go/probe_unix.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sdk/go/probe_unix.go b/sdk/go/probe_unix.go index f38925190..cabde0f7b 100644 --- a/sdk/go/probe_unix.go +++ b/sdk/go/probe_unix.go @@ -2,7 +2,10 @@ package sandbox -import "syscall" +import ( + "errors" + "syscall" +) const canProbe = true @@ -10,6 +13,6 @@ const canProbe = true func (c *agentConn) peek(fd uintptr) bool { var b [1]byte _, _, err := syscall.Recvfrom(int(fd), b[:], syscall.MSG_PEEK) - c.peeked = err == syscall.EAGAIN + c.peeked = errors.Is(err, syscall.EAGAIN) return true } From f97186ef6472c8fa52507a7d5f9cd37d6eb1d114 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:05 +0800 Subject: [PATCH 022/127] silkdtest: use the wire kind constants, inline the single-use buffer view --- sdk/go/silkd/silkdtest/fake.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/sdk/go/silkd/silkdtest/fake.go b/sdk/go/silkd/silkdtest/fake.go index 4c699a604..2d0fff1d6 100644 --- a/sdk/go/silkd/silkdtest/fake.go +++ b/sdk/go/silkd/silkdtest/fake.go @@ -36,8 +36,7 @@ func NewFake(root string) *Fake { return &Fake{Root: root, sessions: map[string]bool{}, branches: []string{"main"}, current: "main"} } -// ServeConn serves RPCs on an already-open connection (e.g. after an HTTP -// hijack) until the peer closes it. +// ServeConn serves RPCs on an already-open connection (e.g. after an HTTP hijack) until the peer closes it. func (f *Fake) ServeConn(conn net.Conn) { serveConn(conn, bufio.NewReader(conn), f.serve) } @@ -118,9 +117,9 @@ func (f *Fake) fsList(conn net.Conn, path string) { var entries []wire.DirEntry for _, e := range ents { info, _ := e.Info() - kind := "file" + kind := wire.FileKindFile if e.IsDir() { - kind = "dir" + kind = wire.FileKindDir } var size uint64 if info != nil { @@ -138,9 +137,9 @@ func (f *Fake) fsStat(conn net.Conn, path string) { errFrame(conn, wire.KindNotFound, err.Error()) return } - kind := "file" + kind := wire.FileKindFile if info.IsDir() { - kind = "dir" + kind = wire.FileKindDir } send(conn, &wire.Stat{Info: wire.FileInfo{ Kind: kind, @@ -225,8 +224,7 @@ func (f *Fake) fsPull(conn net.Conn, path string) { return } _ = tw.Close() - data := buf.Bytes() - sendChunked(conn, data) + sendChunked(conn, buf.Bytes()) } func (f *Fake) sessionCreate(conn net.Conn, req *wire.SessionCreate) { @@ -337,7 +335,7 @@ func (f *Fake) fsWatch(conn net.Conn, r *bufio.Reader, req *wire.FsWatch) { return } send(conn, wire.Ready{}) - send(conn, &wire.Event{Kind: "created", Path: req.Path + "/silkdtest-event"}) + send(conn, &wire.Event{Kind: wire.EventCreated, Path: req.Path + "/silkdtest-event"}) _, _ = io.Copy(io.Discard, r) } From ff6f73e6c10fc0e2a1633514f3d43113fd7a5e60 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:05 +0800 Subject: [PATCH 023/127] sdk: one-line godoc for one-sentence contracts, drop docs that repeat a neighbor --- sdk/go/drain.go | 3 +-- sdk/go/options.go | 1 - sdk/go/pools.go | 2 +- sdk/go/port.go | 2 +- sdk/go/sandbox.go | 20 ++++++-------------- sdk/go/session.go | 6 ++---- sdk/go/silkd/conn.go | 4 +--- sdk/go/tree.go | 4 +--- 8 files changed, 13 insertions(+), 29 deletions(-) diff --git a/sdk/go/drain.go b/sdk/go/drain.go index 20d382696..fa4fa82e5 100644 --- a/sdk/go/drain.go +++ b/sdk/go/drain.go @@ -5,8 +5,7 @@ import ( "net/http" ) -// Drain closes the entry node (root token): new claims/forks/branches are -// refused, live claims run to their leases; poll Info until Claimed is zero. +// Drain closes the entry node (root token): new claims/forks/branches are refused, live claims run to their leases; poll Info until Claimed is zero. func (c *Client) Drain(ctx context.Context) (*NodeInfo, error) { return c.doJSONPtr[NodeInfo](ctx, http.MethodPost, c.addr, "/v1/drain", nil, c.apiToken, "drain") } diff --git a/sdk/go/options.go b/sdk/go/options.go index e30475499..46c4dce18 100644 --- a/sdk/go/options.go +++ b/sdk/go/options.go @@ -109,7 +109,6 @@ func WithArchiveOnExpire() Option { return func(r *claimRequest) { r.OnExpire = "archive" } } -// ttlSeconds rounds a lease up to whole wire seconds. func ttlSeconds(d time.Duration) int { return int((d + time.Second - 1) / time.Second) } diff --git a/sdk/go/pools.go b/sdk/go/pools.go index 396a1e9c9..f0c75cbc2 100644 --- a/sdk/go/pools.go +++ b/sdk/go/pools.go @@ -6,7 +6,7 @@ import ( "net/http" ) -// PoolSpec is a desired warm pool for SetPools; egress, warmup, capture_trim, storage, reader, replicate_seconds, egress_upstream_env and tun are config-owned, so it has no field for them. +// PoolSpec is a desired warm pool for SetPools; config-owned pool fields are not settable here. type PoolSpec struct { Template string `json:"template"` Net NetShape `json:"net,omitempty"` diff --git a/sdk/go/port.go b/sdk/go/port.go index 35f3cb0e1..3dd33d7b3 100644 --- a/sdk/go/port.go +++ b/sdk/go/port.go @@ -110,7 +110,7 @@ func (s *Sandbox) ProxyPort(ctx context.Context, localAddr string, port uint16) return l, nil } -// openStream turns a call into a raw byte stream, shared by DialPort and Lsp.Request. +// openStream turns a call into a raw byte stream. func (s *Sandbox) openStream(ctx context.Context, req wire.Request) (_ *PortConn, err error) { defer func() { s.rebind(ctx, err) }() conn, l, err := s.call(ctx, req) diff --git a/sdk/go/sandbox.go b/sdk/go/sandbox.go index 98b21466f..7b85e326c 100644 --- a/sdk/go/sandbox.go +++ b/sdk/go/sandbox.go @@ -97,8 +97,7 @@ type renewResponse struct { Deadline time.Time `json:"deadline"` } -// Sandbox is a claimed sandbox handle; all data-plane calls dial its owning -// node directly. +// Sandbox is a claimed sandbox handle; all data-plane calls dial its owning node directly. type Sandbox struct { ID string Deadline time.Time @@ -108,10 +107,8 @@ type Sandbox struct { VolumesWant []Volume // TemplateDigest is the content identity of the promoted-template export this sandbox was cloned from; empty otherwise. TemplateDigest string - // FromCheckpoint names the checkpoint this sandbox branched from; empty otherwise. FromCheckpoint string - // NetRoute is the guest's network path at claim, as the node reports it. NetRoute string @@ -123,22 +120,19 @@ type Sandbox struct { owner string } -// Owner returns the data-plane address of the node that owns the sandbox — -// on a cluster the node a redirected claim landed on. +// Owner returns the data-plane address of the node that owns the sandbox, on a cluster the node a redirected claim landed on. func (s *Sandbox) Owner() string { s.mu.Lock() defer s.mu.Unlock() return s.owner } -// Token is the per-sandbox bearer secret — persist it with ID to reattach -// later via Lookup (treat it like a credential). +// Token is the per-sandbox bearer secret; persist it with ID to reattach later via Lookup, and treat it like a credential. func (s *Sandbox) Token() string { return s.token } -// Exec runs argv to completion and returns its stdout; a non-zero exit -// surfaces as *ExitError carrying stderr, alongside the partial stdout. +// Exec runs argv to completion and returns its stdout; a non-zero exit surfaces as *ExitError carrying stderr, alongside the partial stdout. func (s *Sandbox) Exec(ctx context.Context, argv ...string) (string, error) { var stdout, stderr strings.Builder code, err := s.Run(ctx, Cmd{Argv: argv, Stdout: &stdout, Stderr: &stderr}) @@ -151,8 +145,7 @@ func (s *Sandbox) Exec(ctx context.Context, argv ...string) (string, error) { return stdout.String(), nil } -// Run executes cmd in the sandbox, streaming stdio over one relayed silkd -// connection, and returns the exit code; a ctx that ends first kills the command. +// Run executes cmd in the sandbox, streaming stdio over one relayed silkd connection, and returns the exit code; a ctx that ends first kills the command. func (s *Sandbox) Run(ctx context.Context, cmd Cmd) (int, error) { if len(cmd.Argv) == 0 { return 0, fmt.Errorf("empty argv") @@ -217,8 +210,7 @@ func (s *Sandbox) Renew(ctx context.Context, ttl time.Duration) (time.Time, erro return rr.Deadline, nil } -// Close releases the sandbox on its node; releasing one already gone is not -// an error. It takes no ctx so it stays defer-friendly — bounded internally. +// Close releases the sandbox on its node, bounded internally and without a ctx to stay defer-friendly; releasing one already gone is not an error. func (s *Sandbox) Close() error { s.pool.drain() ctx, cancel := context.WithTimeout(context.Background(), releaseTimeout) diff --git a/sdk/go/session.go b/sdk/go/session.go index 71a29e452..332c89cd7 100644 --- a/sdk/go/session.go +++ b/sdk/go/session.go @@ -7,16 +7,14 @@ import ( "github.com/cocoonstack/sandbox/protocol/wire" ) -// Session is a persistent shell in the sandbox: cwd, env, and shell state -// survive across Exec calls until Close. +// Session is a persistent shell in the sandbox: cwd, env, and shell state survive across Exec calls until Close. type Session struct { ID string sb *Sandbox } -// Exec runs argv in the session and returns its combined output; state (cwd, -// env, shell variables) persists to the next call. +// Exec runs argv in the session and returns its combined output; state (cwd, env, shell variables) persists to the next call. func (sess *Session) Exec(ctx context.Context, argv ...string) (string, error) { var out strings.Builder code, err := sess.sb.Run(ctx, Cmd{Argv: argv, Session: sess.ID, Stdout: &out, Stderr: &out}) diff --git a/sdk/go/silkd/conn.go b/sdk/go/silkd/conn.go index 6d6931683..18d74c107 100644 --- a/sdk/go/silkd/conn.go +++ b/sdk/go/silkd/conn.go @@ -11,9 +11,7 @@ import ( "github.com/cocoonstack/sandbox/protocol/wire" ) -// Conn frames a byte stream (typically the upgraded relay connection) into -// silkd RPCs. Reads are single-consumer; Send is safe for concurrent use so -// a stdin pump can interleave with the caller. +// Conn frames a byte stream (typically the upgraded relay connection) into silkd RPCs; reads are single-consumer, and Send is safe for concurrent use so a stdin pump can interleave. type Conn struct { wmu sync.Mutex wbuf []byte diff --git a/sdk/go/tree.go b/sdk/go/tree.go index a0efd448f..1f0376a43 100644 --- a/sdk/go/tree.go +++ b/sdk/go/tree.go @@ -7,9 +7,7 @@ import ( "github.com/cocoonstack/sandbox/protocol/wire" ) -// Push extracts a tar stream under dest in the sandbox — the no-network lane's -// way to move a project in. tarStream is a reader of tar bytes (e.g. from -// archive/tar); silkd runs `tar -x` under dest. +// Push extracts a tar stream under dest in the sandbox, the no-network lane's way to move a project in. func (s *Sandbox) Push(ctx context.Context, dest string, tarStream io.Reader) error { return s.uploadRPC(ctx, &wire.FsPush{Dest: dest}, tarStream) } From 4d79374a9c300dce3a8209e65b4fc9c7e4252796 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:38:10 +0800 Subject: [PATCH 024/127] e2e: drop the guestserver sink done flag --- e2e/cmd/guestserver/main.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/e2e/cmd/guestserver/main.go b/e2e/cmd/guestserver/main.go index fa4958eef..66bb4f6f4 100644 --- a/e2e/cmd/guestserver/main.go +++ b/e2e/cmd/guestserver/main.go @@ -20,9 +20,8 @@ import ( // halfCloseSink greets, shuts its write side, and keeps reading what the client sends after. type halfCloseSink struct { - mu sync.Mutex - got string - done bool + mu sync.Mutex + got string } func (h *halfCloseSink) serve(addr string) { @@ -54,16 +53,13 @@ func (h *halfCloseSink) greetThenRead(conn net.Conn) { } rest, _ := io.ReadAll(tcp) h.mu.Lock() - h.got, h.done = string(rest), true + h.got = string(rest) h.mu.Unlock() } func (h *halfCloseSink) read() string { h.mu.Lock() defer h.mu.Unlock() - if !h.done { - return "" - } return h.got } From aec08c90640622f2e1c1aad144fade9214bce01f Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:38:10 +0800 Subject: [PATCH 025/127] e2e: volumesmoke closes unconditionally, Close is idempotent --- e2e/cmd/volumesmoke/main.go | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/e2e/cmd/volumesmoke/main.go b/e2e/cmd/volumesmoke/main.go index 0a80d5bcb..f667a1fe1 100644 --- a/e2e/cmd/volumesmoke/main.go +++ b/e2e/cmd/volumesmoke/main.go @@ -69,12 +69,9 @@ func run(addr, token, template, volume, rwVolume, probe string, live bool) error _ = sbA.Close() return fmt.Errorf("claim at %s: %w", mountB, err) } - released := false defer func() { - if !released { - _ = sbA.Close() - _ = sbB.Close() - } + _ = sbA.Close() + _ = sbB.Close() }() if want := []sandbox.Volume{{Name: volume, Mount: mountA}}; !slices.Equal(sbA.Volumes, want) { return fmt.Errorf("first claim volumes %+v, want %+v", sbA.Volumes, want) @@ -114,7 +111,6 @@ func run(addr, token, template, volume, rwVolume, probe string, live bool) error if err := errors.Join(sbA.Close(), sbB.Close()); err != nil { return fmt.Errorf("release volume claims: %w", err) } - released = true fmt.Printf("VOLUME PASS concurrent_read_bytes=%d read_only=true released=true\n", len(outputs[0])) if live { if err := runLive(ctx, client, template, volume, probe, outputs[0]); err != nil { @@ -206,12 +202,7 @@ func runWritable(ctx context.Context, client *sandbox.Client, template, volume s return fmt.Errorf("writable claim: %w", err) } claimRW := time.Since(start) - live := writer - defer func() { - if live != nil { - _ = live.Close() - } - }() + defer func() { _ = writer.Close() }() if want := []sandbox.Volume{{Name: volume, Mount: mount, Mode: "rw"}}; !slices.Equal(writer.Volumes, want) { return fmt.Errorf("writable claim volumes %+v, want %+v", writer.Volumes, want) } @@ -231,14 +222,13 @@ func runWritable(ctx context.Context, client *sandbox.Client, template, volume s if err = writer.Close(); err != nil { return fmt.Errorf("release writable claim: %w", err) } - live = nil reader, err := client.New(ctx, template, sandbox.WithNetwork(sandbox.NetNone), sandbox.WithVolumes(sandbox.Volume{Name: volume, Mount: mount})) if err != nil { return fmt.Errorf("read-only claim after the writer released: %w", err) } - live = reader + defer func() { _ = reader.Close() }() out, err := reader.Exec(ctx, "cat", path.Join(mount, file)) if err != nil { return fmt.Errorf("read back the written file: %w", err) @@ -249,7 +239,6 @@ func runWritable(ctx context.Context, client *sandbox.Client, template, volume s if err = reader.Close(); err != nil { return fmt.Errorf("release read-only claim: %w", err) } - live = nil fmt.Printf("VOLUME RW PASS volume=%s claim=%.1fms durable=true busy_refused=true\n", volume, float64(claimRW.Microseconds())/1000) return nil From 2cf2917873aefe1d7a3df7e2b4b889dd5f9c42f1 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:38:10 +0800 Subject: [PATCH 026/127] e2e: harness.Poll replaces six hand-rolled retry loops --- e2e/cmd/androidsmoke/main.go | 20 +++++++++----------- e2e/cmd/browsersmoke/main.go | 23 ++++++++++++----------- e2e/cmd/meshsmoke/main.go | 14 +++++--------- e2e/cmd/portsmoke/main.go | 16 +++++++++------- e2e/cmd/smoke/main.go | 25 +++++++++++-------------- e2e/internal/harness/harness.go | 19 +++++++++++++++++++ e2e/internal/harness/harness_test.go | 27 +++++++++++++++++++++++++++ 7 files changed, 92 insertions(+), 52 deletions(-) diff --git a/e2e/cmd/androidsmoke/main.go b/e2e/cmd/androidsmoke/main.go index aef2bcc4a..e5d9aaeb4 100644 --- a/e2e/cmd/androidsmoke/main.go +++ b/e2e/cmd/androidsmoke/main.go @@ -102,18 +102,16 @@ func initTree(ctx context.Context, sb *sandbox.Sandbox) error { } func adbHandshake(ctx context.Context, sb *sandbox.Sandbox) error { - deadline := time.Now().Add(adbWait) - for { - reply, err := dialAdb(ctx, sb) - if err == nil { - fmt.Printf(" adb: port %d answered %s to CNXN\n", adbPort, reply) - return nil - } - if time.Now().After(deadline) { - return fmt.Errorf("adb port %d: %w", adbPort, err) - } - time.Sleep(2 * time.Second) + var reply string + err := harness.Poll(ctx, adbWait, 2*time.Second, func() (err error) { + reply, err = dialAdb(ctx, sb) + return err + }) + if err != nil { + return fmt.Errorf("adb port %d: %w", adbPort, err) } + fmt.Printf(" adb: port %d answered %s to CNXN\n", adbPort, reply) + return nil } // dialAdb sends one ADB CNXN; a CNXN or AUTH reply proves a live adbd. diff --git a/e2e/cmd/browsersmoke/main.go b/e2e/cmd/browsersmoke/main.go index fd8405da5..fe9afe4b3 100644 --- a/e2e/cmd/browsersmoke/main.go +++ b/e2e/cmd/browsersmoke/main.go @@ -73,21 +73,22 @@ func run(addr, token, template string) error { } func waitCDP(ctx context.Context, sb *sandbox.Sandbox) error { - deadline := time.Now().Add(cdpWait) - for { + err := harness.Poll(ctx, cdpWait, 2*time.Second, func() error { ver, err := cdpJSON(ctx, sb, "GET", "/json/version") - if err == nil { - if b, _ := ver["Browser"].(string); strings.Contains(b, "Chrome") { - fmt.Printf(" cdp: /json/version → %s (protocol %v)\n", b, ver["Protocol-Version"]) - return nil - } - err = fmt.Errorf("unexpected browser %v", ver["Browser"]) + if err != nil { + return err } - if time.Now().After(deadline) { - return fmt.Errorf("cdp never answered: %w", err) + b, _ := ver["Browser"].(string) + if !strings.Contains(b, "Chrome") { + return fmt.Errorf("unexpected browser %v", ver["Browser"]) } - time.Sleep(2 * time.Second) + fmt.Printf(" cdp: /json/version → %s (protocol %v)\n", b, ver["Protocol-Version"]) + return nil + }) + if err != nil { + return fmt.Errorf("cdp never answered: %w", err) } + return nil } // openTarget proves CDP accepts writes, not just liveness. diff --git a/e2e/cmd/meshsmoke/main.go b/e2e/cmd/meshsmoke/main.go index 0c8fb2ff3..ead451382 100644 --- a/e2e/cmd/meshsmoke/main.go +++ b/e2e/cmd/meshsmoke/main.go @@ -60,16 +60,12 @@ func run(addr, peer, token, template string) error { // Poll for gossip to reach the entry node; a premature claim cold-fails at A. var clone *sandbox.Sandbox - deadline := time.Now().Add(15 * time.Second) - for { + err = harness.Poll(ctx, 15*time.Second, 500*time.Millisecond, func() (err error) { clone, err = client.New(ctx, "mesh-tpl", sandbox.WithNetwork(sandbox.NetNone)) - if err == nil { - break - } - if time.Now().After(deadline) { - return fmt.Errorf("name-based claim of mesh-tpl via %s: %w", addr, err) - } - time.Sleep(500 * time.Millisecond) + return err + }) + if err != nil { + return fmt.Errorf("name-based claim of mesh-tpl via %s: %w", addr, err) } defer func() { _ = clone.Close() }() out, err := clone.Exec(ctx, "cat", "/root/marker") diff --git a/e2e/cmd/portsmoke/main.go b/e2e/cmd/portsmoke/main.go index b655f85ac..67f0b3ebd 100644 --- a/e2e/cmd/portsmoke/main.go +++ b/e2e/cmd/portsmoke/main.go @@ -120,15 +120,17 @@ func startGuestListener(ctx context.Context, sb *sandbox.Sandbox, listener strin if _, err := sb.Spawn(ctx, sandbox.Cmd{Argv: argv}); err != nil { return fmt.Errorf("spawn listener: %w", err) } - deadline := time.Now().Add(listenerReadyWait) - for time.Now().Before(deadline) { - out, execErr := sb.Exec(ctx, "sh", "-c", fmt.Sprintf("ss -ltn 'sport = :%d' | grep -c LISTEN", guestPort)) - if execErr == nil && strings.TrimSpace(out) != "0" { - return nil + bound := harness.Poll(ctx, listenerReadyWait, 500*time.Millisecond, func() error { + out, err := sb.Exec(ctx, "sh", "-c", fmt.Sprintf("ss -ltn 'sport = :%d' | grep -c LISTEN", guestPort)) + if err == nil && strings.TrimSpace(out) == "0" { + return errors.New("not listening") } - time.Sleep(500 * time.Millisecond) + return err + }) + if bound != nil { + return fmt.Errorf("listener never bound 127.0.0.1:%d", guestPort) } - return fmt.Errorf("listener never bound 127.0.0.1:%d", guestPort) + return nil } func stepHTTP(ctx context.Context, rt *harness.PortRelay, _ *sandbox.Sandbox) error { diff --git a/e2e/cmd/smoke/main.go b/e2e/cmd/smoke/main.go index 2bdf30025..4d8bbe3cd 100644 --- a/e2e/cmd/smoke/main.go +++ b/e2e/cmd/smoke/main.go @@ -510,16 +510,14 @@ func smokeTree(ctx context.Context, sb *sandbox.Sandbox) error { } // The staging unlink is asynchronous, so settle with a budget. var staging string - var stagingErr error - for range 20 { - staging, stagingErr = sb.Exec(ctx, "sh", "-c", "ls -a /root/tree") - if stagingErr == nil && !strings.Contains(staging, ".silkd-push-") { - break + if err := harness.Poll(ctx, 2*time.Second, 100*time.Millisecond, func() (err error) { + staging, err = sb.Exec(ctx, "sh", "-c", "ls -a /root/tree") + if err == nil && strings.Contains(staging, ".silkd-push-") { + err = errors.New("staging present") } - time.Sleep(100 * time.Millisecond) - } - if stagingErr != nil || strings.Contains(staging, ".silkd-push-") { - return fmt.Errorf("staging left behind: %q, %v", staging, stagingErr) + return err + }); err != nil { + return fmt.Errorf("staging left behind: %q, %v", staging, err) } var pulled bytes.Buffer @@ -535,12 +533,11 @@ func smokeTree(ctx context.Context, sb *sandbox.Sandbox) error { // smokePortForward proves sshd's banner arrives through DialPort and a dead port fails with not_found. func smokePortForward(ctx context.Context, sb *sandbox.Sandbox) error { // The step proves the relay, not sshd readiness, so the positive probe retries. - pc, err := sb.DialPort(ctx, 22) - for attempt := 0; err != nil && attempt < 100; attempt++ { - time.Sleep(100 * time.Millisecond) + var pc *sandbox.PortConn + if err := harness.Poll(ctx, 10*time.Second, 100*time.Millisecond, func() (err error) { pc, err = sb.DialPort(ctx, 22) - } - if err != nil { + return err + }); err != nil { return err } defer func() { _ = pc.Close() }() diff --git a/e2e/internal/harness/harness.go b/e2e/internal/harness/harness.go index ebf9e6ec8..e6743012e 100644 --- a/e2e/internal/harness/harness.go +++ b/e2e/internal/harness/harness.go @@ -10,6 +10,7 @@ import ( "net/http" "strconv" "strings" + "time" sandbox "github.com/cocoonstack/sandbox/sdk/go" ) @@ -93,6 +94,24 @@ func Claim(ctx context.Context, addr, token, template string, opts ...sandbox.Op return client, sb, nil } +func Poll(ctx context.Context, budget, every time.Duration, try func() error) error { + deadline := time.Now().Add(budget) + for { + err := try() + if err == nil { + return nil + } + if time.Now().After(deadline) { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(every): + } + } +} + // HTTPOverPort sends one HTTP/1.1 request over the port relay and returns a 200 body; the Host stays "localhost" because Chrome's DevTools allowlist accepts it but not proxied vhosts. func HTTPOverPort(ctx context.Context, sb *sandbox.Sandbox, port uint16, method, path string, body []byte) ([]byte, error) { pc, err := sb.DialPort(ctx, port) diff --git a/e2e/internal/harness/harness_test.go b/e2e/internal/harness/harness_test.go index 755334d1d..29d3292d0 100644 --- a/e2e/internal/harness/harness_test.go +++ b/e2e/internal/harness/harness_test.go @@ -2,6 +2,8 @@ package harness import ( "bufio" + "context" + "errors" "fmt" "io" "net" @@ -51,6 +53,31 @@ func TestPortRelayPreservesGreetingAndHalfClose(t *testing.T) { } } +func TestPoll(t *testing.T) { + errTry := errors.New("not yet") + + calls := 0 + if err := Poll(t.Context(), time.Second, time.Millisecond, func() error { + calls++ + if calls < 3 { + return errTry + } + return nil + }); err != nil || calls != 3 { + t.Errorf("success: err %v calls %d, want nil and 3", err, calls) + } + + if err := Poll(t.Context(), 5*time.Millisecond, time.Millisecond, func() error { return errTry }); !errors.Is(err, errTry) { + t.Errorf("exhausted: err %v, want the last try error", err) + } + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if err := Poll(ctx, time.Minute, time.Minute, func() error { return errTry }); !errors.Is(err, context.Canceled) { + t.Errorf("canceled: err %v, want context.Canceled", err) + } +} + func serveGuestFirstHalfClose(listener net.Listener) error { conn, err := listener.Accept() if err != nil { From 4890606831083f337a01ef400307d20d9838886b Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:39:47 +0800 Subject: [PATCH 027/127] docs: shorten the README intro to a pitch and link; go-test runs the benchmarks once The architecture diagram and lane summary live in docs/index.md. --- README.md | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 52c096aba..ab913bc52 100644 --- a/README.md +++ b/README.md @@ -7,20 +7,8 @@ for Go and Python. Warm claims are sub-millisecond; a pool miss clones from a golden snapshot in tens of milliseconds; cold boot is ~0.2–0.4 s on bare metal. **Documentation: [cocoonstack.github.io/sandbox](https://cocoonstack.github.io/sandbox/)** -(source in [`docs/`](docs/)). - -## Architecture - -``` -SDK (Go/Python) sandboxd (per node) guest microVM -client.New/new ─ HTTP ─► claim: warm pool / golden clone Cloud Hypervisor -exec/files/… ─ HTTP upgrade (kept) ─► byte relay ─ vsock ─► silkd :2048 - memberlist mesh: warm-count gossip, - MOVED-style redirect to the owning node -``` - -Cloud Hypervisor serves both network lanes. `net=none` has no NIC and uses -vsock-only I/O (hardened default); `net=egress` attaches a bridge/CNI NIC. +(source in [`docs/`](docs/)); the architecture overview is in +[`docs/index.md`](docs/index.md). ## Quick start @@ -140,7 +128,7 @@ The cmdline keys `sandbox-init` consumes are in the make help # every target; the ones below are the daily set make lint test # Rust: boot/init + silkd (fmt --check, clippy -D warnings, tests) make go-lint # Go: protocol/wire + sandboxd + sdk/go + e2e + mcp, GOOS linux AND darwin -make go-test # Go: go test -race across the Go modules +make go-test # Go: go test -race plus one pass of every benchmark, across the Go modules make sh-lint # shellcheck every tracked shell script make sandboxd # build dist/sandboxd make bench # claim-tier + data-plane benchmarks; removes every sbx-* VM From 064a82d5cdbe9bb6a38803d7f70ec086a6d8b08b Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:39:47 +0800 Subject: [PATCH 028/127] docs: state each shared fact once; correct meshsmoke, usage node and dirty-marker wording Token model stays in sandboxd-api.md, the 429 causes get one Admission refusals section, the reader wire behavior and operator view split between the API and deploy pages, the systemd unit is linked rather than copied, and the archive_failover plaintext-env fact stays in security.md and the deploy table. The dirty-marker text now describes what UnmountVolume and DetachVolume check; the usage node note says a rename lands the journal again. --- docs/cluster.md | 2 +- docs/deploy.md | 64 ++++++++++----------------------- docs/egress.md | 5 ++- docs/sandboxd-api.md | 85 +++++++++++++++++++++++++------------------- docs/security.md | 28 ++++++--------- 5 files changed, 79 insertions(+), 105 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index aa6eb6405..5fd6d4e7a 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -168,7 +168,7 @@ cap; to take no claims, cordon the node. | State | New claim, fork, branch | Existing claims | Root live move in | Adoption of archived claims | Refill and pool gossip | Survives restart and reboot | |---|---|---|---|---|---|---| -| serving | yes, under `max_claims`, `max_running`, the tenant's `max_claims`, `disk_reserve` and `memory_limit`; a 429 for a claim without volumes redirects to a peer | yes; a wake is bound by `max_running` and `disk_reserve`, and an archived claim's refused wake moves to a peer with room | yes, under the same caps and `move_concurrency` | yes; no claim cap binds, `max_running` binds at the wake | yes | — | +| serving | yes, unless an [admission refusal](sandboxd-api.md#admission-refusals) applies; a 429 for a claim without volumes redirects to a peer | yes; a wake is bound by `max_running` and `disk_reserve`, and an archived claim's refused wake moves to a peer with room | yes, under the same caps and `move_concurrency` | yes; no claim cap binds, `max_running` binds at the wake | yes | — | | cordoned | no: 429 `node cordoned`; peers stop redirecting here | yes | yes, under the same caps and `move_concurrency` | no; its own claims stay | no; warm VMs are destroyed | yes, as `/cordoned` | | draining | no: 429 `node draining`; peers stop redirecting here | yes | no: admission and commit refuse | no | no; warm VMs are destroyed | no | | draining with `evacuate=1` | no, as draining | yes until the claim's turn, then 503 until a peer holds it and 404 after; a claim that cannot move stays and serves | no | no | no | no: a restart stops the evacuation | diff --git a/docs/deploy.md b/docs/deploy.md index 9be870b2b..0eab49b15 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -129,8 +129,8 @@ the intersection of the two. | `replicate_seconds` (pool entry) | 0 (off) | snapshot every running claim of this pool to the checkpoint store this often and point the claim's registry row at the snapshot, so another node restores the claim, memory and processes included, when this node is lost (see [running claims across node loss](cluster.md#running-claims-across-node-loss)). Each round pauses the guest for the length of a memory and disk capture. A restore loses what ran after the last committed replica: [the bound](cluster.md#running-claims-across-node-loss). The claim's row carries no retention deadline while it runs. Requires `archive_failover` and a `none`-lane pool, both checked at config load; a claim with volumes is skipped. Config-owned like `egress`: `PUT /v1/pools` rejects it. Restart only: a claim is replicated by its pool's setting from the moment it is claimed, so a reload that changes it is refused (a pool restarted with 0 starts retiring its replicas while the registry remains enabled; see [Disabling archive failover](#disabling-archive-failover)) | | `storage` (pool entry) | unset (cocoon's default, 10G) | size of the copy-on-write disk the pool's golden is cold-booted with, in cocoon's own `--storage` spelling (`"40G"`, `"40GiB"`, `"40Gi"`: Docker/Kubernetes units, all binary) and passed to it verbatim; every clone, fork, checkpoint branch and promoted template of the pool inherits it. At least cocoon's default of 10G. Equal sizes spelled differently build the same golden; a changed size rebuilds the golden on the next build, and claims already out keep their disk. Config-owned like `egress`: `PUT /v1/pools` rejects it | | `tun` (pool entry) | `false` | turns the [advanced network mode](advanced-network.md#configuration) on for the pool's claims; needs the node's `tun` block and the `none` lane. Config-owned like `egress`: `PUT /v1/pools` rejects it | -| `max_claims` | 0 (unlimited) | node-wide cap on live claims (to take none, [cordon](sandboxd-api.md#post-v1cordon) the node); claim/fork/branch requests beyond it answer 429 with the pool state unharmed (on a cluster a non-volume claim first tries a redirect to a warm peer, else to a peer that serves the pool; a volume claim answers the 429 with no redirect) | -| `max_running` | 0 (off) | node-wide cap on claims that hold a VM: not hibernated, not archived, not failed, with a wake in flight counted; claim, fork, branch and wake beyond it answer 429 and leave the claim as it was; a claim takes the same redirect as `max_claims`. Hibernated and archived claims hold no slot, so `max_claims` can follow disk while this follows RAM and CPU. Warm and reader VMs are not counted: keep `max_running` × the largest size plus every pool's warm target × its size within the node's RAM. A hibernated claim's wake cannot move to another node, so its 429 stays until a running claim sleeps or goes; with `archive_failover`, an archived claim's refused wake [moves to a peer with room](cluster.md#waking-an-archived-claim-on-a-full-node). `sandboxd_wakes_refused_total` counts the refusals. Lowering the cap stops nothing that runs. A [reload](#reloading-the-config) applies a change | +| `max_claims` | 0 (unlimited) | node-wide cap on live claims (to take none, [cordon](sandboxd-api.md#post-v1cordon) the node); claim/fork/branch requests beyond it answer 429 ([admission refusals](sandboxd-api.md#admission-refusals)) | +| `max_running` | 0 (off) | node-wide cap on claims that hold a VM: not hibernated, not archived, not failed, with a wake in flight counted; claim, fork, branch and wake beyond it answer 429 and leave the claim as it was ([admission refusals](sandboxd-api.md#admission-refusals)). Hibernated and archived claims hold no slot, so `max_claims` can follow disk while this follows RAM and CPU. Warm and reader VMs are not counted: keep `max_running` × the largest size plus every pool's warm target × its size within the node's RAM. A hibernated claim's wake cannot move to another node, so its 429 stays until a running claim sleeps or goes; with `archive_failover`, an archived claim's refused wake [moves to a peer with room](cluster.md#waking-an-archived-claim-on-a-full-node). `sandboxd_wakes_refused_total` counts the refusals. Lowering the cap stops nothing that runs. A [reload](#reloading-the-config) applies a change | | `move_concurrency` | 2 | node-wide cap on [live moves](cluster.md#moving-a-running-claim) in flight, counting moves this node sends and reservations it holds for moves to it; a move past it answers 429. Moves need `meta_store` with a `cell`, for the move record, and `mesh.cluster_key`, but not `archive_failover` or an S3 `checkpoint_store`: a node that sets this without them refuses to load, and one that leaves it unset without them cannot move (409). A move also needs a cocoon with live migration (v0.7.7 or newer; a master build is judged by the newest tag it reports), the Cloud Hypervisor fork at `35b098c65260` or newer (a VM started before the upgrade cannot move), a run dir that supports reflink such as XFS, nodes of the same CPU model, and the same path strings in cocoon's `root_dir` and `run_dir` on both nodes ([same paths](cluster.md#moving-a-running-claim)). Lowering the cap stops no move in flight. A [reload](#reloading-the-config) applies a change | | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token; a config reload records as op `config_reload` with `changed`, a runtime volume change as op `volumes` with `added` or `removed`, and each ended flow of the [advanced network mode](advanced-network.md) as op `tun_flow` with `path` (`tun`), `tenant`, `proto` (`tcp`, `udp`, `icmp`), `dst_addr`, `decision` (`allow`, `deny`), `sent` and `received` bytes, `start`, `end` and the exit `profile`; a guest DNS-relay query records as op `dns` ([fields](egress.md#name-resolution-dns)), a cordon and its lift as ops `cordon` and `uncordon`, and a guest-port relay as op `port` with `port`. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields | | `disk_reserve` | unset | `{"free": "100G", "dirs": ["/srv/sbx/run"]}`: while the filesystem of any listed directory (default `data_dir`) has less than `free` available, checked every 2 s, the node refuses new claims, forks, checkpoint claims, wakes of hibernated or archived sandboxes, hibernates, checkpoint captures and promotes with 429, and parks refill; the idle sweep and a lease-end archive leave their sandboxes running and retry on the next tick; `/v1/info` reports it as at capacity and the mesh sees no pools on it. Running sandboxes keep running. On an XFS project quota the free space is the project's. List cocoon's `run_dir` when it sits on another filesystem, since the copy-on-write disks grow there. Size `free` to at least two sandboxes' worst-case growth (copy-on-write disk plus memory file each) and keep the refusal above your disk alert's warning level, so the warning comes first: on a 640G project with 40G disks and 8G memory files, `100G` refuses at 84% used | @@ -183,9 +183,9 @@ connections to finish, and the next connection wakes the claim. A reader is a warm VM taken from the claim's own pool, so it boots the claim's image with the claim's layers, and it has no NIC (egress-lane claims never hibernate). Taking one removes a VM from the pool until the refill replaces it, -and an empty pool never cold-boots one: the call wakes the claim. A reader is never shared: it -serves one claim, and only the claim's own token reaches it. The guest's own -kernel does the parsing, as it does for the claim: +and an empty pool never cold-boots one: the call wakes the claim. Only the +claim's own token reaches its reader. The guest's own kernel does the parsing, +as it does for the claim: - a hibernated claim's copy-on-write disk is copied by cocoon (`vm disk attach --from-snapshot`) into the reader VM and hot-attached read-write; cocoon uses @@ -217,8 +217,8 @@ is destroyed. A wake waits for a reader only while cocoon copies the disk: milliseconds with a reflink, the time to copy the whole disk without one. A reader serves only the sleep it was built from: a later hibernate or archive gets a fresh one. The claim's own code runs in its reader with root, as it did -in the claim, and the reader serves one claim and is destroyed afterwards, so -the claim's bytes never reach a shared VM or the host kernel. +in the claim, and a reader is never shared, so the claim's bytes never reach a +shared VM or the host kernel. Whenever a reader cannot be built (no warm VM in the claim's pool, a failed attach, an archive without @@ -580,24 +580,11 @@ docs cover validation semantics and the per-VM knobs. ### Auth model -Three token kinds. The root `api_token` has full access — operators and -single-tenant deployments need nothing else. Tenant tokens, added through -the [tenant API](sandboxd-api.md#tenants-v1tenants) and kept in +A root `api_token` for operators, tenant tokens added through the +[tenant API](sandboxd-api.md#tenants-v1tenants) and kept in `/tenants.json` (or the [shared tenant database](#shared-tenant-database)), -create and manage their own resources: claims, forks, checkpoints, -promoted templates, and preview URLs are stamped with the tenant name; -sandbox and checkpoint listings filter to the caller's tenant, and a tenant -can delete only its own checkpoints and templates (root sees everything). -The [operator surfaces](sandboxd-api.md) stay root-only; a -tenant token there is authenticated but not authorized, so it answers 403 (a -wrong token stays 401). Per-sandbox -tokens are unchanged: whoever holds a sandbox's token drives that sandbox. -Fork children inherit the parent's tenant and count against its -`max_claims` (0 = unlimited, a per-node cap, so a tenant's cluster limit is -`max_claims` × nodes); a tenant at its cap gets 429 exactly like a node at -`max_claims`, and every usage event of a tenant's claim carries the tenant for -per-tenant billing. `config.json` has no `tenants` field: a node starts with -no tenants until the API adds them. +and one token per sandbox. What each may call is in the +[API reference](sandboxd-api.md#sandboxd-http-api). ## Running @@ -630,26 +617,9 @@ count): sandboxd then retires it and builds a new golden, and until that is ready, claims for that pool that find no warm VM cold-boot. If clones of the new golden fail the same way, it is retired again at most once every 30 s. -A minimal systemd unit (shipped as -[`packaging/sandboxd.service`](https://github.com/cocoonstack/sandbox/blob/main/packaging/sandboxd.service)): - -```ini -[Unit] -Description=sandboxd -Wants=network-online.target -After=network-online.target - -[Service] -ExecStart=/usr/local/bin/sandboxd -config /etc/sandboxd/config.json -Restart=on-failure -RestartSec=5s -KillMode=process -TimeoutStopSec=27min -Environment=SANDBOXD_LOG_LEVEL=info - -[Install] -WantedBy=multi-user.target -``` +The systemd unit is +[`packaging/sandboxd.service`](https://github.com/cocoonstack/sandbox/blob/main/packaging/sandboxd.service); +install it as shipped. Its `KillMode` and `TimeoutStopSec` are explained below. Each record carries its syslog level, so alerting keys on systemd priority: `journalctl -u sandboxd -p err` lists the errors and `-p warning` the @@ -853,7 +823,9 @@ only; computing durations or prices stays with the consumer. until it catches up. The journal append is unchanged, so a claim, release, hibernate or wake never waits on the database. - **Node:** rows carry the node's mesh `node_id` (its `bind` when unset), or - its host name without a mesh. The name must be unique within the cell. + its host name without a mesh. The name must be unique within the cell. Keep + the name stable: a renamed node lands its whole journal again under the new + name. - **Schema:** sandboxd creates the table at boot when it is missing; create it ahead with the same DDL when the node's role has no `CREATE` right: @@ -1130,7 +1102,7 @@ some of them on a node: |---|---|---| | `demo`, `smoke` | claim tiers and the verb surface | `sandboxd-e2e.sh` | | `volumesmoke` | shared read-only, writable, and live volumes | `sandboxd-e2e.sh` | -| `meshsmoke` | template routing on a two-node mesh without a shared checkpoint store, because [a shared store never routes](cluster.md#templates-on-a-cluster) | | +| `meshsmoke` | template routing on a two-node mesh without a shared checkpoint store, because [a shared store resolves templates locally](cluster.md#templates-on-a-cluster) | | | `crossnode` | a checkpoint claimed on another node through the shared store | | | `coldproof` | a cold claim, by the guest's own clock | | | `lifecycle` | idle → hibernate → archive → wake | `archive-e2e.sh` | diff --git a/docs/egress.md b/docs/egress.md index 96baec91b..410eba6f2 100644 --- a/docs/egress.md +++ b/docs/egress.md @@ -384,9 +384,8 @@ gateway key per user, while the guest holds a placeholder. sets. Processes captured live in a snapshot (sessions, detached execs, units) keep the source's guest values in their own environment; a `guest: false` value never enters the guest, so no snapshot, checkpoint or - store object holds one. With [`archive_failover`](deploy.md#configuration), - the row of an archived or replicated claim carries the env, `guest: false` - values included, in plaintext in the shared PostgreSQL. + store object holds one; the `archive_failover` row is the exception + ([security](security.md#deployment-assumptions)). - **Audit.** An injection is recorded as the secret's name; the value is never logged, listed or returned. diff --git a/docs/sandboxd-api.md b/docs/sandboxd-api.md index 717d8ed44..47bcaae80 100644 --- a/docs/sandboxd-api.md +++ b/docs/sandboxd-api.md @@ -42,6 +42,24 @@ they reach its guest, so they share these answers: - 503 `tenant store down; retry` for an archived claim whose tenant the node has not cached while the meta store is down +## Admission refusals + +A claim, fork or checkpoint branch answers 429 when the node refuses it for +capacity, with the pool state unharmed. The causes: + +- the node is at [`max_claims`](deploy.md#configuration) or `max_running` +- the tenant the claim counts against is at its own `max_claims`: the caller's + for a claim or branch, the parent's for a fork +- the node is draining ([`POST /v1/drain`](#post-v1drain)) or cordoned + ([`POST /v1/cordon`](#post-v1cordon)) +- the disk is under `disk_reserve` +- memory is over `memory_limit` + +On a cluster, a claim without `volumes`, `no_redirect` or `require_promoted` +that meets one of these redirects to a peer with warm capacity or, failing +that, one that serves the pool; see [`POST /v1/claim`](#post-v1claim). Wake and +the guest-reaching verbs have their own 429 causes, listed with each. + ## POST /v1/claim Auth: `Authorization: Bearer ` (when configured). @@ -223,9 +241,8 @@ VM closes the devices. Redirects (mutually exclusive with the fields above) name peers to retry at — sent on a warm miss with warm peers, when the node lacks a golden for the key but gossip names a template owner, and when a claim without `volumes`, -`no_redirect` or `require_promoted` meets a 429 (the node at `max_claims` or -`max_running`, the calling tenant at its cap, the node draining or cordoned, the disk under -`disk_reserve`, or memory over `memory_limit`) and a peer reports warm capacity or, failing that, serves the pool: +`no_redirect` or `require_promoted` meets an [admission refusal](#admission-refusals) +and a peer reports warm capacity or, failing that, serves the pool: ```json {"redirect": ["10.0.0.6:7777", "10.0.0.7:7777"], @@ -258,9 +275,7 @@ busy — a live writer excludes every other claim for that name, live readers exclude a writer), or a `ro` claim against a writable image left dirty by an unclean `rw` release (needs recovery — one `rw` claim must replay and cleanly release before `ro` claims resume; see -[deploy](deploy.md#writable-dataset-volumes)); 429 node at `max_claims` or `max_running`, the -calling tenant at its own `max_claims`, the node draining or cordoned, the disk under -`disk_reserve`, or memory over `memory_limit`; 500 provisioning failed. +[deploy](deploy.md#writable-dataset-volumes)); 429 an [admission refusal](#admission-refusals); 500 provisioning failed. ## GET /v1/volumes @@ -424,19 +439,20 @@ a safe per-volume reason. Guest transport failures remain server errors; timeouts return 504 and cancellation returns 408. A mounted writable attachment marks the image dirty before the guest can write. -Live removal and release clear the marker only when the unmount observes and -normally unmounts the exact writable whole-device filesystem and finds no -residual device mounts. These checks read silkd's own mount namespace, so a -mount of the device in another namespace, or one moved there, is not seen. -Live removal also needs a confirmed host detach. Both -mount and superblock options must be writable; ext4's `emergency_ro` after a -filesystem error counts as read-only. A missing device or pre-unmounted -filesystem does not prove clean unmount. The marker remains on uncertainty and -blocks subsequent readers. Release waits up to 30 seconds for each writable -volume's unmount, and a guest that does not answer within that keeps every -remaining marker. Attach-only writers have no automatic dirty-marker -protection. The caller owns mounting, flushing, unmounting, recovery, and -fencing before reuse. +Live removal and release clear the marker only when the unmount of the mount at +the volume's path succeeds, the mount was the filesystem's root and writable in +both its mount and superblock options (ext4's `emergency_ro` after a filesystem +error counts as read-only), and no other mount of its device remains. Live +removal finds the device by its serial and refuses any other mount of it; +release identifies the mount by its path alone and does not check which device +it is. These checks read silkd's own mount namespace, so a mount of the device +in another namespace, or one moved there, is not seen. Live removal also needs +a confirmed host detach. A missing device or pre-unmounted filesystem does not +prove clean unmount. The marker remains on uncertainty and blocks subsequent +readers. Release waits up to 30 seconds for each writable volume's unmount, and +a guest that does not answer within that keeps every remaining marker. +Attach-only writers have no automatic dirty-marker protection. The caller owns +mounting, flushing, unmounting, recovery, and fencing before reuse. ### Partial failure and recovery @@ -599,8 +615,7 @@ an unknown `on_expire`; 401 bad api token; 403 the parent's tenant was removed (sandbox token); 404 unknown id or wrong sandbox token; 409 egress-lane or volume parent (neither forks, checkpoints, or promotes; see [egress](egress.md)), a failed parent, or an archived parent (`POST /wake` or an exec wakes it first); -429 node or the parent's tenant at `max_claims`, the node at `max_running`, the node draining or cordoned, the disk -under `disk_reserve`, or memory over `memory_limit`; 503 `lease lost; retry` for a replicated parent while its node's lease is overdue, `tenant store down; retry` for a sandbox token whose tenant the node has not cached while the meta store is down. +429 an [admission refusal](#admission-refusals); 503 `lease lost; retry` for a replicated parent while its node's lease is overdue, `tenant store down; retry` for a sandbox token whose tenant the node has not cached while the meta store is down. ## POST /v1/sandboxes/{id}/promote @@ -753,8 +768,10 @@ reach nothing on a pool with a policy. The API references a class and never defines one. A claim keeps the class its tenant had when it was made. A change applies to the next request: a removed or rotated token gets 401 at -once. Lowering `max_claims` below the live count refuses new claims and evicts -none. A removed tenant's claims keep running and stay listed for root +once. `max_claims` is a per-node cap (0 = unlimited), so a tenant's cluster +limit is `max_claims` × nodes; every usage event of a tenant's claim carries the +tenant, for per-tenant billing. Lowering `max_claims` below the live count +refuses new claims and evicts none. A removed tenant's claims keep running and stay listed for root (`GET /v1/sandboxes?tenant=`) until they expire or root releases them, but they cannot extend: renew and fork answer 403 to the sandbox's own token, a hibernated one is never archived, one archived before the removal is not @@ -953,9 +970,7 @@ resolves every record) and answers with a redirect shaped like a warm-miss the claim. See the [placement lifecycle](cluster.md#checkpoints-on-a-cluster). 404 for an unknown checkpoint (locally, and after redirect and heal both -miss), 429 node -or calling tenant at `max_claims`, the node at `max_running`, the node draining or cordoned, the disk under -`disk_reserve`, or memory over `memory_limit`, 503 when the node's +miss), 429 an [admission refusal](#admission-refusals), 503 when the node's concurrent-heal cap is already full — retryable, and the response carries a `Retry-After` hint. @@ -1300,12 +1315,12 @@ claim of a replicating pool that is already open when the lease turns overdue is closed, whether or not it has a replica yet. A hibernated or archived sandbox wakes when the upgrade request arrives, before the `101`, -unless its last sleep is reader-safe, which the pool's [`reader`](deploy.md#reading-sleeping-claims-without-waking-them) setting provides. +unless its last sleep is reader-safe, which the pool's [`reader`](deploy.md#reading-sleeping-claims-without-waking-them) setting provides; +what a reader is, what it needs and when the call wakes instead are there. With it, the node answers the `101` before it touches the sandbox, reads the first frame, and serves `fs_stat`, `fs_list`, `fs_read` and `fs_pull` from a -reader VM, a warm VM of the sandbox's pool that mounts a copy of the sandbox's disk read-only: the sandbox stays -hibernated or archived, no `wake` usage event is recorded, and the read stamps -neither the idle nor the archive clock. An `info` frame wakes the sandbox, since a +reader VM: the sandbox stays hibernated or archived, no `wake` usage event is +recorded, and the read stamps neither the idle nor the archive clock. An `info` frame wakes the sandbox, since a reader would answer with its own processes and uptime. The first frame on the connection that is anything else, or any later one, wakes the sandbox and hands the same connection to its silkd with the unread bytes replayed, so a client sees one @@ -1314,13 +1329,9 @@ an archive), the connection's next read moves to a fresh reader of the new sleep when it wakes, the connection moves to the woken sandbox, so reads never come from a stale sleep. Pipelining a request behind a read that has not yet sent its terminal frame is unsupported: when that request moves the connection, the -reader closes and the rest of the read's reply is lost. Files -and symlinks resolve inside the sandbox's own filesystem as of the guest `sync` -that runs right before the last hibernate's pause; a write the guest made in -the instant between that `sync` and the pause is visible only after a wake. A -hibernate whose `sync` failed, or one made before the pool had `reader`, -is not served by a reader: its reads wake the sandbox. When no reader can be built (no warm VM in the pool, a failed attach, an archive from an older cocoon) the call wakes the sandbox as if the -feature were off. Because the `101` has gone out by then, a failure on this path +reader closes and the rest of the read's reply is lost. A read that no reader +can serve wakes the sandbox as if the feature were off. Because the `101` has +gone out by then, a failure on this path (unknown sandbox, a refused wake, an unreachable guest, an overdue node lease) arrives as a silkd `error` frame (`not_found` for an unknown sandbox, else `internal`) rather than an HTTP status, and the connection ends. A connection diff --git a/docs/security.md b/docs/security.md index f0527a24d..70d8a1f93 100644 --- a/docs/security.md +++ b/docs/security.md @@ -49,9 +49,8 @@ exposing any part of a deployment beyond a single trusted host. proxy's answers, and every credentialed call is journaled. Git auth tokens travel as in-memory headers, never guest disk. Secrets come from the host environment or a claim's `guest: false` env, never the config - file. That env stays in the node-local claim record; with - `archive_failover`, the row of an archived or replicated claim also carries - it, `guest: false` values included, in plaintext in the shared PostgreSQL. + file. That env stays in the node-local claim record, except in the + `archive_failover` row ([assumption 4](#deployment-assumptions)). - **Sandbox identity.** Every clone is reseeded (entropy, machine-id), so branches and forks never share an identity with their source. @@ -84,21 +83,14 @@ all of them. ## Token model -Three credentials, in descending scope ([API reference](sandboxd-api.md)): -the root `api_token` (operator surfaces, full access), tenant tokens -(resource-creating verbs, everything stamped and quota'd per tenant), and -per-sandbox tokens (that sandbox only — holding a handle amplifies to -nothing node-level). Two capability tokens ride on top: preview URLs are -HMAC-signed, expire with the claim's lease (an archived claim kept forever -has none, so its URL runs for the TTL it was minted with), and die with the -sandbox (no revocation list to leak); a checkpoint id is the unguessable capability to -branch it. On a cluster, deleting a checkpoint does not revoke that -capability fleet-wide the instant it runs: the delete is best-effort — -broadcast to every peer the node currently sees — so a peer that is offline -or partitioned at that moment keeps its own replica branchable until -`checkpoint_ttl_hours` ages it out ([placement lifecycle](cluster.md#checkpoints-on-a-cluster)). -Tenants are isolated at the API layer — listings filter, deletes answer 404 -rather than confirming existence, and operator surfaces answer tenants 403. +Three credentials in descending scope: the root `api_token`, tenant tokens +(isolated at the API layer) and per-sandbox tokens (that sandbox only, nothing +node-level); what each may call is in the +[API reference](sandboxd-api.md#sandboxd-http-api). Two capabilities ride on +top: a preview URL is HMAC-signed with no revocation list, and a checkpoint id +is the capability to branch it, which a delete does not revoke fleet-wide the +instant it runs +([why](cluster.md#delete-is-eventual-not-a-fleet-wide-revocation)). ## Dataset volumes From 1c9bccc180b16d24a33b6829b8f3cda1b60afa5b Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:39:47 +0800 Subject: [PATCH 029/127] docs: SDK differences, Python-only text out of the Go page, narrower fixture claim, silkdtest path --- docs/sdk-python.md | 7 +++++-- docs/sdk.md | 17 ++++------------- docs/silkd.md | 6 ++++-- protocol/README.md | 6 ++++-- sdk/go/README.md | 2 +- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/docs/sdk-python.md b/docs/sdk-python.md index b12a8a83d..254b042a6 100644 --- a/docs/sdk-python.md +++ b/docs/sdk-python.md @@ -17,7 +17,10 @@ their siblings), the claim env (`WithEnv`, `Env`/`SetEnv`/`PatchEnv`) and setting claim `metadata` or `on_expire`, and reading a claim's `net_route` remain Go-only, `Template.new` takes no `claim_ref`, and file and tar payloads are whole `bytes` values rather than streams (`read_file`/`pull` return them, `write_file`/`push` take them; the -wire is chunked, the caller's copy is not). Wire fidelity is +wire is chunked, the caller's copy is not). Two behaviors differ as well: +`Session.close()` and `Lsp.stop()` swallow `not_found` where Go returns it, and +a failed session command puts its merged output in `ExitError.stdout` with +`stderr` empty, where Go's `ExitError.Stderr` holds it. Wire fidelity is pinned by the shared protocol fixture corpus that the Rust guest, Go, and Python all round-trip in CI. @@ -209,7 +212,7 @@ Responses include `volumes_want`, `volumes`, `states`, and `converging`. `VolumePhase` and `VOLUME_ATTACHED`, `VOLUME_PENDING`, and `VOLUME_DETACHING` name the three states. A pending state with `converging=False` needs an explicit retry. `VolumeError`, exported from `cocoonsandbox` and `cocoonsandbox.errors`, -retains a complete failure view in `.response`. Complete replies update both +is a subclass of `APIError` and retains a complete failure view in `.response`. Complete replies update both public sets (`sb.volumes_want`, `sb.volumes`); incomplete errors preserve the last view. Serialize calls on one handle. See the [replacement tutorial](live-volumes.md) and [API recovery contract](sandboxd-api.md#partial-failure-and-recovery). diff --git a/docs/sdk.md b/docs/sdk.md index f89020ed1..60e42dc69 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -246,14 +246,8 @@ view, err = sb.PatchVolumes(ctx, sandbox.VolumePatch{Remove: []string{"imagenet" view, err = sb.SetVolumes(ctx, []sandbox.Volume{}) ``` -Python exposes the same operations and response fields: - -```python -view = sb.set_volumes([{"name": "imagenet", "mount": "/datasets/training"}]) -view = sb.get_volumes() -view = sb.patch_volumes(remove=["imagenet"]) -view = sb.set_volumes([]) # None is rejected. -``` +The [Python SDK](sdk-python.md#changing-live-volumes) exposes the same +operations and response fields. `Volume.AttachOnly` leaves mounting and unmounting to the caller: the entry in `Sandbox.Volumes` carries an empty `Mount`; see @@ -267,15 +261,12 @@ writer exclusion and dirty-marker limits. Each response contains `volumes_want`, `volumes`, `states`, and `converging`. Go exposes them as `SandboxVolumesResponse.VolumesWant`, `.Volumes`, `.States`, and `.Converging`. `VolumePhase` has `VolumeAttached`, `VolumePending`, and -`VolumeDetaching` constants. Python exposes `VolumePhase` and the corresponding -`VOLUME_ATTACHED`, `VOLUME_PENDING`, and `VOLUME_DETACHING` constants. Each state -can include an `error`. Methods use the client API token on the owning node +`VolumeDetaching` constants. Each state can include an `error`. Methods use the client API token on the owning node and update both public sets (`Sandbox.VolumesWant`, `Sandbox.Volumes`) from each complete response, including a partial failure. Serialize changes to one handle as with `Renew`. Returned views do not share the handle's slices or dictionaries. Go returns `*sandbox.VolumeError` with a `Response` alongside the returned view. -It wraps `*sandbox.APIError`, so `errors.As` can inspect the status. Python raises -`VolumeError`, a subclass of `APIError`, with `.response`. A response is absent +It wraps `*sandbox.APIError`, so `errors.As` can inspect the status. A response is absent when the reply carries no complete view (a malformed request, authentication, an unknown sandbox, or a non-JSON proxy reply). In that case the handle retains its last known sets. Retry an interrupted operation with the diff --git a/docs/silkd.md b/docs/silkd.md index ecf69e16c..cf89d6182 100644 --- a/docs/silkd.md +++ b/docs/silkd.md @@ -42,8 +42,10 @@ connection-bound verbs are `fs_watch`, `pty_open`, `lsp_request`, and `port_forward`. The authoritative wire contract is the shared fixture corpus in -`protocol/wire/fixtures/v1`, round-tripped by the Rust, Go, and Python test -suites — a frame only one side can parse fails CI. +`protocol/wire/fixtures/v1`, round-tripped by the Rust and Go test suites; the +Python suite round-trips the request frames and checks that each response +decodes to the fixture's `type` tag and `data` — a frame the Rust or Go side +cannot parse fails CI. ## Verbs diff --git a/protocol/README.md b/protocol/README.md index aa7d4a02e..b3a55aabf 100644 --- a/protocol/README.md +++ b/protocol/README.md @@ -5,10 +5,12 @@ implementations of the silkd protocol: - silkd (Rust, guest) parses/emits these in `silkd/src/proto.rs` tests. - `protocol/wire` (Go, host) round-trips the corpus in its tests; it is the one Go implementation, consumed by both the Go SDK and sandboxd. -- the Python SDK round-trips it in `sdk/python/tests/test_fixtures.py`. +- the Python SDK round-trips the request frames and checks only that each + response decodes to the fixture's `type` tag (and `data`), in + `sdk/python/tests/test_fixtures.py`. `req_*.json` are client→server frames, `resp_*.json` server→client. A frame -that only one side can round-trip is a protocol drift bug. The Rust side also +that the Rust or Go side cannot round-trip is a protocol drift bug. The Rust side also pins the corpus size, so adding a verb without its fixture fails CI. `enums.json` pins each wire enum's full value set (error/event/file kinds, diff --git a/sdk/go/README.md b/sdk/go/README.md index 3374ada84..3b7c2e18b 100644 --- a/sdk/go/README.md +++ b/sdk/go/README.md @@ -26,7 +26,7 @@ out, _ := sb.Exec(ctx, "echo", "hello") - `upgrade.go` / `keepalive.go` / `endpoint.go` — the relay upgrade, idle connection reuse, endpoint and TLS parsing - `silkd/` — the conn/stream layer over the relay; the frame types live in `protocol/wire`, whose tests round-trip `protocol/wire/fixtures/` (drift - against the Rust guest fails CI); `silkdtest/` is an in-process fake guest + against the Rust guest fails CI); `silkd/silkdtest/` is an in-process fake guest for consumers' tests Module path `github.com/cocoonstack/sandbox/sdk/go`; no third-party From 5d61f8f099782f274c85b62d428253b5a0a9680e Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:39:47 +0800 Subject: [PATCH 030/127] docs: kernel range of the bare-metal rows; e2b-ci builds from base --- docs/e2b.md | 6 ++++-- docs/performance.md | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/e2b.md b/docs/e2b.md index 90b103cf4..c7acfb536 100644 --- a/docs/e2b.md +++ b/docs/e2b.md @@ -2,7 +2,7 @@ The `e2b-rt` flavor carries [e2b](https://e2b.dev)'s `envd` alongside silkd, so an **unmodified e2b SDK** can drive a cocoon microVM; `e2b-ci` adds e2b's code -interpreter on top, for `@e2b/code-interpreter` and `e2b_code_interpreter`. It is the guest half of a +interpreter, for `@e2b/code-interpreter` and `e2b_code_interpreter`. It is the guest half of a compatibility layer whose other two halves live in [sandbox-operator](https://github.com/cocoonstack/sandbox-operator): the apiserver's e2b REST surface answers `Sandbox.create()`, and `envd-proxy` @@ -78,7 +78,9 @@ for the truth. commit's archive, checks its sha256, and applies the one local change listed in `os-image/e2b-ci/PATCHES.md`; e2b's license ships in the image under `/usr/share/doc/e2b-code-interpreter/`. Moving to a newer recipe is a bump of the -two `CI_RECIPE_*` args plus a refresh of the constraints files. On top of `e2b-rt`'s contents it runs a Jupyter server on +two `CI_RECIPE_*` args plus a refresh of the constraints files. It builds from +`base`, not on top of `e2b-rt`, with the same `envd` pin, guard unit and `user` +account as `e2b-rt`, and it runs a Jupyter server on loopback 8888 and the code-interpreter API on 49999, both enabled at boot, and the guard drops off-guest traffic to 49983, 49999 and 8888. Only the python kernel ships: the recipe installs its javascript, R and Java kernels from diff --git a/docs/performance.md b/docs/performance.md index deb5009cd..a6234549f 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -6,7 +6,7 @@ Reproduce with `make bench` (tier definitions and the dated results log: [Benchmarks](benchmarks.md)), `scripts/sandboxd-e2e.sh` (claim/verb latencies print per run), and `scripts/boot-bench.sh` (boot phases). -**Bare metal** = a 16-core AMD (SVM) node, Ubuntu 24.04, kernel 6.x, local +**Bare metal** = a 16-core AMD (SVM) node, Ubuntu 24.04, kernel 6.17–7.0, local NVMe, KVM. **Nested** = a cloud VM with nested virtualization, never comparable with bare-metal claims ([why](benchmarks.md#the-three-claim-tiers)). From 976d68d696f07e0a8898c18b01fe7d84bdb293e4 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:39:47 +0800 Subject: [PATCH 031/127] docs: flavor READMEs keep build notes and link the guides --- os-image/android/README.md | 16 ++-------------- os-image/browser/README.md | 21 +++------------------ 2 files changed, 5 insertions(+), 32 deletions(-) diff --git a/os-image/android/README.md b/os-image/android/README.md index 3957b54af..00718840b 100644 --- a/os-image/android/README.md +++ b/os-image/android/README.md @@ -3,9 +3,8 @@ Android guest for UI-automation sandboxes: the cocoon project's official redroid image (`ghcr.io/cocoonstack/cocoon/android:15.0`, upstream redroid 15 rootfs + cocoon boot chain, built from `cocoon/os-image/android/`) -plus silkd, claimable like any other template. Remote access is adb on -port 5555 — no VNC in this lineage. The intended pool shape is -`net: none`, `size: xlarge` (4 CPU / 8G). +plus silkd. Claim shape, adb access and behavior are in +[docs/android.md](../../docs/android.md); this file holds the build notes. ## Build @@ -30,14 +29,3 @@ no NIC, 4 CPU / 8G). Do not revert to the previous third-party VNC android pin: that build ships a broken dexpreopt boot-image chain that crash-loops zygote before the framework ever completes. - -## Constraints - -- **Boot chain**: the image boots its embedded 6.8 kernel via Android - `/init`, not the sandbox boot chain (PVH kernel + overlay-root init). - Use Cloud Hypervisor with `net: none`; snapshot/restore of a booted Android - is covered by the `androidsmoke` acceptance round. -- **silkd on Android**: the musl-static binary runs against an Android - userspace (`/system/bin/sh`, no `/bin/sh`); exec behavior is covered by - the `androidsmoke` acceptance round, and the session verbs need bash, which - the image lacks. diff --git a/os-image/browser/README.md b/os-image/browser/README.md index 244d75517..4fecd6399 100644 --- a/os-image/browser/README.md +++ b/os-image/browser/README.md @@ -1,12 +1,8 @@ # browser flavor -Headless-browser guest for CDP automation sandboxes: `base:24.04` plus a -pinned [Chrome for Testing](https://googlechromelabs.github.io/chrome-for-testing/) -build launched at boot with its DevTools (CDP) endpoint on guest loopback -`9222`. An agent claims it and connects any CDP client (Playwright -`connectOverCDP`, Puppeteer) through the silkd port relay. The intended -pool shape is `size: large` (4 CPU / 4G) — a persistent Chromium idles at -~150–250 MB RSS. +Headless-browser guest: `base:24.04` plus a pinned Chrome for Testing build +launched at boot. Claim shape, CDP access and behavior are in +[docs/browser.md](../../docs/browser.md); this file holds the build notes. ## Build @@ -25,14 +21,3 @@ with machine-readable pinned download URLs per version. Version bumps: update `CHROME_VERSION` + `CHROME_SHA256` from `last-known-good-versions-with-downloads.json`, same operational load as `COCOON_AGENT_VERSION`. - -## CDP on loopback is intentional - -Chromium M113+ binds `--remote-debugging-port` to `127.0.0.1` only. In -this stack that is exactly right: silkd's forwarder dials guest loopback, -so CDP is fully reachable through an authorized claim over vsock and never -exposed on a guest NIC — including on the no-network lane. - -`chromium.service` is a product service, not a readiness gate: the claim -still returns the moment silkd answers; Chromium comes up a beat later and -is polled by the workload (`e2e/cmd/browsersmoke` shows the pattern). From 6a44ccfaacc48e3c18dc3eeb2087fc60a8b79f89 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:40:28 +0800 Subject: [PATCH 032/127] docs: a checkpoint branches only for its own tenant --- docs/cluster.md | 4 ++-- docs/security.md | 6 +++--- scripts/archive-e2e.sh | 0 3 files changed, 5 insertions(+), 5 deletions(-) mode change 100755 => 100644 scripts/archive-e2e.sh diff --git a/docs/cluster.md b/docs/cluster.md index 5fd6d4e7a..d4c332df8 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -696,8 +696,8 @@ that broadcast keeps its copy until the checkpoint TTL ages it out. A healed replica carries the source checkpoint's original `CreatedAt`, so it becomes eligible for expiry at the same instant everywhere; each node then removes it on its own hourly sweep, which is independently phased and retries on -failure. The window in which a deleted checkpoint stays branchable by an -id-holder is therefore normally `checkpoint_ttl_hours` plus the wait for the +failure. The window in which a deleted checkpoint stays branchable by its +tenant is therefore normally `checkpoint_ttl_hours` plus the wait for the next hourly sweep; a sweep that fails retries on a later one, so persistent sweep failure extends retention until one succeeds — the TTL is the eligibility point, not a hard ceiling. This is why heal *requires* a nonzero, fleet-matching diff --git a/docs/security.md b/docs/security.md index 70d8a1f93..f77d2ee51 100644 --- a/docs/security.md +++ b/docs/security.md @@ -86,9 +86,9 @@ all of them. Three credentials in descending scope: the root `api_token`, tenant tokens (isolated at the API layer) and per-sandbox tokens (that sandbox only, nothing node-level); what each may call is in the -[API reference](sandboxd-api.md#sandboxd-http-api). Two capabilities ride on -top: a preview URL is HMAC-signed with no revocation list, and a checkpoint id -is the capability to branch it, which a delete does not revoke fleet-wide the +[API reference](sandboxd-api.md#sandboxd-http-api). A preview URL is a +capability on top, HMAC-signed with no revocation list. A checkpoint branches +only for its own tenant, and a delete does not remove it fleet-wide the instant it runs ([why](cluster.md#delete-is-eventual-not-a-fleet-wide-revocation)). diff --git a/scripts/archive-e2e.sh b/scripts/archive-e2e.sh old mode 100755 new mode 100644 From 9dcec8dccde18c83de2a194a79f77b4e94fc6568 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:40:36 +0800 Subject: [PATCH 033/127] docs: a request pipelined behind fs_watch ends the watch without a terminal frame --- docs/silkd.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/silkd.md b/docs/silkd.md index cf89d6182..1bedac49e 100644 --- a/docs/silkd.md +++ b/docs/silkd.md @@ -57,7 +57,7 @@ cannot parse fails CI. | fs | `fs_write {path, mode?}` (+`data`/`data_end` frames) / `fs_read` / `fs_list` / `fs_stat` / `fs_mkdir {parents?}` / `fs_rm {recursive?}` / `fs_rename {from, to}` | `fs_read` streams `data{data}`… → `done` for a regular file and answers `bad_request` for a directory, device or FIFO, `fs_list` streams 4096-entry `entries{entries}` batches → `done`, `fs_stat` answers one `stat{info}`, and the mutating verbs terminate with `done`. Streaming runs both directions; write commits atomically via temp+rename and inherits an overwritten file's mode and, under root, its owner | | tree | `fs_push {dest}` (+tar as `data` frames) / `fs_pull {path}` | whole trees as tar streams through the guest tar: `fs_pull` streams `data{data}`… → `done`, `fs_push` terminates with `done` | | search | `fs_find {path, pattern, glob?}` → `match{file, line, content}`… → `done` / `fs_replace {files, pattern, replacement}` → `replaced{file, replacements}`… → `done` | regex as data, no shell quoting; `glob` is anchored `*`/`?` wildcards over file names; find skips binary and >8 MiB files, and replace skips the same 8 MiB bound with a zero count. One match whose frame would pass the 8 MiB cap ends the find with an `error`. Replace is atomic per file, not per list: it stops with an `error` at the first missing, unreadable or non-UTF-8 file, and the files before it stay rewritten | -| watch | `fs_watch {path, recursive?}` | `ready` once armed (events after it are guaranteed captured), then `event{kind, path}` until the client disconnects; watcher or delivery-queue overflow errors arrive as a terminal `error` instead of silently losing events | +| watch | `fs_watch {path, recursive?}` | `ready` once armed (events after it are guaranteed captured), then `event{kind, path}` until the client disconnects or sends another request, which ends the watch with no terminal frame; watcher or delivery-queue overflow errors arrive as a terminal `error` instead of silently losing events | | pty | `pty_open {cols, rows, cwd?, env?, user?}` / `pty_resize {pid, cols, rows}` | `pty_open` answers `started{pid}`, then a shell under a pseudo-terminal: output as `stdout` frames, input as `stdin` frames, `exit{code}` terminal; `pty_resize` answers `done`. PTYs register in the proc table like any exec | | git | `git_clone {url, path, branch?, depth?, auth?}` / `git_status {path}` / `git_add {path, files}` / `git_commit {path, message, author}` / `git_push {path, auth?}` / `git_pull {path, auth?}` / `git_branch {path, action, name?}` | `git_status_result` (porcelain v2), `git_commit_result{hash}`, `git_branches{current, branches}` for `git_branch {action: "list"}`; every other verb terminates with `done`. A status past about 1 MiB of entries carries `truncated: true` with the head of the list, so one frame never exceeds the cap. `auth` is injected as an in-memory header, never written to guest disk | | port | `port_forward {port}` | relays guest TCP 127.0.0.1:port over this connection: `ready` once connected, then `data` both ways. Each direction ends on its own: `data_end` half-closes the guest socket, `done` reports the guest closing its write side, and the RPC ends once both have. A client that only drops the connection leaves the relay open until the guest side closes or writes. Works on both lanes — the no-network lane's only way in | From 07712d376133cf51c84b7968fe55b3305d38dcc8 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:35:31 +0800 Subject: [PATCH 034/127] metastore: add CAS One and All, and route the pg registries and sources through them --- sandboxd/archives/pg/pg.go | 34 ++++-------------------- sandboxd/metastore/cas.go | 29 +++++++++++++++++++++ sandboxd/moves/pg/pg.go | 26 +++---------------- sandboxd/poolset/pg/pg.go | 12 ++++++--- sandboxd/tenants/pg/pg.go | 53 ++++++++++++-------------------------- 5 files changed, 61 insertions(+), 93 deletions(-) diff --git a/sandboxd/archives/pg/pg.go b/sandboxd/archives/pg/pg.go index 33afe230f..15b37bcc4 100644 --- a/sandboxd/archives/pg/pg.go +++ b/sandboxd/archives/pg/pg.go @@ -4,7 +4,6 @@ package pg import ( "context" _ "embed" - "errors" "time" "github.com/jackc/pgx/v5" @@ -91,7 +90,7 @@ func (r *Registry) TakeOwned(ctx context.Context, owner string, expect map[strin for id, epoch := range expect { ids, epochs = append(ids, id), append(epochs, epoch) } - recs, err := r.collect(ctx, `UPDATE sandboxd_archives a SET epoch = nextval('sandboxd_archives_epoch'), updated_at = now() + recs, err := cas.All(ctx, r.pool, scan, `UPDATE sandboxd_archives a SET epoch = nextval('sandboxd_archives_epoch'), updated_at = now() FROM unnest($2::text[], $3::bigint[]) AS t(id, epoch) WHERE a.cell = $1 AND a.id = t.id AND a.epoch = t.epoch AND a.owner = $4 AND NOT a.consumed RETURNING a.id, a.tenant, a.token_sha256, a.owner, a.epoch, a.consumed, a.checkpoint, a.deadline, NULL::jsonb`, r.cell, ids, epochs, owner) @@ -116,26 +115,17 @@ func (r *Registry) Delete(ctx context.Context, id string, epoch int64) error { } func (r *Registry) Get(ctx context.Context, id string) (archives.Record, bool, error) { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - rec, err := scan(r.pool.QueryRow(qctx, `SELECT id, tenant, token_sha256, owner, epoch, consumed, checkpoint, deadline, claim - FROM sandboxd_archives WHERE cell = $2 AND id = $1`, id, r.cell)) - switch { - case errors.Is(err, pgx.ErrNoRows): - return archives.Record{}, false, nil - case err != nil: - return archives.Record{}, false, cas.Down(err) - } - return rec, true, nil + return cas.One(ctx, r.pool, scan, `SELECT id, tenant, token_sha256, owner, epoch, consumed, checkpoint, deadline, claim + FROM sandboxd_archives WHERE cell = $2 AND id = $1`, id, r.cell) } func (r *Registry) Sweepable(ctx context.Context, now time.Time, owner string) ([]archives.Record, error) { - return r.collect(ctx, `SELECT id, tenant, token_sha256, owner, epoch, consumed, checkpoint, deadline, NULL::jsonb + return cas.All(ctx, r.pool, scan, `SELECT id, tenant, token_sha256, owner, epoch, consumed, checkpoint, deadline, NULL::jsonb FROM sandboxd_archives WHERE cell = $2 AND (consumed OR owner = $3 OR deadline < $1 AND `+leaseExpired+`)`, now, r.cell, owner) } func (r *Registry) Rows(ctx context.Context, ids []string) (map[string]archives.Record, error) { - recs, err := r.collect(ctx, `SELECT id, tenant, token_sha256, owner, epoch, consumed, checkpoint, deadline, NULL::jsonb + recs, err := cas.All(ctx, r.pool, scan, `SELECT id, tenant, token_sha256, owner, epoch, consumed, checkpoint, deadline, NULL::jsonb FROM sandboxd_archives WHERE cell = $1 AND id = ANY($2)`, r.cell, ids) if err != nil { return nil, err @@ -164,20 +154,6 @@ func (r *Registry) Close() { r.pool.Close() } -func (r *Registry) collect(ctx context.Context, sql string, args ...any) ([]archives.Record, error) { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - rows, err := r.pool.Query(qctx, sql, args...) - if err != nil { - return nil, cas.Down(err) - } - recs, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (archives.Record, error) { return scan(row) }) - if err != nil { - return nil, cas.Down(err) - } - return recs, nil -} - func scan(row pgx.Row) (archives.Record, error) { var ( rec archives.Record diff --git a/sandboxd/metastore/cas.go b/sandboxd/metastore/cas.go index d43132145..4ead2f197 100644 --- a/sandboxd/metastore/cas.go +++ b/sandboxd/metastore/cas.go @@ -9,6 +9,8 @@ import ( "github.com/jackc/pgx/v5/pgxpool" ) +type ScanFunc[T any] func(pgx.Row) (T, error) + // CAS runs a registry's compare-and-set queries and answers with its sentinels. type CAS struct { Lost error @@ -44,6 +46,33 @@ func (c CAS) Delete(ctx context.Context, pool *pgxpool.Pool, sql string, args .. return nil } +func (c CAS) One[T any](ctx context.Context, pool *pgxpool.Pool, scan ScanFunc[T], sql string, args ...any) (T, bool, error) { + qctx, cancel := context.WithTimeout(ctx, QueryTimeout) + defer cancel() + v, err := scan(pool.QueryRow(qctx, sql, args...)) + switch { + case errors.Is(err, pgx.ErrNoRows): + return *new(T), false, nil + case err != nil: + return *new(T), false, c.Down(err) + } + return v, true, nil +} + +func (c CAS) All[T any](ctx context.Context, pool *pgxpool.Pool, scan ScanFunc[T], sql string, args ...any) ([]T, error) { + qctx, cancel := context.WithTimeout(ctx, QueryTimeout) + defer cancel() + rows, err := pool.Query(qctx, sql, args...) + if err != nil { + return nil, c.Down(err) + } + out, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (T, error) { return scan(row) }) + if err != nil { + return nil, c.Down(err) + } + return out, nil +} + // Down wraps err in Unavailable. func (c CAS) Down(err error) error { return fmt.Errorf("%w: %w", c.Unavailable, err) diff --git a/sandboxd/moves/pg/pg.go b/sandboxd/moves/pg/pg.go index beebc9b10..5ce7b9c9c 100644 --- a/sandboxd/moves/pg/pg.go +++ b/sandboxd/moves/pg/pg.go @@ -4,7 +4,6 @@ package pg import ( "context" _ "embed" - "errors" "time" "github.com/jackc/pgx/v5" @@ -59,17 +58,8 @@ func (r *Registry) Advance(ctx context.Context, id string, expect int64, from, t } func (r *Registry) Get(ctx context.Context, id string) (moves.Record, bool, error) { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - rec, err := scan(r.pool.QueryRow(qctx, `SELECT id, from_node, to_node, state, epoch, updated_at - FROM sandboxd_moves WHERE cell = $1 AND id = $2`, r.cell, id)) - switch { - case errors.Is(err, pgx.ErrNoRows): - return moves.Record{}, false, nil - case err != nil: - return moves.Record{}, false, cas.Down(err) - } - return rec, true, nil + return cas.One(ctx, r.pool, scan, `SELECT id, from_node, to_node, state, epoch, updated_at + FROM sandboxd_moves WHERE cell = $1 AND id = $2`, r.cell, id) } func (r *Registry) Delete(ctx context.Context, id string, epoch int64) error { @@ -77,18 +67,8 @@ func (r *Registry) Delete(ctx context.Context, id string, epoch int64) error { } func (r *Registry) Stale(ctx context.Context, before time.Time) ([]moves.Record, error) { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - rows, err := r.pool.Query(qctx, `SELECT id, from_node, to_node, state, epoch, updated_at + return cas.All(ctx, r.pool, scan, `SELECT id, from_node, to_node, state, epoch, updated_at FROM sandboxd_moves WHERE cell = $1 AND updated_at < $2`, r.cell, before) - if err != nil { - return nil, cas.Down(err) - } - recs, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (moves.Record, error) { return scan(row) }) - if err != nil { - return nil, cas.Down(err) - } - return recs, nil } // Close releases the connection pool. diff --git a/sandboxd/poolset/pg/pg.go b/sandboxd/poolset/pg/pg.go index cb75f27fe..e55cc3afa 100644 --- a/sandboxd/poolset/pg/pg.go +++ b/sandboxd/poolset/pg/pg.go @@ -20,8 +20,12 @@ const ( channel = "sandboxd_pool_sets" ) -//go:embed schema.sql -var schema string +var ( + //go:embed schema.sql + schema string + + cas = metastore.CAS{Unavailable: poolset.ErrUnavailable} +) var _ poolset.Shared = (*Source)(nil) @@ -54,7 +58,7 @@ func (s *Source) Load(ctx context.Context) (*poolset.Set, int64, error) { case errors.Is(err, pgx.ErrNoRows): return nil, 0, nil case err != nil: - return nil, 0, fmt.Errorf("%w: %w", poolset.ErrUnavailable, err) + return nil, 0, cas.Down(err) } if err = json.Unmarshal(raw, &set.Pools); err != nil { return nil, 0, fmt.Errorf("decode cell %q pool set: %w", s.cell, err) @@ -76,7 +80,7 @@ func (s *Source) Store(ctx context.Context, set poolset.Set) (int64, error) { version = sandboxd_pool_sets.version + 1, updated_at = now() RETURNING version) SELECT version, pg_notify($4, $1) FROM up`, s.cell, set.ConfigSeed, raw, channel).Scan(&version, nil); err != nil { - return 0, fmt.Errorf("%w: %w", poolset.ErrUnavailable, err) + return 0, cas.Down(err) } return version, nil } diff --git a/sandboxd/tenants/pg/pg.go b/sandboxd/tenants/pg/pg.go index ba21160fe..18a5bb44d 100644 --- a/sandboxd/tenants/pg/pg.go +++ b/sandboxd/tenants/pg/pg.go @@ -36,8 +36,12 @@ const ( uniqueViolation = "23505" ) -//go:embed schema.sql -var schema string +var ( + //go:embed schema.sql + schema string + + cas = metastore.CAS{Unavailable: tenants.ErrUnavailable} +) type entry struct { rec *tenants.TenantRecord @@ -104,13 +108,13 @@ func (s *Source) Resolve(ctx context.Context, sum [sha256.Size]byte) (*tenants.T case !s.cold.Allow(): return nil, fmt.Errorf("%w: cold lookups over %d/s", tenants.ErrUnavailable, coldLookups) } - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - r, err := scanRecord(s.pool.QueryRow(qctx, `SELECT name, token_sha256, max_claims, egress_class FROM sandboxd_tenants WHERE token_sha256 = $1`, sum[:])) + r, ok, err := cas.One(ctx, s.pool, scanRecord, `SELECT name, token_sha256, max_claims, egress_class FROM sandboxd_tenants WHERE token_sha256 = $1`, sum[:]) s.mu.Lock() defer s.mu.Unlock() switch { - case errors.Is(err, pgx.ErrNoRows): + case err != nil: + return nil, err + case !ok: if s.seq == start { if len(s.misses) >= maxCached { clear(s.misses) @@ -118,8 +122,6 @@ func (s *Source) Resolve(ctx context.Context, sum [sha256.Size]byte) (*tenants.T s.misses[sum] = time.Now().Add(negativeTTL) } return nil, tenants.ErrUnknown - case err != nil: - return nil, unavailable(err) } if s.fresh(r.Name, start) { s.cache(&r, sum) @@ -161,17 +163,10 @@ func (s *Source) Lookup(ctx context.Context, name string) (tenants.TenantRecord, } func (s *Source) List(ctx context.Context, after string, limit int) ([]tenants.TenantRecord, error) { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - rows, err := s.pool.Query(qctx, `SELECT name, max_claims, egress_class FROM sandboxd_tenants WHERE name > $1 ORDER BY name LIMIT $2`, after, limit) - if err != nil { - return nil, unavailable(err) - } - out, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (tenants.TenantRecord, error) { + return cas.All(ctx, s.pool, func(row pgx.Row) (tenants.TenantRecord, error) { var r tenants.TenantRecord return r, row.Scan(&r.Name, &r.MaxClaims, &r.EgressClass) - }) - return out, unavailable(err) + }, `SELECT name, max_claims, egress_class FROM sandboxd_tenants WHERE name > $1 ORDER BY name LIMIT $2`, after, limit) } func (s *Source) Put(ctx context.Context, r tenants.TenantRecord) (tenants.Change, error) { @@ -258,13 +253,13 @@ func (s *Source) Classes(ctx context.Context) (map[string]int, error) { defer cancel() rows, err := s.pool.Query(qctx, `SELECT egress_class, count(*) FROM sandboxd_tenants WHERE egress_class <> '' GROUP BY egress_class`) if err != nil { - return nil, unavailable(err) + return nil, cas.Down(err) } out := map[string]int{} var class string var n int if _, err = pgx.ForEachRow(rows, []any{&class, &n}, func() error { out[class] = n; return nil }); err != nil { - return nil, unavailable(err) + return nil, cas.Down(err) } return out, nil } @@ -304,16 +299,7 @@ func (s *Source) fetch(name string) { } func (s *Source) lookupNow(ctx context.Context, name string) (tenants.TenantRecord, bool, error) { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - r, err := scanRecord(s.pool.QueryRow(qctx, `SELECT name, token_sha256, max_claims, egress_class FROM sandboxd_tenants WHERE name = $1`, name)) - switch { - case errors.Is(err, pgx.ErrNoRows): - return tenants.TenantRecord{}, false, nil - case err != nil: - return tenants.TenantRecord{}, false, unavailable(err) - } - return r, true, nil + return cas.One(ctx, s.pool, scanRecord, `SELECT name, token_sha256, max_claims, egress_class FROM sandboxd_tenants WHERE name = $1`, name) } // write runs f in one transaction that notifies every node to evict key, then evicts it here at once. @@ -332,7 +318,7 @@ func (s *Source) write(ctx context.Context, key string, f func(tx pgx.Tx) error) if errors.Is(err, tenants.ErrInvalid) || errors.Is(err, tenants.ErrUnknown) { return err } - return unavailable(err) + return cas.Down(err) } s.evict(key) return nil @@ -420,10 +406,3 @@ func scanRecord(row pgx.Row) (tenants.TenantRecord, error) { r.TokenSHA256 = hex.EncodeToString(sum) return r, nil } - -func unavailable(err error) error { - if err == nil { - return nil - } - return fmt.Errorf("%w: %w", tenants.ErrUnavailable, err) -} From e95908f79ad3ab26e6ed620dd27eddc8dbdec42e Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:35:58 +0800 Subject: [PATCH 035/127] metastoretest: name the schema with rand.Text --- sandboxd/metastore/metastoretest/metastoretest.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sandboxd/metastore/metastoretest/metastoretest.go b/sandboxd/metastore/metastoretest/metastoretest.go index e4166739e..5726a7033 100644 --- a/sandboxd/metastore/metastoretest/metastoretest.go +++ b/sandboxd/metastore/metastoretest/metastoretest.go @@ -4,7 +4,6 @@ package metastoretest import ( "context" "crypto/rand" - "encoding/hex" "os" "strings" "testing" @@ -19,9 +18,7 @@ func PGSchema(t *testing.T) string { if url == "" { t.Skip("SANDBOXD_TEST_PG_URL unset") } - raw := make([]byte, 6) - _, _ = rand.Read(raw) - name := "t_" + hex.EncodeToString(raw) + name := "t_" + strings.ToLower(rand.Text()) exec := func(ctx context.Context, sql string) error { conn, err := pgx.Connect(ctx, url) if err != nil { From 554eab213e878a360fea686f25cf7e5719c58aee Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:35:58 +0800 Subject: [PATCH 036/127] movestest: rename Contract to Run like archivestest --- sandboxd/moves/movestest/movestest.go | 4 ++-- sandboxd/moves/movestest/movestest_test.go | 2 +- sandboxd/moves/pg/pg_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sandboxd/moves/movestest/movestest.go b/sandboxd/moves/movestest/movestest.go index 46cb3c3f6..c16e28312 100644 --- a/sandboxd/moves/movestest/movestest.go +++ b/sandboxd/moves/movestest/movestest.go @@ -115,8 +115,8 @@ func (m *Mem) write(rec moves.Record) int64 { return rec.Epoch } -// Contract checks the behavior every backend shares. -func Contract(t *testing.T, open Opener) { +// Run checks the behavior every backend shares. +func Run(t *testing.T, open Opener) { t.Run("begin get", func(t *testing.T) { testBeginGet(t, open(t)) }) t.Run("every allowed advance", func(t *testing.T) { testAdvance(t, open(t)) }) t.Run("advance is a compare-and-set", func(t *testing.T) { testAdvanceFences(t, open(t)) }) diff --git a/sandboxd/moves/movestest/movestest_test.go b/sandboxd/moves/movestest/movestest_test.go index 9c1d74dfb..8d9977c16 100644 --- a/sandboxd/moves/movestest/movestest_test.go +++ b/sandboxd/moves/movestest/movestest_test.go @@ -7,5 +7,5 @@ import ( ) func TestMemContract(t *testing.T) { - Contract(t, func(*testing.T) moves.Registry { return NewMem() }) + Run(t, func(*testing.T) moves.Registry { return NewMem() }) } diff --git a/sandboxd/moves/pg/pg_test.go b/sandboxd/moves/pg/pg_test.go index 528c2ba86..8c15d19e8 100644 --- a/sandboxd/moves/pg/pg_test.go +++ b/sandboxd/moves/pg/pg_test.go @@ -11,7 +11,7 @@ import ( ) func TestRegistryContract(t *testing.T) { - movestest.Contract(t, func(t *testing.T) moves.Registry { return open(t, metastoretest.PGSchema(t), "c1") }) + movestest.Run(t, func(t *testing.T) moves.Registry { return open(t, metastoretest.PGSchema(t), "c1") }) } func TestCellsKeepTheirOwnRecords(t *testing.T) { From 73dbd8a921d5e061dbdbc0bea656a200388fb2ae Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:09 +0800 Subject: [PATCH 037/127] ca: drop the unreachable second exists check behind writeCAFiles' stat --- sandboxd/ca.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/sandboxd/ca.go b/sandboxd/ca.go index 64df934d7..3d388bcf1 100644 --- a/sandboxd/ca.go +++ b/sandboxd/ca.go @@ -107,9 +107,6 @@ func writeKeyMaterial(path string, data []byte, perm os.FileMode, force bool) er flags |= os.O_EXCL } f, err := os.OpenFile(path, flags, perm) //nolint:gosec // operator-chosen -out path - if errors.Is(err, os.ErrExist) { - return fmt.Errorf("%s exists; refusing to overwrite (use -force)", path) - } if err != nil { return err } From d642750f39a9b855abac6740dfa4785969f22c7e Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:24 +0800 Subject: [PATCH 038/127] server: normalize the preview base once in NewPreviewServer --- sandboxd/config/config.go | 2 +- sandboxd/server/preview.go | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 188900813..64c14c220 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -954,7 +954,7 @@ func validateShards(names []string, field, kind string) error { return nil } -// namesHost reads addr the way Mint does and reports whether it carries a host a client could dial. +// namesHost reads addr the way NewPreviewServer does and reports whether it carries a host a client could dial. func namesHost(addr string) bool { if !strings.Contains(addr, "://") { addr = "http://" + addr diff --git a/sandboxd/server/preview.go b/sandboxd/server/preview.go index 683dda87b..8e97a2cba 100644 --- a/sandboxd/server/preview.go +++ b/sandboxd/server/preview.go @@ -47,7 +47,10 @@ type PreviewServer struct { // NewPreviewServer serves preview URLs signed with secret and minted under base. func NewPreviewServer(secret, base, owner string, mgr PreviewManager) *PreviewServer { - p := &PreviewServer{secret: []byte(secret), base: base, owner: owner, mgr: mgr} + if !strings.Contains(base, "://") { + base = "http://" + base + } + p := &PreviewServer{secret: []byte(secret), base: strings.TrimRight(base, "/"), owner: owner, mgr: mgr} p.transport = &http.Transport{ // A pooled guest conn skips PreviewDial, and with it wakeResolved and the Transition lock DisableKeepAlives: true, @@ -72,11 +75,7 @@ func (p *PreviewServer) Mint(id string, port uint16, ttl time.Duration) string { payload, _ := json.Marshal(claims) enc := base64.RawURLEncoding.EncodeToString(payload) token := enc + "." + p.sign(enc) - base := p.base - if !strings.Contains(base, "://") { - base = "http://" + base - } - return fmt.Sprintf("%s/p/%s/", strings.TrimRight(base, "/"), token) + return fmt.Sprintf("%s/p/%s/", p.base, token) } // Handler serves preview requests and health checks on preview_listen. From 0fda44e3ce501aee26eda442fc05a725c3d5d9c1 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:34 +0800 Subject: [PATCH 039/127] server: build the exec frame with slices.Concat --- sandboxd/server/exec.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sandboxd/server/exec.go b/sandboxd/server/exec.go index a48c936ae..d7bcc2f6f 100644 --- a/sandboxd/server/exec.go +++ b/sandboxd/server/exec.go @@ -7,6 +7,7 @@ import ( "io" "net" "net/http" + "slices" "strings" "time" @@ -76,10 +77,8 @@ func (s *Server) handleExec(w http.ResponseWriter, r *http.Request) { if s.mgr.AuditEnabled() { s.mgr.Audit(ctx, id, frame) } - frame = append(frame, '\n') stdinClose, _ := wire.EncodeRequest(wire.StdinClose{}) - frame = append(frame, stdinClose...) - frame = append(frame, '\n') + frame = slices.Concat(frame, []byte{'\n'}, stdinClose, []byte{'\n'}) if _, err := guest.Write(frame); err != nil { writeErr(w, http.StatusBadGateway, "guest agent unreachable") return From 7bf6a31d680000cece1ab26365132ba894933b4a Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:37:09 +0800 Subject: [PATCH 040/127] server: fold relay registration, deadline reset, keepalive and the 101 into openRelay --- sandboxd/server/reader.go | 8 +------- sandboxd/server/relay.go | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/sandboxd/server/reader.go b/sandboxd/server/reader.go index c6abe8caa..12dd40424 100644 --- a/sandboxd/server/reader.go +++ b/sandboxd/server/reader.go @@ -11,7 +11,6 @@ import ( "net/http" "slices" "sync/atomic" - "time" "github.com/projecteru2/core/log" @@ -195,16 +194,11 @@ func (s *Server) serveSleeping(ctx context.Context, w http.ResponseWriter, id, t if !ok { return } - release, ok := s.trackRelay(client, io.NopCloser(nil)) + release, ok := s.openRelay(client, io.NopCloser(nil), switchingProtocols) if !ok { return } defer release() - _ = client.SetDeadline(time.Time{}) - keepAlive(client) - if _, err := client.Write(switchingProtocols); err != nil { - return - } rt := &router{s: s, ctx: ctx, id: id, token: token, client: client, br: bufio.NewReaderSize(clientReader(client, clientBuf), routeBuf)} guest, done, ok := rt.run() if !ok { diff --git a/sandboxd/server/relay.go b/sandboxd/server/relay.go index 7193e3118..e752bf146 100644 --- a/sandboxd/server/relay.go +++ b/sandboxd/server/relay.go @@ -109,23 +109,16 @@ func (s *Server) audited(ctx context.Context, id string, clientR io.Reader) io.R } func (s *Server) splice(client net.Conn, clientR io.Reader, guest net.Conn, hello []byte, ends relayEnds) { - release, ok := s.trackRelay(client, guest) + release, ok := s.openRelay(client, guest, hello) if !ok { return } defer release() - - // The http server may have armed a header-read deadline on this conn - _ = client.SetDeadline(time.Time{}) - keepAlive(client) - if _, err := client.Write(hello); err != nil { - return - } copyBoth(client, clientR, guest, ends, s.noSplice) } -// trackRelay registers a hijacked pair for CloseRelays; !ok means the server is already draining. -func (s *Server) trackRelay(client net.Conn, guest io.Closer) (func(), bool) { +// openRelay registers a hijacked pair for CloseRelays and sends hello; !ok means the server is draining or the client is gone. +func (s *Server) openRelay(client net.Conn, guest io.Closer, hello []byte) (func(), bool) { s.relayMu.Lock() if s.relayClosed { s.relayMu.Unlock() @@ -136,14 +129,22 @@ func (s *Server) trackRelay(client net.Conn, guest io.Closer) (func(), bool) { s.relays[client] = guest s.relayWG.Add(1) s.relayMu.Unlock() - return func() { + release := func() { _ = client.Close() _ = guest.Close() s.relayMu.Lock() delete(s.relays, client) s.relayMu.Unlock() s.relayWG.Done() - }, true + } + // The http server may have armed a header-read deadline on this conn + _ = client.SetDeadline(time.Time{}) + keepAlive(client) + if _, err := client.Write(hello); err != nil { + release() + return nil, false + } + return release, true } func (s *Server) retargetRelay(client net.Conn, peer io.Closer) { From eb47d3b2188bc191fff2b0eb6ace777371ddff2a Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:37:32 +0800 Subject: [PATCH 041/127] server: answer a down tenant store through writePoolErr --- sandboxd/server/server.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index a5b1740c4..fa7473325 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -918,8 +918,7 @@ func (s *Server) requireToken(next http.HandlerFunc) http.HandlerFunc { tenant, err := s.resolveScope(r) switch { case errors.Is(err, pool.ErrTenantStoreDown): - w.Header().Set("Retry-After", retryAfterSeconds) - writeErr(w, http.StatusServiceUnavailable, "tenant store down; retry") + writePoolErr(w, err) return case err != nil: writeErr(w, http.StatusUnauthorized, "invalid api token") From eb2624940a2f77208986fe8adee0f95af6602796 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:37:53 +0800 Subject: [PATCH 042/127] types: tag VolumePatch slices omitempty like the other slices --- sandboxd/types/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index 0d81b5407..218e0f546 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -32,8 +32,8 @@ type VolumeRegistration struct { // VolumePatch is the wire body of PATCH /v1/sandboxes/{id}/volumes. type VolumePatch struct { - Add []Volume `json:"add,omitzero"` - Remove []string `json:"remove,omitzero"` + Add []Volume `json:"add,omitempty"` + Remove []string `json:"remove,omitempty"` } // VolumePhase is how far one held volume has converged. From 15be936afbb1b56b4ad938ce0d9306cc9148059b Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:38:00 +0800 Subject: [PATCH 043/127] usage: drop read's godoc that restates the loop --- sandboxd/usage/ship.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sandboxd/usage/ship.go b/sandboxd/usage/ship.go index 207941d3e..348d1ff83 100644 --- a/sandboxd/usage/ship.go +++ b/sandboxd/usage/ship.go @@ -145,7 +145,6 @@ func (s *shipper) locate(ctx context.Context, live, backup *os.File) (*os.File, return found, s.at.end } -// read decodes up to shipBatch lines of files, the first from off, into s.node's next records. func (s *shipper) read(ctx context.Context, files []*os.File, off int64) ([]Record, anchor, int) { var ( recs []Record From cc53aaa98c649438926ae7a0585650a70c340ae0 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:38:38 +0800 Subject: [PATCH 044/127] outbound: drop nil checks on Flow, TunBytes and TunState that outboundOptions always sets --- sandboxd/outbound/outbound.go | 2 +- sandboxd/outbound/tun.go | 21 +++++---------------- 2 files changed, 6 insertions(+), 17 deletions(-) diff --git a/sandboxd/outbound/outbound.go b/sandboxd/outbound/outbound.go index 8443a79f3..697dea90d 100644 --- a/sandboxd/outbound/outbound.go +++ b/sandboxd/outbound/outbound.go @@ -248,7 +248,7 @@ func (h *Host) Reloaded() { } for _, t := range off { h.dropTun(t) - h.tunDown(t.sb.ID, h.tunState(t.sb).Sock) + h.tunDown(t.sb.ID, h.o.TunState(t.sb).Sock) } } diff --git a/sandboxd/outbound/tun.go b/sandboxd/outbound/tun.go index d34d93b74..fcd2ad2d8 100644 --- a/sandboxd/outbound/tun.go +++ b/sandboxd/outbound/tun.go @@ -91,7 +91,7 @@ func (h *Host) armTun(ctx context.Context, sb *types.Sandbox, d *doors, v *View, if h.o.Router == nil || sock == "" { return } - st := h.tunState(sb) + st := h.o.TunState(sb) if !v.Tun(key) || !st.On || !st.Addr.IsValid() { if v.Tun(key) || st.Addr.IsValid() { h.tunDown(id, sock) @@ -111,7 +111,7 @@ func (h *Host) armTun(ctx context.Context, sb *types.Sandbox, d *doors, v *View, tc := tunnet.Claim{ ID: id, Addr: t.addr, - Enabled: func() bool { return h.o.View().Tun(key) && h.tunLive(id, t) && h.tunState(sb).On }, + Enabled: func() bool { return h.o.View().Tun(key) && h.tunLive(id, t) && h.o.TunState(sb).On }, Profile: profile, Allow: func(dst netip.Addr, port uint16) bool { if c.current().EvalFlow(port) != egress.DecisionAllow { @@ -124,9 +124,7 @@ func (h *Host) armTun(ctx context.Context, sb *types.Sandbox, d *doors, v *View, Used: t.used, Touch: sb.TouchAt, Record: func(r tunnet.FlowRecord) { - if h.o.Flow != nil { - h.o.Flow(evCtx, id, tenant, profile, r) - } + h.o.Flow(evCtx, id, tenant, profile, r) if usage && r.Allowed && r.Up+r.Down > 0 { dst := netip.AddrPortFrom(r.Dst, r.Port).String() h.o.Transfer(evCtx, id, tenant, egress.Event{Host: dst, Port: r.Port, Upstream: "tun/" + cmp.Or(profile, "default")}, r.Up, r.Down) @@ -140,7 +138,7 @@ func (h *Host) armTun(ctx context.Context, sb *types.Sandbox, d *doors, v *View, d.tun = t } h.mu.Unlock() - if live && !h.tunState(sb).On { + if live && !h.o.TunState(sb).On { h.mu.Lock() mine := d.tun == t if mine { @@ -189,13 +187,6 @@ func (h *Host) serveTun(ctx context.Context, t *tunDoor) { } } -func (h *Host) tunState(sb *types.Sandbox) TunState { - if h.o.TunState == nil { - return TunState{On: !sb.TunOff, Addr: sb.TunAddr, Profile: sb.TunProfile, Bytes: sb.TunBytes, Sock: sb.VsockSocket} - } - return h.o.TunState(sb) -} - func (h *Host) tunDown(id, sock string) { go func() { ctx, cancel := context.WithTimeout(context.Background(), tunVerbTimeout) @@ -220,7 +211,5 @@ func (h *Host) dropTun(t *tunDoor) { } _ = t.ln.Close() h.o.Router.Disarm(t.addr) - if h.o.TunBytes != nil { - h.o.TunBytes(t.sb, t.used.Load()) - } + h.o.TunBytes(t.sb, t.used.Load()) } From dca01a95dfb4a6fffe14aa715e2e5f5f1723ed39 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:39:00 +0800 Subject: [PATCH 045/127] tunnet: fold adjust's carry with fold --- sandboxd/tunnet/packet.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/sandboxd/tunnet/packet.go b/sandboxd/tunnet/packet.go index 3c6eb1c8f..f289bdd78 100644 --- a/sandboxd/tunnet/packet.go +++ b/sandboxd/tunnet/packet.go @@ -260,13 +260,10 @@ func rewrite(frame []byte, at int, to [4]byte) { // adjust moves a ones'-complement sum from one address to another (RFC 1624). func adjust(sum uint16, from, to [4]byte) uint16 { - s := uint32(sum) + s := uint64(sum) for i := 0; i < 4; i += 2 { - s += uint32(^binary.BigEndian.Uint16(from[i:])) - s += uint32(binary.BigEndian.Uint16(to[i:])) + s += uint64(^binary.BigEndian.Uint16(from[i:])) + s += uint64(binary.BigEndian.Uint16(to[i:])) } - for s>>16 != 0 { - s = s&0xffff + s>>16 - } - return uint16(s) + return fold(s) } From 4eaada4b31b78e21619f639875b8d217043bb8b7 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:39:02 +0800 Subject: [PATCH 046/127] tunnet: drop the unreferenced OffloadUSO6 --- sandboxd/tunnet/packet.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sandboxd/tunnet/packet.go b/sandboxd/tunnet/packet.go index f289bdd78..ecc619d22 100644 --- a/sandboxd/tunnet/packet.go +++ b/sandboxd/tunnet/packet.go @@ -26,7 +26,6 @@ const ( OffloadTSO6 = 0x04 OffloadTSOECN = 0x08 OffloadUSO4 = 0x20 - OffloadUSO6 = 0x40 // IPv4 protocols the tun path carries. ProtoICMP = 1 From 90f2e90230f0dafec6860eeb9daf7a8faa7b2ca4 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:39:14 +0800 Subject: [PATCH 047/127] egress, tunnet: drop comments that repeat a godoc or label a section --- sandboxd/egress/secrets.go | 1 - sandboxd/tunnet/packet.go | 1 - sandboxd/tunnet/router.go | 1 - 3 files changed, 3 deletions(-) diff --git a/sandboxd/egress/secrets.go b/sandboxd/egress/secrets.go index 06fc3a465..087e72dd5 100644 --- a/sandboxd/egress/secrets.go +++ b/sandboxd/egress/secrets.go @@ -11,7 +11,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -// nonInjectable lists the secret headers the transport owns or strips. var nonInjectable = slices.Concat([]string{"Host", "Content-Length"}, hopHeaders) // SecretSpec declares a credential the proxy injects, valued from the env ValueEnv names (a claim's host-only entry first, then the node's), or from the claim env Name when ValueEnv is empty. diff --git a/sandboxd/tunnet/packet.go b/sandboxd/tunnet/packet.go index ecc619d22..8bd8c7755 100644 --- a/sandboxd/tunnet/packet.go +++ b/sandboxd/tunnet/packet.go @@ -27,7 +27,6 @@ const ( OffloadTSOECN = 0x08 OffloadUSO4 = 0x20 - // IPv4 protocols the tun path carries. ProtoICMP = 1 ProtoTCP = 6 ProtoUDP = 17 diff --git a/sandboxd/tunnet/router.go b/sandboxd/tunnet/router.go index 7ac357daa..9bdcd983a 100644 --- a/sandboxd/tunnet/router.go +++ b/sandboxd/tunnet/router.go @@ -19,7 +19,6 @@ import ( ) const ( - // Drop reasons, one counter each. DropNone Drop = iota DropMalformed DropNotIPv4 From c6225c94048143d223481f8525497a609c10e858 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:39:42 +0800 Subject: [PATCH 048/127] outbound: read resolv.conf with os.ReadFile and strings.Lines --- sandboxd/outbound/dns.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/sandboxd/outbound/dns.go b/sandboxd/outbound/dns.go index c07f4c169..7ef462b23 100644 --- a/sandboxd/outbound/dns.go +++ b/sandboxd/outbound/dns.go @@ -1,7 +1,6 @@ package outbound import ( - "bufio" "cmp" "context" "encoding/binary" @@ -413,15 +412,10 @@ func writeDNSMsg(w io.Writer, msg []byte) error { // readNameservers reads path's nameserver lines as host:53 addresses, the loopback default when it names none. func readNameservers(path string) []string { - f, err := os.Open(path) //nolint:gosec // the node's resolver config - if err != nil { - return []string{"127.0.0.1:53"} - } - defer func() { _ = f.Close() }() + data, _ := os.ReadFile(path) //nolint:gosec // the node's resolver config var out []string - sc := bufio.NewScanner(f) - for sc.Scan() { - fields := strings.Fields(sc.Text()) + for line := range strings.Lines(string(data)) { + fields := strings.Fields(line) if len(fields) >= 2 && fields[0] == "nameserver" { if ip, err := netip.ParseAddr(fields[1]); err == nil { out = append(out, netip.AddrPortFrom(ip, 53).String()) From 97dcc69eb49759ac63cec3d9ee983d17ea320509 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:39:44 +0800 Subject: [PATCH 049/127] outbound: cancel the resolver at its one caller --- sandboxd/outbound/dns.go | 4 ---- sandboxd/outbound/doors.go | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/sandboxd/outbound/dns.go b/sandboxd/outbound/dns.go index 7ef462b23..bd693f390 100644 --- a/sandboxd/outbound/dns.go +++ b/sandboxd/outbound/dns.go @@ -100,10 +100,6 @@ func (r *resolver) serve(ln net.Listener, datagram bool) { } } -func (r *resolver) close() { - r.cancel() -} - func (r *resolver) serveConn(conn net.Conn, datagram bool) { defer func() { _ = conn.Close() }() stop := context.AfterFunc(r.ctx, func() { _ = conn.Close() }) diff --git a/sandboxd/outbound/doors.go b/sandboxd/outbound/doors.go index 3d28b0478..9afd52208 100644 --- a/sandboxd/outbound/doors.go +++ b/sandboxd/outbound/doors.go @@ -57,7 +57,7 @@ func (d *doors) close() { } } if d.resolver != nil { - d.resolver.close() + d.resolver.cancel() } } From 5fa2f4ee38967bcf34f35c72a4ae8b5c6eb29175 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:40:03 +0800 Subject: [PATCH 050/127] tunnet/pg: inline the single-caller scan and exec helpers --- sandboxd/tunnet/pg/pg.go | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/sandboxd/tunnet/pg/pg.go b/sandboxd/tunnet/pg/pg.go index 4feef9f68..d82f4768b 100644 --- a/sandboxd/tunnet/pg/pg.go +++ b/sandboxd/tunnet/pg/pg.go @@ -60,7 +60,9 @@ func (s *Store) Assign(ctx context.Context, claimID string) (netip.Addr, error) hosts := int64(1)<<(32-s.subnet.Bits()) - 2 for range assignTries { var addr string - err := s.scan(ctx, &addr, assignSQL, s.cell, claimID, s.subnet.Addr().String(), hosts, s.reuse.Seconds()) + qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) + err := s.pool.QueryRow(qctx, assignSQL, s.cell, claimID, s.subnet.Addr().String(), hosts, s.reuse.Seconds()).Scan(&addr) + cancel() switch { case err == nil: return netip.ParseAddr(addr) @@ -73,29 +75,19 @@ func (s *Store) Assign(ctx context.Context, claimID string) (netip.Addr, error) // Release stamps addr released; the stamp starts its reuse window. func (s *Store) Release(ctx context.Context, claimID string, addr netip.Addr) error { - return s.exec(ctx, `UPDATE sandboxd_tun_addrs SET released_at = now() - WHERE cell = $1 AND address = $2 AND claim = $3 AND released_at IS NULL`, s.cell, addr.String(), claimID) -} - -func (s *Store) Close() { - s.pool.Close() -} - -func (s *Store) scan(ctx context.Context, dest any, sql string, args ...any) error { - qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) - defer cancel() - return s.pool.QueryRow(qctx, sql, args...).Scan(dest) -} - -func (s *Store) exec(ctx context.Context, sql string, args ...any) error { qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) defer cancel() - if _, err := s.pool.Exec(qctx, sql, args...); err != nil { + if _, err := s.pool.Exec(qctx, `UPDATE sandboxd_tun_addrs SET released_at = now() + WHERE cell = $1 AND address = $2 AND claim = $3 AND released_at IS NULL`, s.cell, addr.String(), claimID); err != nil { return down(err) } return nil } +func (s *Store) Close() { + s.pool.Close() +} + func down(err error) error { return fmt.Errorf("tun address store: %w", err) } From faa33b9070cb152502c5d4fb7ad3238e9b03bd39 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:40:36 +0800 Subject: [PATCH 051/127] egress: intersect the pool and tenant decisions in one helper --- sandboxd/egress/policy.go | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/sandboxd/egress/policy.go b/sandboxd/egress/policy.go index db154fd9c..3a740eec7 100644 --- a/sandboxd/egress/policy.go +++ b/sandboxd/egress/policy.go @@ -238,39 +238,25 @@ func Compose(pool, tenant Policy) PolicyEvaluator { func (c composite) Eval(host, method string, port uint16) (Rule, Decision) { rule, pd := c.pool.Eval(host, method, port) - if pd == DecisionAllow { - if _, td := c.tenant.Eval(host, method, port); td == DecisionAllow { - return rule, DecisionAllow - } - } - return Rule{}, DecisionDeny + _, td := c.tenant.Eval(host, method, port) + return intersect(rule, pd, td) } func (c composite) EvalHost(host string, port uint16) (Rule, Decision) { rule, pd := c.pool.EvalHost(host, port) - if pd == DecisionAllow { - if _, td := c.tenant.EvalHost(host, port); td == DecisionAllow { - return rule, DecisionAllow - } - } - return Rule{}, DecisionDeny + _, td := c.tenant.EvalHost(host, port) + return intersect(rule, pd, td) } func (c composite) EvalInner(host, method string, port uint16) (Rule, Decision) { rule, pd := c.pool.EvalInner(host, method, port) - if pd == DecisionAllow { - if _, td := c.tenant.Eval(host, method, port); td == DecisionAllow { - return rule, DecisionAllow - } - } - return Rule{}, DecisionDeny + _, td := c.tenant.Eval(host, method, port) + return intersect(rule, pd, td) } func (c composite) EvalFlow(port uint16) Decision { - if c.pool.EvalFlow(port) == DecisionAllow && c.tenant.EvalFlow(port) == DecisionAllow { - return DecisionAllow - } - return DecisionDeny + _, d := intersect(Rule{}, c.pool.EvalFlow(port), c.tenant.EvalFlow(port)) + return d } // ServesSocks is the pool's call: the tenant's rules already gate every tunnel through Eval. @@ -285,3 +271,10 @@ func (c composite) ServesDNS() bool { func (c composite) InterceptsHost(pattern string) bool { return c.pool.InterceptsHost(pattern) && slices.ContainsFunc(c.tenant.Allow, func(r Rule) bool { return r.Covers(pattern) }) } + +func intersect(rule Rule, pd, td Decision) (Rule, Decision) { + if pd == DecisionAllow && td == DecisionAllow { + return rule, DecisionAllow + } + return Rule{}, DecisionDeny +} From fe47733a30f1e217e8d4c9bc83f22adbf4e00648 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:41:58 +0800 Subject: [PATCH 052/127] engine: map an unexpected silkd frame to its error in frameErr --- sandboxd/engine/silkd.go | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/sandboxd/engine/silkd.go b/sandboxd/engine/silkd.go index a5dc6e262..9b1e7c5ad 100644 --- a/sandboxd/engine/silkd.go +++ b/sandboxd/engine/silkd.go @@ -94,7 +94,7 @@ func (e *Engine) silkdStream(ctx context.Context, vsockSocket string, req wire.R case *wire.Done: return nil case *wire.ErrorResp: - return fmt.Errorf("silkd %w", resp) + return frameErr(resp) default: if err := onFrame(resp); err != nil { return err @@ -108,7 +108,7 @@ func (e *Engine) silkdCollect[F wire.Response, E any](ctx context.Context, vsock err := e.silkdStream(ctx, vsockSocket, req, func(frame wire.Response) error { resp, ok := frame.(F) if !ok { - return fmt.Errorf("unexpected silkd frame %q", frame.RespType()) + return frameErr(frame) } out = append(out, items(resp)...) if len(out) > limit { @@ -147,10 +147,7 @@ func (e *Engine) silkdOnce[T wire.Response](ctx context.Context, vsockSocket str if _, ok := frame.(T); ok { return nil } - if resp, ok := frame.(*wire.ErrorResp); ok { - return fmt.Errorf("silkd %w", resp) - } - return fmt.Errorf("unexpected silkd frame %q", frame.RespType()) + return frameErr(frame) } func (e *Engine) silkdWriteFile(ctx context.Context, vsockSocket, path string, mode uint32, data []byte) error { @@ -177,14 +174,10 @@ func (e *Engine) silkdWriteFile(ctx context.Context, vsockSocket, path string, m if err != nil { return errors.Join(sendErr, err) } - switch resp := frame.(type) { - case *wire.Done: + if _, ok := frame.(*wire.Done); ok { return nil - case *wire.ErrorResp: - return fmt.Errorf("silkd %w", resp) - default: - return fmt.Errorf("unexpected silkd frame %q", resp.RespType()) } + return frameErr(frame) } func (e *Engine) silkdExec(ctx context.Context, vsockSocket string, argv ...string) error { @@ -220,10 +213,8 @@ func (e *Engine) silkdOutput(ctx context.Context, vsockSocket string, argv ...st return nil, &guestExitError{code: resp.Code, output: string(bytes.TrimSpace(out))} } return stdout, nil - case *wire.ErrorResp: - return nil, fmt.Errorf("silkd %w", resp) default: - return nil, fmt.Errorf("unexpected silkd frame %q", resp.RespType()) + return nil, frameErr(resp) } } } @@ -234,3 +225,10 @@ func appendCapped(out, data []byte) []byte { } return out } + +func frameErr(frame wire.Response) error { + if resp, ok := frame.(*wire.ErrorResp); ok { + return fmt.Errorf("silkd %w", resp) + } + return fmt.Errorf("unexpected silkd frame %q", frame.RespType()) +} From b29a16a24de5a1adf0cea0862defb8b5ab63004a Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:42:19 +0800 Subject: [PATCH 053/127] engine: build clone and restore argv with slices.Concat --- sandboxd/engine/engine.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index 95873c923..5bbd0292f 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -14,6 +14,7 @@ import ( "os" "os/exec" "runtime" + "slices" "strconv" "strings" "sync" @@ -390,7 +391,7 @@ func (e *Engine) probe(ctx context.Context, vsockSocket string, port int, timeou } func (e *Engine) cloneArgs(name string, key types.PoolKey, source ...string) []string { - return append(append([]string{"vm", "clone"}, source...), e.cloneFlags(name, key, e.restoreArgs())...) + return slices.Concat([]string{"vm", "clone"}, source, e.cloneFlags(name, key, e.restoreArgs())) } func (e *Engine) cloneFlags(name string, key types.PoolKey, restore []string) []string { @@ -401,8 +402,7 @@ func (e *Engine) cloneFlags(name string, key types.PoolKey, restore []string) [] } func (e *Engine) restoreCmdArgs(vmName, snapRef string) []string { - args := append([]string{"vm", "restore", argOutput, formatJSON}, e.restoreArgs()...) - return append(args, vmName, snapRef) + return slices.Concat([]string{"vm", "restore", argOutput, formatJSON}, e.restoreArgs(), []string{vmName, snapRef}) } func (e *Engine) directIOArg() string { From 7960d3bbc45b95afab7380687bfee774a0565204 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:42:50 +0800 Subject: [PATCH 054/127] tunnet: join the freed-file close error with cmp.Or --- sandboxd/tunnet/addrs.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sandboxd/tunnet/addrs.go b/sandboxd/tunnet/addrs.go index 38bd075f9..177947db2 100644 --- a/sandboxd/tunnet/addrs.go +++ b/sandboxd/tunnet/addrs.go @@ -2,6 +2,7 @@ package tunnet import ( "bytes" + "cmp" "context" "encoding/binary" "errors" @@ -142,9 +143,7 @@ func (l *Local) Release(_ context.Context, claimID string, addr netip.Addr) erro return fmt.Errorf("open %s: %w", freedFile, err) } _, err = f.Write(appendFreed(nil, addr, now)) - if closeErr := f.Close(); err == nil { - err = closeErr - } + err = cmp.Or(err, f.Close()) if err != nil { return fmt.Errorf("append %s: %w", freedFile, err) } From 52e92760b576312f07c97fe2b608d4e06f9cb421 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:42:50 +0800 Subject: [PATCH 055/127] mesh: assign the secret key unconditionally --- sandboxd/mesh/mesh.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sandboxd/mesh/mesh.go b/sandboxd/mesh/mesh.go index 4a23002e5..6b58fa41c 100644 --- a/sandboxd/mesh/mesh.go +++ b/sandboxd/mesh/mesh.go @@ -57,9 +57,7 @@ func New(ctx context.Context, cfg *memberlist.Config, self NodeState, secretKey cfg.Name = self.NodeID cfg.Delegate = (*delegate)(m) cfg.Events = (*eventDelegate)(m) - if len(secretKey) > 0 { - cfg.SecretKey = secretKey - } + cfg.SecretKey = secretKey ml, err := memberlist.Create(cfg) if err != nil { return nil, fmt.Errorf("create memberlist: %w", err) From b0590a0ff0ca182468371ea07111166865b517d8 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:43:15 +0800 Subject: [PATCH 056/127] review: move DialFunc next to DialUpstream and lead probe.go's const block with ProbeHeader (exported above unexported) --- sandboxd/egress/proxy.go | 3 --- sandboxd/egress/upstream.go | 3 +++ sandboxd/store/peer/probe.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sandboxd/egress/proxy.go b/sandboxd/egress/proxy.go index 928cc859b..9e4666cc3 100644 --- a/sandboxd/egress/proxy.go +++ b/sandboxd/egress/proxy.go @@ -132,9 +132,6 @@ type connPools struct { routed map[routeKey]*http.Transport } -// DialFunc opens the upstream connection for a permitted request. -type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error) - // Proxy is one sandbox's forward proxy, gated by Policy and audited per request. type Proxy struct { policy Evaluator diff --git a/sandboxd/egress/upstream.go b/sandboxd/egress/upstream.go index 69841f07d..ea883a34d 100644 --- a/sandboxd/egress/upstream.go +++ b/sandboxd/egress/upstream.go @@ -27,6 +27,9 @@ type Router interface { Dial(ctx context.Context, route, network, addr string) (net.Conn, error) } +// DialFunc opens the upstream connection for a permitted request. +type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error) + // UpstreamAllow is the set of proxies a claim may route through: exact host names and IP prefixes. type UpstreamAllow struct { hosts []string diff --git a/sandboxd/store/peer/probe.go b/sandboxd/store/peer/probe.go index 1aa94c260..d30a03914 100644 --- a/sandboxd/store/peer/probe.go +++ b/sandboxd/store/peer/probe.go @@ -17,6 +17,9 @@ import ( ) const ( + // ProbeHeader carries the base64 probe MAC when a ProbeKey is configured. + ProbeHeader = "X-Cocoon-Probe" + probeTimeout = time.Second fanOutTimeout = 2 * time.Second maxRedirectOwners = 3 @@ -26,9 +29,6 @@ const ( // probeGrace bounds how long a fan-out waits for more owners once it has its first. probeGrace = 150 * time.Millisecond - // ProbeHeader carries the base64 probe MAC when a ProbeKey is configured. - ProbeHeader = "X-Cocoon-Probe" - // probeBucketSeconds is the coarse time step the probe MAC is bound to. probeBucketSeconds = 30 ) From 3d655b349a42b746816b412710f34e953424ec9d Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:34:46 +0800 Subject: [PATCH 057/127] pool: stamp audit frames with one generic wrapper --- sandboxd/pool/telemetry.go | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/sandboxd/pool/telemetry.go b/sandboxd/pool/telemetry.go index c339348e5..b9b3b5294 100644 --- a/sandboxd/pool/telemetry.go +++ b/sandboxd/pool/telemetry.go @@ -160,6 +160,12 @@ type tunFlowFrame struct { Profile string `json:"profile"` } +type stamped[F any] struct { + Time time.Time `json:"t"` + ID string `json:"id"` + Frame F `json:",embed"` +} + func (m *Manager) Counters() Counters { c := &m.counters warm, clone, cold := c.claimWarmLatency.snapshot(), c.claimCloneLatency.snapshot(), c.claimColdLatency.snapshot() @@ -225,11 +231,7 @@ func (m *Manager) recordAudit(ctx context.Context, id string, frame auditFrame) if m.audit == nil { return } - m.appendAudit(ctx, struct { - Time time.Time `json:"t"` - ID string `json:"id"` - auditFrame - }{time.Now(), id, frame}) + m.appendAudit(ctx, stamped[auditFrame]{time.Now(), id, frame}) } func (m *Manager) appendAudit(ctx context.Context, event any) { @@ -247,15 +249,10 @@ func (m *Manager) recordEgress(ctx context.Context, id, tenant string, ev egress decision = "rate_limited" } if m.audit != nil { - m.appendAudit(ctx, struct { - Time time.Time `json:"t"` - ID string `json:"id"` - egressFrame - }{ - Time: time.Now(), ID: id, + m.appendAudit(ctx, stamped[egressFrame]{time.Now(), id, egressFrame{ Op: egressEvent, Dest: ev.Host, Method: ev.Method, Port: ev.Port, Decision: decision, Secret: ev.Injected, Upstream: ev.Upstream, Tenant: tenant, Src: ev.Local, Dst: ev.Remote, - }) + }}) } m.recordUsage(ctx, usage.Event{Event: egressEvent, ID: id, Tenant: tenant, Reference: ev.Host, Upstream: ev.Upstream, Secret: ev.Injected}) } @@ -264,11 +261,7 @@ func (m *Manager) recordDNS(ctx context.Context, id, tenant string, ev outbound. if m.audit == nil { return } - m.appendAudit(ctx, struct { - Time time.Time `json:"t"` - ID string `json:"id"` - dnsFrame - }{time.Now(), id, dnsFrame{Op: dnsEvent, Name: ev.Name, Type: ev.Type, Answers: ev.Answers, Decision: ev.Decision, Tenant: tenant}}) + m.appendAudit(ctx, stamped[dnsFrame]{time.Now(), id, dnsFrame{Op: dnsEvent, Name: ev.Name, Type: ev.Type, Answers: ev.Answers, Decision: ev.Decision, Tenant: tenant}}) } func (m *Manager) recordTunFlow(ctx context.Context, id, tenant, profile string, r tunnet.FlowRecord) { @@ -279,15 +272,10 @@ func (m *Manager) recordTunFlow(ctx context.Context, id, tenant, profile string, if r.Allowed { decision = "allow" } - m.appendAudit(ctx, struct { - Time time.Time `json:"t"` - ID string `json:"id"` - tunFlowFrame - }{ - Time: r.End, ID: id, + m.appendAudit(ctx, stamped[tunFlowFrame]{r.End, id, tunFlowFrame{ Op: tunFlowEvent, Path: tunFlowPath, Tenant: tenant, Proto: protoName(r.Proto), Dst: netip.AddrPortFrom(r.Dst, r.Port), Decision: decision, Sent: r.Up, Received: r.Down, Start: r.Start, End: r.End, Profile: cmp.Or(profile, "default"), - }) + }}) } func (m *Manager) recordUsage(ctx context.Context, ev usage.Event) { From aba3c89f9fd1b3373ad43f0bf995be8ec045d527 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:35:50 +0800 Subject: [PATCH 058/127] pool: name the template visibility predicate once --- sandboxd/pool/pool.go | 4 ++++ sandboxd/pool/template.go | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 041b290ef..ddeeb43ea 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -750,6 +750,10 @@ func tenantOwns(tenant, owner string) bool { return tenant == "" || tenant == owner } +func templateVisible(tenant, owner string) bool { + return owner == "" || tenantOwns(tenant, owner) +} + func hasVolumes(sb *types.Sandbox) bool { return len(sb.Volumes) > 0 || len(sb.VolumesWant) > 0 } diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 243490655..40438bc75 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -195,7 +195,7 @@ func (m *Manager) HasPoolGolden(key types.PoolKey) bool { return ok && p.goldenDir != "" } -// HasPromotedTemplate applies resolveGolden's owner test, so routing does not promise another tenant's template. +// HasPromotedTemplate reports whether tenant may clone the promoted template held under key. func (m *Manager) HasPromotedTemplate(ctx context.Context, key types.PoolKey, tenant string) bool { if m.pooled(key) { return false @@ -217,7 +217,7 @@ func (m *Manager) HasPromotedTemplate(ctx context.Context, key types.PoolKey, te } owner = rec.Tenant } - return owner == "" || tenantOwns(tenant, owner) + return templateVisible(tenant, owner) } func (m *Manager) loadTemplates(ctx context.Context) { @@ -393,7 +393,7 @@ func (m *Manager) resolveGolden(ctx context.Context, key types.PoolKey, tenant s cleanup() return goldenResolution{}, fmt.Errorf("decode template metadata: %w", err) } - if rec.Tenant != "" && !tenantOwns(tenant, rec.Tenant) { + if !templateVisible(tenant, rec.Tenant) { cleanup() return goldenResolution{}, nil } From 5e5c378a994b7ec2404593f1883586f6e8aecd5e Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:35:50 +0800 Subject: [PATCH 059/127] pool: drop a promote comment that restates WithoutCancel --- sandboxd/pool/template.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index 40438bc75..83b45fdec 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -66,7 +66,6 @@ func (m *Manager) Promote(ctx context.Context, id string, cred Cred, template, t if err := m.checkTemplateOwner(ctx, store.TemplateID(key.Hash()), tenant); err != nil { return types.PoolKey{}, "", err } - // A started promote must finish uncanceled ctx = context.WithoutCancel(ctx) staging, err := m.tpls.Stage(store.TemplateID(key.Hash())) if err != nil { From a6062bec44b9022b9d6b95f67c72734d34813759 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:12 +0800 Subject: [PATCH 060/127] pool: drop a comment that restates the pools.json commit --- sandboxd/pool/setpools.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sandboxd/pool/setpools.go b/sandboxd/pool/setpools.go index 13363e408..539a560ec 100644 --- a/sandboxd/pool/setpools.go +++ b/sandboxd/pool/setpools.go @@ -87,7 +87,6 @@ func (m *Manager) applyPools(ctx context.Context, desired map[types.PoolKey]conf runCtx := context.WithoutCancel(ctx) m.destroyAll(runCtx, trim).Wait() m.refillOnce(runCtx) - // Persist the applied set so a restart rebuilds from it, not the config seed. return m.poolStore.commit(seq, poolset.Set{ConfigSeed: m.configSeedHash, Pools: slices.Collect(maps.Values(desired))}) } From 04b2b6d5caba9fc6f7e5be3f1eaa7da0f6908d33 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:37:19 +0800 Subject: [PATCH 061/127] pool: build a pool from its spec in one call --- sandboxd/pool/egress_test.go | 2 +- sandboxd/pool/pool.go | 4 +--- sandboxd/pool/poolstore.go | 4 +--- sandboxd/pool/promote_test.go | 2 +- sandboxd/pool/setpools.go | 4 +--- sandboxd/pool/watermark.go | 6 ++++-- 6 files changed, 9 insertions(+), 13 deletions(-) diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index f94f56641..13626a708 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -112,7 +112,7 @@ func TestEffectivePolicyKeepsTheLayerPinnedAtClaim(t *testing.T) { cfg.EgressClasses = []config.EgressClass{{Name: "desk", Egress: &egress.Policy{Allow: []egress.Rule{{Host: "b.test"}}}}} }) - m.pools = map[types.PoolKey]*pool{testKey: newPool(testKey)} + m.pools = map[types.PoolKey]*pool{testKey: newPool(config.PoolSpec{PoolKey: testKey})} if _, ok := m.view.Load().out.Resolve(&types.Sandbox{Key: testKey, Tenant: "acme", EgressClass: "desk", Layer: types.LayerUnpooled}); !ok { t.Error("a claim made on an unpooled key lost its tenant policy when the key gained a pool") } diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index ddeeb43ea..793c34730 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -466,9 +466,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg m.cfg = cfg m.view.Store(newConfigView(cfg, secrets)) for _, spec := range cfg.Pools { - p := newPool(spec.PoolKey) - p.applySpec(spec) - m.pools[spec.PoolKey] = p + m.pools[spec.PoolKey] = newPool(spec) } ca, caErr := outbound.LoadCA(cfg) if caErr != nil { diff --git a/sandboxd/pool/poolstore.go b/sandboxd/pool/poolstore.go index b510f4555..bac0eac82 100644 --- a/sandboxd/pool/poolstore.go +++ b/sandboxd/pool/poolstore.go @@ -109,9 +109,7 @@ func (m *Manager) adoptPools(ctx context.Context) error { if err := spec.ValidateLimits(); err != nil { return fmt.Errorf("restore pool %q from %s: %w", spec.Template, from, err) } - p := newPool(spec.PoolKey) - p.applySpec(spec) - m.pools[spec.PoolKey] = p + m.pools[spec.PoolKey] = newPool(spec) } logger.Infof(ctx, "restored %d API-applied pools from %s", len(set.Pools), from) return nil diff --git a/sandboxd/pool/promote_test.go b/sandboxd/pool/promote_test.go index ae47f8981..dea500ea1 100644 --- a/sandboxd/pool/promote_test.go +++ b/sandboxd/pool/promote_test.go @@ -261,7 +261,7 @@ func TestResolveGoldenNeverServesAPooledKeyFromATemplate(t *testing.T) { t.Fatalf("Promote: %v", err) } m.mu.Lock() - m.pools[key] = newPool(key) + m.pools[key] = newPool(config.PoolSpec{PoolKey: key}) m.mu.Unlock() golden, err := m.resolveGolden(t.Context(), key, "") if err != nil { diff --git a/sandboxd/pool/setpools.go b/sandboxd/pool/setpools.go index 539a560ec..f6b746856 100644 --- a/sandboxd/pool/setpools.go +++ b/sandboxd/pool/setpools.go @@ -78,9 +78,7 @@ func (m *Manager) applyPools(ctx context.Context, desired map[types.PoolKey]conf if p := m.pools[key]; p != nil { continue } - p := newPool(key) - p.applySpec(spec) - m.pools[key] = p + m.pools[key] = newPool(spec) } m.mu.Unlock() diff --git a/sandboxd/pool/watermark.go b/sandboxd/pool/watermark.go index b8f68592a..2b0534ab2 100644 --- a/sandboxd/pool/watermark.go +++ b/sandboxd/pool/watermark.go @@ -59,8 +59,10 @@ type pool struct { goldenRetiredAt time.Time } -func newPool(key types.PoolKey) *pool { - return &pool{key: key, hash: key.Hash()} +func newPool(spec config.PoolSpec) *pool { + p := &pool{key: spec.PoolKey, hash: spec.Hash()} + p.applySpec(spec) + return p } func (p *pool) settled() bool { From af140b549ee875505685e8be55cce32815f6684e Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:37:56 +0800 Subject: [PATCH 062/127] pool: state the pool lock rule once on the type --- sandboxd/pool/watermark.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sandboxd/pool/watermark.go b/sandboxd/pool/watermark.go index 2b0534ab2..5b5815d9c 100644 --- a/sandboxd/pool/watermark.go +++ b/sandboxd/pool/watermark.go @@ -24,6 +24,7 @@ const ( warmShrinkDwell = rateDecayTau ) +// pool fields are guarded by the manager mutex, except claimClones and the immutable key and hash. type pool struct { key types.PoolKey hash string @@ -120,7 +121,6 @@ func (p *pool) popWarm() *types.Sandbox { return sb } -// trimWarm shrinks p.warm to at most target and returns the trimmed VM names; callers hold m.mu. func (p *pool) trimWarm(target int) []string { var trim []string for len(p.warm) > target { @@ -131,7 +131,7 @@ func (p *pool) trimWarm(target int) []string { return trim } -// noteArrival counts one claim; a bin that has spanned rateBin folds its arrivals per second into the rate EWMA. Caller holds the manager mutex. +// noteArrival counts one claim; a bin that has spanned rateBin folds its arrivals per second into the rate EWMA. func (p *pool) noteArrival(now time.Time) { switch elapsed := now.Sub(p.binStart); { case p.binStart.IsZero(): @@ -145,7 +145,6 @@ func (p *pool) noteArrival(now time.Time) { p.lastArrival = now } -// noteLead folds one provision duration into the lead EWMA. Caller holds the manager mutex. func (p *pool) noteLead(d time.Duration) { if p.lead == 0 { p.lead = d @@ -154,7 +153,7 @@ func (p *pool) noteLead(d time.Duration) { p.lead = time.Duration(ewmaAlpha*float64(d) + (1-ewmaAlpha)*float64(p.lead)) } -// effectiveTarget raises the floor toward warmMax by rate × lead. Caller holds the manager mutex. +// effectiveTarget raises the floor toward warmMax by rate × lead. func (p *pool) effectiveTarget(now time.Time) int { if p.warmMax <= 0 { return p.floor @@ -173,7 +172,7 @@ func (p *pool) effectiveTarget(now time.Time) int { return max(p.floor, min(dynamic, p.warmMax)) } -// shrink trims the pool to its target once the warm count has stayed above it for warmShrinkDwell, returning the trimmed VM names. Caller holds the manager mutex. +// shrink trims the pool to its target once the warm count has stayed above it for warmShrinkDwell, returning the trimmed VM names. func (p *pool) shrink(now time.Time) []string { target := p.effectiveTarget(now) switch { From 00c11f4017732c3366ce881e149b7b946891ac15 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:38:15 +0800 Subject: [PATCH 063/127] pool: create the tun release retry map with the manager --- sandboxd/pool/pool.go | 1 + sandboxd/pool/tun.go | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 793c34730..431448479 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -420,6 +420,7 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg claimed: map[string]*types.Sandbox{}, claimedVMs: map[string]*types.Sandbox{}, pendingRemovals: map[string]pendingRemoval{}, + tunReleases: map[string]netip.Addr{}, moving: map[string]*outMove{}, incoming: map[string]*inMove{}, heldTenants: map[string]int{}, diff --git a/sandboxd/pool/tun.go b/sandboxd/pool/tun.go index 119ffa62b..f2b8dbaa1 100644 --- a/sandboxd/pool/tun.go +++ b/sandboxd/pool/tun.go @@ -140,9 +140,6 @@ func (m *Manager) releaseTun(ctx context.Context, id string, addr netip.Addr) { if err := m.tunAddrs.Release(context.WithoutCancel(ctx), id, addr); err != nil { log.WithFunc("pool.releaseTun").Warnf(ctx, "release overlay address %s of %s, retrying hourly: %v", addr, id, err) m.mu.Lock() - if m.tunReleases == nil { - m.tunReleases = map[string]netip.Addr{} - } m.tunReleases[id] = addr m.mu.Unlock() } From f7d2e00f205503d0d303ad3f7baa3a3e910e1d2e Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:38:24 +0800 Subject: [PATCH 064/127] pool: one logger in resolveMoveIn --- sandboxd/pool/move_in.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sandboxd/pool/move_in.go b/sandboxd/pool/move_in.go index a26aab64d..faf7cb611 100644 --- a/sandboxd/pool/move_in.go +++ b/sandboxd/pool/move_in.go @@ -225,6 +225,7 @@ func (m *Manager) settleMoveIn(ctx context.Context, sb *types.Sandbox) { // False while the registry is down. func (m *Manager) resolveMoveIn(ctx context.Context, sb *types.Sandbox) bool { + logger := log.WithFunc("pool.resolveMoveIn") sb.Transition.Lock() defer sb.Transition.Unlock() m.mu.Lock() @@ -235,7 +236,7 @@ func (m *Manager) resolveMoveIn(ctx context.Context, sb *types.Sandbox) bool { } rec, ok, err := m.moves.Get(ctx, sb.ID) if err != nil { - log.WithFunc("pool.resolveMoveIn").Warnf(ctx, "resolve moved claim %s, retrying: %v", sb.ID, err) + logger.Warnf(ctx, "resolve moved claim %s, retrying: %v", sb.ID, err) return false } toHere := ok && rec.To == m.nodeID @@ -247,14 +248,14 @@ func (m *Manager) resolveMoveIn(ctx context.Context, sb *types.Sandbox) bool { } vm, found, inspectErr := m.eng.Inspect(ctx, sb.VMName) if inspectErr != nil { - log.WithFunc("pool.resolveMoveIn").Warnf(ctx, "inspect moved claim %s, retrying: %v", sb.ID, inspectErr) + logger.Warnf(ctx, "inspect moved claim %s, retrying: %v", sb.ID, inspectErr) return false } m.startMoveIn(ctx, sb, rec.From, committed, found && vm.State != vmStateRunning) return true case toHere && (rec.State == moves.Pending || rec.State == moves.Final): if _, err = m.moves.Advance(ctx, sb.ID, rec.Epoch, rec.State, moves.Aborted); err != nil { - log.WithFunc("pool.resolveMoveIn").Warnf(ctx, "abort the move of %s, retrying: %v", sb.ID, err) + logger.Warnf(ctx, "abort the move of %s, retrying: %v", sb.ID, err) return false } } From 090036be7eb28c62a0e3ae35b814f49d26485773 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:39:07 +0800 Subject: [PATCH 065/127] pool: journal the move record with new(expr) --- sandboxd/pool/move_out.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index ac6823574..1f6354a51 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -213,8 +213,7 @@ func (m *Manager) moveBlocker(sb *types.Sandbox) string { } func (m *Manager) journalMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMove) error { - journal := mv.MoveOut - live, err := m.commitIfLive(ctx, sb, func() { sb.MoveOut = &journal }) + live, err := m.commitIfLive(ctx, sb, func() { sb.MoveOut = new(mv.MoveOut) }) if !live { return ErrUnknownSandbox } From 3fd32d86fa44a80722d7818baec913d04bfafbe5 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:39:35 +0800 Subject: [PATCH 066/127] pool: filter failed archive marks in the reap keep loop --- sandboxd/pool/reap.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/sandboxd/pool/reap.go b/sandboxd/pool/reap.go index 9ce89de44..e2eea3214 100644 --- a/sandboxd/pool/reap.go +++ b/sandboxd/pool/reap.go @@ -4,7 +4,6 @@ import ( "cmp" "context" "net/netip" - "slices" "time" "github.com/projecteru2/core/log" @@ -80,17 +79,14 @@ func (m *Manager) reapOnce(ctx context.Context) { logger := log.WithFunc("pool.reapOnce") failed := m.ckMarks.markAll(purges) - expired = slices.DeleteFunc(expired, func(v victim) bool { - markErr, bad := failed[v.ck] - if bad { - logger.Errorf(ctx, markErr, "mark archive ck %s; keeping %s", v.ck, v.id) - } - return bad - }) m.mu.Lock() keep := expired[:0] var purgeCks, purged []string for _, v := range expired { + if markErr, bad := failed[v.ck]; bad { + logger.Errorf(ctx, markErr, "mark archive ck %s; keeping %s", v.ck, v.id) + continue + } if v.action == reapPurge { if m.claimed[v.id] != v.sb || v.sb.ArchiveCk != v.ck { continue From a5bed27b3642a9fe30d01e9c778592d78aa2e823 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:41:18 +0800 Subject: [PATCH 067/127] pool: read the hypervisor statm under the manager proc root --- sandboxd/pool/stats.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sandboxd/pool/stats.go b/sandboxd/pool/stats.go index 1b41ee2fa..90b87d460 100644 --- a/sandboxd/pool/stats.go +++ b/sandboxd/pool/stats.go @@ -47,7 +47,6 @@ func (m *Manager) Stats(ctx context.Context, id string) (SandboxStats, bool) { return st, true } -// vmResidentBytes reports the resident set of the hypervisor process serving vmName. func (m *Manager) vmResidentBytes(ctx context.Context, vmName string) (int64, bool) { if vmName == "" { return 0, false @@ -56,11 +55,11 @@ func (m *Manager) vmResidentBytes(ctx context.Context, vmName string) (int64, bo if err != nil || !ok || vm.PID <= 0 { return 0, false } - return residentBytes(vm.PID) + return residentBytes(m.procDir, vm.PID) } -func residentBytes(pid int) (int64, bool) { - b, err := os.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "statm")) +func residentBytes(proc string, pid int) (int64, bool) { + b, err := os.ReadFile(filepath.Join(proc, strconv.Itoa(pid), "statm")) //nolint:gosec // an engine pid under the proc root if err != nil { return 0, false } From ce1a53bb004ee19242ee0090af862aa39764362f Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:41:25 +0800 Subject: [PATCH 068/127] pool: drop the removeStaleVM godoc that restates its signature --- sandboxd/pool/reconcile.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index f98fa1c26..d65cb5dcf 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -221,7 +221,6 @@ func (m *Manager) sweepStaleVMs(ctx context.Context, live map[string]types.VMRec return removed } -// removeStaleVM reclaims one unowned VM, reporting whether it is gone. func (m *Manager) removeStaleVM(ctx context.Context, name string, rec types.VMRecord) bool { logger := log.WithFunc("pool.removeStaleVM") if rec.State == vmStateCreating { From f1c31731466cb618aead5fe97ccf2b065a180437 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:35:43 +0800 Subject: [PATCH 069/127] silkd: borrow request fields in verbs that only read them --- silkd/src/find.rs | 6 ++--- silkd/src/fs.rs | 37 +++++++++++------------------ silkd/src/git.rs | 56 ++++++++++++++++++++++---------------------- silkd/src/server.rs | 46 +++++++++++++++++++++++------------- silkd/src/session.rs | 4 ++-- silkd/src/tree.rs | 12 +++++----- 6 files changed, 83 insertions(+), 78 deletions(-) diff --git a/silkd/src/find.rs b/silkd/src/find.rs index 55b4961d0..66e5e0c70 100644 --- a/silkd/src/find.rs +++ b/silkd/src/find.rs @@ -151,7 +151,7 @@ pub async fn find( w: &mut W, path: String, pattern: String, - glob: Option, + glob: Option<&str>, ) -> std::io::Result<()> where R: AsyncBufRead + Unpin, @@ -224,7 +224,7 @@ async fn find_bounded( w: &mut W, path: String, pattern: String, - glob: Option, + glob: Option<&str>, budget_bytes: usize, ) -> std::io::Result<()> where @@ -236,7 +236,7 @@ where Ok(re) => re, Err(e) => return proto::error_frame(w, ErrorKind::BadRequest, e).await, }; - let name_re = match glob.as_deref().filter(|g| !g.is_empty()).map(glob_regex) { + let name_re = match glob.filter(|g| !g.is_empty()).map(glob_regex) { None => None, Some(Ok(re)) => Some(re), Some(Err(e)) => return proto::error_frame(w, ErrorKind::BadRequest, e.to_string()).await, diff --git a/silkd/src/fs.rs b/silkd/src/fs.rs index ae28b38ba..58d7ee314 100644 --- a/silkd/src/fs.rs +++ b/silkd/src/fs.rs @@ -17,17 +17,12 @@ use crate::proto::{ pub const LIST_BATCH: usize = 4096; /// Streams client `data` frames into `path` via a temp file renamed only on clean completion. -pub async fn write( - reader: &mut R, - w: &mut W, - path: String, - mode: Option, -) -> io::Result<()> +pub async fn write(reader: &mut R, w: &mut W, path: &str, mode: Option) -> io::Result<()> where R: AsyncBufRead + Unpin, W: AsyncWrite + Unpin, { - let path = Path::new(&path); + let path = Path::new(path); let (tmp, mut file) = match create_tmp(path, mode).await { Ok(created) => created, Err(e) => return err_frame(w, &e, "create").await, @@ -129,8 +124,8 @@ pub async fn write_atomic(path: &Path, bytes: &[u8]) -> io::Result<()> { } /// Reports metadata for `path` (following symlinks). -pub async fn stat(w: &mut W, path: String) -> io::Result<()> { - let meta = match fs::metadata(&path).await { +pub async fn stat(w: &mut W, path: &str) -> io::Result<()> { + let meta = match fs::metadata(path).await { Ok(m) => m, Err(e) => return err_frame(w, &e, "stat").await, }; @@ -150,15 +145,11 @@ pub async fn stat(w: &mut W, path: String) -> io::Result< } /// Creates a directory, with parents when `parents` is set. -pub async fn mkdir( - w: &mut W, - path: String, - parents: bool, -) -> io::Result<()> { +pub async fn mkdir(w: &mut W, path: &str, parents: bool) -> io::Result<()> { let res = if parents { - fs::create_dir_all(&path).await + fs::create_dir_all(path).await } else { - fs::create_dir(&path).await + fs::create_dir(path).await }; match res { Ok(()) => proto::write_frame(w, &Response::Done).await, @@ -167,17 +158,17 @@ pub async fn mkdir( } /// Removes a file, or a directory (recursively when `recursive` is set). -pub async fn rm(w: &mut W, path: String, recursive: bool) -> io::Result<()> { - let meta = match fs::symlink_metadata(&path).await { +pub async fn rm(w: &mut W, path: &str, recursive: bool) -> io::Result<()> { + let meta = match fs::symlink_metadata(path).await { Ok(m) => m, Err(e) => return err_frame(w, &e, "rm").await, }; let res = if meta.is_dir() && recursive { - fs::remove_dir_all(&path).await + fs::remove_dir_all(path).await } else if meta.is_dir() { - fs::remove_dir(&path).await + fs::remove_dir(path).await } else { - fs::remove_file(&path).await + fs::remove_file(path).await }; match res { Ok(()) => proto::write_frame(w, &Response::Done).await, @@ -186,8 +177,8 @@ pub async fn rm(w: &mut W, path: String, recursive: bool) } /// Renames `from` to `to` within the guest filesystem. -pub async fn rename(w: &mut W, from: String, to: String) -> io::Result<()> { - match fs::rename(&from, &to).await { +pub async fn rename(w: &mut W, from: &str, to: &str) -> io::Result<()> { + match fs::rename(from, to).await { Ok(()) => proto::write_frame(w, &Response::Done).await, Err(e) => err_frame(w, &e, "rename").await, } diff --git a/silkd/src/git.rs b/silkd/src/git.rs index 0f6e0e778..67aa46c0a 100644 --- a/silkd/src/git.rs +++ b/silkd/src/git.rs @@ -17,34 +17,34 @@ const NO_EGRESS: &str = "no network on this sandbox; use fs.push to move a repos /// Clones `url` into `path` (optionally a branch, shallow depth), then Done. pub async fn clone( w: &mut W, - url: String, - path: String, - branch: Option, + url: &str, + path: &str, + branch: Option<&str>, depth: Option, - auth: Option, + auth: Option<&str>, ) -> std::io::Result<()> { if !crate::net::has_egress() { return proto::error_frame(w, ErrorKind::Unimplemented, NO_EGRESS).await; } let depth_arg; let mut args: Vec<&str> = vec!["clone"]; - if let Some(b) = &branch { + if let Some(b) = branch { args.extend(["--branch", b]); } if let Some(d) = depth { depth_arg = d.to_string(); args.extend(["--depth", &depth_arg]); } - args.push(&url); - args.push(&path); + args.push(url); + args.push(path); // clone has no repo dir yet, so run it from cwd (-C "."). - let out = git(".", auth.as_deref(), &args).await?; + let out = git(".", auth, &args).await?; terminal(w, "clone", &out).await } /// Reports branch, ahead/behind, and per-file status from porcelain v2. -pub async fn status(w: &mut W, path: String) -> std::io::Result<()> { - let out = git(&path, None, &["status", "--porcelain=v2", "--branch"]).await?; +pub async fn status(w: &mut W, path: &str) -> std::io::Result<()> { + let out = git(path, None, &["status", "--porcelain=v2", "--branch"]).await?; if !out.status.success() { return terminal(w, "status", &out).await; } @@ -54,35 +54,35 @@ pub async fn status(w: &mut W, path: String) -> std::io:: /// Stages `files` under `path`. pub async fn add( w: &mut W, - path: String, - files: Vec, + path: &str, + files: &[String], ) -> std::io::Result<()> { let mut args = vec!["add", "--"]; args.extend(files.iter().map(String::as_str)); - let out = git(&path, None, &args).await?; + let out = git(path, None, &args).await?; terminal(w, "add", &out).await } /// Commits staged changes and returns the new commit hash. pub async fn commit( w: &mut W, - path: String, - message: String, - author: String, + path: &str, + message: &str, + author: &str, ) -> std::io::Result<()> { // a fresh guest has no committer identity, so derive one from the author. - let (name, email) = split_author(&author); - let mut cmd = git_cmd(&path, None); + let (name, email) = split_author(author); + let mut cmd = git_cmd(path, None); cmd.env("GIT_COMMITTER_NAME", name) .env("GIT_COMMITTER_EMAIL", email); let out = cmd - .args(["commit", "--author", &author, "-m", &message]) + .args(["commit", "--author", author, "-m", message]) .output() .await?; if !out.status.success() { return terminal(w, "commit", &out).await; } - let rev = git(&path, None, &["rev-parse", "HEAD"]).await?; + let rev = git(path, None, &["rev-parse", "HEAD"]).await?; if !rev.status.success() { return terminal(w, "rev-parse", &rev).await; } @@ -93,26 +93,26 @@ pub async fn commit( /// Runs `push` or `pull` on the current branch behind the egress guard. pub async fn net_verb( w: &mut W, - path: String, - auth: Option, + path: &str, + auth: Option<&str>, verb: &str, ) -> std::io::Result<()> { if !crate::net::has_egress() { return proto::error_frame(w, ErrorKind::Unimplemented, NO_EGRESS).await; } - let out = git(&path, auth.as_deref(), &[verb]).await?; + let out = git(path, auth, &[verb]).await?; terminal(w, verb, &out).await } /// Lists, creates, deletes, or checks out a branch. pub async fn branch( w: &mut W, - path: String, + path: &str, op: GitBranchOp, - name: Option, + name: Option<&str>, ) -> std::io::Result<()> { - let args: Vec<&str> = match (op, &name) { - (GitBranchOp::List, _) => return list_branches(w, &path).await, + let args: Vec<&str> = match (op, name) { + (GitBranchOp::List, _) => return list_branches(w, path).await, (_, None) => { return proto::error_frame(w, ErrorKind::BadRequest, "branch op needs a name").await; } @@ -120,7 +120,7 @@ pub async fn branch( (GitBranchOp::Delete, Some(name)) => vec!["branch", "-D", name], (GitBranchOp::Checkout, Some(name)) => vec!["checkout", name], }; - let out = git(&path, None, &args).await?; + let out = git(path, None, &args).await?; terminal(w, "branch", &out).await } diff --git a/silkd/src/server.rs b/silkd/src/server.rs index a44297d9b..036f10438 100644 --- a/silkd/src/server.rs +++ b/silkd/src/server.rs @@ -86,7 +86,7 @@ impl State { Request::Attach { pid } => self.attach(&mut reader, writer, pid).await, Request::Exec(e) => return self.exec(e, reader, writer).await, Request::SessionCreate { id, cwd, env } => { - match self.sessions.create(id, cwd, &env).await { + match self.sessions.create(id, cwd.as_deref(), &env).await { Ok(id) => proto::write_frame(writer, &Response::SessionCreated { id }).await, Err(e) => { let kind = crate::exec::spawn_kind(&e); @@ -110,20 +110,20 @@ impl State { proto::error_frame(writer, ErrorKind::NotFound, "no such session").await } } - Request::FsWrite { path, mode } => fs::write(&mut reader, writer, path, mode).await, + Request::FsWrite { path, mode } => fs::write(&mut reader, writer, &path, mode).await, Request::FsRead { path } => fs::read(writer, path).await, Request::FsList { path } => fs::list(writer, path).await, - Request::FsStat { path } => fs::stat(writer, path).await, - Request::FsMkdir { path, parents } => fs::mkdir(writer, path, parents).await, - Request::FsRm { path, recursive } => fs::rm(writer, path, recursive).await, - Request::FsRename { from, to } => fs::rename(writer, from, to).await, - Request::FsPush { dest } => tree::push(&mut reader, writer, dest).await, - Request::FsPull { path } => tree::pull(writer, path).await, + Request::FsStat { path } => fs::stat(writer, &path).await, + Request::FsMkdir { path, parents } => fs::mkdir(writer, &path, parents).await, + Request::FsRm { path, recursive } => fs::rm(writer, &path, recursive).await, + Request::FsRename { from, to } => fs::rename(writer, &from, &to).await, + Request::FsPush { dest } => tree::push(&mut reader, writer, &dest).await, + Request::FsPull { path } => tree::pull(writer, &path).await, Request::FsFind { path, pattern, glob, - } => find::find(&mut reader, writer, path, pattern, glob).await, + } => find::find(&mut reader, writer, path, pattern, glob.as_deref()).await, Request::FsReplace { files, pattern, @@ -168,18 +168,32 @@ impl State { branch, depth, auth, - } => git::clone(writer, url, path, branch, depth, auth).await, - Request::GitStatus { path } => git::status(writer, path).await, - Request::GitAdd { path, files } => git::add(writer, path, files).await, + } => { + git::clone( + writer, + &url, + &path, + branch.as_deref(), + depth, + auth.as_deref(), + ) + .await + } + Request::GitStatus { path } => git::status(writer, &path).await, + Request::GitAdd { path, files } => git::add(writer, &path, &files).await, Request::GitCommit { path, message, author, - } => git::commit(writer, path, message, author).await, - Request::GitPush { path, auth } => git::net_verb(writer, path, auth, "push").await, - Request::GitPull { path, auth } => git::net_verb(writer, path, auth, "pull").await, + } => git::commit(writer, &path, &message, &author).await, + Request::GitPush { path, auth } => { + git::net_verb(writer, &path, auth.as_deref(), "push").await + } + Request::GitPull { path, auth } => { + git::net_verb(writer, &path, auth.as_deref(), "pull").await + } Request::GitBranch { path, action, name } => { - git::branch(writer, path, action, name).await + git::branch(writer, &path, action, name.as_deref()).await } Request::Stdin { .. } | Request::StdinClose diff --git a/silkd/src/session.rs b/silkd/src/session.rs index 7d12b1159..93b7e5666 100644 --- a/silkd/src/session.rs +++ b/silkd/src/session.rs @@ -31,7 +31,7 @@ impl Table { pub async fn create( &self, id: Option, - cwd: Option, + cwd: Option<&str>, env: &HashMap, ) -> io::Result { let id = match id { @@ -47,7 +47,7 @@ impl Table { let mut cmd = Command::new("bash"); cmd.env_clear().envs(sysutil::base_env().iter().copied()); sysutil::apply_claim_env(&mut cmd); - if let Some(dir) = &cwd { + if let Some(dir) = cwd { cmd.current_dir(dir); } let mut child = cmd diff --git a/silkd/src/tree.rs b/silkd/src/tree.rs index 497f1dba8..87688493b 100644 --- a/silkd/src/tree.rs +++ b/silkd/src/tree.rs @@ -21,28 +21,28 @@ const DIR_FLAGS: OFlag = OFlag::O_DIRECTORY .union(OFlag::O_CLOEXEC); /// Extracts a client tar stream into `dest`, creating it; a stream failure leaves `dest` unchanged. -pub async fn push(reader: &mut R, w: &mut W, dest: String) -> io::Result<()> +pub async fn push(reader: &mut R, w: &mut W, dest: &str) -> io::Result<()> where R: AsyncBufRead + Unpin, W: AsyncWrite + Unpin, { - if let Err(e) = tokio::fs::create_dir_all(&dest).await { + if let Err(e) = tokio::fs::create_dir_all(dest).await { return err_frame(w, &e, "mkdir dest").await; } let name = format!(".silkd-push-{}", sysutil::tmp_suffix()); - let (dest_fd, staging) = match stage_dir(&dest, &name) { + let (dest_fd, staging) = match stage_dir(dest, &name) { Ok(fds) => fds, Err(e) => return err_frame(w, &e, "stage push").await, }; let res = push_staged(reader, w, staging, dest_fd).await; // a failed push can leave a large partial tree, so cleanup runs on every exit path. - let _ = tokio::fs::remove_dir_all(Path::new(&dest).join(&name)).await; + let _ = tokio::fs::remove_dir_all(Path::new(dest).join(&name)).await; res } /// Streams `path` back as a tar carrying the entry under its own basename. -pub async fn pull(w: &mut W, path: String) -> io::Result<()> { - let p = Path::new(&path); +pub async fn pull(w: &mut W, path: &str) -> io::Result<()> { + let p = Path::new(path); let (parent, name) = match (p.parent(), p.file_name()) { (Some(par), Some(n)) => (par, n), _ => return proto::error_frame(w, ErrorKind::BadRequest, "invalid path").await, From b03e2604510eba238e878ad69a4266e039532c74 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:05 +0800 Subject: [PATCH 070/127] silkd: share take_pipe for every piped child stdio handle --- silkd/src/lsp.rs | 12 ++++++++---- silkd/src/session.rs | 9 ++------- silkd/src/sysutil.rs | 5 +++++ silkd/src/tree.rs | 15 +++++++-------- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/silkd/src/lsp.rs b/silkd/src/lsp.rs index de90636b2..6e8f2a5fc 100644 --- a/silkd/src/lsp.rs +++ b/silkd/src/lsp.rs @@ -64,10 +64,14 @@ impl Broker { Ok(c) => c, Err(e) => return proto::err_frame(w, &e, "spawn language server").await, }; - let Some(stdio) = child.stdout.take().zip(child.stdin.take()) else { - let _ = child.start_kill(); - return proto::error_frame(w, ErrorKind::Internal, "language server stdio not piped") - .await; + let stdio = sysutil::take_pipe(child.stdout.take(), "stdout") + .and_then(|out| Ok((out, sysutil::take_pipe(child.stdin.take(), "stdin")?))); + let stdio = match stdio { + Ok(stdio) => stdio, + Err(e) => { + let _ = child.start_kill(); + return proto::err_frame(w, &e, "spawn language server").await; + } }; let server = Server { child, diff --git a/silkd/src/session.rs b/silkd/src/session.rs index 93b7e5666..58a44eb62 100644 --- a/silkd/src/session.rs +++ b/silkd/src/session.rs @@ -60,8 +60,8 @@ impl Table { .kill_on_drop(true) .spawn()?; let pid = child.id().unwrap_or(0); - let stdin = take_pipe(child.stdin.take(), "stdin")?; - let stdout = take_pipe(child.stdout.take(), "stdout")?; + let stdin = sysutil::take_pipe(child.stdin.take(), "stdin")?; + let stdout = sysutil::take_pipe(child.stdout.take(), "stdout")?; let session = Arc::new(Session { pid, io: tokio::sync::Mutex::new(Io { @@ -278,11 +278,6 @@ async fn emit(out: &mut Option<&mut W>, frame: &mut Vec(pipe: Option, name: &str) -> io::Result { - pipe.ok_or_else(|| io::Error::other(format!("child {name} not piped"))) -} - /// POSIX single-quote quoting: wraps `s` in '…' onto `out`, reopening around embedded quotes. fn shell_quote_into(out: &mut String, s: &str) { out.push('\''); diff --git a/silkd/src/sysutil.rs b/silkd/src/sysutil.rs index 01927695c..9da88cb30 100644 --- a/silkd/src/sysutil.rs +++ b/silkd/src/sysutil.rs @@ -200,6 +200,11 @@ pub async fn wait_code(child: &mut tokio::process::Child) -> i32 { child.wait().await.map_or(-1, exit_code) } +/// Unwraps a piped child stdio handle; absent means the spawn contract broke. +pub fn take_pipe(pipe: Option, name: &str) -> std::io::Result { + pipe.ok_or_else(|| std::io::Error::other(format!("child {name} not piped"))) +} + /// Opens tun device `name` with a virtio-net header leading every packet. #[cfg(target_os = "linux")] pub fn tun_open(name: &str) -> std::io::Result { diff --git a/silkd/src/tree.rs b/silkd/src/tree.rs index 87688493b..7fd1b8ded 100644 --- a/silkd/src/tree.rs +++ b/silkd/src/tree.rs @@ -68,8 +68,9 @@ pub async fn pull(w: &mut W, path: &str) -> io::Result<() Ok(pair) => pair, Err(e) => return err_frame(w, &e, "spawn tar").await, }; - let Some(mut out) = child.stdout.take() else { - return err_frame(w, &io::Error::other("tar stdout not piped"), "spawn tar").await; + let mut out = match sysutil::take_pipe(child.stdout.take(), "stdout") { + Ok(out) => out, + Err(e) => return err_frame(w, &e, "spawn tar").await, }; if let Err(e) = proto::stream_data_frames(&mut out, w).await? { @@ -99,8 +100,9 @@ where Ok(pair) => pair, Err(e) => return err_frame(w, &e, "spawn tar").await, }; - let Some(mut sink) = child.stdin.take() else { - return err_frame(w, &io::Error::other("tar stdin not piped"), "spawn tar").await; + let mut sink = match sysutil::take_pipe(child.stdin.take(), "stdin") { + Ok(sink) => sink, + Err(e) => return err_frame(w, &e, "spawn tar").await, }; let feed = proto::feed_data_frames(reader, &mut sink).await; @@ -132,10 +134,7 @@ fn spawn_tar( mut cmd: Command, ) -> io::Result<(tokio::process::Child, tokio::task::JoinHandle>)> { let mut child = cmd.stderr(Stdio::piped()).kill_on_drop(true).spawn()?; - let stderr = child - .stderr - .take() - .ok_or_else(|| io::Error::other("tar stderr not piped"))?; + let stderr = sysutil::take_pipe(child.stderr.take(), "stderr")?; Ok((child, tokio::spawn(drain(stderr)))) } From 187ffa980c04e942c4f91d9799755b03d897b80c Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:11 +0800 Subject: [PATCH 071/127] review: declare server.rs type aliases above the types that use them --- silkd/src/server.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/silkd/src/server.rs b/silkd/src/server.rs index 036f10438..31927885c 100644 --- a/silkd/src/server.rs +++ b/silkd/src/server.rs @@ -11,6 +11,9 @@ use crate::proc::{Chunk, Table, write_chunk}; use crate::proto::{self, ErrorKind, ExecReq, ProcInfo, Request, Response}; use crate::{exec, find, forward, fs, git, lsp, pty, session, tree, tun, watch}; +type Parsed = serde_json::Result; +type Feeder = JoinHandle<(R, Option)>; + /// Shared daemon state handed to every connection. pub struct State { pub table: Table, @@ -343,9 +346,6 @@ impl Default for State { } } -type Parsed = serde_json::Result; -type Feeder = JoinHandle<(R, Option)>; - fn spawn_feeder(reader: R) -> (mpsc::Receiver, Feeder) where R: AsyncBufRead + Unpin + Send + 'static, From d0752b18b9205e7cfc51c3250fc000ae50ee1915 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:23 +0800 Subject: [PATCH 072/127] silkd: drop crate paths that an import already covers --- silkd/src/fs.rs | 3 ++- silkd/src/server.rs | 6 +++--- silkd/src/sysutil.rs | 3 +-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/silkd/src/fs.rs b/silkd/src/fs.rs index 58d7ee314..24e0265e2 100644 --- a/silkd/src/fs.rs +++ b/silkd/src/fs.rs @@ -12,6 +12,7 @@ use tokio::sync::mpsc; use crate::proto::{ self, BULK_CHUNK, DirEntry, ErrorKind, FileInfo, FileKind, Response, err_frame, }; +use crate::sysutil; /// Entries per `entries` frame; a worst-case 1.6KiB entry keeps a full batch under MAX_FRAME. pub const LIST_BATCH: usize = 4096; @@ -214,7 +215,7 @@ fn scan_dir(path: &str, tx: &mpsc::Sender>) -> io::Result<()> { fn tmp_name(path: &Path) -> PathBuf { let mut tmp = path.as_os_str().to_os_string(); tmp.push(".silkd-"); - tmp.push(crate::sysutil::tmp_suffix()); + tmp.push(sysutil::tmp_suffix()); tmp.push(".tmp"); PathBuf::from(tmp) } diff --git a/silkd/src/server.rs b/silkd/src/server.rs index 31927885c..1093f5552 100644 --- a/silkd/src/server.rs +++ b/silkd/src/server.rs @@ -9,7 +9,7 @@ use tokio::task::JoinHandle; use crate::proc::{Chunk, Table, write_chunk}; use crate::proto::{self, ErrorKind, ExecReq, ProcInfo, Request, Response}; -use crate::{exec, find, forward, fs, git, lsp, pty, session, tree, tun, watch}; +use crate::{exec, find, forward, fs, git, lsp, pty, session, sysutil, tree, tun, watch}; type Parsed = serde_json::Result; type Feeder = JoinHandle<(R, Option)>; @@ -92,7 +92,7 @@ impl State { match self.sessions.create(id, cwd.as_deref(), &env).await { Ok(id) => proto::write_frame(writer, &Response::SessionCreated { id }).await, Err(e) => { - let kind = crate::exec::spawn_kind(&e); + let kind = exec::spawn_kind(&e); proto::error_frame(writer, kind, format!("session create: {e}")).await } } @@ -278,7 +278,7 @@ impl State { }; // an exited pid may already be recycled, so signalling it would hit an unrelated process. if proc.exit_code().is_none() { - crate::sysutil::signal_group(proc.pid, signal.unwrap_or(libc::SIGKILL)); + sysutil::signal_group(proc.pid, signal.unwrap_or(libc::SIGKILL)); } proto::write_frame(w, &Response::Done).await } diff --git a/silkd/src/sysutil.rs b/silkd/src/sysutil.rs index 9da88cb30..0d39517b3 100644 --- a/silkd/src/sysutil.rs +++ b/silkd/src/sysutil.rs @@ -130,8 +130,7 @@ pub fn start_in(cmd: &mut Command, dir: &OwnedFd) { // SAFETY: fchdir is async-signal-safe and `dir` outlives the spawn that runs this hook. unsafe { cmd.pre_exec(move || { - nix::unistd::fchdir(std::os::fd::BorrowedFd::borrow_raw(fd)) - .map_err(std::io::Error::from) + nix::unistd::fchdir(BorrowedFd::borrow_raw(fd)).map_err(std::io::Error::from) }) }; } From d614421745252c978fea0c729515342835c4e957 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:34 +0800 Subject: [PATCH 073/127] silkd: cut comments that restate the code or its callee's doc --- silkd/src/exec.rs | 2 +- silkd/src/git.rs | 2 +- silkd/src/proto.rs | 2 +- silkd/src/server.rs | 1 - silkd/src/session.rs | 1 - 5 files changed, 3 insertions(+), 5 deletions(-) diff --git a/silkd/src/exec.rs b/silkd/src/exec.rs index 5d7ba6da1..b421fa22b 100644 --- a/silkd/src/exec.rs +++ b/silkd/src/exec.rs @@ -251,7 +251,7 @@ fn effective_path<'a>( }) } -/// Finds a bare program on path like execvp, minus its fall-through past an entry whose exec fails, so std can posix_spawn an absolute path; a relative entry leaves the search to execvp. +/// Resolves a bare program on an all-absolute `path`, so std can posix_spawn it. fn resolve_program(name: &str, path: Option<&str>) -> Option { if name.is_empty() || name.contains('/') { return None; diff --git a/silkd/src/git.rs b/silkd/src/git.rs index 67aa46c0a..1ec4cf828 100644 --- a/silkd/src/git.rs +++ b/silkd/src/git.rs @@ -37,7 +37,7 @@ pub async fn clone( } args.push(url); args.push(path); - // clone has no repo dir yet, so run it from cwd (-C "."). + // clone has no repo dir yet. let out = git(".", auth, &args).await?; terminal(w, "clone", &out).await } diff --git a/silkd/src/proto.rs b/silkd/src/proto.rs index 8dde830ac..f9f4da1a1 100644 --- a/silkd/src/proto.rs +++ b/silkd/src/proto.rs @@ -12,7 +12,7 @@ use tokio::io::{ AsyncBufRead, AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, }; -/// Mirrors cocoon-agent's frame cap so a malformed peer can't OOM us. +/// Every client rejects a larger frame, and the cap keeps a malformed peer from OOMing us. pub const MAX_FRAME: usize = 8 * 1024 * 1024; /// Read size for exec, pty and session output. diff --git a/silkd/src/server.rs b/silkd/src/server.rs index 1093f5552..56771b903 100644 --- a/silkd/src/server.rs +++ b/silkd/src/server.rs @@ -218,7 +218,6 @@ impl State { R: AsyncBufRead + Unpin + Send + 'static, W: AsyncWrite + Unpin, { - // validated ahead of the session/process split so an empty argv errors on both paths. if e.argv.is_empty() { proto::error_frame(writer, ErrorKind::BadRequest, "argv must not be empty").await?; return Ok((reader, None)); diff --git a/silkd/src/session.rs b/silkd/src/session.rs index 58a44eb62..957f985ba 100644 --- a/silkd/src/session.rs +++ b/silkd/src/session.rs @@ -89,7 +89,6 @@ impl Table { io.converse::(&init, None).await?; } - // reserve the id under the table lock so a concurrent create with the same id sees AlreadyExists. let mut map = sysutil::lock(&self.inner); if map.contains_key(&id) { return Err(io::Error::new( From 33b126fa7eaeb1a563b0aa3b94df2c733d851616 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:40 +0800 Subject: [PATCH 074/127] silkd: name the watch event queue depth --- silkd/src/watch.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/silkd/src/watch.rs b/silkd/src/watch.rs index c68594984..ae4decec8 100644 --- a/silkd/src/watch.rs +++ b/silkd/src/watch.rs @@ -7,8 +7,9 @@ use tokio::sync::mpsc; use crate::proto::{self, ErrorKind, EventKind, Response}; const OVERFLOW_MESSAGE: &str = "watch event queue overflow"; +const EVENT_QUEUE: usize = 256; -/// Events written per syscall; batching keeps the queue far from its 256-slot cap. +/// Events written per syscall; batching keeps the queue far from its cap. const EVENT_BATCH: usize = 64; /// Watches `path`, writing `ready`, ordered events, or a terminal error until disconnect. @@ -22,7 +23,7 @@ where R: AsyncBufRead + Unpin, W: AsyncWrite + Unpin, { - let (tx, mut rx) = mpsc::channel::(256); + let (tx, mut rx) = mpsc::channel::(EVENT_QUEUE); // arming walks the tree and adds a watch per directory, so it must not hold a runtime worker let armed = tokio::task::spawn_blocking(move || arm(tx, &path, recursive)).await; let _watcher = match armed { From e9cdf9692024a7f794b7d8bd164bd0401a2bb9ac Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:53 +0800 Subject: [PATCH 075/127] boot/init: zip record_serial's slots with ids like record_devices --- boot/init/src/boot.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/boot/init/src/boot.rs b/boot/init/src/boot.rs index 3bf563181..2ea72e700 100644 --- a/boot/init/src/boot.rs +++ b/boot/init/src/boot.rs @@ -248,9 +248,9 @@ fn record_devices(ids: &[&str], found: &mut [Option]) { } fn record_serial(ids: &[&str], found: &mut [Option], serial: &str, device: &str) { - for (i, id) in ids.iter().enumerate() { - if found[i].is_none() && *id == serial && Path::new(device).exists() { - found[i] = Some(device.into()); + for (slot, id) in found.iter_mut().zip(ids) { + if slot.is_none() && *id == serial && Path::new(device).exists() { + *slot = Some(device.into()); } } } From b3f42a294cbabd81bd747b03924aa918eb85fa45 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:53 +0800 Subject: [PATCH 076/127] boot/init: name the cmdline flag parser for both flags it reads --- boot/init/src/cfg.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/boot/init/src/cfg.rs b/boot/init/src/cfg.rs index 94784bf8d..4912e6497 100644 --- a/boot/init/src/cfg.rs +++ b/boot/init/src/cfg.rs @@ -61,8 +61,8 @@ pub fn parse(cmdline: &str) -> Result, String> { } } "cocoon.hostname" if !val.is_empty() => cfg.hostname = Some(val), - "sandbox.debug" => cfg.debug = debug_token(val), - "sandbox.trace" => cfg.trace = debug_token(val), + "sandbox.debug" => cfg.debug = flag_token(val), + "sandbox.trace" => cfg.trace = flag_token(val), "ip" => { if let Some(param) = parse_ip_param(val) { cfg.ips.push(param); @@ -85,7 +85,7 @@ pub fn parse(cmdline: &str) -> Result, String> { pub fn debug_requested(cmdline: &str) -> bool { params(cmdline) .rfind(|&(key, _)| key == "sandbox.debug") - .is_some_and(|(_, val)| debug_token(val)) + .is_some_and(|(_, val)| flag_token(val)) } /// Overlay mount data; index-based layer mountpoints keep a serial string out of lowerdir parsing. @@ -118,7 +118,7 @@ fn params(cmdline: &str) -> impl DoubleEndedIterator { .map(|tok| tok.split_once('=').unwrap_or((tok, ""))) } -fn debug_token(val: &str) -> bool { +fn flag_token(val: &str) -> bool { val.is_empty() || val == "1" } From 99f7062f770ba9a738cde98f4efee5692c69c5f5 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:39:58 +0800 Subject: [PATCH 077/127] sdk/python: close a client's parked control connections --- sdk/python/cocoonsandbox/client.py | 12 ++++++++++-- sdk/python/tests/test_client.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index bc088e425..0b40718fa 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -14,7 +14,7 @@ from typing import Any, TypeVar, cast from .checkpoint import Checkpoint -from .conn import Conn, dial_agent, is_quiet, remaining_timeout +from .conn import Conn, _Closeable, dial_agent, is_quiet, remaining_timeout from .endpoint import _endpoint_url from .errors import APIError, SandboxTimeout, _error_message, _require_field from .sandbox import Sandbox @@ -27,7 +27,7 @@ _IDLE_CONNS = 8 -class Client: +class Client(_Closeable): """Talks to one sandboxd node (and, transparently, its cluster).""" def __init__( @@ -146,6 +146,14 @@ def info(self) -> dict[str, Any]: """The node's pool/claim counters, as served by GET /v1/info.""" return self._request(self.addr, "GET", "/v1/info", None, "info") + def close(self) -> None: + """Closes the parked control connections; a later call dials afresh.""" + with self._idle_lock: + while self._idle: + _, idle = self._idle.popitem() + for conn, _ in idle: + conn.close() + def _claim_from( self, addr: str, diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 1572ace17..736a5fed0 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -344,6 +344,17 @@ def test_a_parked_connection_the_node_closed_is_not_reused(kept_node, fd_floor): assert KeptNode.conns == 2 +def test_close_releases_the_parked_connections(kept_node): + KeptNode.routes[("POST", "/v1/claim")] = lambda body, path: (200, {"id": "sb_1", "token": "tok"}) + with Client(kept_node) as client: + client.new("rt:24.04") + assert KeptNode.closed == 0 + wait_until(lambda: KeptNode.closed == 1, "close left the parked connection open") + client.close() + assert client.new("rt:24.04").id == "sb_1" + assert KeptNode.conns == 2 + + @pytest.fixture def kept_node(): server = ThreadingHTTPServer(("127.0.0.1", 0), KeptNode) From 8d22700218f9ca7577efe8cd3174f5cca42653be Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:40:16 +0800 Subject: [PATCH 078/127] sdk/openai: hand the claimed sandbox to the new session instead of a second client --- sdk/openai/cocoonsandbox_openai/adapter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sdk/openai/cocoonsandbox_openai/adapter.py b/sdk/openai/cocoonsandbox_openai/adapter.py index 0a2587e98..ca0ff7bcd 100644 --- a/sdk/openai/cocoonsandbox_openai/adapter.py +++ b/sdk/openai/cocoonsandbox_openai/adapter.py @@ -162,7 +162,9 @@ async def create( with contextlib.suppress(Exception): # the construction error is the one worth reading await asyncio.to_thread(sb.close) raise - return self._wrap_session(CocoonSandboxSession.from_state(state)) + session = CocoonSandboxSession.from_state(state) + session._sb = sb + return self._wrap_session(session) async def delete(self, session: SandboxSession) -> SandboxSession: inner = session._inner From 022abe9ccd00f8b0b139007531a200999bd9ce5b Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:41:02 +0800 Subject: [PATCH 079/127] sdk/python: keep module-private names underscored and the retry status set sorted --- sdk/python/cocoonsandbox/client.py | 14 +++++++------- sdk/python/cocoonsandbox/conn.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index 0b40718fa..c98672496 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -21,7 +21,7 @@ from .template import _claim_body, _template_query from .volumes import Volume -T = TypeVar("T") +_T = TypeVar("_T") _PEERS_TIMEOUT = 5.0 _IDLE_CONNS = 8 @@ -297,8 +297,8 @@ def _retry_miss(exc: APIError) -> bool: def _try_each( - candidates: Iterable[str], call: Callable[[str], T], retry: Callable[[APIError], bool] = _retry_miss -) -> T: + candidates: Iterable[str], call: Callable[[str], _T], retry: Callable[[APIError], bool] = _retry_miss +) -> _T: last_error = None for addr in candidates: try: @@ -311,7 +311,7 @@ def _try_each( def _retry_transient(exc: APIError) -> bool: - return exc.status in (0, 401, 404, 429, 503, 500) + return exc.status in (0, 401, 404, 429, 500, 503) def _claim_refused(exc: APIError) -> bool: @@ -341,8 +341,8 @@ def attempt(addr: str) -> tuple[str, dict[str, Any]]: raise APIError(verb, origin_exc.status, combined) from origin_exc -def _scatter(addrs: Sequence[str], probe: Callable[[str], T]) -> T: - results: queue.Queue[tuple[T | None, Exception | None]] = queue.Queue(maxsize=len(addrs)) +def _scatter(addrs: Sequence[str], probe: Callable[[str], _T]) -> _T: + results: queue.Queue[tuple[_T | None, Exception | None]] = queue.Queue(maxsize=len(addrs)) def run(addr: str) -> None: try: @@ -356,6 +356,6 @@ def run(addr: str) -> None: for _ in addrs: value, error = results.get() if error is None: - return cast(T, value) + return cast(_T, value) last_error = error raise cast(Exception, last_error) diff --git a/sdk/python/cocoonsandbox/conn.py b/sdk/python/cocoonsandbox/conn.py index e8d7a3ca3..697d8cf2f 100644 --- a/sdk/python/cocoonsandbox/conn.py +++ b/sdk/python/cocoonsandbox/conn.py @@ -15,7 +15,7 @@ from .errors import APIError, ProtocolError, SandboxError, SandboxTimeout, SilkdError, _error_message, _mark_gone from .frames import MAX_FRAME, decode_response, encode_request -KEEP_ALIVE_CONNS = 8 +_KEEP_ALIVE_CONNS = 8 _FD_SETSIZE = 1024 _READ_BUFFER = 64 * 1024 _TCP_KEEPALIVE = {"TCP_KEEPIDLE": 15, "TCP_KEEPALIVE": 15, "TCP_KEEPINTVL": 15, "TCP_KEEPCNT": 9} @@ -146,7 +146,7 @@ def take(self) -> Conn | None: def park(self, conn: Conn) -> None: with self._lock: - if self._idle > 0 and len(self._parked) < KEEP_ALIVE_CONNS: + if self._idle > 0 and len(self._parked) < _KEEP_ALIVE_CONNS: self._parked.append((time.monotonic() + self._idle, conn)) if self._sweep is None: self._arm(self._idle) From 8443ec4abbe3c1eebb67c3fb5f17f0884992b32b Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:41:02 +0800 Subject: [PATCH 080/127] sdk/python: drop the endpoint scheme parameter only tests passed --- sdk/python/cocoonsandbox/endpoint.py | 4 ++-- sdk/python/tests/test_endpoint.py | 17 ++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/sdk/python/cocoonsandbox/endpoint.py b/sdk/python/cocoonsandbox/endpoint.py index 42c53e6e5..062e57be5 100644 --- a/sdk/python/cocoonsandbox/endpoint.py +++ b/sdk/python/cocoonsandbox/endpoint.py @@ -5,11 +5,11 @@ import urllib.parse -def _endpoint_url(addr: str, scheme: str = "http") -> urllib.parse.SplitResult: +def _endpoint_url(addr: str) -> urllib.parse.SplitResult: if any(ord(c) <= 32 or ord(c) == 127 for c in addr): raise ValueError("sandboxd endpoint contains whitespace or a control character") if "://" not in addr: - addr = f"{scheme}://{addr}" + addr = f"http://{addr}" endpoint = urllib.parse.urlsplit(addr) if ( endpoint.scheme not in ("http", "https") diff --git a/sdk/python/tests/test_endpoint.py b/sdk/python/tests/test_endpoint.py index 89869730e..37cbd52b0 100644 --- a/sdk/python/tests/test_endpoint.py +++ b/sdk/python/tests/test_endpoint.py @@ -9,18 +9,17 @@ @pytest.mark.parametrize( - ("addr", "scheme", "want"), + ("addr", "want"), [ - ("localhost:7777", "http", "http://localhost:7777"), - ("localhost:443", "https", "https://localhost:443"), - ("https://example.com/", "http", "https://example.com"), - ("http://example.com", "https", "http://example.com"), - ("[::1]:7777", "https", "https://[::1]:7777"), - ("https://[::1]", "http", "https://[::1]"), + ("localhost:7777", "http://localhost:7777"), + ("https://example.com/", "https://example.com"), + ("http://example.com", "http://example.com"), + ("[::1]:7777", "http://[::1]:7777"), + ("https://[::1]", "https://[::1]"), ], ) -def test_endpoint_url(addr: str, scheme: str, want: str) -> None: - assert _endpoint_url(addr, scheme).geturl() == want +def test_endpoint_url(addr: str, want: str) -> None: + assert _endpoint_url(addr).geturl() == want @pytest.mark.parametrize( From ba58c984a86d18f160f61ec442020df62c4e3023 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:41:14 +0800 Subject: [PATCH 081/127] sdk/python: leave the callback rule to _deliver's docstring --- sdk/python/cocoonsandbox/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/cocoonsandbox/errors.py b/sdk/python/cocoonsandbox/errors.py index 0112610dd..221bf30ff 100644 --- a/sdk/python/cocoonsandbox/errors.py +++ b/sdk/python/cocoonsandbox/errors.py @@ -78,6 +78,6 @@ def _error_message(raw: bytes) -> str: def _mark_gone(exc: _E, tag: tuple[str, str]) -> _E: - """Tags an owner's 404 or refused dial with its (sandbox id, owner address); callback errors stay untagged.""" + """Tags an owner's 404 or refused dial with its (sandbox id, owner address).""" exc._owner_gone = tag return exc From a7a7be0227e7171a53bf14dec074606f27d7bfb6 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:41:19 +0800 Subject: [PATCH 082/127] sdk/python: time the e2e steps and port deadline on the monotonic clock --- sdk/python/e2e.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/python/e2e.py b/sdk/python/e2e.py index 53ee44301..cf05c40ec 100644 --- a/sdk/python/e2e.py +++ b/sdk/python/e2e.py @@ -149,7 +149,7 @@ def _promote(): @step("port") def _port(): - deadline = time.time() + 5 + deadline = time.monotonic() + 5 while True: try: conn = sb.dial_port(22) @@ -158,20 +158,20 @@ def _port(): assert banner.startswith(b"SSH-"), banner return except SilkdError: - if time.time() > deadline: + if time.monotonic() > deadline: raise time.sleep(0.3) try: for name, fn in steps: - start = time.time() + start = time.monotonic() try: fn() except BaseException: print(f" {name:<18} FAIL") traceback.print_exc() raise - print(f" {name:<18} ok ({int((time.time() - start) * 1000)}ms)") + print(f" {name:<18} ok ({int((time.monotonic() - start) * 1000)}ms)") finally: with contextlib.suppress(Exception): sb.close() From 5d6f63af0b88208be8e0a53fac50359246108824 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:41:33 +0800 Subject: [PATCH 083/127] sdk/langchain: derive the exec tool's cutoff sentence from CALL_TIMEOUT --- sdk/langchain/cocoonsandbox_langchain/toolkit.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/langchain/cocoonsandbox_langchain/toolkit.py b/sdk/langchain/cocoonsandbox_langchain/toolkit.py index f4fd34a94..b4527bb48 100644 --- a/sdk/langchain/cocoonsandbox_langchain/toolkit.py +++ b/sdk/langchain/cocoonsandbox_langchain/toolkit.py @@ -12,7 +12,6 @@ from langchain_core.tools import StructuredTool, ToolException from pydantic import BaseModel, Field -# one call's wall clock over the claim, the dial and the command; the tool description states it CALL_TIMEOUT = 300 @@ -70,7 +69,7 @@ def get_tools(self) -> list[StructuredTool]: "Returns stdout; a non-empty stderr is appended as a 'stderr:' " "line and a non-zero status as an 'exit code: N' line; a " "command that prints nothing and exits 0 returns '(no output)'. " - "The call is cut off after 5 minutes and the reply says so. " + f"The call is cut off after {CALL_TIMEOUT // 60} minutes and the reply says so. " "Files and installed packages persist across calls; environment " "variables and the working directory do not.", ExecInput, From 2685b9c5cd2d71c115aa8d580e8784c4bdd5b919 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:43:30 +0800 Subject: [PATCH 084/127] sdk/python: raise the floor to 3.10 and lint under the root ruff config CI tests 3.12 only and both adapters already require 3.10; the local ruff blocks pinned py39 and kept cocoonsandbox first-party, so the fixes are the root config's import grouping and the TimeoutError alias. --- docs/openai-adapter.md | 4 ++-- sdk/python/cocoonsandbox/sandbox.py | 2 +- sdk/python/pyproject.toml | 9 +-------- sdk/python/tests/conftest.py | 4 ++-- sdk/python/tests/test_checkpoint.py | 1 - sdk/python/tests/test_client.py | 3 +-- sdk/python/tests/test_endpoint.py | 1 - sdk/python/tests/test_fault_matrix.py | 1 - sdk/python/tests/test_fixtures.py | 1 - sdk/python/tests/test_frames.py | 1 - sdk/python/tests/test_hardening.py | 3 +-- sdk/python/tests/test_keepalive.py | 3 +-- sdk/python/tests/test_proc.py | 3 +-- sdk/python/tests/test_stream.py | 5 ++--- sdk/python/tests/test_upgrade.py | 3 +-- sdk/python/tests/test_volume_controller.py | 3 +-- sdk/python/tests/test_volumes.py | 3 +-- sdk/python/tests/test_wire_binding.py | 3 +-- 18 files changed, 16 insertions(+), 37 deletions(-) diff --git a/docs/openai-adapter.md b/docs/openai-adapter.md index 8c4e27f37..33fa22f15 100644 --- a/docs/openai-adapter.md +++ b/docs/openai-adapter.md @@ -44,5 +44,5 @@ ref, network lane (empty for the node's default, or `none`/`egress`), and `ttl_seconds`, which defaults to one hour: nothing renews a lease, and a session outlives the node's 5-minute default. The session state is JSON-serializable, so a run can be resumed against the same sandbox after a -process restart. Requires Python 3.10+ (the Agents SDK floor); the -underlying `cocoonsandbox` stays 3.9+. +process restart. Requires Python 3.10+, the floor of both the Agents SDK +and `cocoonsandbox`. diff --git a/sdk/python/cocoonsandbox/sandbox.py b/sdk/python/cocoonsandbox/sandbox.py index cfe517daf..ee50e49e3 100644 --- a/sdk/python/cocoonsandbox/sandbox.py +++ b/sdk/python/cocoonsandbox/sandbox.py @@ -566,7 +566,7 @@ def _proxy_accept_loop(self, listener: socket.socket, port: int) -> None: while True: try: local, _ = listener.accept() - except socket.timeout: + except TimeoutError: continue except OSError: return diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 10e279959..870a50941 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" name = "cocoonstack-sandbox" version = "0.1.23" description = "Python SDK for the cocoon sandbox control plane" -requires-python = ">=3.9" +requires-python = ">=3.10" license = { text = "Apache-2.0" } readme = "README.md" @@ -22,10 +22,3 @@ pythonpath = ["."] [tool.mypy] strict = true files = ["cocoonsandbox", "examples"] - -[tool.ruff] -line-length = 120 -target-version = "py39" - -[tool.ruff.lint] -select = ["E", "F", "W", "I", "UP", "B", "SIM"] diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index fbb169ed6..8a1852191 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -6,11 +6,11 @@ import socket import threading import time +from collections.abc import Callable from http.server import BaseHTTPRequestHandler, HTTPServer -from typing import BinaryIO, Callable +from typing import BinaryIO import pytest - from cocoonsandbox import Client, Sandbox diff --git a/sdk/python/tests/test_checkpoint.py b/sdk/python/tests/test_checkpoint.py index 5792db126..1b573ecee 100644 --- a/sdk/python/tests/test_checkpoint.py +++ b/sdk/python/tests/test_checkpoint.py @@ -1,7 +1,6 @@ """Checkpoint claim redirect, mirroring the Client.new redirect in test_fault_matrix.py.""" import pytest - from cocoonsandbox import APIError, Client diff --git a/sdk/python/tests/test_client.py b/sdk/python/tests/test_client.py index 736a5fed0..85c81f5af 100644 --- a/sdk/python/tests/test_client.py +++ b/sdk/python/tests/test_client.py @@ -5,9 +5,8 @@ from http.server import ThreadingHTTPServer import pytest -from conftest import FakeNode, sandbox_at, wait_until - from cocoonsandbox import APIError, Client, Template +from conftest import FakeNode, sandbox_at, wait_until class KeptNode(FakeNode): diff --git a/sdk/python/tests/test_endpoint.py b/sdk/python/tests/test_endpoint.py index 37cbd52b0..763c2fdef 100644 --- a/sdk/python/tests/test_endpoint.py +++ b/sdk/python/tests/test_endpoint.py @@ -2,7 +2,6 @@ import threading import pytest - from cocoonsandbox import Client, SandboxTimeout from cocoonsandbox.conn import dial_agent from cocoonsandbox.endpoint import _endpoint_url diff --git a/sdk/python/tests/test_fault_matrix.py b/sdk/python/tests/test_fault_matrix.py index 4ec5b225b..87dd5ab46 100644 --- a/sdk/python/tests/test_fault_matrix.py +++ b/sdk/python/tests/test_fault_matrix.py @@ -5,7 +5,6 @@ import time import pytest - from cocoonsandbox import APIError, Client diff --git a/sdk/python/tests/test_fixtures.py b/sdk/python/tests/test_fixtures.py index f34cb9c69..059bff8d1 100644 --- a/sdk/python/tests/test_fixtures.py +++ b/sdk/python/tests/test_fixtures.py @@ -5,7 +5,6 @@ import pathlib import pytest - from cocoonsandbox import frames FIXTURES = pathlib.Path(__file__).parents[3] / "protocol" / "wire" / "fixtures" / "v1" diff --git a/sdk/python/tests/test_frames.py b/sdk/python/tests/test_frames.py index 29a93f75c..a180cd25a 100644 --- a/sdk/python/tests/test_frames.py +++ b/sdk/python/tests/test_frames.py @@ -4,7 +4,6 @@ import json import pytest - from cocoonsandbox import frames diff --git a/sdk/python/tests/test_hardening.py b/sdk/python/tests/test_hardening.py index 286736e03..1f6a3dec0 100644 --- a/sdk/python/tests/test_hardening.py +++ b/sdk/python/tests/test_hardening.py @@ -7,8 +7,6 @@ import time import pytest -from conftest import sandbox_at - from cocoonsandbox import ( APIError, Client, @@ -23,6 +21,7 @@ ) from cocoonsandbox.conn import Conn, dial_agent, remaining_timeout from cocoonsandbox.frames import BULK_CHUNK +from conftest import sandbox_at def test_dial_agent_rejects_control_chars_in_identity(): diff --git a/sdk/python/tests/test_keepalive.py b/sdk/python/tests/test_keepalive.py index 00fb19bc4..c932c1cdd 100644 --- a/sdk/python/tests/test_keepalive.py +++ b/sdk/python/tests/test_keepalive.py @@ -7,10 +7,9 @@ import threading import pytest -from conftest import accept_upgrade, sandbox_at, wait_until - from cocoonsandbox import APIError, Client, SandboxError, SilkdError from cocoonsandbox.errors import _mark_gone +from conftest import accept_upgrade, sandbox_at, wait_until INPUT_OPS = ("stdin", "stdin_close", "data", "data_end") diff --git a/sdk/python/tests/test_proc.py b/sdk/python/tests/test_proc.py index dda18cde2..f6f12e48f 100644 --- a/sdk/python/tests/test_proc.py +++ b/sdk/python/tests/test_proc.py @@ -3,10 +3,9 @@ import threading import pytest -from conftest import FakeConn, sandbox_at - from cocoonsandbox import ProtocolError from cocoonsandbox.frames import BULK_CHUNK +from conftest import FakeConn, sandbox_at class BlockingStdinConn(FakeConn): diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py index a565a72a8..4d587cc04 100644 --- a/sdk/python/tests/test_stream.py +++ b/sdk/python/tests/test_stream.py @@ -7,11 +7,10 @@ import time import pytest -from conftest import accept_upgrade, sandbox_at - from cocoonsandbox import ProtocolError, Sandbox, SandboxTimeout from cocoonsandbox import conn as conn_module from cocoonsandbox import sandbox as sandbox_module +from conftest import accept_upgrade, sandbox_at TIMEOUT = 0.2 @@ -92,7 +91,7 @@ def test_dial_is_still_bounded_by_the_client_timeout(): assert time.monotonic() - started < 3 * TIMEOUT -@pytest.mark.parametrize(("exc", "want"), [(socket.timeout(), SandboxTimeout), (ConnectionResetError(), ProtocolError)]) +@pytest.mark.parametrize(("exc", "want"), [(TimeoutError(), SandboxTimeout), (ConnectionResetError(), ProtocolError)]) def test_handshake_failures_are_typed(exc, want): assert isinstance(conn_module._handshake_error("agent upgrade", exc), want) diff --git a/sdk/python/tests/test_upgrade.py b/sdk/python/tests/test_upgrade.py index 4ac25de7f..6f0b11920 100644 --- a/sdk/python/tests/test_upgrade.py +++ b/sdk/python/tests/test_upgrade.py @@ -7,9 +7,8 @@ import time import pytest -from conftest import sandbox_at - from cocoonsandbox import APIError, SandboxTimeout +from conftest import sandbox_at REPLIES = { "info": {"type": "info", "version": "fake", "uptime_secs": 0, "procs": 0}, diff --git a/sdk/python/tests/test_volume_controller.py b/sdk/python/tests/test_volume_controller.py index e0215f72d..58d35a8ae 100644 --- a/sdk/python/tests/test_volume_controller.py +++ b/sdk/python/tests/test_volume_controller.py @@ -1,9 +1,8 @@ from types import SimpleNamespace import pytest -from conftest import FakeNode - from cocoonsandbox import Client, VolumeError +from conftest import FakeNode from examples.volume_controller import VolumeHooks, hibernate_volume, resume_volume PATH = "/v1/sandboxes/sb_1" diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index e7ddabb5f..2ae8b371d 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -3,9 +3,8 @@ from collections import UserDict import pytest -from conftest import FakeNode, sandbox_at - from cocoonsandbox import APIError, VolumeError +from conftest import FakeNode, sandbox_at PATH = "/v1/sandboxes/sb_1/volumes" VIEW = { diff --git a/sdk/python/tests/test_wire_binding.py b/sdk/python/tests/test_wire_binding.py index a9525541e..168f7fcf7 100644 --- a/sdk/python/tests/test_wire_binding.py +++ b/sdk/python/tests/test_wire_binding.py @@ -7,11 +7,10 @@ import threading import pytest -from conftest import FakeConn, sandbox_at - from cocoonsandbox import Lsp, Pty, Session from cocoonsandbox.conn import Conn from cocoonsandbox.frames import PROTO_VERSION +from conftest import FakeConn, sandbox_at FIXTURES = pathlib.Path(__file__).parents[3] / "protocol" / "wire" / "fixtures" / "v1" From 88ee4782c61553bbc85eaa0bfe60b9ab4baf167b Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:44:00 +0800 Subject: [PATCH 085/127] sdk/openai: share the faked-session setup and merge the failed-construction tests --- sdk/openai/tests/test_adapter.py | 63 ++++++++++---------------------- 1 file changed, 20 insertions(+), 43 deletions(-) diff --git a/sdk/openai/tests/test_adapter.py b/sdk/openai/tests/test_adapter.py index 2939cdeba..62d96a03a 100644 --- a/sdk/openai/tests/test_adapter.py +++ b/sdk/openai/tests/test_adapter.py @@ -92,19 +92,13 @@ async def go(): def test_exec_maps_stdio_and_exit(node, monkeypatch): class FakeSandbox: - def __init__(self, **kw): - self.id = "sb_1" - def run(self, argv, on_stdout=None, on_stderr=None, timeout=None): assert argv == ["echo", "hi"] and timeout is None on_stdout(b"hi\n") return 0 async def go(): - client = CocoonSandboxClient() - session = await client.create(options=CocoonSandboxClientOptions(addr=node)) - inner = session._inner - monkeypatch.setattr(inner, "_sandbox", lambda: FakeSandbox()) + inner = await _session_with(node, monkeypatch, FakeSandbox()) result = await inner._exec_internal("echo", "hi") assert result.exit_code == 0 and result.stdout == b"hi\n" assert result.ok() @@ -124,10 +118,7 @@ def run(self, argv, on_stdout=None, on_stderr=None, timeout=None): return 0 async def go(): - client = CocoonSandboxClient() - session = await client.create(options=CocoonSandboxClientOptions(addr=node)) - inner = session._inner - monkeypatch.setattr(inner, "_sandbox", lambda: FakeSandbox()) + inner = await _session_with(node, monkeypatch, FakeSandbox()) with pytest.raises(TimeoutError): await inner._exec_internal("sleep", "9", timeout=1.5) result = await inner._exec_internal("sleep", "9") @@ -142,10 +133,7 @@ def read_file(self, path): raise SilkdError("not_found", path) async def go(): - client = CocoonSandboxClient() - session = await client.create(options=CocoonSandboxClientOptions(addr=node)) - inner = session._inner - monkeypatch.setattr(inner, "_sandbox", lambda: FakeSandbox()) + inner = await _session_with(node, monkeypatch, FakeSandbox()) with pytest.raises(FileNotFoundError): await inner.read(Path("/nope")) @@ -158,10 +146,7 @@ def stat(self, path): raise ProtocolError("dial 127.0.0.1:1: connection refused") async def go(): - client = CocoonSandboxClient() - session = await client.create(options=CocoonSandboxClientOptions(addr=node)) - inner = session._inner - monkeypatch.setattr(inner, "_sandbox", lambda: FakeSandbox()) + inner = await _session_with(node, monkeypatch, FakeSandbox()) assert await inner.running() is False asyncio.run(go()) @@ -182,10 +167,7 @@ def push(self, dest, data): calls["push"] = (dest, data) async def go(): - client = CocoonSandboxClient() - session = await client.create(options=CocoonSandboxClientOptions(addr=node)) - inner = session._inner - monkeypatch.setattr(inner, "_sandbox", lambda: FakeSandbox()) + inner = await _session_with(node, monkeypatch, FakeSandbox()) await inner.write(Path("/workspace/a.txt"), io.BytesIO(b"body")) assert calls["write"] == ("/workspace/a.txt", b"body") tar = await inner.persist_workspace() @@ -198,15 +180,20 @@ async def go(): asyncio.run(go()) -def test_create_keeps_the_construction_error_when_the_release_also_fails(node, monkeypatch): +@pytest.mark.parametrize("release_fails", [False, True]) +def test_create_releases_the_claim_and_keeps_the_construction_error(node, monkeypatch, release_fails): + released = [] + def bad_snapshot(*args, **kwargs): raise ValueError("bad snapshot") - def bad_close(self): - raise RuntimeError("node gone") + def close(self): + released.append(self.id) + if release_fails: + raise RuntimeError("node gone") monkeypatch.setattr("cocoonsandbox_openai.adapter.resolve_snapshot", bad_snapshot) - monkeypatch.setattr("cocoonsandbox.Sandbox.close", bad_close) + monkeypatch.setattr("cocoonsandbox.Sandbox.close", close) async def go(): client = CocoonSandboxClient() @@ -214,24 +201,14 @@ async def go(): await client.create(options=CocoonSandboxClientOptions(addr=node)) asyncio.run(go()) + assert released == ["sb_1"] -def test_create_releases_the_claim_when_state_construction_fails(node, monkeypatch): - released = [] - - def bad_snapshot(*args, **kwargs): - raise ValueError("bad snapshot") - - monkeypatch.setattr("cocoonsandbox_openai.adapter.resolve_snapshot", bad_snapshot) - monkeypatch.setattr("cocoonsandbox.Sandbox.close", lambda self: released.append(self.id)) - - async def go(): - client = CocoonSandboxClient() - with pytest.raises(ValueError): - await client.create(options=CocoonSandboxClientOptions(addr=node)) - - asyncio.run(go()) - assert released == ["sb_1"] +async def _session_with(node, monkeypatch, fake): + session = await CocoonSandboxClient().create(options=CocoonSandboxClientOptions(addr=node)) + inner = session._inner + monkeypatch.setattr(inner, "_sandbox", lambda: fake) + return inner def _silkd_pull_tar(root: str, files: dict[str, bytes]) -> bytes: From 4c1fbe86947fc76ac02ace1273a4a91b145c4962 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:48:19 +0800 Subject: [PATCH 086/127] sdk/python: fold copy-paste tests into parametrized cases and shared fixtures --- sdk/python/tests/conftest.py | 24 +++- sdk/python/tests/test_endpoint.py | 2 + sdk/python/tests/test_fault_matrix.py | 129 ++++++--------------- sdk/python/tests/test_fixtures.py | 4 +- sdk/python/tests/test_keepalive.py | 36 ++---- sdk/python/tests/test_stream.py | 59 +++------- sdk/python/tests/test_volume_controller.py | 2 + sdk/python/tests/test_volumes.py | 36 +++--- sdk/python/tests/test_wire_binding.py | 8 +- 9 files changed, 100 insertions(+), 200 deletions(-) diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 8a1852191..859deb2d9 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -2,6 +2,7 @@ import json import os +import pathlib import resource import socket import threading @@ -13,6 +14,8 @@ import pytest from cocoonsandbox import Client, Sandbox +FIXTURES = pathlib.Path(__file__).parents[3] / "protocol" / "wire" / "fixtures" / "v1" + class FakeNode(BaseHTTPRequestHandler): routes = {} @@ -63,12 +66,6 @@ def __init__(self, frames): self._frames = list(frames) self.sent = [] - def __enter__(self): - return self - - def __exit__(self, *exc): - pass - def send(self, op, **fields): self.sent.append((op, fields)) @@ -162,6 +159,21 @@ def answer(): server.close() +@pytest.fixture +def script_server(): + servers = [] + + def start(script, *args) -> str: + server = socket.create_server(("127.0.0.1", 0)) + servers.append(server) + threading.Thread(target=script, args=(server, *args), daemon=True).start() + return f"127.0.0.1:{server.getsockname()[1]}" + + yield start + for server in servers: + server.close() + + @pytest.fixture def black_hole(): sock = socket.socket() diff --git a/sdk/python/tests/test_endpoint.py b/sdk/python/tests/test_endpoint.py index 763c2fdef..6d6cfdb2e 100644 --- a/sdk/python/tests/test_endpoint.py +++ b/sdk/python/tests/test_endpoint.py @@ -1,3 +1,5 @@ +"""Endpoint origin parsing and the TLS handshake timeout.""" + import socket import threading diff --git a/sdk/python/tests/test_fault_matrix.py b/sdk/python/tests/test_fault_matrix.py index 87dd5ab46..48b4c72bb 100644 --- a/sdk/python/tests/test_fault_matrix.py +++ b/sdk/python/tests/test_fault_matrix.py @@ -8,9 +8,17 @@ from cocoonsandbox import APIError, Client -def test_claim_redirect_skips_dead_candidate(spawn_node, dead_addr): +@pytest.mark.parametrize( + ("status", "message"), + [(0, ""), (404, "gone"), (403, "tenant not allowed")], + ids=["dead", "404", "403_definitive"], +) +def test_claim_redirect_skips_a_failed_candidate(spawn_node, dead_addr, status, message): + failed = ( + spawn_node({("POST", "/v1/claim"): lambda body, path: (status, {"error": message})}) if status else dead_addr + ) good = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"id": "sb_ok", "token": "t"})}) - entry = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"redirect": [dead_addr, good]})}) + entry = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"redirect": [failed, good]})}) sb = Client(entry).new("rt:24.04") assert sb.id == "sb_ok" and sb.owner == good @@ -41,13 +49,6 @@ def drop_reply(): assert not good_calls and len(entry_calls) == 1 -def test_claim_redirect_skips_404_candidate(spawn_node): - lost = spawn_node({("POST", "/v1/claim"): lambda body, path: (404, {"error": "gone"})}) - good = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"id": "sb_2", "token": "t"})}) - entry = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"redirect": [lost, good]})}) - assert Client(entry).new("rt:24.04").id == "sb_2" - - def test_claim_redirect_all_candidates_fail_then_origin_fallback_fails(spawn_node): a = spawn_node({("POST", "/v1/claim"): lambda body, path: (404, {"error": "gone a"})}) b = spawn_node({("POST", "/v1/claim"): lambda body, path: (404, {"error": "gone b"})}) @@ -68,28 +69,29 @@ def entry_claim(body, path): assert len(calls) == 2 and calls[1]["no_redirect"] is True -def test_claim_redirect_all_candidates_fail_then_origin_heals(spawn_node): - full_calls = [] +@pytest.mark.parametrize(("status", "message"), [(429, "full"), (401, "invalid api token")]) +def test_claim_redirect_retryable_candidate_falls_back_to_origin(spawn_node, status, message): + candidate_calls = [] - def full_claim(body, path): - full_calls.append(body) - return 429, {"error": "full"} + def candidate_claim(body, path): + candidate_calls.append(body) + return status, {"error": message} - full = spawn_node({("POST", "/v1/claim"): full_claim}) + candidate = spawn_node({("POST", "/v1/claim"): candidate_claim}) calls = [] def entry_claim(body, path): calls.append(body) if len(calls) == 1: - return 200, {"redirect": [full]} + return 200, {"redirect": [candidate]} return 200, {"id": "sb_healed", "token": "t"} entry = spawn_node({("POST", "/v1/claim"): entry_claim}) sb = Client(entry).new("rt:24.04") assert sb.id == "sb_healed" and sb.owner == entry assert len(calls) == 2 and calls[1]["no_redirect"] is True - assert len(full_calls) == 1 + assert len(candidate_calls) == 1 @pytest.mark.parametrize(("status", "message"), [(409, "no egress"), (403, "tenant not allowed")]) @@ -109,37 +111,6 @@ def entry_claim(body, path): assert len(calls) == 1 -def test_claim_redirect_401_candidate_falls_back_to_origin(spawn_node): - stale_calls = [] - - def stale_claim(body, path): - stale_calls.append(body) - return 401, {"error": "invalid api token"} - - stale = spawn_node({("POST", "/v1/claim"): stale_claim}) - - calls = [] - - def entry_claim(body, path): - calls.append(body) - if len(calls) == 1: - return 200, {"redirect": [stale]} - return 200, {"id": "sb_local", "token": "t"} - - entry = spawn_node({("POST", "/v1/claim"): entry_claim}) - sb = Client(entry).new("rt:24.04") - assert sb.id == "sb_local" and sb.owner == entry - assert len(calls) == 2 and calls[1]["no_redirect"] is True - assert len(stale_calls) == 1 - - -def test_claim_redirect_candidate_definitive_error_still_tries_next_candidate(spawn_node): - forbidden = spawn_node({("POST", "/v1/claim"): lambda body, path: (403, {"error": "tenant not allowed"})}) - good = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"id": "sb_3", "token": "t"})}) - entry = spawn_node({("POST", "/v1/claim"): lambda body, path: (200, {"redirect": [forbidden, good]})}) - assert Client(entry).new("rt:24.04").id == "sb_3" - - def test_delete_template_skips_dead_owner(spawn_node, dead_addr): seen = [] @@ -182,27 +153,14 @@ def ok(body, path): assert touched == ["bad"] -def test_lookup_dead_peer_resolves_fast(spawn_node, dead_addr): - owner = spawn_node({("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (200, {"owner_addr": "10.0.0.9:7777"})}) - entry = spawn_node( - { - ("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (404, {"error": "not here"}), - ("GET", "/v1/peers"): lambda body, path: (200, {"peers": [dead_addr, owner]}), - } - ) - start = time.monotonic() - sb = Client(entry, timeout=10.0).lookup("sb_1", "tok") - elapsed = time.monotonic() - start - assert sb.owner == "10.0.0.9:7777" - assert elapsed < 2.0, elapsed - - -def test_lookup_hung_peer_resolves_fast(spawn_node, black_hole): +@pytest.mark.parametrize("peer", ["dead_addr", "black_hole"]) +def test_lookup_unreachable_peer_resolves_fast(spawn_node, request, peer): + unreachable = request.getfixturevalue(peer) owner = spawn_node({("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (200, {"owner_addr": "10.0.0.9:7777"})}) entry = spawn_node( { ("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (404, {"error": "not here"}), - ("GET", "/v1/peers"): lambda body, path: (200, {"peers": [black_hole, owner]}), + ("GET", "/v1/peers"): lambda body, path: (200, {"peers": [unreachable, owner]}), } ) start = time.monotonic() @@ -224,7 +182,8 @@ def test_lookup_all_miss(spawn_node, dead_addr): assert exc.value.status == 404 and "no owner found" in exc.value.message -def test_close_releases_a_sandbox_that_moved(spawn_node): +@pytest.mark.parametrize("retired", [False, True], ids=["old_node_404", "old_node_retired"]) +def test_close_releases_a_sandbox_that_moved(spawn_node, dead_addr, retired): released = [] def release(body, path): @@ -235,16 +194,20 @@ def release(body, path): { ("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (200, {}), ("POST", "/v1/sandboxes/sb_1/release"): release, + ("GET", "/v1/peers"): lambda body, path: (200, {"peers": []}), } ) - old = spawn_node( - { - ("POST", "/v1/sandboxes/sb_1/release"): lambda body, path: (404, {"error": "unknown sandbox"}), - ("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (404, {"error": "not here"}), - ("GET", "/v1/peers"): lambda body, path: (200, {"peers": [adopter]}), - } - ) - Client(old).attach(old, "sb_1", "tok").close() + if retired: + entry, old = adopter, dead_addr + else: + entry = old = spawn_node( + { + ("POST", "/v1/sandboxes/sb_1/release"): lambda body, path: (404, {"error": "unknown sandbox"}), + ("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (404, {"error": "not here"}), + ("GET", "/v1/peers"): lambda body, path: (200, {"peers": [adopter]}), + } + ) + Client(entry).attach(old, "sb_1", "tok").close() assert released @@ -269,24 +232,6 @@ def test_a_released_sandbox_still_answers_404(spawn_node): assert exc.value.status == 404 and sb.owner == old -def test_close_follows_a_move_off_a_retired_node(spawn_node, dead_addr): - released = [] - - def release(body, path): - released.append(path) - return (204, {}) - - adopter = spawn_node( - { - ("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (200, {}), - ("POST", "/v1/sandboxes/sb_1/release"): release, - ("GET", "/v1/peers"): lambda body, path: (200, {"peers": []}), - } - ) - Client(adopter).attach(dead_addr, "sb_1", "tok").close() - assert released - - def test_close_reports_a_retired_node_when_no_node_holds_the_sandbox(spawn_node, dead_addr): entry = spawn_node( { diff --git a/sdk/python/tests/test_fixtures.py b/sdk/python/tests/test_fixtures.py index 059bff8d1..145f807e9 100644 --- a/sdk/python/tests/test_fixtures.py +++ b/sdk/python/tests/test_fixtures.py @@ -2,12 +2,10 @@ import base64 import json -import pathlib import pytest from cocoonsandbox import frames - -FIXTURES = pathlib.Path(__file__).parents[3] / "protocol" / "wire" / "fixtures" / "v1" +from conftest import FIXTURES def fixture_files(prefix): diff --git a/sdk/python/tests/test_keepalive.py b/sdk/python/tests/test_keepalive.py index c932c1cdd..536e997bd 100644 --- a/sdk/python/tests/test_keepalive.py +++ b/sdk/python/tests/test_keepalive.py @@ -183,33 +183,15 @@ def test_an_upload_the_daemon_rejects_at_open_raises_its_error_frame(relay): sb.write_file("/missing/f", bytes(64 << 20)) -def test_a_moved_sandbox_follows_its_new_owner_on_the_next_call(spawn_node): - agent = FakeAgent() - try: - directory = spawn_node( - {("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (200, {"owner_addr": agent.addr})} - ) - old = spawn_node({("GET", "/v1/peers"): lambda body, path: (200, {"peers": [directory]})}) - sb = Client(old).attach(old, "sb_1", "tok") - assert sb.stat("/")["kind"] == "dir" - assert sb.owner == agent.addr - assert sb.exec("echo", "42") == "42\n" - finally: - agent.stop() - - -def test_a_handle_follows_its_sandbox_off_a_retired_node(spawn_node, dead_addr): - agent = FakeAgent() - try: - directory = spawn_node( - {("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (200, {"owner_addr": agent.addr})} - ) - entry = spawn_node({("GET", "/v1/peers"): lambda body, path: (200, {"peers": [directory]})}) - sb = Client(entry).attach(dead_addr, "sb_1", "tok") - assert sb.stat("/")["kind"] == "dir" - assert sb.owner == agent.addr - finally: - agent.stop() +@pytest.mark.parametrize("retired", [False, True], ids=["owner_404", "connection_refused"]) +def test_a_handle_follows_its_moved_sandbox_on_the_next_call(spawn_node, dead_addr, relay, retired): + agent, _ = relay() + directory = spawn_node({("GET", "/v1/sandboxes/sb_1/owner"): lambda body, path: (200, {"owner_addr": agent.addr})}) + entry = spawn_node({("GET", "/v1/peers"): lambda body, path: (200, {"peers": [directory]})}) + sb = Client(entry).attach(dead_addr if retired else entry, "sb_1", "tok") + assert sb.stat("/")["kind"] == "dir" + assert sb.owner == agent.addr + assert sb.exec("echo", "42") == "42\n" @pytest.mark.parametrize("retired", [False, True], ids=["owner_404", "connection_refused"]) diff --git a/sdk/python/tests/test_stream.py b/sdk/python/tests/test_stream.py index 4d587cc04..bae509584 100644 --- a/sdk/python/tests/test_stream.py +++ b/sdk/python/tests/test_stream.py @@ -3,7 +3,6 @@ import contextlib import json import socket -import threading import time import pytest @@ -15,50 +14,32 @@ TIMEOUT = 0.2 -def test_port_stream_outlives_the_client_timeout(): - server = socket.create_server(("127.0.0.1", 0)) - addr = f"127.0.0.1:{server.getsockname()[1]}" +def test_port_stream_outlives_the_client_timeout(script_server): ops: list[str] = [] - threading.Thread(target=serve_port_forward, args=(server, 3 * TIMEOUT, ops), daemon=True).start() - sb = short_timeout_sandbox(addr) - try: - with sb.dial_port(5000) as port: - assert port.recv() == b"late" - assert port.recv() == b"" - finally: - server.close() + sb = short_timeout_sandbox(script_server(serve_port_forward, 3 * TIMEOUT, ops)) + with sb.dial_port(5000) as port: + assert port.recv() == b"late" + assert port.recv() == b"" assert ops == ["port_forward"] -def test_run_timeout_cuts_a_silent_command_and_kills_it(): - server = socket.create_server(("127.0.0.1", 0)) - addr = f"127.0.0.1:{server.getsockname()[1]}" +def test_run_timeout_cuts_a_silent_command_and_kills_it(script_server): kills: list[dict] = [] - threading.Thread(target=serve_started_then_hang, args=(server, 5 * TIMEOUT, kills), daemon=True).start() - sb = short_timeout_sandbox(addr) + sb = short_timeout_sandbox(script_server(serve_started_then_hang, 5 * TIMEOUT, kills)) started = time.monotonic() - try: - with pytest.raises(TimeoutError): - sb.run(["sleep", "9"], timeout=TIMEOUT) - finally: - server.close() + with pytest.raises(TimeoutError): + sb.run(["sleep", "9"], timeout=TIMEOUT) assert time.monotonic() - started < 3 * TIMEOUT assert kills == [{"v": 1, "op": "kill", "pid": 7}], kills -def test_run_timeout_bounds_the_kill_it_sends(monkeypatch): +def test_run_timeout_bounds_the_kill_it_sends(monkeypatch, script_server): monkeypatch.setattr(sandbox_module, "_KILL_WAIT_SECONDS", TIMEOUT) - server = socket.create_server(("127.0.0.1", 0)) - addr = f"127.0.0.1:{server.getsockname()[1]}" kills: list[dict] = [] - threading.Thread(target=serve_started_then_trickle_the_kill_reply, args=(server, kills), daemon=True).start() - sb = short_timeout_sandbox(addr) + sb = short_timeout_sandbox(script_server(serve_started_then_trickle_the_kill_reply, kills)) started = time.monotonic() - try: - with pytest.raises(TimeoutError): - sb.run(["sleep", "9"], timeout=TIMEOUT) - finally: - server.close() + with pytest.raises(TimeoutError): + sb.run(["sleep", "9"], timeout=TIMEOUT) assert time.monotonic() - started < 4 * TIMEOUT assert kills == [{"v": 1, "op": "kill", "pid": 7}], kills @@ -77,17 +58,11 @@ def test_run_rejects_a_non_positive_timeout(): sb.run(["true"], timeout=0) -def test_dial_is_still_bounded_by_the_client_timeout(): - server = socket.create_server(("127.0.0.1", 0)) - addr = f"127.0.0.1:{server.getsockname()[1]}" - threading.Thread(target=serve_silence, args=(server,), daemon=True).start() - sb = short_timeout_sandbox(addr) +def test_dial_is_still_bounded_by_the_client_timeout(script_server): + sb = short_timeout_sandbox(script_server(serve_silence)) started = time.monotonic() - try: - with pytest.raises(SandboxTimeout): - sb.dial_port(5000) - finally: - server.close() + with pytest.raises(SandboxTimeout): + sb.dial_port(5000) assert time.monotonic() - started < 3 * TIMEOUT diff --git a/sdk/python/tests/test_volume_controller.py b/sdk/python/tests/test_volume_controller.py index 58d35a8ae..e41a09663 100644 --- a/sdk/python/tests/test_volume_controller.py +++ b/sdk/python/tests/test_volume_controller.py @@ -1,3 +1,5 @@ +"""The example volume controller's hibernate and resume flows against a fake node.""" + from types import SimpleNamespace import pytest diff --git a/sdk/python/tests/test_volumes.py b/sdk/python/tests/test_volumes.py index 2ae8b371d..e8f233380 100644 --- a/sdk/python/tests/test_volumes.py +++ b/sdk/python/tests/test_volumes.py @@ -16,12 +16,7 @@ def test_volume_mapping_is_json_encoded(spawn_node): - seen = [] - - def reply(body, path): - seen.append(body) - return 200, VIEW - + seen, reply = recording() sb = sandbox_at(spawn_node({("PUT", PATH): reply})) sb.set_volumes([UserDict({"name": "data", "mount": "/data"})]) assert seen == [{"volumes": [{"name": "data", "mount": "/data"}]}] @@ -40,12 +35,7 @@ def reply(body, path): ], ) def test_volume_requests_use_owner_and_api_token(spawn_node, method, body, call): - seen = [] - - def reply(body, path): - seen.append(body) - return 200, VIEW - + seen, reply = recording() addr = spawn_node({(method, PATH): reply}) sb = sandbox_at(addr, api_token="operator-token") sb._client.addr = "unused.invalid:7777" @@ -103,12 +93,7 @@ def test_validation_error_preserves_last_view(spawn_node): def test_explicit_empty_set_and_omitted_patch(spawn_node): - seen = [] - - def reply(body, path): - seen.append(body) - return 200, {"volumes_want": [], "volumes": [], "converging": False, "states": []} - + seen, reply = recording({"volumes_want": [], "volumes": [], "converging": False, "states": []}) sb = sandbox_at(spawn_node({("PUT", PATH): reply, ("PATCH", PATH): reply})) with pytest.raises(TypeError, match="explicit"): sb.set_volumes(None) @@ -199,13 +184,18 @@ def test_volume_mutations_encode_explicit_device_fields(spawn_node): {"name": "device", "mount": "", "mode": "rw", "attach_only": True}, {"name": "mounted", "attach_only": False}, ] + seen, reply = recording() + sb = sandbox_at(spawn_node({("PUT", PATH): reply, ("PATCH", PATH): reply})) + sb.set_volumes(entries) + sb.patch_volumes(add=entries) + assert seen == [{"volumes": entries}, {"add": entries}] + + +def recording(view=VIEW): seen = [] def reply(body, path): seen.append(body) - return 200, VIEW + return 200, view - sb = sandbox_at(spawn_node({("PUT", PATH): reply, ("PATCH", PATH): reply})) - sb.set_volumes(entries) - sb.patch_volumes(add=entries) - assert seen == [{"volumes": entries}, {"add": entries}] + return seen, reply diff --git a/sdk/python/tests/test_wire_binding.py b/sdk/python/tests/test_wire_binding.py index 168f7fcf7..7fcf7146d 100644 --- a/sdk/python/tests/test_wire_binding.py +++ b/sdk/python/tests/test_wire_binding.py @@ -2,7 +2,6 @@ import contextlib import json -import pathlib import socket import threading @@ -10,9 +9,7 @@ from cocoonsandbox import Lsp, Pty, Session from cocoonsandbox.conn import Conn from cocoonsandbox.frames import PROTO_VERSION -from conftest import FakeConn, sandbox_at - -FIXTURES = pathlib.Path(__file__).parents[3] / "protocol" / "wire" / "fixtures" / "v1" +from conftest import FIXTURES, FakeConn, sandbox_at CASES = [ ( @@ -96,9 +93,6 @@ def recv(self): return {"type": "git_branches", "branches": [], "current": ""} return {"type": "done"} - def recv_until(self, *terminal): - yield self.recv() - @pytest.mark.parametrize("stem,replies,invoke", CASES, ids=lambda c: c if isinstance(c, str) else "") def test_call_site_matches_fixture(monkeypatch, stem, replies, invoke): From ae311674da15b276e91ca3fa159847f9555c1308 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:48:20 +0800 Subject: [PATCH 087/127] sdk/langchain: parametrize the guest-error tool tests --- sdk/langchain/tests/test_toolkit.py | 30 +++++++++++++---------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/sdk/langchain/tests/test_toolkit.py b/sdk/langchain/tests/test_toolkit.py index f792230d2..4e882fcfd 100644 --- a/sdk/langchain/tests/test_toolkit.py +++ b/sdk/langchain/tests/test_toolkit.py @@ -109,15 +109,22 @@ def claim(deadline=None): assert kit.get_tools()[0].invoke({"command": "echo hi"}) == f"cut off after {CALL_TIMEOUT}s" -def test_a_guest_error_comes_back_as_a_tool_error(monkeypatch): +@pytest.mark.parametrize( + ("name", "method", "args", "message"), + [ + ("sandbox_write_file", "write_file", {"path": "/nope/a.txt", "content": "body"}, "no such directory"), + ("sandbox_read_file", "read_file", {"path": "/nope"}, "no such file"), + ], +) +def test_a_guest_error_comes_back_as_a_tool_error(monkeypatch, name, method, args, message): kit, fake = hooked(monkeypatch) - def refused(path, data): - raise SilkdError("not_found", "no such directory") + def refused(*_): + raise SilkdError("not_found", message) - fake.write_file = refused - tool = next(t for t in kit.get_tools() if t.name == "sandbox_write_file") - assert "no such directory" in tool.invoke({"path": "/nope/a.txt", "content": "body"}) + setattr(fake, method, refused) + tool = next(t for t in kit.get_tools() if t.name == name) + assert message in tool.invoke(args) def test_default_lease_outlives_an_agent_run(monkeypatch): @@ -160,17 +167,6 @@ def test_a_failed_release_never_hides_the_block_error(monkeypatch): kit.get_tools()[0].invoke({"command": "x"}) -def test_read_file_reports_a_missing_path_as_a_tool_error(monkeypatch): - kit, fake = hooked(monkeypatch) - - def missing(path): - raise SilkdError("not_found", "no such file") - - fake.read_file = missing - tool = next(t for t in kit.get_tools() if t.name == "sandbox_read_file") - assert "no such file" in tool.invoke({"path": "/nope"}) - - def test_a_stalled_agent_upgrade_comes_back_as_a_tool_error(): server = socket.create_server(("127.0.0.1", 0)) addr = f"127.0.0.1:{server.getsockname()[1]}" From b47e6a2b1ee7625e20a2462d6ae8cfb4c40c6de8 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:54 +0800 Subject: [PATCH 088/127] pool: make the consume-vs-unfence epoch race deterministic --- sandboxd/pool/lease_test.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/sandboxd/pool/lease_test.go b/sandboxd/pool/lease_test.go index b1cbba6d8..ed68205eb 100644 --- a/sandboxd/pool/lease_test.go +++ b/sandboxd/pool/lease_test.go @@ -6,7 +6,6 @@ import ( "fmt" "net" "os" - "sync" "sync/atomic" "testing" "testing/synctest" @@ -531,17 +530,14 @@ func TestLeaseConsumeDoesNotRaceAnUnfenceRewritingTheEpoch(t *testing.T) { if err != nil { t.Fatalf("rows: %v", err) } - var unfence sync.WaitGroup - unfence.Go(func() { + a.archives = &hookRegistry{Registry: c.reg, beforeConsume: func() { if _, takeErr := a.takeOwnRows(t.Context(), map[string]*types.Sandbox{sb.ID: sb}, rows); takeErr != nil { t.Errorf("take own rows: %v", takeErr) } - }) - time.Sleep(20 * time.Millisecond) + }} sb.Transition.Lock() consumed, err := a.consumeArchive(t.Context(), sb) sb.Transition.Unlock() - unfence.Wait() if err != nil || consumed == 0 { t.Errorf("consume after the unfence took the row: epoch %d, %v", consumed, err) } From ff15d260cfd03814c5cf702b838ca562c4edb00c Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:40:18 +0800 Subject: [PATCH 089/127] pool: run the claim-beside-repromote test on the synctest clock --- sandboxd/pool/promote_test.go | 68 ++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/sandboxd/pool/promote_test.go b/sandboxd/pool/promote_test.go index dea500ea1..9d900a28e 100644 --- a/sandboxd/pool/promote_test.go +++ b/sandboxd/pool/promote_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "slices" "testing" + "testing/synctest" "time" "github.com/cocoonstack/sandbox/sandboxd/config" @@ -601,40 +602,43 @@ func TestRejectedLabelWritesLeakNoRecLock(t *testing.T) { } func TestClaimFromATemplateRunsBesideItsRepromote(t *testing.T) { - m := newTestManager(t, newFakeEngine()) - parent := mustClaim(t, m, testKey) - key, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:x", "") - if err != nil { - t.Fatalf("Promote: %v", err) - } - slow := &slowPrepareStore{Store: m.tpls, entered: make(chan struct{}), release: make(chan struct{})} - m.tpls = slow - repromoted := make(chan error, 1) - go func() { - _, _, promoteErr := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:x", "") - repromoted <- promoteErr - }() - <-slow.entered - - ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) - defer cancel() - claimed := make(chan error, 1) - go func() { - _, claimErr := claimAny(ctx, m, key, ClaimOptions{}) - claimed <- claimErr - }() - select { - case err = <-claimed: + synctest.Test(t, func(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + parent := mustClaim(t, m, testKey) + key, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:x", "") if err != nil { - t.Errorf("claim during a re-promote: %v, want a clone of the committed template", err) + t.Fatalf("Promote: %v", err) } - case <-time.After(2 * time.Second): - t.Error("a claim with a 100 ms context was still blocked 2 s into a re-promote") - } - close(slow.release) - if err = <-repromoted; err != nil { - t.Fatalf("re-promote: %v", err) - } + slow := &slowPrepareStore{Store: m.tpls, entered: make(chan struct{}), release: make(chan struct{})} + m.tpls = slow + repromoted := make(chan error, 1) + go func() { + _, _, promoteErr := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:x", "") + repromoted <- promoteErr + }() + <-slow.entered + + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + claimed := make(chan error, 1) + go func() { + _, claimErr := claimAny(ctx, m, key, ClaimOptions{}) + claimed <- claimErr + }() + synctest.Wait() + select { + case err = <-claimed: + if err != nil { + t.Errorf("claim during a re-promote: %v, want a clone of the committed template", err) + } + default: + t.Error("a claim with a 100 ms context was still blocked during a re-promote") + } + close(slow.release) + if err = <-repromoted; err != nil { + t.Fatalf("re-promote: %v", err) + } + }) } type slowPrepareStore struct { From 7636ee580477f5aeafcf919c50ca76108ed3587e Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:41:56 +0800 Subject: [PATCH 090/127] pool: order the racing lookup behind the wake adoption with synctest --- sandboxd/pool/failover_test.go | 64 ++++++++++++++++------------------ 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 42066c7f0..e5aa5cbb4 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -1860,40 +1860,38 @@ func TestFailoverAReleaseFindsItsUnsettledArchiveTakenElsewhere(t *testing.T) { } func TestALookupRacingAWakeAdoptionTakesTheRowOnce(t *testing.T) { - c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) - sb := mustClaim(t, a, testKey) - mustArchive(t, a, sb) - rec, _, err := c.reg.Get(t.Context(), sb.ID) - if err != nil { - t.Fatalf("row: %v", err) - } - if _, err = c.reg.Release(t.Context(), sb.ID, rec.Epoch, "a"); err != nil { - t.Fatalf("release: %v", err) - } - reg := &hookRegistry{Registry: c.reg} - b.archives = reg - lookup := make(chan bool, 1) - reg.afterTake = func() { - go func() { - adopted, _ := b.AdoptArchived(t.Context(), sb.ID, sb.Token) - lookup <- adopted - }() - select { - case <-lookup: - lookup <- true - case <-time.After(200 * time.Millisecond): + synctest.Test(t, func(t *testing.T) { + c := newFailoverCluster(t) + a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + mustArchive(t, a, sb) + rec, _, err := c.reg.Get(t.Context(), sb.ID) + if err != nil { + t.Fatalf("row: %v", err) } - } - if adopted, adoptErr := b.AdoptForWake(t.Context(), sb.ID, sb.Token); adoptErr != nil || !adopted { - t.Fatalf("wake adoption: %v %v", adopted, adoptErr) - } - if !<-lookup { - t.Error("the racing lookup did not find the adopted claim") - } - if got := reg.takes.Load(); got != 1 { - t.Errorf("row taken %d times, want once", got) - } + if _, err = c.reg.Release(t.Context(), sb.ID, rec.Epoch, "a"); err != nil { + t.Fatalf("release: %v", err) + } + reg := &hookRegistry{Registry: c.reg} + b.archives = reg + lookup := make(chan bool, 1) + reg.afterTake = func() { + go func() { + adopted, _ := b.AdoptArchived(t.Context(), sb.ID, sb.Token) + lookup <- adopted + }() + synctest.Wait() + } + if adopted, adoptErr := b.AdoptForWake(t.Context(), sb.ID, sb.Token); adoptErr != nil || !adopted { + t.Fatalf("wake adoption: %v %v", adopted, adoptErr) + } + if !<-lookup { + t.Error("the racing lookup did not find the adopted claim") + } + if got := reg.takes.Load(); got != 1 { + t.Errorf("row taken %d times, want once", got) + } + }) } type failoverCluster struct { From 760fa671dbebeeaeed16a1caa519bfdb14f27f41 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:43:46 +0800 Subject: [PATCH 091/127] pool: assert the kept claims of an evacuation unconditionally --- sandboxd/pool/drain_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sandboxd/pool/drain_test.go b/sandboxd/pool/drain_test.go index 376ab667e..dbc070de4 100644 --- a/sandboxd/pool/drain_test.go +++ b/sandboxd/pool/drain_test.go @@ -232,7 +232,7 @@ func TestUndrainStopsAnEvacuationBeforeItsHandOff(t *testing.T) { close(stall) synctest.Wait() - assertEvacuation(t, a, 0, 0) + assertEvacuation(t, a, 0, 1) if _, g := a.Info(); g.Draining { t.Error("still draining after undrain") } @@ -273,7 +273,7 @@ func TestAnUndrainDuringAnEvacuationsDrainStopsIt(t *testing.T) { wg.Wait() synctest.Wait() - assertEvacuation(t, a, 0, 0) + assertEvacuation(t, a, 0, 1) held, ok := a.byID(sb.ID) if !ok || held.HibernateSnap != "" || held.ArchiveCk != "" { t.Errorf("held %v, want the claim still running after the undrain", ok) @@ -747,7 +747,7 @@ func TestEvacuationDeadlines(t *testing.T) { func assertEvacuation(t *testing.T, m *Manager, left, kept int) { t.Helper() - if _, g := m.Info(); g.Evacuating != left || g.Draining && g.Claimed-g.Evacuating != kept { + if _, g := m.Info(); g.Evacuating != left || g.Claimed-g.Evacuating != kept { t.Errorf("evacuation left %d of %d claims, want %d left and %d kept", g.Evacuating, g.Claimed, left, kept) } } From ce24e4f4cb4069f7f161b4286ab3adddc31749a1 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:45:14 +0800 Subject: [PATCH 092/127] pool: mustPromote helper for the successful promotes --- sandboxd/pool/promote_test.go | 154 +++++++++++----------------------- 1 file changed, 48 insertions(+), 106 deletions(-) diff --git a/sandboxd/pool/promote_test.go b/sandboxd/pool/promote_test.go index 9d900a28e..81957e89b 100644 --- a/sandboxd/pool/promote_test.go +++ b/sandboxd/pool/promote_test.go @@ -24,10 +24,7 @@ func TestPromoteThenClaimClonesFromTemplate(t *testing.T) { m := newTestManager(t, eng) parent := mustClaim(t, m, testKey) - gotKey, gotDigest, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:x", "") - if err != nil { - t.Fatalf("Promote: %v", err) - } + gotKey, gotDigest := mustPromote(t, m, parent, "tpl:x", "") if len(eng.snapSaves) != 1 || !slices.Contains(eng.snapRemoves, eng.snapSaves[0]) { t.Errorf("snapSaves=%v snapRemoves=%v, want one transient snapshot dropped", eng.snapSaves, eng.snapRemoves) } @@ -71,23 +68,14 @@ func TestRepromoteContentDigestTracksExportBytes(t *testing.T) { m := newTestManager(t, eng) parent := mustClaim(t, m, testKey) - _, firstDigest, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:digest", "") - if err != nil { - t.Fatalf("first Promote: %v", err) - } - _, secondDigest, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:digest", "") - if err != nil { - t.Fatalf("same-byte Promote: %v", err) - } + _, firstDigest := mustPromote(t, m, parent, "tpl:digest", "") + _, secondDigest := mustPromote(t, m, parent, "tpl:digest", "") if secondDigest != firstDigest { t.Errorf("same export bytes changed digest: %q then %q", firstDigest, secondDigest) } eng.exportContent = []byte("generation two") - thirdKey, thirdDigest, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:digest", "") - if err != nil { - t.Fatalf("changed-byte Promote: %v", err) - } + thirdKey, thirdDigest := mustPromote(t, m, parent, "tpl:digest", "") if thirdDigest == firstDigest { t.Errorf("changed export bytes kept digest %q", thirdDigest) } @@ -106,9 +94,7 @@ func TestPromoteHibernatedUsesWakeImage(t *testing.T) { parent := mustClaim(t, m, testKey) hibernateClaim(t, m, parent) - if _, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:hib", ""); err != nil { - t.Fatalf("Promote: %v", err) - } + mustPromote(t, m, parent, "tpl:hib", "") if len(eng.snapSaves) != 0 { t.Errorf("snapSaves %v, want none — the hibernate image is the source", eng.snapSaves) } @@ -145,20 +131,14 @@ func TestTemplateDigestGuardReadsTheSharedRecord(t *testing.T) { eng.exportContent = []byte("generation one") m := newTestManager(t, eng) parent := mustClaim(t, m, testKey) - key, stale, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:shared", "") - if err != nil { - t.Fatalf("Promote: %v", err) - } + key, stale := mustPromote(t, m, parent, "tpl:shared", "") peer := newTestManager(t, newFakeEngine()) peer.tpls = m.tpls id := store.TemplateID(key.Hash()) peer.tplSet[id] = TemplateInfo{Key: key, ContentDigest: stale} eng.exportContent = []byte("generation two") - _, latest, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:shared", "") - if err != nil { - t.Fatalf("re-Promote: %v", err) - } + _, latest := mustPromote(t, m, parent, "tpl:shared", "") if err := peer.SetTemplateLabels(t.Context(), key, types.Metadata{"a": "b"}, "", stale); !errors.Is(err, ErrTemplateReplaced) { t.Errorf("labels with the replaced digest: %v, want ErrTemplateReplaced", err) } @@ -181,10 +161,7 @@ func TestDeleteTemplateRefusesPooledUnknownAndReplacedTemplates(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 0}) parent := mustClaim(t, m, testKey) - _, digest, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:del", "") - if err != nil { - t.Fatalf("Promote: %v", err) - } + _, digest := mustPromote(t, m, parent, "tpl:del", "") key := tplKey("tpl:del") if err := m.DeleteTemplate(t.Context(), testKey, "", ""); !errors.Is(err, ErrPooledTemplate) { @@ -225,9 +202,7 @@ func TestTemplateRecordLockEvictsWithTheRecord(t *testing.T) { if hasRecLock(m, id) { t.Error("recLocks kept an entry for a template that was never promoted") } - if _, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, key.Template, ""); err != nil { - t.Fatalf("Promote: %v", err) - } + mustPromote(t, m, parent, key.Template, "") if err := m.DeleteTemplate(t.Context(), key, "", ""); err != nil { t.Fatalf("DeleteTemplate: %v", err) } @@ -257,10 +232,7 @@ func TestReconcileSweepsGoldenTmpDirs(t *testing.T) { func TestResolveGoldenNeverServesAPooledKeyFromATemplate(t *testing.T) { m := newTestManager(t, newFakeEngine()) parent := mustClaim(t, m, testKey) - key, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:pooled", "") - if err != nil { - t.Fatalf("Promote: %v", err) - } + key, _ := mustPromote(t, m, parent, "tpl:pooled", "") m.mu.Lock() m.pools[key] = newPool(config.PoolSpec{PoolKey: key}) m.mu.Unlock() @@ -294,10 +266,7 @@ func TestTemplateTenantScopedDelete(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) parent := claimTenant(t, m, "acme") - key, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:tenant", "acme") - if err != nil { - t.Fatalf("Promote: %v", err) - } + key, _ := mustPromote(t, m, parent, "tpl:tenant", "acme") if err := m.DeleteTemplate(t.Context(), key, "beta", ""); !errors.Is(err, ErrUnknownTemplate) { t.Errorf("cross-tenant delete: %v, want ErrUnknownTemplate", err) @@ -305,9 +274,7 @@ func TestTemplateTenantScopedDelete(t *testing.T) { if err := m.DeleteTemplate(t.Context(), key, "acme", ""); err != nil { t.Errorf("own delete: %v", err) } - if _, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:tenant", "acme"); err != nil { - t.Fatalf("re-promote: %v", err) - } + mustPromote(t, m, parent, "tpl:tenant", "acme") if err := m.DeleteTemplate(t.Context(), key, "", ""); err != nil { t.Errorf("root delete: %v", err) } @@ -331,9 +298,7 @@ func TestPromoteFailsClosedOnMetaError(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) a := claimTenant(t, m, "acme") - if _, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "shared:v1", "acme"); err != nil { - t.Fatalf("promote: %v", err) - } + mustPromote(t, m, a, "shared:v1", "acme") key := tplKey("shared:v1") meta := filepath.Join(m.dataDir, "checkpoints", store.TemplateID(key.Hash()), store.MetaFile) if err := os.Chmod(meta, 0o000); err != nil { @@ -349,9 +314,7 @@ func TestPromoteRefusesCrossTenantOverwrite(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) a := claimTenant(t, m, "acme") - if _, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "shared:v1", "acme"); err != nil { - t.Fatalf("acme promote: %v", err) - } + mustPromote(t, m, a, "shared:v1", "acme") b := claimTenant(t, m, "beta") if _, _, err := m.Promote(t.Context(), b.ID, Cred{Token: b.Token}, "shared:v1", "beta"); !errors.Is(err, ErrTemplateOwned) { t.Errorf("beta overwrite: %v, want ErrTemplateOwned", err) @@ -367,15 +330,9 @@ func TestTemplateClaimIsTenantScoped(t *testing.T) { m := newTestManager(t, eng) a := claimTenant(t, m, "acme") - private, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "acme-private", "acme") - if err != nil { - t.Fatalf("acme promote: %v", err) - } + private, _ := mustPromote(t, m, a, "acme-private", "acme") r := claimTenant(t, m, "") - shared, _, err := m.Promote(t.Context(), r.ID, Cred{Token: r.Token}, "ops-shared", "") - if err != nil { - t.Fatalf("root promote: %v", err) - } + shared, _ := mustPromote(t, m, r, "ops-shared", "") if _, err := m.ClaimProvision(t.Context(), private, ClaimOptions{RequirePromoted: true, TTL: time.Hour, Tenant: "beta"}); !errors.Is(err, ErrUnknownTemplate) { t.Errorf("beta claiming acme's template: %v, want ErrUnknownTemplate", err) @@ -404,10 +361,7 @@ func TestTemplateClaimIsTenantScoped(t *testing.T) { func TestHasPromotedTemplateIsTenantScoped(t *testing.T) { m := newTestManager(t, newFakeEngine()) a := claimTenant(t, m, "acme") - key, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "acme-private", "acme") - if err != nil { - t.Fatalf("promote: %v", err) - } + key, _ := mustPromote(t, m, a, "acme-private", "acme") if m.HasPromotedTemplate(t.Context(), key, "beta") { t.Error("beta sees acme's template as a local golden") @@ -422,10 +376,7 @@ func TestHasPromotedTemplateIsTenantScoped(t *testing.T) { func TestTemplateHashesAreTenantScoped(t *testing.T) { m := tenantManager(t, t.TempDir(), tenants.TenantSpec{Name: "acme", Token: "acme-tok"}) a := claimTenant(t, m, "acme") - key, _, err := m.Promote(t.Context(), a.ID, Cred{Token: a.Token}, "acme-private", "acme") - if err != nil { - t.Fatalf("promote: %v", err) - } + key, _ := mustPromote(t, m, a, "acme-private", "acme") if err := m.DeleteTenant(t.Context(), "acme"); err != nil { t.Fatalf("remove tenant: %v", err) } @@ -453,9 +404,7 @@ func TestTemplateHashesSortedForMeshCompare(t *testing.T) { m := newTestManager(t, eng) parent := mustClaim(t, m, testKey) for _, name := range []string{"tpl:a", "tpl:b", "tpl:c", "tpl:d"} { - if _, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, name, ""); err != nil { - t.Fatalf("Promote %s: %v", name, err) - } + mustPromote(t, m, parent, name, "") } hashes := m.TemplateHashes() if len(hashes) != 8 { @@ -471,21 +420,15 @@ func TestTemplatesListWhatTheNodeHoldsAcrossARestart(t *testing.T) { dir := t.TempDir() m := newTestManagerAt(t, eng, dir) parent := mustClaim(t, m, testKey) - keyA, digestA, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:a", "") - if err != nil { - t.Fatalf("Promote tpl:a: %v", err) - } + keyA, digestA := mustPromote(t, m, parent, "tpl:a", "") acme := claimTenant(t, m, "acme") - keyB, digestB, err := m.Promote(t.Context(), acme.ID, Cred{Token: acme.Token}, "tpl:b", "acme") - if err != nil { - t.Fatalf("Promote tpl:b: %v", err) - } + keyB, digestB := mustPromote(t, m, acme, "tpl:b", "acme") oldID := store.TemplateID(tplKey("tpl:old").Hash()) staging := stageTemplate(t, m, oldID) - if err = os.WriteFile(filepath.Join(staging, store.MetaFile), []byte(`{"id":"`+oldID+`","created_at":"2026-01-01T00:00:00Z"}`), 0o600); err != nil { + if err := os.WriteFile(filepath.Join(staging, store.MetaFile), []byte(`{"id":"`+oldID+`","created_at":"2026-01-01T00:00:00Z"}`), 0o600); err != nil { t.Fatalf("write keyless meta: %v", err) } - if _, err = storetest.PublishDigested(t.Context(), m.tpls, staging, oldID); err != nil { + if _, err := storetest.PublishDigested(t.Context(), m.tpls, staging, oldID); err != nil { t.Fatalf("publish keyless record: %v", err) } @@ -508,7 +451,7 @@ func TestTemplatesListWhatTheNodeHoldsAcrossARestart(t *testing.T) { if hashes := restarted.TemplateHashes(); len(hashes) != 6 { t.Errorf("gossip %v, want the keyless record still routed", hashes) } - if err = restarted.DeleteTemplate(t.Context(), keyA, "", ""); err != nil { + if err := restarted.DeleteTemplate(t.Context(), keyA, "", ""); err != nil { t.Fatalf("DeleteTemplate: %v", err) } if got = restarted.Templates(); len(got) != 1 || got[0].Key != keyB { @@ -525,10 +468,7 @@ func TestTemplateLabelsPersistUntilTheTemplateIsRepromotedOrDeleted(t *testing.T dir := t.TempDir() m := newTestManagerAt(t, eng, dir) parent := mustClaim(t, m, testKey) - key, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:l", "") - if err != nil { - t.Fatalf("Promote: %v", err) - } + key, _ := mustPromote(t, m, parent, "tpl:l", "") labelsOf := func(m *Manager) types.Metadata { t.Helper() i := slices.IndexFunc(m.Templates(), func(ti TemplateInfo) bool { return ti.Key == key }) @@ -538,10 +478,10 @@ func TestTemplateLabelsPersistUntilTheTemplateIsRepromotedOrDeleted(t *testing.T return m.Templates()[i].Labels } want := types.Metadata{"v1": "sha256:aa"} - if err = m.SetTemplateLabels(t.Context(), key, want, "", "sha256:observed-elsewhere"); !errors.Is(err, ErrTemplateReplaced) || labelsOf(m) != nil { + if err := m.SetTemplateLabels(t.Context(), key, want, "", "sha256:observed-elsewhere"); !errors.Is(err, ErrTemplateReplaced) || labelsOf(m) != nil { t.Errorf("labels with another digest: %v, want ErrTemplateReplaced and no labels written", err) } - if err = m.SetTemplateLabels(t.Context(), key, want, "", ""); err != nil { + if err := m.SetTemplateLabels(t.Context(), key, want, "", ""); err != nil { t.Fatalf("SetTemplateLabels: %v", err) } if got := labelsOf(m); !maps.Equal(got, want) { @@ -551,38 +491,34 @@ func TestTemplateLabelsPersistUntilTheTemplateIsRepromotedOrDeleted(t *testing.T if got := labelsOf(restarted); !maps.Equal(got, want) { t.Errorf("labels after restart %v, want %v", got, want) } - if err = m.SetTemplateLabels(t.Context(), key, types.Metadata{"x": "y"}, "acme", ""); !errors.Is(err, ErrUnknownTemplate) { + if err := m.SetTemplateLabels(t.Context(), key, types.Metadata{"x": "y"}, "acme", ""); !errors.Is(err, ErrUnknownTemplate) { t.Errorf("a tenant labeling the operator's template: %v, want ErrUnknownTemplate", err) } - if err = m.SetTemplateLabels(t.Context(), tplKey("tpl:none"), want, "", ""); !errors.Is(err, ErrUnknownTemplate) { + if err := m.SetTemplateLabels(t.Context(), tplKey("tpl:none"), want, "", ""); !errors.Is(err, ErrUnknownTemplate) { t.Errorf("an unknown template: %v, want ErrUnknownTemplate", err) } - if _, _, err = m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:l", ""); err != nil { - t.Fatalf("re-promote: %v", err) - } + mustPromote(t, m, parent, "tpl:l", "") if got := labelsOf(newTestManagerAt(t, eng, dir)); got != nil { t.Errorf("labels %v survived a re-promote, want none", got) } - if err = m.SetTemplateLabels(t.Context(), key, want, "", ""); err != nil { + if err := m.SetTemplateLabels(t.Context(), key, want, "", ""); err != nil { t.Fatalf("SetTemplateLabels again: %v", err) } - if err = m.SetTemplateLabels(t.Context(), key, types.Metadata{}, "", ""); err != nil || labelsOf(m) != nil { + if err := m.SetTemplateLabels(t.Context(), key, types.Metadata{}, "", ""); err != nil || labelsOf(m) != nil { t.Errorf("empty labels: %v, labels %v; want them cleared", err, labelsOf(m)) } - if err = m.SetTemplateLabels(t.Context(), key, want, "", ""); err != nil { + if err := m.SetTemplateLabels(t.Context(), key, want, "", ""); err != nil { t.Fatalf("SetTemplateLabels before delete: %v", err) } - if err = m.DeleteTemplate(t.Context(), key, "", ""); err != nil { + if err := m.DeleteTemplate(t.Context(), key, "", ""); err != nil { t.Fatalf("DeleteTemplate: %v", err) } - if _, _, err = m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:l", ""); err != nil { - t.Fatalf("promote after delete: %v", err) - } + mustPromote(t, m, parent, "tpl:l", "") if got := labelsOf(newTestManagerAt(t, eng, dir)); got != nil { t.Errorf("labels %v outlived the deleted template", got) } pooled := newTestManagerAt(t, eng, t.TempDir(), config.PoolSpec{PoolKey: key}) - if err = pooled.SetTemplateLabels(t.Context(), key, want, "", ""); !errors.Is(err, ErrPooledTemplate) { + if err := pooled.SetTemplateLabels(t.Context(), key, want, "", ""); !errors.Is(err, ErrPooledTemplate) { t.Errorf("a pooled key: %v, want ErrPooledTemplate", err) } } @@ -605,10 +541,7 @@ func TestClaimFromATemplateRunsBesideItsRepromote(t *testing.T) { synctest.Test(t, func(t *testing.T) { m := newTestManager(t, newFakeEngine()) parent := mustClaim(t, m, testKey) - key, _, err := m.Promote(t.Context(), parent.ID, Cred{Token: parent.Token}, "tpl:x", "") - if err != nil { - t.Fatalf("Promote: %v", err) - } + key, _ := mustPromote(t, m, parent, "tpl:x", "") slow := &slowPrepareStore{Store: m.tpls, entered: make(chan struct{}), release: make(chan struct{})} m.tpls = slow repromoted := make(chan error, 1) @@ -627,7 +560,7 @@ func TestClaimFromATemplateRunsBesideItsRepromote(t *testing.T) { }() synctest.Wait() select { - case err = <-claimed: + case err := <-claimed: if err != nil { t.Errorf("claim during a re-promote: %v, want a clone of the committed template", err) } @@ -635,7 +568,7 @@ func TestClaimFromATemplateRunsBesideItsRepromote(t *testing.T) { t.Error("a claim with a 100 ms context was still blocked during a re-promote") } close(slow.release) - if err = <-repromoted; err != nil { + if err := <-repromoted; err != nil { t.Fatalf("re-promote: %v", err) } }) @@ -653,6 +586,15 @@ func (s *slowPrepareStore) PrepareDigested(ctx context.Context, staging, id, lin return s.Store.PrepareDigested(ctx, staging, id, lineage) } +func mustPromote(t *testing.T, m *Manager, sb *types.Sandbox, name, tenant string) (types.PoolKey, string) { + t.Helper() + key, digest, err := m.Promote(t.Context(), sb.ID, Cred{Token: sb.Token}, name, tenant) + if err != nil { + t.Fatalf("promote %s: %v", name, err) + } + return key, digest +} + func stageTemplate(t *testing.T, m *Manager, id string) string { t.Helper() staging, err := m.tpls.Stage(id) From ab7b95e06215148d550bc0ac65f396f05700adc2 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:46:19 +0800 Subject: [PATCH 093/127] pool: pair, mustFlush and hookRegistry helpers for the failover tests --- sandboxd/pool/cordon_test.go | 4 +- sandboxd/pool/drain_test.go | 24 ++-- sandboxd/pool/failover_test.go | 216 ++++++++++++------------------ sandboxd/pool/interrupted_test.go | 4 +- sandboxd/pool/pool_test.go | 7 + sandboxd/pool/replica_test.go | 32 ++--- sandboxd/pool/tun_test.go | 4 +- 7 files changed, 117 insertions(+), 174 deletions(-) diff --git a/sandboxd/pool/cordon_test.go b/sandboxd/pool/cordon_test.go index 3a33bd546..db60366dd 100644 --- a/sandboxd/pool/cordon_test.go +++ b/sandboxd/pool/cordon_test.go @@ -175,9 +175,7 @@ func TestACordonedNodeNeverAdoptsButKeepsItsOwnRowAcrossARestart(t *testing.T) { mustArchive(t, a, lost) own := mustClaim(t, b, testKey) mustArchive(t, b, own) - if err := b.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, b) if err := b.Cordon(t.Context()); err != nil { t.Fatalf("cordon: %v", err) } diff --git a/sandboxd/pool/drain_test.go b/sandboxd/pool/drain_test.go index dbc070de4..b81aa0c27 100644 --- a/sandboxd/pool/drain_test.go +++ b/sandboxd/pool/drain_test.go @@ -62,7 +62,7 @@ func TestDrainRefusesClaimsTrimsWarmAndUndrainRefills(t *testing.T) { func TestEvacuateHandsClaimsToAPeerWhileItsLeaseIsLive(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) running, asleep, archived := mustClaim(t, a, testKey), mustClaim(t, a, testKey), mustClaim(t, a, testKey) hibernateClaim(t, a, asleep) mustArchive(t, a, archived) @@ -95,7 +95,7 @@ func TestEvacuateHandsClaimsToAPeerWhileItsLeaseIsLive(t *testing.T) { func TestEvacuateHandOffRefusedByAConcurrentOwnerWriteKeepsTheClaim(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.archives = &hookRegistry{Registry: c.reg, beforeRelease: func() { @@ -127,7 +127,7 @@ func TestEvacuateHandOffRefusedByAConcurrentOwnerWriteKeepsTheClaim(t *testing.T func TestEvacuateHandOffOfARowAPeerTookDropsTheClaim(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.archives = &hookRegistry{Registry: c.reg, beforeRelease: func() { @@ -198,7 +198,7 @@ func TestEvacuateWithoutArchiveFailoverSkipsEveryClaim(t *testing.T) { func TestEvacuateWaitsOutAnArchiveInFlight(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.mu.Lock() @@ -220,7 +220,7 @@ func TestEvacuateWaitsOutAnArchiveInFlight(t *testing.T) { func TestUndrainStopsAnEvacuationBeforeItsHandOff(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) stall := stallExports(a) sb := mustClaim(t, a, testKey) @@ -405,7 +405,7 @@ func TestAnEnvWriteThatLostTheClaimAnswersUnknown(t *testing.T) { func TestAnEnvWriteDuringTheArchiveReachesItsRow(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) written := make(chan error, 1) @@ -507,7 +507,7 @@ func TestEvacuateKeepsAClaimArchivedWithoutARow(t *testing.T) { func TestEvacuateMovesTheCheapestClaimsFirst(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) running, hibernated, archived := mustClaim(t, a, testKey), mustClaim(t, a, testKey), mustClaim(t, a, testKey) hibernateClaim(t, a, hibernated) mustArchive(t, a, archived) @@ -580,7 +580,7 @@ func TestEvacuateWithNoPeerToAdoptKeepsTheClaim(t *testing.T) { func TestEvacuateConvergesWhenAPeersAnswerIsLost(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) a.WithPeerAdopt(func(ctx context.Context, id, token string, _ bool) bool { if _, err := b.AdoptArchived(ctx, id, token); err != nil { @@ -605,7 +605,7 @@ func TestEvacuateConvergesWhenAPeersAnswerIsLost(t *testing.T) { func TestEvacuateConvergesWhenTheReleaseReplyIsLost(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) a.archives = &hookRegistry{Registry: c.reg, loseReleaseReply: true} @@ -626,7 +626,7 @@ func TestEvacuateConvergesWhenTheReleaseReplyIsLost(t *testing.T) { func TestARepeatEvacuateResumesAHandOffAnOutageCutShort(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) a.archives = &hookRegistry{Registry: c.reg, loseReleaseReply: true} @@ -649,7 +649,7 @@ func TestARepeatEvacuateResumesAHandOffAnOutageCutShort(t *testing.T) { func TestARepeatEvacuateWithNoAdopterLeavesAReleasedRowReleased(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) a.archives = &hookRegistry{Registry: c.reg, loseReleaseReply: true} a.Evacuate(t.Context()) @@ -673,7 +673,7 @@ func TestARepeatEvacuateWithNoAdopterLeavesAReleasedRowReleased(t *testing.T) { func TestACrashBeforeThePeersAnswerLeavesARowAnyNodeSettles(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) adopted, purged := mustClaim(t, a, testKey), mustClaim(t, a, testKey) for _, sb := range []*types.Sandbox{adopted, purged} { mustArchive(t, a, sb) diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index e5aa5cbb4..14742ac64 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -72,7 +72,7 @@ func TestFailoverRowWriteFailureKeepsTheClaimHibernated(t *testing.T) { func TestFailoverEnvRotationReachesTheAdopter(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := archivedWithEnv(t, a) if rowEnv(t, c, sb) != "old" { t.Fatalf("archive row env %q, want old", rowEnv(t, c, sb)) @@ -150,7 +150,7 @@ func TestFailoverEnvRotationSurvivesALostReply(t *testing.T) { func TestFailoverEnvRotationOfAClaimAdoptedElsewhereDropsIt(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := archivedWithEnv(t, a) c.expire("a") mustAdopt(t, b, sb, "adopt") @@ -243,7 +243,7 @@ func TestFailoverOwnerWakeWithTheRegistryDownFailsClosed(t *testing.T) { func TestFailoverAdoptionMovesTheClaimAndTheOldOwnerLetsGo(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk @@ -273,7 +273,7 @@ func TestFailoverAdoptionMovesTheClaimAndTheOldOwnerLetsGo(t *testing.T) { func TestFailoverNoAdoptionWhileTheOwnerIsAlive(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) @@ -285,7 +285,7 @@ func TestFailoverNoAdoptionWhileTheOwnerIsAlive(t *testing.T) { func TestFailoverADrainingNodeDoesNotAdopt(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) c.expire("a") @@ -303,7 +303,7 @@ func TestFailoverADrainingNodeDoesNotAdopt(t *testing.T) { func TestFailoverAdoptionIgnoresClaimCapsAndAWakeStillChecksMaxRunning(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) for _, m := range []*Manager{a, b} { if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "acme", Token: "tok-acme", MaxClaims: 1}); err != nil { t.Fatalf("put tenant: %v", err) @@ -339,7 +339,7 @@ func TestFailoverAdoptionIgnoresClaimCapsAndAWakeStillChecksMaxRunning(t *testin func TestFailoverAdoptionNeedsTheToken(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) c.expire("a") @@ -493,9 +493,7 @@ func TestFailoverRestartDropsAClaimWhoseInterruptedArchiveWasAdopted(t *testing. a := c.node(t, "a", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) a.mu.Lock() vm, snap := sb.VMName, sb.HibernateSnap a.mu.Unlock() @@ -527,9 +525,7 @@ func TestFailoverRestartWithTheRegistryDownKeepsEverything(t *testing.T) { a := c.node(t, "a", dirA, keepAnHour) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.reg.SetDown(true) restarted := c.node(t, "a", dirA, keepAnHour) @@ -566,7 +562,7 @@ func TestFailoverReleaseDropsTheRowThenTheCheckpoint(t *testing.T) { func TestFailoverReleaseAfterAdoptionKeepsTheCheckpoint(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) c.expire("a") @@ -640,7 +636,7 @@ func TestFailoverSweepPurgesADepartedOwnersArchiveOnce(t *testing.T) { func TestFailoverAdoptionMidWakeFencesTheOwnerOut(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk @@ -730,9 +726,7 @@ func TestFailoverRestartFencesWithOneQuery(t *testing.T) { mustArchive(t, a, sb) ids = append(ids, sb.ID) } - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) restarted := c.node(t, "a", dirA, keepAnHour) counted := &countingRegistry{Registry: c.reg} restarted.archives = counted @@ -799,9 +793,7 @@ func TestFailoverRestartAfterAConsumeWithoutCommitKeepsTheClaim(t *testing.T) { epoch := c.take(t, sb.ID, "a") _, _ = a.commitIfLive(t.Context(), sb, func() { sb.ArchiveEpoch = epoch }) c.consume(t, sb.ID, epoch) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) restarted := c.restartReconciled(t, a) held, _ := restarted.byID(sb.ID) @@ -819,9 +811,7 @@ func TestFailoverAnArchiveWokenElsewhereIsDropped(t *testing.T) { a, b := c.node(t, "a", dirA, keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.expire("a") mustAdopt(t, b, sb, "adopt") if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { @@ -843,7 +833,7 @@ func TestFailoverAnArchiveWokenElsewhereIsDropped(t *testing.T) { func TestFailoverALeftoverDeleteMarkerSparesTheAdoptersCheckpoint(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk @@ -864,16 +854,14 @@ func TestFailoverALeftoverDeleteMarkerSparesTheAdoptersCheckpoint(t *testing.T) func TestFailoverRestartClearsTheMarkerOfAnAdoptedClaim(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk if err := a.ckMarks.mark(ck); err != nil { t.Fatalf("mark: %v", err) } - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.expire("a") mustAdopt(t, b, sb, "adopt") c.renew(t, "a") @@ -886,7 +874,7 @@ func TestFailoverRestartClearsTheMarkerOfAnAdoptedClaim(t *testing.T) { func TestFailoverARepeatedAdoptionOnOneNodeKeepsOneClaim(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) for _, m := range []*Manager{a, b} { if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "acme", Token: "tok-acme"}); err != nil { t.Fatalf("put tenant: %v", err) @@ -927,12 +915,10 @@ func TestFailoverARepeatedAdoptionOnOneNodeKeepsOneClaim(t *testing.T) { func TestFailoverRestartInsideAnAdoptersWakeDropsTheClaim(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) ck := sb.ArchiveCk c.expire("a") mustAdopt(t, b, sb, "adopt") @@ -961,12 +947,10 @@ func TestFailoverRestartInsideAnAdoptersWakeDropsTheClaim(t *testing.T) { func TestFailoverRestartAfterAnAdopterReleasedDropsTheClaim(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.expire("a") mustAdopt(t, b, sb, "adopt") b.ckpts = &failingDeleteStore{Store: b.ckpts, attempts: make(chan string, 8)} @@ -1020,12 +1004,10 @@ func TestFailoverReleaseDuringAnArchiveLeavesNoRow(t *testing.T) { func TestFailoverRestartDropsAClaimAdoptedWhileItsInterruptedArchiveSettles(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) const ck = "ck_00000000000000cc" seedArchiveCk(t, a, ck, sb.ID) if _, err := a.putArchive(t.Context(), sb, ck); err != nil { @@ -1055,9 +1037,7 @@ func TestFailoverRestartKeepsAClaimWhoseInterruptedArchiveRowVanished(t *testing a := c.node(t, "a", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) const ck = "ck_00000000000000dd" seedArchiveCk(t, a, ck, sb.ID) epoch, err := a.putArchive(t.Context(), sb, ck) @@ -1091,9 +1071,7 @@ func TestFailoverRestartLeavesItsOwnTombstoneToTheSweep(t *testing.T) { if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) before, _, _ := c.reg.Get(t.Context(), sb.ID) restarted := c.restartReconciled(t, a) @@ -1107,7 +1085,7 @@ func TestFailoverRestartLeavesItsOwnTombstoneToTheSweep(t *testing.T) { func TestFailoverSweepLeavesADepartedOwnersTombstoneUntilItReturns(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) d := c.node(t, "d", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) @@ -1214,7 +1192,7 @@ func TestFailoverATombstoneIsNotAdopted(t *testing.T) { func TestFailoverADropClearsTheMarkerBeforeTheCheckpointIsUnpinned(t *testing.T) { for range 20 { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk @@ -1249,7 +1227,7 @@ func TestFailoverADropClearsTheMarkerBeforeTheCheckpointIsUnpinned(t *testing.T) func TestFailoverReleaseAfterALostTakeReplyLeavesNoLiveRow(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.archives = &hookRegistry{Registry: c.reg, loseTakeReply: true} @@ -1264,7 +1242,7 @@ func TestFailoverReleaseAfterALostTakeReplyLeavesNoLiveRow(t *testing.T) { func TestFailoverReleaseAfterALostRearmReplyLeavesNoLiveRow(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.archives = &hookRegistry{Registry: c.reg, loseConsumeReply: true} @@ -1286,7 +1264,7 @@ func TestFailoverReleaseAfterALostRearmReplyLeavesNoLiveRow(t *testing.T) { func TestFailoverARepeatedAdoptionOfAReleasedClaimIsRefused(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) c.expire("a") @@ -1328,7 +1306,7 @@ func TestFailoverSweepDropsTheRowOfAnArchiveThatNeverCommitted(t *testing.T) { func TestFailoverSweepRearmsATombstoneItsArchivedClaimHolds(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) a.archives = &hookRegistry{Registry: c.reg, loseConsumeReply: true} @@ -1350,7 +1328,7 @@ func TestFailoverSweepRearmsATombstoneItsArchivedClaimHolds(t *testing.T) { func TestFailoverSweepLeavesADepartedOwnersWakeTombstone(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) epoch := c.take(t, sb.ID, "a") @@ -1365,15 +1343,13 @@ func TestFailoverSweepLeavesADepartedOwnersWakeTombstone(t *testing.T) { func TestFailoverRestartAfterAConsumeWhileAPeerSweptKeepsTheClaim(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) epoch := c.take(t, sb.ID, "a") _, _ = a.commitIfLive(t.Context(), sb, func() { sb.ArchiveEpoch = epoch }) c.consume(t, sb.ID, epoch) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.expire("a") b.sweepOrphanArchives(t.Context()) c.renew(t, "a") @@ -1426,7 +1402,7 @@ func TestFailoverArchiveExportsNothingForARowAnotherNodeHolds(t *testing.T) { func TestFailoverAWakeKeepsItsFenceAgainstThisNodesOwnAdoption(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk @@ -1459,7 +1435,7 @@ func TestFailoverAWakeKeepsItsFenceAgainstThisNodesOwnAdoption(t *testing.T) { func TestFailoverConcurrentAdoptionsOnOneNodeAllAnswerForIt(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) for _, m := range []*Manager{a, b} { if err := m.PutTenant(t.Context(), tenants.TenantSpec{Name: "acme", Token: "tok-acme"}); err != nil { t.Fatalf("put tenant: %v", err) @@ -1501,7 +1477,7 @@ func TestFailoverConcurrentAdoptionsOnOneNodeAllAnswerForIt(t *testing.T) { func TestFailoverACoalescedAdoptionStillNeedsTheToken(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) c.expire("a") @@ -1552,7 +1528,7 @@ func TestFailoverArchivePersistsItsIntentBeforeThePut(t *testing.T) { func TestFailoverRestartDropsAClaimWhoseInterruptedArchiveWasConsumedElsewhere(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) a.mu.Lock() @@ -1652,7 +1628,7 @@ func TestFailoverAnAbandonKeepsTheExportWhileItsIntentIsDurable(t *testing.T) { func TestFailoverAnOwnerReturningWithItsWakeCommittedPurgesItsTombstone(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) ck := sb.ArchiveCk @@ -1660,9 +1636,7 @@ func TestFailoverAnOwnerReturningWithItsWakeCommittedPurgesItsTombstone(t *testi if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.expire("a") b.sweepOrphanArchives(t.Context()) c.renew(t, "a") @@ -1679,7 +1653,7 @@ func TestFailoverAnOwnerReturningWithItsWakeCommittedPurgesItsTombstone(t *testi func TestFailoverRestartDropsAClaimConsumedWhileItSettlesItsInterruptedArchive(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) ck := interruptArchive(t, a, sb, true) @@ -1727,7 +1701,7 @@ func TestFailoverRestartUndoesItsOwnInterruptedArchiveAndItsIntent(t *testing.T) func TestFailoverReleaseAfterALostArchivePutReplyLeavesNothingAdoptable(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) c.archiveLosingPutReply(t, a, sb) @@ -1834,7 +1808,7 @@ func TestFailoverAWakeWithoutARegistryKeepsItsPendingIntent(t *testing.T) { func TestFailoverAReleaseFindsItsUnsettledArchiveTakenElsewhere(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) hibernateClaim(t, a, sb) a.mu.Lock() @@ -1862,7 +1836,7 @@ func TestFailoverAReleaseFindsItsUnsettledArchiveTakenElsewhere(t *testing.T) { func TestALookupRacingAWakeAdoptionTakesTheRowOnce(t *testing.T) { synctest.Test(t, func(t *testing.T) { c := newFailoverCluster(t) - a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + a, b := c.pair(t) sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) rec, _, err := c.reg.Get(t.Context(), sb.ID) @@ -1951,6 +1925,11 @@ func (c *failoverCluster) node(t *testing.T, name, dataDir string, deleteAfter i return m } +func (c *failoverCluster) pair(t *testing.T) (a, b *Manager) { + t.Helper() + return c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) +} + func (c *failoverCluster) restart(t *testing.T, old *Manager) *Manager { t.Helper() return c.nodeOn(t, fakeOf(old), old.nodeID, old.dataDir, keepAnHour) @@ -2024,79 +2003,40 @@ type hookRegistry struct { } func (r *hookRegistry) Put(ctx context.Context, rec archives.Record) (int64, error) { - if r.failPut { - return 0, archives.ErrUnavailable - } - if hook := r.beforePut; hook != nil { - r.beforePut = nil - hook() - } - epoch, err := r.Registry.Put(ctx, rec) - if r.losePutReply && err == nil { - r.losePutReply = false - return 0, archives.ErrUnavailable - } - return epoch, err + return r.write(func() (int64, error) { return r.Registry.Put(ctx, rec) }) } func (r *hookRegistry) Replace(ctx context.Context, rec archives.Record, expect int64) (int64, error) { r.replaces++ - if r.failPut { - return 0, archives.ErrUnavailable - } - if hook := r.beforePut; hook != nil { - r.beforePut = nil - hook() - } - epoch, err := r.Registry.Replace(ctx, rec, expect) - if r.losePutReply && err == nil { - r.losePutReply = false - return 0, archives.ErrUnavailable - } - return epoch, err + return r.write(func() (int64, error) { return r.Registry.Replace(ctx, rec, expect) }) } func (r *hookRegistry) Take(ctx context.Context, id string, expect int64, owner string) (int64, error) { - if hook := r.beforeTake; hook != nil { - r.beforeTake = nil - hook() - } + fire(&r.beforeTake) epoch, err := r.Registry.Take(ctx, id, expect, owner) if err == nil { r.takes.Add(1) - if hook := r.afterTake; hook != nil { - r.afterTake = nil - hook() - } + fire(&r.afterTake) } - if r.loseTakeReply && err == nil { - r.loseTakeReply = false + if err == nil && lost(&r.loseTakeReply) { return 0, archives.ErrUnavailable } return epoch, err } func (r *hookRegistry) Release(ctx context.Context, id string, expect int64, owner string) (int64, error) { - if hook := r.beforeRelease; hook != nil { - r.beforeRelease = nil - hook() - } + fire(&r.beforeRelease) released, err := r.Registry.Release(ctx, id, expect, owner) - if r.loseReleaseReply && err == nil { - r.loseReleaseReply = false + if err == nil && lost(&r.loseReleaseReply) { return 0, archives.ErrUnavailable } return released, err } func (r *hookRegistry) Delete(ctx context.Context, id string, epoch int64) error { - if hook := r.beforeDelete; hook != nil { - r.beforeDelete = nil - hook() - } + fire(&r.beforeDelete) err := r.Registry.Delete(ctx, id, epoch) - if err == nil && r.loseDeleteReply { - r.loseDeleteReply = false + if err == nil && lost(&r.loseDeleteReply) { return archives.ErrUnavailable } return err @@ -2104,26 +2044,44 @@ func (r *hookRegistry) Delete(ctx context.Context, id string, epoch int64) error func (r *hookRegistry) Rows(ctx context.Context, ids []string) (map[string]archives.Record, error) { rows, err := r.Registry.Rows(ctx, ids) - if hook := r.afterRows; hook != nil { - r.afterRows = nil - hook() - } + fire(&r.afterRows) return rows, err } func (r *hookRegistry) Consume(ctx context.Context, id string, expect int64) (int64, error) { - if hook := r.beforeConsume; hook != nil { - r.beforeConsume = nil - hook() - } + fire(&r.beforeConsume) epoch, err := r.Registry.Consume(ctx, id, expect) - if r.loseConsumeReply && err == nil { - r.loseConsumeReply = false + if err == nil && lost(&r.loseConsumeReply) { + return 0, archives.ErrUnavailable + } + return epoch, err +} + +func (r *hookRegistry) write(do func() (int64, error)) (int64, error) { + if r.failPut { + return 0, archives.ErrUnavailable + } + fire(&r.beforePut) + epoch, err := do() + if err == nil && lost(&r.losePutReply) { return 0, archives.ErrUnavailable } return epoch, err } +func fire(hook *func()) { + if h := *hook; h != nil { + *hook = nil + h() + } +} + +func lost(flag *bool) bool { + was := *flag + *flag = false + return was +} + type unreadableStore struct { store.Store } diff --git a/sandboxd/pool/interrupted_test.go b/sandboxd/pool/interrupted_test.go index 78185591a..5929b9a05 100644 --- a/sandboxd/pool/interrupted_test.go +++ b/sandboxd/pool/interrupted_test.go @@ -365,9 +365,7 @@ func TestAFailedCaptureKeepsItsIntentWhileSandboxdStops(t *testing.T) { } waitStarted(t, eng, 1) waitSettled(t, sb) - if err := m.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, m) if tt.intent(journaled(t, m)[sb.ID]) == "" || sb.Failed != "" { t.Errorf("journal lost the intent or failed the claim: %q", sb.Failed) } diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index bc3ea3a05..126daf10b 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -2289,6 +2289,13 @@ func hibernateClaim(t *testing.T, m *Manager, sb *types.Sandbox) { } } +func mustFlush(t *testing.T, m *Manager) { + t.Helper() + if err := m.FlushClaims(); err != nil { + t.Fatalf("flush: %v", err) + } +} + func mustAdopt(t *testing.T, m *Manager, sb *types.Sandbox, step string) { t.Helper() if adopted, err := m.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || !adopted { diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 917d79349..2322862cb 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -193,9 +193,7 @@ func TestReplicaRestartSettlesALostPutReply(t *testing.T) { a.mu.Lock() pending := sb.ReplicaPending a.mu.Unlock() - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) restarted := c.restartReconciled(t, a) held, ok := restarted.byID(sb.ID) @@ -795,9 +793,7 @@ func TestReplicaRestartKeepsItsRowAndStaysFencedUntilALeaseRenews(t *testing.T) sb := mustClaim(t, a, testKey) replicateNow(t, a, sb) ck := a.replicaOf(sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) before, _, _ := c.reg.Get(t.Context(), sb.ID) restarted := c.restartReconciled(t, a) @@ -821,9 +817,7 @@ func TestReplicaRestartAdoptsTheRowsCheckpointOfAnInterruptedRound(t *testing.T) sb := mustClaim(t, a, testKey) replicateNow(t, a, sb) first := a.replicaOf(sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) interrupted, _, publishErr := a.publishCheckpoint(t.Context(), sb, store.CheckpointID(randHex(8)), "replica", sb.Tenant, "", true) if publishErr != nil { t.Fatalf("publish: %v", publishErr) @@ -906,9 +900,7 @@ func TestReconcileWithTheRegistryDownRecordsOnlyTheEndNoOtherNodeCanHold(t *test a := c.node(t, "a", t.TempDir(), keepAnHour) replicated, plain := mustClaim(t, a, testKey), mustClaim(t, a, plainKey) replicateNow(t, a, replicated) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) eng := fakeOf(a) delete(eng.vms, replicated.VMName) delete(eng.vms, plain.VMName) @@ -930,9 +922,7 @@ func TestReplicaRestartDropsAClaimAdoptedElsewhere(t *testing.T) { sb := mustClaim(t, a, testKey) replicateNow(t, a, sb) ck := a.replicaOf(sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.expire("a") mustAdopt(t, b, sb, "adopt") @@ -952,9 +942,7 @@ func TestReplicaRestartDropsAClaimTheAdopterWokeAndFinished(t *testing.T) { a, b := c.node(t, "a", dirA, keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) replicateNow(t, a, sb) - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) c.expire("a") mustAdopt(t, b, sb, "adopt") if _, _, _, err := b.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { @@ -1075,9 +1063,7 @@ func TestReplicaRestartKeepsTheCheckpointOfARoundItCannotSettleYet(t *testing.T) a.mu.Lock() pending := sb.ReplicaPending a.mu.Unlock() - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) restarted := c.restart(t, a) restarted.ckpts = &unreadableStore{Store: restarted.ckpts} @@ -1371,9 +1357,7 @@ func TestARunningClaimThatMovedClosesItsDoors(t *testing.T) { func restartWithoutVM(t *testing.T, c *failoverCluster, a *Manager, sb *types.Sandbox, deleteAfter int) *Manager { t.Helper() - if err := a.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, a) eng := fakeOf(a) delete(eng.vms, sb.VMName) restarted := c.nodeOn(t, eng, a.nodeID, a.dataDir, deleteAfter) diff --git a/sandboxd/pool/tun_test.go b/sandboxd/pool/tun_test.go index 749c7dae1..da0172ef4 100644 --- a/sandboxd/pool/tun_test.go +++ b/sandboxd/pool/tun_test.go @@ -297,9 +297,7 @@ func TestFlushClaimsPersistsTheLiveTunBytes(t *testing.T) { t.Fatal(err) } waitFor(t, func() bool { used, _ := m.out.TunUsed(sb.ID); return used > 0 }) - if err := m.FlushClaims(); err != nil { - t.Fatalf("flush: %v", err) - } + mustFlush(t, m) if got := journaled(t, m)[sb.ID].TunBytes; got != int64(len(pkt)-tunnet.VnetHdrLen) { t.Errorf("journaled tun bytes %d, want the %d sent", got, len(pkt)-tunnet.VnetHdrLen) } From 218fa1b19c1dac99845b7413ed318f611e709350 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:47:22 +0800 Subject: [PATCH 094/127] pool: evacuatingCalls and counts helpers in the move tests --- sandboxd/pool/move_test.go | 79 +++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 44 deletions(-) diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 0936dd33f..de31d5811 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -35,7 +35,7 @@ func TestMoveHandsARunningClaimToThePeerWithBothNodesCountingIt(t *testing.T) { vm := sb.VMName var during [][2][3]int mc.p.roundHook = func(int) error { - during = append(during, [2][3]int{moveCounts(mc.s, "acme"), moveCounts(mc.t, "acme")}) + during = append(during, mc.counts("acme")) return nil } mc.p.commitHook = mc.p.roundHook @@ -52,7 +52,7 @@ func TestMoveHandsARunningClaimToThePeerWithBothNodesCountingIt(t *testing.T) { t.Errorf("step %d counts source/target %v, want 1/1/1 on both", i, counts) } } - if got := [2][3]int{moveCounts(mc.s, "acme"), moveCounts(mc.t, "acme")}; got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { + if got := mc.counts("acme"); got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { t.Errorf("counts after the move %v, want 0/0/0 on the source and 1/1/1 on the target", got) } moved, ok := mc.t.byID(sb.ID) @@ -204,7 +204,7 @@ func TestMoveWhoseCommitAnswerIsLostResolvesThroughTheRecord(t *testing.T) { if moved, ok := mc.t.byID(sb.ID); !ok || moved.MoveIn { t.Errorf("target claim %+v %v, want it started", moved, ok) } - if got := [2][3]int{moveCounts(mc.s, "acme"), moveCounts(mc.t, "acme")}; got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { + if got := mc.counts("acme"); got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { t.Errorf("counts %v, want the claim on the target only", got) } } @@ -536,7 +536,7 @@ func TestMoveNeedsNoArchiveRegistry(t *testing.T) { if moved, ok := mc.t.byID(sb.ID); !ok || moved.MoveIn { t.Errorf("target claim %+v %v, want it started", moved, ok) } - if got := [2][3]int{moveCounts(mc.s, "acme"), moveCounts(mc.t, "acme")}; got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { + if got := mc.counts("acme"); got != [2][3]int{{0, 0, 0}, {1, 1, 1}} { t.Errorf("counts %v, want the claim on the target only", got) } } @@ -816,18 +816,7 @@ func TestMoveFinalRoundAnswersUserCallsAtOnce(t *testing.T) { cred := Cred{Token: sb.Token} ctx := t.Context() mc.p.commitHook = func(int) error { - _, forkErr := mc.s.Fork(ctx, sb.ID, cred, 1, time.Minute, "", "") - _, ckErr := mc.s.Checkpoint(ctx, sb.ID, cred, "ck", "") - _, _, promoteErr := mc.s.Promote(ctx, sb.ID, cred, "tpl", "") - for op, err := range map[string]error{ - "fork": forkErr, "checkpoint": ckErr, "promote": promoteErr, - "hibernate": mc.s.Hibernate(ctx, sb.ID, cred), - "release": mc.s.Release(ctx, sb.ID, cred), - } { - if !errors.Is(err, ErrEvacuating) { - t.Errorf("%s during the final round: %v, want ErrEvacuating", op, err) - } - } + assertEvacuating(t, evacuatingCalls(ctx, mc.s, sb.ID, cred), "during the final round") return nil } if _, err := mc.s.Move(ctx, sb.ID, "b", time.Minute); err != nil { @@ -1147,22 +1136,12 @@ func TestMoveReceivedClaimWaitingOutAnOutageIsNotHibernatedOrServed(t *testing.T mc.t.mu.Lock() mc.t.evac = nil mc.t.mu.Unlock() - _, forkErr := mc.t.Fork(ctx, sb.ID, cred, 1, time.Minute, "", "") - _, ckErr := mc.t.Checkpoint(ctx, sb.ID, cred, "ck", "") - _, _, promoteErr := mc.t.Promote(ctx, sb.ID, cred, "tpl", "") - _, renewErr := mc.t.Renew(ctx, sb.ID, cred, time.Hour, "") - _, dialErr := mc.t.DialPort(ctx, sb.ID, cred, 80) - for op, err := range map[string]error{ - "fork": forkErr, "checkpoint": ckErr, "promote": promoteErr, "renew": renewErr, "dial": dialErr, - "hibernate": mc.t.Hibernate(ctx, sb.ID, cred), - "release": mc.t.Release(ctx, sb.ID, cred), - "env": mc.t.PatchEnv(ctx, sb.ID, types.EnvPatch{"GH_TOKEN": {Value: "x", Guest: hostOnly}}, ""), - "wake": mc.t.Wake(ctx, sb.ID, cred), - } { - if !errors.Is(err, ErrEvacuating) { - t.Errorf("%s of a received claim: %v, want ErrEvacuating", op, err) - } - } + calls := evacuatingCalls(ctx, mc.t, sb.ID, cred) + _, calls["renew"] = mc.t.Renew(ctx, sb.ID, cred, time.Hour, "") + _, calls["dial"] = mc.t.DialPort(ctx, sb.ID, cred, 80) + calls["env"] = mc.t.PatchEnv(ctx, sb.ID, types.EnvPatch{"GH_TOKEN": {Value: "x", Guest: hostOnly}}, "") + calls["wake"] = mc.t.Wake(ctx, sb.ID, cred) + assertEvacuating(t, calls, "of a received claim") if _, err := mc.t.DialPort(ctx, sb.ID, Cred{Operator: true}, 80); !errors.Is(err, ErrPaused) { t.Errorf("operator dial of a received claim: %v, want ErrPaused", err) } @@ -1472,18 +1451,7 @@ func testMoveHoldsOffCallers(t *testing.T, archived bool) { if round != 2 { return nil } - _, forkErr := mc.s.Fork(t.Context(), sb.ID, cred, 1, time.Minute, "", "") - _, ckErr := mc.s.Checkpoint(t.Context(), sb.ID, cred, "ck", "") - _, _, promoteErr := mc.s.Promote(t.Context(), sb.ID, cred, "tpl", "") - for op, err := range map[string]error{ - "fork": forkErr, "checkpoint": ckErr, "promote": promoteErr, - "hibernate": mc.s.Hibernate(t.Context(), sb.ID, cred), - "release": mc.s.Release(t.Context(), sb.ID, cred), - } { - if !errors.Is(err, ErrEvacuating) { - t.Errorf("%s during a move: %v, want ErrEvacuating", op, err) - } - } + assertEvacuating(t, evacuatingCalls(t.Context(), mc.s, sb.ID, cred), "during a move") mc.s.mu.Lock() replicable := mc.s.replicable(sb) mc.s.mu.Unlock() @@ -1561,6 +1529,10 @@ func (mc *moveCluster) moveNode(t *testing.T, name string) *Manager { return m } +func (mc *moveCluster) counts(tenant string) [2][3]int { + return [2][3]int{moveCounts(mc.s, tenant), moveCounts(mc.t, tenant)} +} + type fakeMovePeer struct { c *failoverCluster srv *httptest.Server @@ -1757,6 +1729,25 @@ func startsOf(m *Manager) []string { return slices.Clone(eng.starts) } +func evacuatingCalls(ctx context.Context, m *Manager, id string, cred Cred) map[string]error { + calls := map[string]error{} + _, calls["fork"] = m.Fork(ctx, id, cred, 1, time.Minute, "", "") + _, calls["checkpoint"] = m.Checkpoint(ctx, id, cred, "ck", "") + _, _, calls["promote"] = m.Promote(ctx, id, cred, "tpl", "") + calls["hibernate"] = m.Hibernate(ctx, id, cred) + calls["release"] = m.Release(ctx, id, cred) + return calls +} + +func assertEvacuating(t *testing.T, calls map[string]error, when string) { + t.Helper() + for op, err := range calls { + if !errors.Is(err, ErrEvacuating) { + t.Errorf("%s %s: %v, want ErrEvacuating", op, when, err) + } + } +} + func moveCounts(m *Manager, tenant string) [3]int { m.mu.Lock() defer m.mu.Unlock() From a06818a216cf8ffd940ac6f3f7a7d40145cf2246 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:47:42 +0800 Subject: [PATCH 095/127] pool: one refused-adoption helper --- sandboxd/pool/drain_test.go | 4 ++-- sandboxd/pool/failover_test.go | 8 ++++---- sandboxd/pool/pool_test.go | 7 ------- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/sandboxd/pool/drain_test.go b/sandboxd/pool/drain_test.go index b81aa0c27..c86fe1cf7 100644 --- a/sandboxd/pool/drain_test.go +++ b/sandboxd/pool/drain_test.go @@ -115,7 +115,7 @@ func TestEvacuateHandOffRefusedByAConcurrentOwnerWriteKeepsTheClaim(t *testing.T if rec, _, _ := c.reg.Get(t.Context(), sb.ID); rec.Owner != "a" { t.Errorf("row owner %q after a refused hand-off, want a", rec.Owner) } - assertNotAdopted(t, b, sb, "adopt a row the hand-off did not release") + mustNotAdopt(t, b, sb, "adopt a row the hand-off did not release") a.Evacuate(t.Context()) synctest.Wait() @@ -243,7 +243,7 @@ func TestUndrainStopsAnEvacuationBeforeItsHandOff(t *testing.T) { if rec, _, _ := c.reg.Get(t.Context(), sb.ID); rec.Owner != "a" { t.Errorf("row owner %q after undrain, want a", rec.Owner) } - assertNotAdopted(t, b, sb, "adopt a claim whose evacuation stopped") + mustNotAdopt(t, b, sb, "adopt a claim whose evacuation stopped") if _, _, _, err := a.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Errorf("wake after undrain: %v", err) } diff --git a/sandboxd/pool/failover_test.go b/sandboxd/pool/failover_test.go index 14742ac64..106a50932 100644 --- a/sandboxd/pool/failover_test.go +++ b/sandboxd/pool/failover_test.go @@ -277,7 +277,7 @@ func TestFailoverNoAdoptionWhileTheOwnerIsAlive(t *testing.T) { sb := mustClaim(t, a, testKey) mustArchive(t, a, sb) - assertNotAdopted(t, b, sb, "adopt from a live owner") + mustNotAdopt(t, b, sb, "adopt from a live owner") if rec, _, _ := c.reg.Get(t.Context(), sb.ID); rec.Owner != "a" || rec.Epoch != 1 { t.Errorf("row %s@%d, want a@1", rec.Owner, rec.Epoch) } @@ -291,7 +291,7 @@ func TestFailoverADrainingNodeDoesNotAdopt(t *testing.T) { c.expire("a") b.Drain(t.Context()) - assertNotAdopted(t, b, sb, "adopt on a draining node") + mustNotAdopt(t, b, sb, "adopt on a draining node") if rec, _, _ := c.reg.Get(t.Context(), sb.ID); rec.Owner != "a" || rec.Epoch != 1 { t.Errorf("row %s@%d, want a@1 left for another node", rec.Owner, rec.Epoch) } @@ -1183,7 +1183,7 @@ func TestFailoverATombstoneIsNotAdopted(t *testing.T) { counted := &countingRegistry{Registry: c.reg} d.archives = counted - assertNotAdopted(t, d, sb, "adopt of a tombstone") + mustNotAdopt(t, d, sb, "adopt of a tombstone") if n := counted.takes.Load(); n != 0 { t.Errorf("adoption tried %d takes on a tombstone, want none", n) } @@ -1275,7 +1275,7 @@ func TestFailoverARepeatedAdoptionOfAReleasedClaimIsRefused(t *testing.T) { t.Fatalf("release: %v", err) } - assertNotAdopted(t, b, sb, "repeated adopt of a released claim") + mustNotAdopt(t, b, sb, "repeated adopt of a released claim") if rec, ok, _ := c.reg.Get(t.Context(), sb.ID); ok { t.Errorf("row %+v outlived the released claim", rec) } diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 126daf10b..876c6ee77 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -2310,13 +2310,6 @@ func mustNotAdopt(t *testing.T, m *Manager, sb *types.Sandbox, step string) { } } -func assertNotAdopted(t *testing.T, m *Manager, sb *types.Sandbox, step string) { - t.Helper() - if adopted, err := m.AdoptArchived(t.Context(), sb.ID, sb.Token); err != nil || adopted { - t.Errorf("%s: %v %v, want refused", step, adopted, err) - } -} - func breakStore(t *testing.T, m *Manager) { t.Helper() m.store.path = filepath.Join(t.TempDir(), "gone", "claims.json") From 0eee22a78b3406a8bb837c4d9216d9c969054fdb Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:48:38 +0800 Subject: [PATCH 096/127] pool: fold the persist-failure rollback tests into a table and share the seed and unlock helpers --- sandboxd/pool/archive_test.go | 179 +++++++++++++------------------- sandboxd/pool/hibernate_test.go | 12 +-- 2 files changed, 77 insertions(+), 114 deletions(-) diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index d964708d0..bef6befd6 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "slices" "strings" + "sync" "testing" "testing/synctest" "time" @@ -168,25 +169,8 @@ func TestReconcileReclaimsOrphanArchiveCk(t *testing.T) { } seedClaims(t, dataDir, claims) m := newTestManagerAt(t, eng, dataDir) - seedCk := func(id, sandboxID string) { - t.Helper() - staging, err := m.ckpts.Stage(id) - if err != nil { - t.Fatalf("stage: %v", err) - } - if err := os.MkdirAll(filepath.Join(staging, store.ExportDir), 0o750); err != nil { - t.Fatalf("mkdir: %v", err) - } - meta := `{"id":"` + id + `","sandbox_id":"` + sandboxID + `","archive":true}` - if err := os.WriteFile(filepath.Join(staging, store.MetaFile), []byte(meta), 0o600); err != nil { - t.Fatalf("meta: %v", err) - } - if err := m.ckpts.Publish(t.Context(), staging, id, ""); err != nil { - t.Fatalf("publish: %v", err) - } - } - seedCk("ck_00000000000000aa", "sb_mine") - seedCk("ck_00000000000000bb", "sb_elsewhere") + seedArchiveCkBy(t, m, "", "ck_00000000000000aa", "sb_mine") + seedArchiveCkBy(t, m, "", "ck_00000000000000bb", "sb_elsewhere") if err := m.Reconcile(t.Context()); err != nil { t.Fatalf("Reconcile: %v", err) @@ -560,12 +544,8 @@ func TestArchiveRemovalCommitPinsCheckpoint(t *testing.T) { } m.store.writeMu.Lock() - locked := true - t.Cleanup(func() { - if locked { - m.store.writeMu.Unlock() - } - }) + unlock := sync.OnceFunc(m.store.writeMu.Unlock) + t.Cleanup(unlock) done := make(chan error, 1) go func() { if action == testArchiveRelease { @@ -587,8 +567,7 @@ func TestArchiveRemovalCommitPinsCheckpoint(t *testing.T) { t.Error("retry deleted the checkpoint before the claims commit") } breakStore(t, m) - m.store.writeMu.Unlock() - locked = false + unlock() if err := <-done; action == testArchiveRelease && err == nil { t.Fatal("release succeeded despite a persist failure") } @@ -662,12 +641,8 @@ func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { id, token, ck := sb.ID, sb.Token, sb.ArchiveCk m.store.writeMu.Lock() - locked := true - t.Cleanup(func() { - if locked { - m.store.writeMu.Unlock() - } - }) + unlock := sync.OnceFunc(m.store.writeMu.Unlock) + t.Cleanup(unlock) woke := make(chan error, 1) go func() { _, _, _, err := m.WakeAgentSocket(t.Context(), id, token) @@ -695,8 +670,7 @@ func TestArchiveDeleteRetryRechecksWakeRollback(t *testing.T) { t.Fatal("retry deleted the checkpoint before wake persistence completed") } breakStore(t, m) - m.store.writeMu.Unlock() - locked = false + unlock() if err := <-woke; err == nil { t.Fatal("wake succeeded despite a persist failure") } @@ -784,78 +758,71 @@ func TestArchivePersistFailureRollsBack(t *testing.T) { }) } -func TestReleaseArchivedRollsBackOnPersistFailure(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng, archivePool(3600)) - sb := mustClaim(t, m, testKey) - id, token := sb.ID, sb.Token - mustArchive(t, m, sb) - ck := sb.ArchiveCk - breakStore(t, m) - - if err := m.Release(t.Context(), id, Cred{Token: token}); err == nil { - t.Fatal("release succeeded despite a persist failure") - } - if _, ok := m.claim(id, token); !ok { - t.Error("release dropped the claim despite a failed persist") - } - if !pinnedHidden(t, m, ck) { - t.Error("kept ck is not pinned after a failed release") - } - healStore(t, m) - }) -} - -func TestWakeArchivedRollsBackOnPersistFailure(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng, archivePool(3600)) - sb := mustClaim(t, m, testKey) - id, token := sb.ID, sb.Token - mustArchive(t, m, sb) - ck := sb.ArchiveCk - breakStore(t, m) - - if _, _, _, err := m.WakeAgentSocket(t.Context(), id, token); err == nil { - t.Fatal("wake succeeded despite a persist failure") - } - m.mu.Lock() - archived := sb.ArchiveCk == ck && sb.VMName == "" - m.mu.Unlock() - if !archived { - t.Error("wake did not roll the record back to archived on a failed persist") - } - if !pinnedHidden(t, m, ck) { - t.Error("kept ck is not pinned after a failed wake") - } - healStore(t, m) - }) -} - -func TestReapPurgeRollsBackOnPersistFailure(t *testing.T) { - synctest.Test(t, func(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng, archivePool(3600)) - sb := mustClaim(t, m, testKey) - id := sb.ID - mustArchive(t, m, sb) - ck := sb.ArchiveCk - expire(m, sb) - breakStore(t, m) +func TestArchivedClaimRollsBackOnPersistFailure(t *testing.T) { + tests := []struct { + name string + setup func(m *Manager, sb *types.Sandbox) + act func(t *testing.T, m *Manager, sb *types.Sandbox, ck string) + }{ + { + name: "release", + act: func(t *testing.T, m *Manager, sb *types.Sandbox, _ string) { + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { + t.Fatal("release succeeded despite a persist failure") + } + if _, ok := m.claim(sb.ID, sb.Token); !ok { + t.Error("release dropped the claim despite a failed persist") + } + }, + }, + { + name: "wake", + act: func(t *testing.T, m *Manager, sb *types.Sandbox, ck string) { + if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err == nil { + t.Fatal("wake succeeded despite a persist failure") + } + m.mu.Lock() + archived := sb.ArchiveCk == ck && sb.VMName == "" + m.mu.Unlock() + if !archived { + t.Error("wake did not roll the record back to archived on a failed persist") + } + }, + }, + { + name: "reap purge", + setup: expire, + act: func(t *testing.T, m *Manager, sb *types.Sandbox, _ string) { + m.reapOnce(t.Context()) + m.mu.Lock() + _, present := m.claimed[sb.ID] + m.mu.Unlock() + if !present { + t.Error("reap dropped the archived claim despite a failed persist") + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + m := newTestManager(t, newFakeEngine(), archivePool(3600)) + sb := mustClaim(t, m, testKey) + mustArchive(t, m, sb) + ck := sb.ArchiveCk + if tt.setup != nil { + tt.setup(m, sb) + } + breakStore(t, m) - m.reapOnce(t.Context()) - m.mu.Lock() - _, present := m.claimed[id] - m.mu.Unlock() - if !present { - t.Error("reap dropped the archived claim despite a failed persist") - } - if !pinnedHidden(t, m, ck) { - t.Error("kept ck is not pinned after a failed reap purge") - } - healStore(t, m) - }) + tt.act(t, m, sb, ck) + if !pinnedHidden(t, m, ck) { + t.Errorf("kept ck is not pinned after a failed %s", tt.name) + } + healStore(t, m) + }) + }) + } } func TestArchiveOnceSkipsInFlight(t *testing.T) { diff --git a/sandboxd/pool/hibernate_test.go b/sandboxd/pool/hibernate_test.go index f206204ee..eeb78dbfc 100644 --- a/sandboxd/pool/hibernate_test.go +++ b/sandboxd/pool/hibernate_test.go @@ -7,6 +7,7 @@ import ( "path/filepath" "slices" "strings" + "sync" "testing" "testing/synctest" "time" @@ -85,12 +86,8 @@ func TestAReleaseRacingAWakeBillsTheWakeFirst(t *testing.T) { sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) m.store.writeMu.Lock() - locked := true - t.Cleanup(func() { - if locked { - m.store.writeMu.Unlock() - } - }) + unlock := sync.OnceFunc(m.store.writeMu.Unlock) + t.Cleanup(unlock) woke := make(chan error, 1) go func() { _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) @@ -111,8 +108,7 @@ func TestAReleaseRacingAWakeBillsTheWakeFirst(t *testing.T) { defer m.mu.Unlock() return m.claimed[sb.ID] == nil }) - m.store.writeMu.Unlock() - locked = false + unlock() <-woke if err := <-released; err != nil { t.Fatalf("release: %v", err) From ef6b205241eb48ab5f5fd6e8cf09d421aeeb84d4 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:53:20 +0800 Subject: [PATCH 097/127] pool: shared test helpers for claims, leases, pending removals, policies, reloads and audit reads; fold the intercept, checkpoint broadcast and disk reserve tests --- sandboxd/pool/archive_test.go | 4 +- sandboxd/pool/checkpoint_heal_test.go | 81 +++++++----------- sandboxd/pool/cordon_test.go | 5 +- sandboxd/pool/disk_test.go | 19 +++-- sandboxd/pool/drain_test.go | 26 +++--- sandboxd/pool/egress_test.go | 10 +-- sandboxd/pool/env_test.go | 8 +- sandboxd/pool/expiry_test.go | 12 +-- sandboxd/pool/intercept_test.go | 113 +++++++++++++------------- sandboxd/pool/interrupted_test.go | 6 +- sandboxd/pool/liveness_test.go | 7 +- sandboxd/pool/move_test.go | 3 +- sandboxd/pool/pool_test.go | 37 +++++++++ sandboxd/pool/reader_test.go | 9 +- sandboxd/pool/reconcile_test.go | 8 +- sandboxd/pool/reload_test.go | 39 +++------ sandboxd/pool/remove_test.go | 20 ++--- sandboxd/pool/replica_test.go | 3 +- sandboxd/pool/running_test.go | 12 ++- sandboxd/pool/telemetry_test.go | 12 +-- sandboxd/pool/tenants_test.go | 8 +- sandboxd/pool/tun_test.go | 6 +- sandboxd/pool/volume_catalog_test.go | 8 +- 23 files changed, 195 insertions(+), 261 deletions(-) diff --git a/sandboxd/pool/archive_test.go b/sandboxd/pool/archive_test.go index bef6befd6..8a0a73268 100644 --- a/sandboxd/pool/archive_test.go +++ b/sandboxd/pool/archive_test.go @@ -372,9 +372,7 @@ func TestArchiveWakeRegrantsTheClaimsLease(t *testing.T) { if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } - m.mu.Lock() - lease := time.Until(sb.Deadline) - m.mu.Unlock() + lease := time.Until(m.deadlineOf(sb)) if lease < time.Hour || lease > 2*time.Hour { t.Errorf("lease after the wake = %v, want the 2h the claim asked for", lease) } diff --git a/sandboxd/pool/checkpoint_heal_test.go b/sandboxd/pool/checkpoint_heal_test.go index 7611abe75..1399fc9ef 100644 --- a/sandboxd/pool/checkpoint_heal_test.go +++ b/sandboxd/pool/checkpoint_heal_test.go @@ -329,61 +329,36 @@ func TestValidateHealedCheckpoint(t *testing.T) { } } -func TestDeleteCheckpointBroadcasts(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng) - src := mustClaim(t, m, testKey) - ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") - if err != nil { - t.Fatalf("checkpoint: %v", err) - } - var calls []string - m.peerDelete = func(_ context.Context, id string) { calls = append(calls, id) } - - if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, "", DeleteFleet); err != nil { - t.Fatalf("DeleteCheckpoint: %v", err) - } - if len(calls) != 1 || calls[0] != ckpt.ID { - t.Errorf("peerDelete calls = %v, want one call for %s", calls, ckpt.ID) - } -} - -func TestDeleteCheckpointNoForwardSkipsBroadcast(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng) - src := mustClaim(t, m, testKey) - ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") - if err != nil { - t.Fatalf("checkpoint: %v", err) - } - called := false - m.peerDelete = func(context.Context, string) { called = true } - - if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, "", DeleteLocal); err != nil { - t.Fatalf("DeleteCheckpoint: %v", err) - } - if called { - t.Error("peerDelete called despite no_forward") - } -} - -func TestDeleteCheckpointSharedStoreSkipsBroadcast(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng) - src := mustClaim(t, m, testKey) - ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") - if err != nil { - t.Fatalf("checkpoint: %v", err) +func TestDeleteCheckpointBroadcastsOnlyAcrossAFleet(t *testing.T) { + tests := []struct { + name string + scope DeleteScope + shared bool + calls int + }{ + {"fleet scope", DeleteFleet, false, 1}, + {"no forward", DeleteLocal, false, 0}, + {"shared store", DeleteFleet, true, 0}, } - m.ckptsShared = true - called := false - m.peerDelete = func(context.Context, string) { called = true } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + src := mustClaim(t, m, testKey) + ckpt, err := m.Checkpoint(t.Context(), src.ID, Cred{Token: src.Token}, "", "") + if err != nil { + t.Fatalf("checkpoint: %v", err) + } + m.ckptsShared = tt.shared + var calls []string + m.peerDelete = func(_ context.Context, id string) { calls = append(calls, id) } - if err := m.DeleteCheckpoint(t.Context(), ckpt.ID, "", DeleteFleet); err != nil { - t.Fatalf("DeleteCheckpoint: %v", err) - } - if called { - t.Error("peerDelete called on a shared-store node") + if err = m.DeleteCheckpoint(t.Context(), ckpt.ID, "", tt.scope); err != nil { + t.Fatalf("DeleteCheckpoint: %v", err) + } + if len(calls) != tt.calls || tt.calls == 1 && calls[0] != ckpt.ID { + t.Errorf("peerDelete calls = %v, want %d for %s", calls, tt.calls, ckpt.ID) + } + }) } } diff --git a/sandboxd/pool/cordon_test.go b/sandboxd/pool/cordon_test.go index db60366dd..40cf55e4f 100644 --- a/sandboxd/pool/cordon_test.go +++ b/sandboxd/pool/cordon_test.go @@ -225,10 +225,7 @@ func TestCordonOutlivesAReloadAndAnUndrain(t *testing.T) { if err := m.Cordon(t.Context()); err != nil { t.Fatalf("cordon: %v", err) } - next := *m.cfg - if _, err := m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload: %v", err) - } + reloadWith(t, m, func(*config.Config) {}) m.Drain(t.Context()) m.Undrain(t.Context()) if _, g := m.Info(); !g.Cordoned || g.Draining { diff --git a/sandboxd/pool/disk_test.go b/sandboxd/pool/disk_test.go index c2d462210..166c20cc8 100644 --- a/sandboxd/pool/disk_test.go +++ b/sandboxd/pool/disk_test.go @@ -2,6 +2,7 @@ package pool import ( "errors" + "path/filepath" "testing" "time" @@ -25,9 +26,7 @@ func TestDiskUnderReserveRefusesClaimsAndWakesUntilItRecovers(t *testing.T) { t.Errorf("gauges %+v, want at capacity for the disk", g) } - next := &config.Config{DataDir: m.dataDir, DiskReserve: &config.DiskReserve{Free: "1K"}, Pools: []config.PoolSpec{{PoolKey: testKey}}} - m.view.Store(newConfigView(next, testSecrets(t))) - m.checkDisk(t.Context()) + setDiskReserve(t, m, &config.DiskReserve{Free: "1K"}) if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Errorf("wake above the reserve: %v", err) } @@ -68,8 +67,7 @@ func TestDiskUnderReserveHoldsHibernatesAndCaptures(t *testing.T) { t.Errorf("promote under the reserve: %v, want ErrQuota", err) } - m.view.Store(newConfigView(&config.Config{DataDir: m.dataDir, DiskReserve: &config.DiskReserve{Free: "1K"}, Pools: []config.PoolSpec{{PoolKey: testKey}}}, testSecrets(t))) - m.checkDisk(t.Context()) + setDiskReserve(t, m, &config.DiskReserve{Free: "1K"}) if err := m.Hibernate(t.Context(), sb.ID, cred); err != nil { t.Errorf("hibernate above the reserve: %v", err) } @@ -78,14 +76,19 @@ func TestDiskUnderReserveHoldsHibernatesAndCaptures(t *testing.T) { func TestAFailedStatfsKeepsTheDiskLowState(t *testing.T) { m := diskReserveManager(t, "1000P") m.checkDisk(t.Context()) - unreadable := &config.Config{DataDir: m.dataDir, DiskReserve: &config.DiskReserve{Free: "1K", Dirs: []string{m.dataDir + "/missing"}}, Pools: []config.PoolSpec{{PoolKey: testKey}}} - m.view.Store(newConfigView(unreadable, testSecrets(t))) - m.checkDisk(t.Context()) + setDiskReserve(t, m, &config.DiskReserve{Free: "1K", Dirs: []string{filepath.Join(m.dataDir, "missing")}}) if _, err := claimAny(t.Context(), m, testKey, ClaimOptions{TTL: time.Hour}); !errors.Is(err, ErrQuota) { t.Errorf("claim after a failed statfs: %v, want ErrQuota while the disk was last seen low", err) } } +func setDiskReserve(t *testing.T, m *Manager, reserve *config.DiskReserve) { + t.Helper() + cfg := &config.Config{DataDir: m.dataDir, DiskReserve: reserve, Pools: []config.PoolSpec{{PoolKey: testKey}}} + m.view.Store(newConfigView(cfg, testSecrets(t))) + m.checkDisk(t.Context()) +} + func diskReserveManager(t *testing.T, free string) *Manager { t.Helper() cfg := &config.Config{DataDir: t.TempDir(), DiskReserve: &config.DiskReserve{Free: free}, Pools: []config.PoolSpec{{PoolKey: testKey}}} diff --git a/sandboxd/pool/drain_test.go b/sandboxd/pool/drain_test.go index c86fe1cf7..251309fa0 100644 --- a/sandboxd/pool/drain_test.go +++ b/sandboxd/pool/drain_test.go @@ -189,8 +189,8 @@ func TestEvacuateWithoutArchiveFailoverSkipsEveryClaim(t *testing.T) { synctest.Wait() assertEvacuation(t, m, 0, 1) - if held, ok := m.byID(sb.ID); !ok || held.HibernateSnap != "" || len(eng.hibernates) != 0 { - t.Errorf("held %v hibernates %v, want the claim left running", ok, eng.hibernates) + if !runningKept(m, sb.ID) || len(eng.hibernates) != 0 { + t.Errorf("hibernates %v, want the claim left running", eng.hibernates) } }) } @@ -274,9 +274,8 @@ func TestAnUndrainDuringAnEvacuationsDrainStopsIt(t *testing.T) { synctest.Wait() assertEvacuation(t, a, 0, 1) - held, ok := a.byID(sb.ID) - if !ok || held.HibernateSnap != "" || held.ArchiveCk != "" { - t.Errorf("held %v, want the claim still running after the undrain", ok) + if !runningKept(a, sb.ID) { + t.Error("want the claim still running after the undrain") } }) } @@ -381,8 +380,8 @@ func TestEvacuateKeepsAClaimOfARemovedTenant(t *testing.T) { synctest.Wait() assertEvacuation(t, m, 0, 1) - if held, ok := m.byID(sb.ID); !ok || held.HibernateSnap != "" || held.ArchiveCk != "" { - t.Errorf("held %v, want the claim left running", ok) + if !runningKept(m, sb.ID) { + t.Error("want the claim left running") } }) } @@ -452,8 +451,8 @@ func TestASecondEvacuateWhileOneRunsChangesNothing(t *testing.T) { a.Undrain(t.Context()) release() synctest.Wait() - if held, ok := a.byID(sb.ID); !ok || held.HibernateSnap != "" || held.ArchiveCk != "" { - t.Errorf("held %v, want the claim left running after undrain", ok) + if !runningKept(a, sb.ID) { + t.Error("want the claim left running after undrain") } }) } @@ -471,8 +470,8 @@ func TestAStoppedEvacuationDoesNotPauseAClaimItHadQueued(t *testing.T) { if err := a.evacuateClaim(t.Context(), sb); err != nil { t.Errorf("evacuateClaim after undrain: %v, want nil", err) } - if held, ok := a.byID(sb.ID); !ok || held.HibernateSnap != "" || held.ArchiveCk != "" { - t.Errorf("held %v, want the claim left running", ok) + if !runningKept(a, sb.ID) { + t.Error("want the claim left running") } release() @@ -752,6 +751,11 @@ func assertEvacuation(t *testing.T, m *Manager, left, kept int) { } } +func runningKept(m *Manager, id string) bool { + held, ok := m.byID(id) + return ok && held.HibernateSnap == "" && held.ArchiveCk == "" +} + func holdMaint(m *Manager) (release func()) { for range cap(m.maintSem) { m.maintSem <- struct{}{} diff --git a/sandboxd/pool/egress_test.go b/sandboxd/pool/egress_test.go index 13626a708..79fb6605e 100644 --- a/sandboxd/pool/egress_test.go +++ b/sandboxd/pool/egress_test.go @@ -252,9 +252,7 @@ func TestBatchArmFailureEndsEveryClaimEvent(t *testing.T) { t.Fatal("finalizeBatch must fail when a batch member cannot arm") } waitFor(t, m.store.synced) - m.mu.Lock() - claimed := len(m.claimed) - m.mu.Unlock() + claimed := claimedCount(m) if claimed != 0 { t.Errorf("rollback left %d claims", claimed) } @@ -293,9 +291,7 @@ func TestRollbackKeepsLockWhenRemoveFails(t *testing.T) { m.rollbackClaim(t.Context(), sbs, false) waitFor(t, m.store.synced) - m.mu.Lock() - claimed := len(m.claimed) - m.mu.Unlock() + claimed := claimedCount(m) if claimed != 0 { t.Errorf("rollback left %d claims", claimed) } @@ -534,7 +530,7 @@ func TestAPromotedTemplatesCloneInterceptsAsItsSourcePool(t *testing.T) { func TestAClaimWithoutEgressGetsNoDoorsWhateverItsPolicy(t *testing.T) { eng := newFakeEngine() eng.sockRoot = outboundtest.SockRoot(t) - pol := &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + pol := hostPolicy() m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: pol}) pin := pinDoor(t, engine.EgressSocketPath(refillWarmVM(t, m).VsockSocket)) warm, err := m.ClaimWarm(t.Context(), testKey, ClaimOptions{NoEgress: true}) diff --git a/sandboxd/pool/env_test.go b/sandboxd/pool/env_test.go index d5b0720e4..4ed0ad221 100644 --- a/sandboxd/pool/env_test.go +++ b/sandboxd/pool/env_test.go @@ -349,9 +349,7 @@ func TestAFailedEnvDeliveryDestroysTheClaim(t *testing.T) { if len(eng.removes) != 1 { t.Errorf("removed %v, want the claim's VM", eng.removes) } - m.mu.Lock() - claimed := len(m.claimed) - m.mu.Unlock() + claimed := claimedCount(m) if claimed != 0 { t.Errorf("%d claims recorded, want none", claimed) } @@ -502,9 +500,7 @@ func TestAClaimWithAnUninterceptedInjectIsRefused(t *testing.T) { if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{Env: injectEnv("k", "other.test")}); !errors.Is(err, ErrBadEnv) { t.Fatalf("claim: %v, want ErrBadEnv", err) } - m.mu.Lock() - claimed := len(m.claimed) - m.mu.Unlock() + claimed := claimedCount(m) if claimed != 0 || len(eng.removedNames()) != 1 { t.Errorf("%d claims left and %v removed, want none kept and the VM destroyed", claimed, eng.removedNames()) } diff --git a/sandboxd/pool/expiry_test.go b/sandboxd/pool/expiry_test.go index 92e652a8c..1e9768781 100644 --- a/sandboxd/pool/expiry_test.go +++ b/sandboxd/pool/expiry_test.go @@ -32,9 +32,7 @@ func TestExpiryArchiveHibernatesAndArchivesARunningClaim(t *testing.T) { if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("round %d: wake: %v", round, err) } - m.mu.Lock() - lease := time.Until(sb.Deadline) - m.mu.Unlock() + lease := time.Until(m.deadlineOf(sb)) if lease < 59*time.Minute { t.Fatalf("round %d: lease after the wake %v, want the claim's hour again", round, lease) } @@ -174,9 +172,7 @@ func TestExpiryArchiveWakeBeforeTheExportGrantsAFreshLease(t *testing.T) { if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } - m.mu.Lock() - lease := time.Until(sb.Deadline) - m.mu.Unlock() + lease := time.Until(m.deadlineOf(sb)) if lease < 59*time.Minute { t.Fatalf("lease after a wake from the expiry hibernate = %v, want the claim's hour again", lease) } @@ -200,9 +196,7 @@ func TestATierClaimWokenPastItsLeaseBeforeTheExportGetsAFreshLease(t *testing.T) if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); err != nil { t.Fatalf("wake: %v", err) } - m.mu.Lock() - lease := time.Until(sb.Deadline) - m.mu.Unlock() + lease := time.Until(m.deadlineOf(sb)) if lease < 59*time.Minute { t.Fatalf("lease after a wake past the lease = %v, want the claim's hour again", lease) } diff --git a/sandboxd/pool/intercept_test.go b/sandboxd/pool/intercept_test.go index 9e6f5c356..e57b9b649 100644 --- a/sandboxd/pool/intercept_test.go +++ b/sandboxd/pool/intercept_test.go @@ -12,38 +12,35 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) -func TestGoldenBuildInstallsCAForInterceptPool(t *testing.T) { - eng := newFakeEngine() - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: interceptPolicy()}) - if m.out.CAFingerprint() == "" { - t.Fatal("egress CA not loaded for an intercept pool") - } - final := buildGoldenAt(t, m, testKey, "") - if len(eng.caInstalls) != 1 { - t.Fatalf("InstallCACert calls = %d, want 1", len(eng.caInstalls)) - } - stamp, err := os.ReadFile(final + goldenStampSuffix) - if err != nil { - t.Fatalf("read golden stamp: %v", err) - } - if want := m.goldenStamp(m.view.Load(), testKey, ""); string(stamp) != want { - t.Errorf("stamp = %q, want %q", stamp, want) - } -} - -func TestGoldenBuildSkipsCAForPlainPool(t *testing.T) { - eng := newFakeEngine() - plain := &egress.Policy{Allow: []egress.Rule{{Host: "api.github.com", Secret: "gh"}}} - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: plain}) - if m.out.CAFingerprint() != "" { - t.Error("egress CA loaded though no pool intercepts") - } - final := buildGoldenAt(t, m, testKey, "") - if len(eng.caInstalls) != 0 { - t.Errorf("InstallCACert called %d times for a plain pool", len(eng.caInstalls)) - } - if stamp, err := os.ReadFile(final + goldenStampSuffix); err != nil || string(stamp) != m.goldenStamp(m.view.Load(), testKey, "") { - t.Errorf("stamp = %q (%v), want one without a CA fingerprint", stamp, err) +func TestGoldenBuildInstallsCAOnlyForAnInterceptPool(t *testing.T) { + tests := []struct { + name string + policy *egress.Policy + caLoaded bool + installs int + }{ + {"intercept pool", interceptPolicy(), true, 1}, + {"plain pool", plainPolicy(), false, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + eng := newFakeEngine() + m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: tt.policy}) + if loaded := m.out.CAFingerprint() != ""; loaded != tt.caLoaded { + t.Fatalf("egress CA loaded = %v, want %v", loaded, tt.caLoaded) + } + final := buildGoldenAt(t, m, testKey, "") + if len(eng.caInstalls) != tt.installs { + t.Fatalf("InstallCACert calls = %d, want %d", len(eng.caInstalls), tt.installs) + } + stamp, err := os.ReadFile(final + goldenStampSuffix) + if err != nil { + t.Fatalf("read golden stamp: %v", err) + } + if want := m.goldenStamp(m.view.Load(), testKey, ""); string(stamp) != want { + t.Errorf("stamp = %q, want %q", stamp, want) + } + }) } } @@ -76,7 +73,7 @@ func TestGoldenCAStampRebuildsOnMismatch(t *testing.T) { } p.goldenDir = "" editEgress(t, m, func(cfg *config.Config) { - cfg.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 1, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "api.github.com", Secret: "gh"}}}}} + cfg.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 1, Egress: plainPolicy()}} }) m.adoptGolden(p, "") if p.goldenDir != "" { @@ -84,30 +81,28 @@ func TestGoldenCAStampRebuildsOnMismatch(t *testing.T) { } } -func TestColdProvisionInstallsCAForInterceptPool(t *testing.T) { - eng := newFakeEngine() - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: interceptPolicy()}) - sb, err := m.provision(t.Context(), testKey, "") - if err != nil { - t.Fatalf("cold provision: %v", err) - } - m.destroy(t.Context(), sb.VMName) - if n := len(eng.caInstalls); n != 1 { - t.Errorf("InstallCACert calls = %d, want 1 (pre-golden cold claim must trust the root)", n) - } -} - -func TestColdProvisionSkipsCAForPlainPool(t *testing.T) { - eng := newFakeEngine() - plain := &egress.Policy{Allow: []egress.Rule{{Host: "api.github.com", Secret: "gh"}}} - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: plain}) - sb, err := m.provision(t.Context(), testKey, "") - if err != nil { - t.Fatalf("cold provision: %v", err) - } - m.destroy(t.Context(), sb.VMName) - if n := len(eng.caInstalls); n != 0 { - t.Errorf("InstallCACert called %d times for a plain pool", n) +func TestColdProvisionInstallsCAOnlyForAnInterceptPool(t *testing.T) { + tests := []struct { + name string + policy *egress.Policy + installs int + }{ + {"intercept pool", interceptPolicy(), 1}, + {"plain pool", plainPolicy(), 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + eng := newFakeEngine() + m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: tt.policy}) + sb, err := m.provision(t.Context(), testKey, "") + if err != nil { + t.Fatalf("cold provision: %v", err) + } + m.destroy(t.Context(), sb.VMName) + if n := len(eng.caInstalls); n != tt.installs { + t.Errorf("InstallCACert calls = %d, want %d", n, tt.installs) + } + }) } } @@ -132,6 +127,10 @@ func TestInterceptPoolAllowsPromote(t *testing.T) { } } +func plainPolicy() *egress.Policy { + return &egress.Policy{Allow: []egress.Rule{{Host: "api.github.com", Secret: "gh"}}} +} + func interceptPolicy() *egress.Policy { return &egress.Policy{Allow: []egress.Rule{{Host: "api.github.com", Secret: "gh", Intercept: egress.InterceptAlways}}} } diff --git a/sandboxd/pool/interrupted_test.go b/sandboxd/pool/interrupted_test.go index 5929b9a05..9608f6fc9 100644 --- a/sandboxd/pool/interrupted_test.go +++ b/sandboxd/pool/interrupted_test.go @@ -305,11 +305,7 @@ func TestAFailedHibernateNeverStartsAStoppedVM(t *testing.T) { m := newTestManager(t, eng) sb := mustClaim(t, m, testKey) eng.hibernateErr = errors.New("signal: killed") - eng.captureHook = func() { - eng.mu.Lock() - defer eng.mu.Unlock() - eng.stopped[sb.VMName] = true - } + eng.captureHook = func() { killVMM(eng, sb.VMName) } if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil { t.Fatal("Hibernate reported success") } diff --git a/sandboxd/pool/liveness_test.go b/sandboxd/pool/liveness_test.go index 5ce7b0129..aadfac64d 100644 --- a/sandboxd/pool/liveness_test.go +++ b/sandboxd/pool/liveness_test.go @@ -10,7 +10,6 @@ import ( "time" "github.com/cocoonstack/sandbox/sandboxd/config" - "github.com/cocoonstack/sandbox/sandboxd/egress" "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/outbound" "github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest" @@ -207,7 +206,7 @@ func TestWakeOfAFailedClaimRearmsEgressAfterARestart(t *testing.T) { eng.sockRoot = outboundtest.SockRoot(t) cfg := &config.Config{ DataDir: t.TempDir(), Bridges: []string{"sbxbr0"}, EgressCA: writeTestEgressCA(t), VMMRestart: types.VMMRestartNone, - Pools: []config.PoolSpec{{PoolKey: testKey, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}}}}, + Pools: []config.PoolSpec{{PoolKey: testKey, Egress: hostPolicy()}}, } first := mustManager(t, cfg, eng) sb := mustClaim(t, first, testKey) @@ -241,7 +240,7 @@ func TestWakeOfAFailedClaimRearmsEgressAfterARestart(t *testing.T) { func TestRestartKeepsAnEgressDoorThatSurvivedTheVMM(t *testing.T) { eng := newFakeEngine() eng.sockRoot = outboundtest.SockRoot(t) - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}}}) + m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Egress: hostPolicy()}) sb := mustClaim(t, m, testKey) killVMM(eng, sb.VMName) @@ -292,7 +291,7 @@ func TestRestartThatCannotArmTheEgressDoorFailsAndSpendsTheBudget(t *testing.T) } eng := newFakeEngine() eng.sockRoot = outboundtest.SockRoot(t) - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}}}) + m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Egress: hostPolicy()}) sb := mustClaim(t, m, testKey) killVMM(eng, sb.VMName) if err := os.Chmod(eng.sockRoot, 0o500); err != nil { diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index de31d5811..bec653d17 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -19,7 +19,6 @@ import ( "time" "github.com/cocoonstack/sandbox/sandboxd/archives" - "github.com/cocoonstack/sandbox/sandboxd/egress" "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/moves" "github.com/cocoonstack/sandbox/sandboxd/moves/movestest" @@ -1509,7 +1508,7 @@ func newMoveCluster(t *testing.T) *moveCluster { func newMoveClusterWith(t *testing.T, edit func(*failoverCluster)) *moveCluster { t.Helper() c := newFailoverCluster(t) - c.egressPol = &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + c.egressPol = hostPolicy() edit(c) reg := movestest.NewMem() c.moves = reg diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 876c6ee77..6e89f2d2c 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -2289,6 +2289,43 @@ func hibernateClaim(t *testing.T, m *Manager, sb *types.Sandbox) { } } +func auditLines(t *testing.T, dir string) []string { + t.Helper() + raw, err := os.ReadFile(filepath.Join(dir, "audit.jsonl")) + if err != nil { + t.Fatalf("read audit journal: %v", err) + } + return strings.Split(strings.TrimSpace(string(raw)), "\n") +} + +func reloadWith(t *testing.T, m *Manager, edit func(*config.Config)) ReloadResult { + t.Helper() + next := *m.cfg + edit(&next) + res, err := m.ReloadConfig(t.Context(), &next) + if err != nil { + t.Fatalf("reload: %v", err) + } + return res +} + +func hostPolicy() *egress.Policy { + return &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} +} + +func pendingOf(m *Manager, name string) bool { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.pendingRemovals[name] + return ok +} + +func claimedCount(m *Manager) int { + m.mu.Lock() + defer m.mu.Unlock() + return len(m.claimed) +} + func mustFlush(t *testing.T, m *Manager) { t.Helper() if err := m.FlushClaims(); err != nil { diff --git a/sandboxd/pool/reader_test.go b/sandboxd/pool/reader_test.go index f8768f200..bfebed0cd 100644 --- a/sandboxd/pool/reader_test.go +++ b/sandboxd/pool/reader_test.go @@ -166,18 +166,13 @@ func TestReaderServesOnlyPoolsThatAskForIt(t *testing.T) { func TestReloadTurnsTheReaderOnForLaterSleeps(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1}) - next := *m.cfg - next.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 1, Reader: true}} - res, err := m.ReloadConfig(t.Context(), &next) - if err != nil { - t.Fatalf("reload: %v", err) - } + res := reloadWith(t, m, func(c *config.Config) { c.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 1, Reader: true}} }) if want := []string{"pools[rt:24.04 none small].reader"}; !slices.Equal(res.Changed, want) { t.Errorf("changed %v, want %v", res.Changed, want) } sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) - if _, _, _, err = m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { + if _, _, _, err := m.WakeAgentSocketOrRead(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrSleeping) { t.Errorf("WakeAgentSocketOrRead after the reload = %v, want ErrSleeping", err) } } diff --git a/sandboxd/pool/reconcile_test.go b/sandboxd/pool/reconcile_test.go index 56335f63a..493a45194 100644 --- a/sandboxd/pool/reconcile_test.go +++ b/sandboxd/pool/reconcile_test.go @@ -72,9 +72,7 @@ func TestReconcileStaleCreateSweep(t *testing.T) { if eng.removed("sbx-stale-1") != tt.wantRemove { t.Errorf("removed=%v, want %v", eng.removed("sbx-stale-1"), tt.wantRemove) } - m.mu.Lock() - _, pending := m.pendingRemovals["sbx-stale-1"] - m.mu.Unlock() + pending := pendingOf(m, "sbx-stale-1") if pending != tt.wantPending { t.Errorf("pending=%v, want %v", pending, tt.wantPending) } @@ -109,9 +107,7 @@ func TestBusyStaleCreateRetryConverges(t *testing.T) { eng.mu.Lock() _, present := eng.vms["sbx-stale-1"] eng.mu.Unlock() - m.mu.Lock() - _, pending := m.pendingRemovals["sbx-stale-1"] - m.mu.Unlock() + pending := pendingOf(m, "sbx-stale-1") removed := eng.removed("sbx-stale-1") if present != tt.wantPresent || removed != tt.wantRemove || pending != tt.wantPending { t.Errorf("present=%v removed=%v pending=%v, want %v %v %v", diff --git a/sandboxd/pool/reload_test.go b/sandboxd/pool/reload_test.go index c96639838..296d9fae6 100644 --- a/sandboxd/pool/reload_test.go +++ b/sandboxd/pool/reload_test.go @@ -30,13 +30,10 @@ var interceptOnKey = types.PoolKey{Template: "mitm:1", Net: types.NetNone, Size: func TestReloadGivesANewKeyItsSettingsBeforeItsGolden(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) - next := *m.cfg argv := []string{"node", "-e", "0"} - next.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 3, Warmup: argv, Storage: "20G"}} - res, err := m.ReloadConfig(t.Context(), &next) - if err != nil { - t.Fatalf("reload: %v", err) - } + res := reloadWith(t, m, func(c *config.Config) { + c.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 3, Warmup: argv, Storage: "20G"}} + }) label := "pools[rt:24.04 none small]" if want := []string{label + ".warmup", label + ".storage"}; !slices.Equal(res.Changed, want) { t.Errorf("changed %v, want %v", res.Changed, want) @@ -67,11 +64,7 @@ func TestReloadRetiresAGoldenWhoseStampChangedAndLeavesClaims(t *testing.T) { waitFor(t, func() bool { return refilledTo(t, m, testKey, 2) }) sb := mustClaim(t, m, testKey) vm := sb.VMName - next := *m.cfg - next.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 2, Storage: "30G"}} - if _, err := m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload: %v", err) - } + reloadWith(t, m, func(c *config.Config) { c.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 2, Storage: "30G"}} }) m.mu.Lock() golden, warm, live := m.pools[testKey].goldenDir, len(m.pools[testKey].warm), m.claimed[sb.ID] m.mu.Unlock() @@ -128,7 +121,7 @@ func TestReloadTurnsInterceptOnOnlyForAKeyItHasNotServedWithTheCALoaded(t *testi func TestReloadRefusesDroppingAPolicyLiveClaimsUse(t *testing.T) { eng := newFakeEngine() eng.sockRoot = outboundtest.SockRoot(t) - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}}}) + m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: hostPolicy()}) if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{NoEgress: true}); err != nil { t.Fatalf("claim without egress: %v", err) } @@ -158,7 +151,7 @@ func TestARestartWithoutAClaimsPoolPolicyCountsTheClaimDenied(t *testing.T) { eng := newFakeEngine() eng.sockRoot = outboundtest.SockRoot(t) dataDir := t.TempDir() - policy := &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + policy := hostPolicy() m := egressManagerAt(t, eng, dataDir, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: policy}) mustClaim(t, m, testKey) m2 := egressManagerAt(t, eng, dataDir, config.PoolSpec{PoolKey: otherKey, Warm: 1, Egress: policy}) @@ -248,11 +241,7 @@ func TestAReloadDropsTheLiveProxiesPooledConnections(t *testing.T) { inflight := make(chan error, 1) go func() { inflight <- get("/slow") }() <-started - next := *m.cfg - next.EgressInternalAllow = []string{"10.8.0.0/16"} - if _, err := m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload: %v", err) - } + reloadWith(t, m, func(c *config.Config) { c.EgressInternalAllow = []string{"10.8.0.0/16"} }) if err := get("/"); err != nil { t.Fatalf("get after the reload: %v", err) } @@ -293,11 +282,7 @@ func TestReloadIsAudited(t *testing.T) { func TestAReloadOfTheEgressLayerChangesTheClusterDigest(t *testing.T) { m := newTestManager(t, newFakeEngine()) before := m.ClusterDigest() - next := *m.cfg - next.EgressInternalAllow = []string{"10.8.0.0/16"} - if _, err := m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload: %v", err) - } + reloadWith(t, m, func(c *config.Config) { c.EgressInternalAllow = []string{"10.8.0.0/16"} }) if m.ClusterDigest() == before { t.Error("the digest still covers the boot config after an egress reload") } @@ -334,14 +319,10 @@ func TestAnInterruptedReloadStillDestroysTheRetiredWarmVMs(t *testing.T) { func TestAReloadThatUnguardsTheNodeClosesAWarmVMsDoorsAtItsClaim(t *testing.T) { eng := newFakeEngine() eng.sockRoot = outboundtest.SockRoot(t) - m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}}}) + m := egressManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 1, Egress: hostPolicy()}) warm := refillWarmVM(t, m) pin := pinDoor(t, engine.EgressSocketPath(warm.VsockSocket)) - next := *m.cfg - next.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 1}} - if _, err := m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload: %v", err) - } + reloadWith(t, m, func(c *config.Config) { c.Pools = []config.PoolSpec{{PoolKey: testKey, Warm: 1}} }) conn, err := net.Dial("unix", pin) if err != nil { t.Fatalf("the unclaimed warm VM's door: %v, want it bound until the claim", err) diff --git a/sandboxd/pool/remove_test.go b/sandboxd/pool/remove_test.go index 3fb30e64f..450bd771a 100644 --- a/sandboxd/pool/remove_test.go +++ b/sandboxd/pool/remove_test.go @@ -44,17 +44,13 @@ func TestRemovalRetryConvergesConfirmedSurvivor(t *testing.T) { eng.removeErrFor = "survivor" m.destroy(t.Context(), "survivor") - m.mu.Lock() - _, pending := m.pendingRemovals["survivor"] - m.mu.Unlock() + pending := pendingOf(m, "survivor") if !pending { t.Fatal("surviving VM was not retained for retry") } m.retryRemovals(t.Context()).Wait() - m.mu.Lock() - _, pending = m.pendingRemovals["survivor"] - m.mu.Unlock() + pending = pendingOf(m, "survivor") if !pending { t.Fatal("failed retry dropped the survivor from the queue") } @@ -63,9 +59,7 @@ func TestRemovalRetryConvergesConfirmedSurvivor(t *testing.T) { eng.removeErrFor = "" eng.mu.Unlock() m.retryRemovals(t.Context()).Wait() - m.mu.Lock() - _, pending = m.pendingRemovals["survivor"] - m.mu.Unlock() + pending = pendingOf(m, "survivor") if pending || !eng.removed("survivor") { t.Fatalf("retry did not converge: pending=%v removed=%v", pending, eng.removed("survivor")) } @@ -93,9 +87,7 @@ func TestRemovalRetryFinishesEgressCleanup(t *testing.T) { eng.removeErrFor = "" eng.mu.Unlock() m.retryRemovals(t.Context()).Wait() - m.mu.Lock() - _, pending := m.pendingRemovals["survivor"] - m.mu.Unlock() + pending := pendingOf(m, "survivor") if unlocked := taps.Unlocked(); pending || !slices.Equal(unlocked, []string{"tap-survivor"}) || !eng.removed("survivor") { t.Fatalf("egress cleanup incomplete: pending=%v unlocked=%v removed=%v", pending, unlocked, eng.removed("survivor")) @@ -126,9 +118,7 @@ func TestStaleRemovalRetryKeepsTapForCleanup(t *testing.T) { eng.removeErrFor = "" eng.mu.Unlock() m.retryRemovals(t.Context()).Wait() - m.mu.Lock() - _, pending := m.pendingRemovals["sbx-stale"] - m.mu.Unlock() + pending := pendingOf(m, "sbx-stale") if pending || !eng.removed("sbx-stale") { t.Fatalf("stale retry did not converge: pending=%v removed=%v", pending, eng.removed("sbx-stale")) } diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 2322862cb..0893bc129 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -10,7 +10,6 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/archives" "github.com/cocoonstack/sandbox/sandboxd/config" - "github.com/cocoonstack/sandbox/sandboxd/egress" "github.com/cocoonstack/sandbox/sandboxd/engine" "github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest" "github.com/cocoonstack/sandbox/sandboxd/store" @@ -1328,7 +1327,7 @@ func TestReplicaRestartMidArchiveKeepsTheClaim(t *testing.T) { func TestARunningClaimThatMovedClosesItsDoors(t *testing.T) { c := newReplicaCluster(t) - c.egressPol = &egress.Policy{Allow: []egress.Rule{{Host: "example.com"}}} + c.egressPol = hostPolicy() ea, eb := newFakeEngine(), newFakeEngine() ea.sockRoot, eb.sockRoot = outboundtest.SockRoot(t), outboundtest.SockRoot(t) a := c.nodeOn(t, ea, "a", t.TempDir(), keepAnHour) diff --git a/sandboxd/pool/running_test.go b/sandboxd/pool/running_test.go index c20da302b..567c63539 100644 --- a/sandboxd/pool/running_test.go +++ b/sandboxd/pool/running_test.go @@ -224,13 +224,11 @@ func TestAReloadAppliesTheClaimCapsWithoutStoppingAClaim(t *testing.T) { func TestAReloadAppliesTheNodeSettings(t *testing.T) { m := newTestManager(t, newFakeEngine()) - next := *m.cfg - next.MaxForkCount, next.ReleaseDelaySeconds, next.VMMRestart = 1, 5, types.VMMRestartNone - next.SyncClaims, next.CheckpointTTLHours = true, 24 - next.IdleHibernateSeconds, next.ArchiveAfterSeconds, next.ArchiveDeleteAfterSeconds = 60, 120, 3600 - if _, err := m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload: %v", err) - } + reloadWith(t, m, func(c *config.Config) { + c.MaxForkCount, c.ReleaseDelaySeconds, c.VMMRestart = 1, 5, types.VMMRestartNone + c.SyncClaims, c.CheckpointTTLHours = true, 24 + c.IdleHibernateSeconds, c.ArchiveAfterSeconds, c.ArchiveDeleteAfterSeconds = 60, 120, 3600 + }) v := m.view.Load() got := []time.Duration{v.releaseDelay, v.ckptTTL, v.idleDefault, v.archiveAfterDefault, v.archiveDeleteDefault} want := []time.Duration{5 * time.Second, 24 * time.Hour, time.Minute, 2 * time.Minute, time.Hour} diff --git a/sandboxd/pool/telemetry_test.go b/sandboxd/pool/telemetry_test.go index 8c2f19d5f..797e82d88 100644 --- a/sandboxd/pool/telemetry_test.go +++ b/sandboxd/pool/telemetry_test.go @@ -612,11 +612,7 @@ func TestEgressAuditNamesTenantAndEndsARelayedFrameCannot(t *testing.T) { Local: netip.MustParseAddrPort("198.51.100.7:40123"), Remote: netip.MustParseAddrPort("93.184.216.34:443"), }) m.Audit(t.Context(), "sb_v", []byte(`{"op":"egress","dest":"x","tenant":"other","src_addr":"198.51.100.7:1","dst_addr":"192.0.2.1:25"}`)) - raw, err := os.ReadFile(filepath.Join(dir, "audit.jsonl")) - if err != nil { - t.Fatalf("read audit journal: %v", err) - } - lines := strings.Split(strings.TrimSpace(string(raw)), "\n") + lines := auditLines(t, dir) if len(lines) != 2 { t.Fatalf("audit lines %q, want 2", lines) } @@ -646,11 +642,7 @@ func TestARelayedFrameCannotForgeOrDropItsAuditLine(t *testing.T) { for _, frame := range frames { m.Audit(t.Context(), "sb_f", []byte(frame)) } - raw, err := os.ReadFile(filepath.Join(dir, "audit.jsonl")) - if err != nil { - t.Fatalf("read audit journal: %v", err) - } - lines := strings.Split(strings.TrimSpace(string(raw)), "\n") + lines := auditLines(t, dir) if len(lines) != len(frames) { t.Fatalf("audit lines %q for %d frames", lines, len(frames)) } diff --git a/sandboxd/pool/tenants_test.go b/sandboxd/pool/tenants_test.go index 0c943ce43..c3dc90c3f 100644 --- a/sandboxd/pool/tenants_test.go +++ b/sandboxd/pool/tenants_test.go @@ -294,11 +294,9 @@ func TestARuntimeTenantTakesItsEgressClassWithoutARestart(t *testing.T) { if status("b.test") != http.StatusOK || status("a.test") != http.StatusForbidden { t.Fatal("the live claim's door does not serve the class it was claimed under") } - next := *m.cfg - next.EgressClasses = []config.EgressClass{{Name: "desk", Egress: &egress.Policy{Allow: []egress.Rule{{Host: "a.test"}}}}} - if _, err = m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload the class: %v", err) - } + reloadWith(t, m, func(c *config.Config) { + c.EgressClasses = []config.EgressClass{{Name: "desk", Egress: &egress.Policy{Allow: []egress.Rule{{Host: "a.test"}}}}} + }) if status("a.test") != http.StatusOK { t.Error("a reload that widens the class did not reach the live claim") } diff --git a/sandboxd/pool/tun_test.go b/sandboxd/pool/tun_test.go index da0172ef4..571f236ea 100644 --- a/sandboxd/pool/tun_test.go +++ b/sandboxd/pool/tun_test.go @@ -219,11 +219,7 @@ func TestAWakeIntoAPoolWithoutTunTellsTheGuestTunDown(t *testing.T) { m := tunManager(t, eng, addrs, config.PoolSpec{PoolKey: testKey, Tun: true, Egress: anyPolicy}) sb := mustClaim(t, m, testKey) hibernateClaim(t, m, sb) - next := *m.cfg - next.Pools = []config.PoolSpec{{PoolKey: testKey, Egress: anyPolicy}} - if _, err := m.ReloadConfig(t.Context(), &next); err != nil { - t.Fatalf("reload: %v", err) - } + reloadWith(t, m, func(c *config.Config) { c.Pools = []config.PoolSpec{{PoolKey: testKey, Egress: anyPolicy}} }) if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { t.Fatalf("wake: %v", err) } diff --git a/sandboxd/pool/volume_catalog_test.go b/sandboxd/pool/volume_catalog_test.go index b3f3a9d75..336c2b65e 100644 --- a/sandboxd/pool/volume_catalog_test.go +++ b/sandboxd/pool/volume_catalog_test.go @@ -107,14 +107,10 @@ func TestRuntimeCatalogChangesAreAudited(t *testing.T) { t.Fatalf("UnregisterVolume: %v", err) } } - raw, err := os.ReadFile(filepath.Join(m.dataDir, "audit.jsonl")) - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - lines := strings.Split(strings.TrimSpace(string(raw)), "\n") + lines := auditLines(t, m.dataDir) want := []auditFrame{{Op: "volumes", Added: []string{"data"}}, {Op: "volumes", Removed: []string{"data"}}, {Op: "volumes", Removed: []string{"data"}}} if len(lines) != len(want) { - t.Fatalf("audit records: %s", raw) + t.Fatalf("audit records: %q", lines) } for i, line := range lines { var got auditFrame From af0df5e6c0b02da79dfbb5484fa820dfafdccd36 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:54:14 +0800 Subject: [PATCH 098/127] review: move misplaced quota, reload and replication test helpers next to their subjects --- sandboxd/pool/pool_test.go | 6 ++++ sandboxd/pool/reload_test.go | 23 +++++++++++++ sandboxd/pool/running_test.go | 41 +++++++++++------------- sandboxd/pool/telemetry_test.go | 48 ---------------------------- sandboxd/pool/tenants_test.go | 29 +++++++++++++++++ sandboxd/pool/volume_replica_test.go | 6 ---- 6 files changed, 76 insertions(+), 77 deletions(-) diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 6e89f2d2c..5655af078 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -2784,3 +2784,9 @@ func liveCatalog(t *testing.T) []config.VolumeSpec { {Name: "other", Path: writeVolumeImage(t, "other.img", "other"), Writable: true}, } } + +func disableReplication(m *Manager) { + view := *m.view.Load() + view.poolReplicate = nil + m.view.Store(&view) +} diff --git a/sandboxd/pool/reload_test.go b/sandboxd/pool/reload_test.go index 296d9fae6..c17a7c77a 100644 --- a/sandboxd/pool/reload_test.go +++ b/sandboxd/pool/reload_test.go @@ -16,6 +16,7 @@ import ( "sync" "sync/atomic" "testing" + "time" "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/egress" @@ -344,3 +345,25 @@ func TestAReloadThatUnguardsTheNodeClosesAWarmVMsDoorsAtItsClaim(t *testing.T) { t.Errorf("claim route %q, want none", route) } } + +func TestAReloadAppliesTheNodeSettings(t *testing.T) { + m := newTestManager(t, newFakeEngine()) + reloadWith(t, m, func(c *config.Config) { + c.MaxForkCount, c.ReleaseDelaySeconds, c.VMMRestart = 1, 5, types.VMMRestartNone + c.SyncClaims, c.CheckpointTTLHours = true, 24 + c.IdleHibernateSeconds, c.ArchiveAfterSeconds, c.ArchiveDeleteAfterSeconds = 60, 120, 3600 + }) + v := m.view.Load() + got := []time.Duration{v.releaseDelay, v.ckptTTL, v.idleDefault, v.archiveAfterDefault, v.archiveDeleteDefault} + want := []time.Duration{5 * time.Second, 24 * time.Hour, time.Minute, 2 * time.Minute, time.Hour} + if !slices.Equal(got, want) || v.maxFork != 1 || v.vmmRestart != types.VMMRestartNone { + t.Errorf("durations %v max fork %d restart %q, want %v, 1 and none", got, v.maxFork, v.vmmRestart, want) + } + if !m.store.sync { + t.Error("sync_claims did not reach the store") + } + parent := mustClaim(t, m, testKey) + if _, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, 2, 0, "", ""); !errors.Is(err, ErrBadCount) { + t.Errorf("fork of 2 over a reloaded limit of 1: %v, want ErrBadCount", err) + } +} diff --git a/sandboxd/pool/running_test.go b/sandboxd/pool/running_test.go index 567c63539..c074214c3 100644 --- a/sandboxd/pool/running_test.go +++ b/sandboxd/pool/running_test.go @@ -3,7 +3,6 @@ package pool import ( "context" "errors" - "slices" "strings" "sync" "sync/atomic" @@ -222,28 +221,6 @@ func TestAReloadAppliesTheClaimCapsWithoutStoppingAClaim(t *testing.T) { } } -func TestAReloadAppliesTheNodeSettings(t *testing.T) { - m := newTestManager(t, newFakeEngine()) - reloadWith(t, m, func(c *config.Config) { - c.MaxForkCount, c.ReleaseDelaySeconds, c.VMMRestart = 1, 5, types.VMMRestartNone - c.SyncClaims, c.CheckpointTTLHours = true, 24 - c.IdleHibernateSeconds, c.ArchiveAfterSeconds, c.ArchiveDeleteAfterSeconds = 60, 120, 3600 - }) - v := m.view.Load() - got := []time.Duration{v.releaseDelay, v.ckptTTL, v.idleDefault, v.archiveAfterDefault, v.archiveDeleteDefault} - want := []time.Duration{5 * time.Second, 24 * time.Hour, time.Minute, 2 * time.Minute, time.Hour} - if !slices.Equal(got, want) || v.maxFork != 1 || v.vmmRestart != types.VMMRestartNone { - t.Errorf("durations %v max fork %d restart %q, want %v, 1 and none", got, v.maxFork, v.vmmRestart, want) - } - if !m.store.sync { - t.Error("sync_claims did not reach the store") - } - parent := mustClaim(t, m, testKey) - if _, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, 2, 0, "", ""); !errors.Is(err, ErrBadCount) { - t.Errorf("fork of 2 over a reloaded limit of 1: %v, want ErrBadCount", err) - } -} - func TestARefusedWakeIsCountedByTheLimitThatRefusedIt(t *testing.T) { m := newTestManager(t, newFakeEngine()) editView(m, func(v *configView) { v.maxRunning = 1 }) @@ -351,6 +328,24 @@ func TestOneHandOffRunsAtATimeAndItsCooldownStartsWhenItFails(t *testing.T) { }) } +func TestQuotaRefusesClaimsPastCap(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 2}) + editView(m, func(v *configView) { v.maxClaims = 1 }) + + first := mustClaim(t, m, testKey) + if _, err := claimAny(t.Context(), m, testKey, ClaimOptions{TTL: time.Hour}); !errors.Is(err, ErrQuota) { + t.Fatalf("claim past cap: %v, want ErrQuota", err) + } + + if err := m.Release(t.Context(), first.ID, Cred{Token: first.Token}); err != nil { + t.Fatalf("Release: %v", err) + } + if _, err := claimAny(t.Context(), m, testKey, ClaimOptions{TTL: time.Hour}); err != nil { + t.Errorf("claim after release: %v", err) + } +} + func wakeClaim(t *testing.T, m *Manager, sb *types.Sandbox) error { t.Helper() _, _, done, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token) diff --git a/sandboxd/pool/telemetry_test.go b/sandboxd/pool/telemetry_test.go index 797e82d88..4c0230011 100644 --- a/sandboxd/pool/telemetry_test.go +++ b/sandboxd/pool/telemetry_test.go @@ -2,7 +2,6 @@ package pool import ( "encoding/json/v2" - "errors" "maps" "net/http" "net/netip" @@ -141,53 +140,6 @@ func TestJournalRotationKeepsOneBackup(t *testing.T) { } } -func TestQuotaRefusesClaimsPastCap(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 2}) - editView(m, func(v *configView) { v.maxClaims = 1 }) - - first := mustClaim(t, m, testKey) - if _, err := claimAny(t.Context(), m, testKey, ClaimOptions{TTL: time.Hour}); !errors.Is(err, ErrQuota) { - t.Fatalf("claim past cap: %v, want ErrQuota", err) - } - - if err := m.Release(t.Context(), first.ID, Cred{Token: first.Token}); err != nil { - t.Fatalf("Release: %v", err) - } - if _, err := claimAny(t.Context(), m, testKey, ClaimOptions{TTL: time.Hour}); err != nil { - t.Errorf("claim after release: %v", err) - } -} - -func TestTenantQuotaBindsPerTenant(t *testing.T) { - eng := newFakeEngine() - m := newTestManager(t, eng) - setTenantCaps(t, m, map[string]int{"acme": 1, "beta": 2}) - - first := claimTenant(t, m, "acme") - if first.Tenant != "acme" { - t.Errorf("tenant %q, want acme", first.Tenant) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{TTL: time.Hour, Tenant: "acme"}); !errors.Is(err, ErrQuota) { - t.Fatalf("acme past its cap: %v, want ErrQuota", err) - } - - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{TTL: time.Hour, Tenant: "beta"}); err != nil { - t.Errorf("beta claim while acme is at cap: %v", err) - } - if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{TTL: time.Hour}); err != nil { - t.Errorf("root claim while acme is at cap: %v", err) - } - counts := m.TenantClaims() - if counts["acme"] != 1 || counts["beta"] != 1 { - t.Errorf("TenantClaims %v, want acme=1 beta=1", counts) - } - if err := m.Release(t.Context(), first.ID, Cred{Token: first.Token}); err != nil { - t.Fatalf("Release: %v", err) - } - claimTenant(t, m, "acme") -} - func TestTenantStampedInJournals(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng) diff --git a/sandboxd/pool/tenants_test.go b/sandboxd/pool/tenants_test.go index c3dc90c3f..cf806de97 100644 --- a/sandboxd/pool/tenants_test.go +++ b/sandboxd/pool/tenants_test.go @@ -389,6 +389,35 @@ func TestTwoNodesOnOneMetaStoreShareTheTenantSet(t *testing.T) { } } +func TestTenantQuotaBindsPerTenant(t *testing.T) { + eng := newFakeEngine() + m := newTestManager(t, eng) + setTenantCaps(t, m, map[string]int{"acme": 1, "beta": 2}) + + first := claimTenant(t, m, "acme") + if first.Tenant != "acme" { + t.Errorf("tenant %q, want acme", first.Tenant) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{TTL: time.Hour, Tenant: "acme"}); !errors.Is(err, ErrQuota) { + t.Fatalf("acme past its cap: %v, want ErrQuota", err) + } + + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{TTL: time.Hour, Tenant: "beta"}); err != nil { + t.Errorf("beta claim while acme is at cap: %v", err) + } + if _, err := m.ClaimProvision(t.Context(), testKey, ClaimOptions{TTL: time.Hour}); err != nil { + t.Errorf("root claim while acme is at cap: %v", err) + } + counts := m.TenantClaims() + if counts["acme"] != 1 || counts["beta"] != 1 { + t.Errorf("TenantClaims %v, want acme=1 beta=1", counts) + } + if err := m.Release(t.Context(), first.ID, Cred{Token: first.Token}); err != nil { + t.Fatalf("Release: %v", err) + } + claimTenant(t, m, "acme") +} + func tenantOf(t *testing.T, m *Manager, token string) (string, bool) { t.Helper() r, err := m.TenantByToken(t.Context(), token) diff --git a/sandboxd/pool/volume_replica_test.go b/sandboxd/pool/volume_replica_test.go index 9206f4260..b6b00e189 100644 --- a/sandboxd/pool/volume_replica_test.go +++ b/sandboxd/pool/volume_replica_test.go @@ -107,9 +107,3 @@ func newReplicaVolumeManager(t *testing.T) (*Manager, *failoverCluster) { c.volumes = []config.VolumeSpec{{Name: "data", Path: writeVolumeImage(t, "data", "original"), Writable: true}} return c.node(t, "a", t.TempDir(), keepAnHour), c } - -func disableReplication(m *Manager) { - view := *m.view.Load() - view.poolReplicate = nil - m.view.Store(&view) -} From 10a0a4d62c8cb6df5b702e0da84bc5d627a64d45 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:38 +0800 Subject: [PATCH 099/127] tunnet/udp: fail from the test goroutine and drop the blind sleep in the exit test --- sandboxd/tunnet/udp/exit_linux_test.go | 61 +++++++++++++++----------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/sandboxd/tunnet/udp/exit_linux_test.go b/sandboxd/tunnet/udp/exit_linux_test.go index 674051b55..4b1753fe0 100644 --- a/sandboxd/tunnet/udp/exit_linux_test.go +++ b/sandboxd/tunnet/udp/exit_linux_test.go @@ -3,6 +3,7 @@ package udp_test import ( "bytes" "encoding/binary" + "fmt" "io" "net" "net/netip" @@ -48,54 +49,55 @@ func TestTrafficCrossesAKernelExit(t *testing.T) { go func() { _ = r.Serve(host, claimAddr) }() go pumpGuest(tun, guest) - inNetns(t, "inet", func() { + inNetns(t, "inet", func() error { l, err := net.Listen("tcp", "10.98.0.2:7000") if err != nil { - t.Fatal(err) + return err } c, err := net.ListenPacket("udp", "10.98.0.2:7001") if err != nil { - t.Fatal(err) + return err } go echoTCP(l) go echoUDP(c) + return nil }) - time.Sleep(300 * time.Millisecond) waitNeigh(t) - var got []byte - inNetns(t, "guest", func() { + inNetns(t, "guest", func() error { c, err := net.DialTimeout("tcp", "10.98.0.2:7000", 5*time.Second) if err != nil { - t.Log(run(t, "sh", "-c", "ip netns exec guest cat /proc/net/snmp | grep -E '^(Ip|Tcp|Udp):'; ip netns exec exit cat /proc/net/snmp | grep -E '^(Ip|Tcp):'; ip netns exec exit conntrack -L 2>&1 | head; ip -n exit -s link show vxlan0 | head -8")) - t.Fatalf("tcp through the exit: %v", err) + diag, _ := runOut("sh", "-c", "ip netns exec guest cat /proc/net/snmp | grep -E '^(Ip|Tcp|Udp):'; ip netns exec exit cat /proc/net/snmp | grep -E '^(Ip|Tcp):'; ip netns exec exit conntrack -L 2>&1 | head; ip -n exit -s link show vxlan0 | head -8") + t.Log(diag) + return fmt.Errorf("tcp through the exit: %w", err) } defer func() { _ = c.Close() }() start := time.Now() go func() { _, _ = c.Write(bytes.Repeat([]byte{0x5a}, bulk)); _ = c.(*net.TCPConn).CloseWrite() }() - got, err = io.ReadAll(c) + got, err := io.ReadAll(c) if err != nil || len(got) != bulk { - t.Fatalf("tcp echo %d bytes, err %v", len(got), err) + return fmt.Errorf("tcp echo %d bytes, err %v", len(got), err) } t.Logf("tcp echo %d MiB both ways in %v", bulk>>20, time.Since(start)) u, err := net.Dial("udp", "10.98.0.2:7001") if err != nil { - t.Fatal(err) + return err } defer func() { _ = u.Close() }() _ = u.SetReadDeadline(time.Now().Add(3 * time.Second)) if _, err = u.Write([]byte("ping")); err != nil { - t.Fatal(err) + return err } b := make([]byte, 64) n, err := u.Read(b) if err != nil || !strings.HasPrefix(string(b[:n]), "10.98.0.1") { - t.Fatalf("udp echo %q %v, want the exit's masquerade address", b[:n], err) + return fmt.Errorf("udp echo %q %v, want the exit's masquerade address", b[:n], err) } other, _ := net.DialTimeout("tcp", otherAddr.String()+":7000", 500*time.Millisecond) if other != nil { t.Error("a claim reached another claim's overlay address") } + return nil }) ct := run(t, "ip", "netns", "exec", "exit", "conntrack", "-L", "-s", claimAddr.String()) if !strings.Contains(ct, "src="+claimAddr.String()) { @@ -143,20 +145,21 @@ NFT func guestTun(t *testing.T) *os.File { t.Helper() var f *os.File - inNetns(t, "guest", func() { + inNetns(t, "guest", func() error { fd, err := unix.Open("/dev/net/tun", unix.O_RDWR|unix.O_CLOEXEC, 0) if err != nil { - t.Fatal(err) + return err } ifr, _ := unix.NewIfreq("tun0") ifr.SetUint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_VNET_HDR) if err = unix.IoctlIfreq(fd, unix.TUNSETIFF, ifr); err != nil { - t.Fatal(err) + return err } if err = unix.IoctlSetInt(fd, unix.TUNSETOFFLOAD, tunnet.OffloadCSUM|tunnet.OffloadTSO4|tunnet.OffloadTSO6|tunnet.OffloadTSOECN); err != nil { - t.Fatal(err) + return err } f = os.NewFile(uintptr(fd), "tun0") + return nil }) run(t, "sh", "-c", "ip -n guest addr add 100.64.0.2/30 dev tun0 && ip -n guest link set tun0 mtu 1450 up && ip -n guest route add default via 100.64.0.1") return f @@ -230,32 +233,38 @@ func waitNeigh(t *testing.T) { } // inNetns runs fn on a thread inside netns name; sockets fn opens stay in that netns. -func inNetns(t *testing.T, name string, fn func()) { +func inNetns(t *testing.T, name string, fn func() error) { t.Helper() - done := make(chan struct{}) + errc := make(chan error, 1) go func() { - defer close(done) runtime.LockOSThread() f, err := os.Open("/var/run/netns/" + name) if err != nil { - t.Error(err) + errc <- err return } defer func() { _ = f.Close() }() if err = unix.Setns(int(f.Fd()), unix.CLONE_NEWNET); err != nil { - t.Error(err) + errc <- err return } - fn() + errc <- fn() }() - <-done + if err := <-errc; err != nil { + t.Fatal(err) + } } func run(t *testing.T, name string, args ...string) string { t.Helper() - out, err := exec.Command(name, args...).CombinedOutput() + out, err := runOut(name, args...) if err != nil { t.Fatalf("%s %v: %v\n%s", name, args, err, out) } - return string(out) + return out +} + +func runOut(name string, args ...string) (string, error) { + out, err := exec.Command(name, args...).CombinedOutput() + return string(out), err } From 268a4a94584256d6ad3b6ea7513dd7266b2836eb Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:36:38 +0800 Subject: [PATCH 100/127] store/s3: wait for the second Fetch to join by its stack, make the budget bound a const assertion --- sandboxd/store/s3/s3_test.go | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/sandboxd/store/s3/s3_test.go b/sandboxd/store/s3/s3_test.go index e7f3b341e..784774ab4 100644 --- a/sandboxd/store/s3/s3_test.go +++ b/sandboxd/store/s3/s3_test.go @@ -15,6 +15,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "runtime" "slices" "strconv" "strings" @@ -27,6 +28,8 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/store/storetest" ) +const _ = uint(store.GenerationGrace - transferBudget - 1) + func TestS3BackendContract(t *testing.T) { _, st := newFakeStore(t) storetest.RunContract(t, st) @@ -139,7 +142,6 @@ func TestFetchReportsMetaWithoutExportAsNotFound(t *testing.T) { func TestFetchOutlivesTheCallerThatStartedIt(t *testing.T) { const id = "ck_00000000000000dd" - metaKey := "ck/" + id + "/" + store.MetaFile fake, st, release := newGatedStore(t, "") publishRecord(t, st, id, metaJSON(id), "payload") fake.resetRequests() @@ -156,10 +158,7 @@ func TestFetchOutlivesTheCallerThatStartedIt(t *testing.T) { _, _, _, err := st.Fetch(t.Context(), id) joined <- err }() - for fake.requestCount(http.MethodGet, metaKey) < 2 { - time.Sleep(time.Millisecond) - } - time.Sleep(50 * time.Millisecond) + waitParkedFetchers(t, 2) hangUp() select { case err := <-started: @@ -802,13 +801,6 @@ func TestDeleteRemovesTheLineageMarker(t *testing.T) { } } -func TestTransferBudgetStaysBelowTheReuseWindow(t *testing.T) { - budget := transferBudget - if budget >= store.GenerationGrace { - t.Errorf("transfer budget %v must stay below the generation grace %v", budget, store.GenerationGrace) - } -} - func TestFetchFileDownloadsOnlyTheRequestedFile(t *testing.T) { const id = "ck_00000000000000aa" meta := metaJSON(id) @@ -1330,6 +1322,24 @@ func newTestStoreAt(t *testing.T, endpoint http.Handler, cfg store.S3Config) *St return st } +func waitParkedFetchers(t *testing.T, want int) { + t.Helper() + buf := make([]byte, 1<<20) + for range 5000 { + parked := 0 + for _, g := range strings.Split(string(buf[:runtime.Stack(buf, true)]), "\n\n") { + if strings.Contains(g, "s3.(*Store).Fetch(") && !strings.Contains(g, "readMeta") { + parked++ + } + } + if parked >= want { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("fewer than %d Fetch callers joined the shared download", want) +} + func assertSingleMetaGet(t *testing.T, fake *fakeS3, metaKey string) { t.Helper() if got := fake.requestCount(http.MethodGet, metaKey); got != 1 { From 0b0a5a8452ae2c9c16bd00a2cea0fc308c9c6767 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:37:29 +0800 Subject: [PATCH 101/127] egress: wait for the handler before asserting no transfer or audit --- sandboxd/egress/proxy_test.go | 3 ++- sandboxd/egress/socks_test.go | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/sandboxd/egress/proxy_test.go b/sandboxd/egress/proxy_test.go index b0e539fd0..2064f5b6d 100644 --- a/sandboxd/egress/proxy_test.go +++ b/sandboxd/egress/proxy_test.go @@ -589,10 +589,11 @@ func TestARefusedDialAnswersItsStatusAndIsAuditedWithNoTransfer(t *testing.T) { if ev := recvEvent(t, events); ev.Decision != tt.decision { t.Errorf("CONNECT audit %+v, want decision %v", ev, tt.decision) } + front.Close() select { case ev := <-transfers: t.Errorf("a refused dial reported a transfer for %s", ev.Host) - case <-time.After(200 * time.Millisecond): + default: } }) } diff --git a/sandboxd/egress/socks_test.go b/sandboxd/egress/socks_test.go index c980cc384..3970816e5 100644 --- a/sandboxd/egress/socks_test.go +++ b/sandboxd/egress/socks_test.go @@ -97,14 +97,18 @@ func TestSocksRefusesWhatItCannotServe(t *testing.T) { t.Run(tt.name, func(t *testing.T) { events := make(chan Event, 4) _, addr := socksProxy(t, Policy{Allow: []Rule{{Host: "*"}}}, nil, fixedDial("127.0.0.1:1"), events, nil) - _, reply := socksExchange(t, addr, tt.greeting, tt.request) + conn, reply := socksExchange(t, addr, tt.greeting, tt.request) if string(reply[:2]) != string(tt.want) { t.Errorf("reply = %v, want prefix %v", reply, tt.want) } + _ = conn.SetReadDeadline(time.Now().Add(5 * time.Second)) + if _, err := io.Copy(io.Discard, conn); errors.Is(err, os.ErrDeadlineExceeded) { + t.Fatalf("the proxy never hung up: %v", err) + } select { case ev := <-events: t.Errorf("refused handshake recorded %+v", ev) - case <-time.After(50 * time.Millisecond): + default: } }) } From fd36ac7f8a6a2600cdf57f99d0d543e011ad3fae Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:38:14 +0800 Subject: [PATCH 102/127] mcp: gate the release race on a channel, checked type assertions, shared node_info server --- mcp/node_info_test.go | 16 ++++++---------- mcp/server_test.go | 38 ++++++++++++++++++++++++++++---------- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/mcp/node_info_test.go b/mcp/node_info_test.go index 562a41af6..609aa94da 100644 --- a/mcp/node_info_test.go +++ b/mcp/node_info_test.go @@ -3,13 +3,14 @@ package main import ( "encoding/json/v2" "net/http" - "net/http/httptest" - "strings" "testing" ) func TestNodeInfoReportsCapacityState(t *testing.T) { - node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + srv := newTestServer(t, func(w http.ResponseWriter, r *http.Request) bool { + if r.URL.Path != "/v1/info" { + return false + } _ = json.MarshalWrite(w, map[string]any{ "pools": []any{}, "claimed": 2, @@ -18,13 +19,8 @@ func TestNodeInfoReportsCapacityState(t *testing.T) { "at_capacity": true, "at_capacity_reason": "not enough memory", }) - })) - t.Cleanup(node.Close) - - srv, err := newServer(strings.TrimPrefix(node.URL, "http://"), "", "rt:24.04") - if err != nil { - t.Fatalf("newServer: %v", err) - } + return true + }) text, err := toolNodeInfo(t.Context(), srv, nil) if err != nil { t.Fatalf("toolNodeInfo: %v", err) diff --git a/mcp/server_test.go b/mcp/server_test.go index a57122dba..9fbfee915 100644 --- a/mcp/server_test.go +++ b/mcp/server_test.go @@ -14,7 +14,6 @@ import ( "sync" "sync/atomic" "testing" - "time" "github.com/cocoonstack/sandbox/protocol/wire" "github.com/cocoonstack/sandbox/sdk/go/silkd/silkdtest" @@ -53,14 +52,14 @@ func TestServeSpeaksMCP(t *testing.T) { t.Fatalf("got %d replies, want 8 (notification unanswered)", len(replies)) } - initResult := replies[1]["result"].(map[string]any) + initResult := as[map[string]any](t, replies[1]["result"]) if initResult["protocolVersion"] != protocolVersion { t.Errorf("initialize protocolVersion = %v", initResult["protocolVersion"]) } - if info := initResult["serverInfo"].(map[string]any); info["version"] != version { + if info := as[map[string]any](t, initResult["serverInfo"]); info["version"] != version { t.Errorf("initialize serverInfo = %v, want version %q", info, version) } - tools := replies[2]["result"].(map[string]any)["tools"].([]any) + tools := as[[]any](t, as[map[string]any](t, replies[2]["result"])["tools"]) if len(tools) != len(toolSpecs()) { t.Errorf("tools/list: %d tools, want %d", len(tools), len(toolSpecs())) } @@ -76,7 +75,7 @@ func TestServeSpeaksMCP(t *testing.T) { t.Errorf("release: %q", got) } - execReply := replies[6]["result"].(map[string]any) + execReply := as[map[string]any](t, replies[6]["result"]) if execReply["isError"] != true { t.Errorf("exec after release: %+v, want isError", execReply) } @@ -114,7 +113,7 @@ func TestExecKeepsOutputWhenTheGuestDrops(t *testing.T) { `{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"create_sandbox","arguments":{}}}`, `{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"exec","arguments":{"sandbox_id":"sb_1","command":"yes"}}}`, ) - execResult := replies[2]["result"].(map[string]any) + execResult := as[map[string]any](t, replies[2]["result"]) text := toolText(t, replies[2]) if execResult["isError"] != true || !strings.Contains(text, `"stdout":"partial"`) || !strings.Contains(text, `"error"`) { t.Errorf("exec reply %v: want isError with the partial stdout and an error field", text) @@ -179,9 +178,13 @@ func TestToolRepliesReplaceInvalidUTF8(t *testing.T) { func TestCloseBoxesWaitsForEveryReleaseOnce(t *testing.T) { var releases atomic.Int32 + var arrived sync.WaitGroup + arrived.Add(3) + gate := make(chan struct{}) node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/release") { - time.Sleep(200 * time.Millisecond) + arrived.Done() + <-gate releases.Add(1) } w.WriteHeader(http.StatusNoContent) @@ -196,14 +199,20 @@ func TestCloseBoxesWaitsForEveryReleaseOnce(t *testing.T) { srv.trackBox(srv.client.Attach(addr, id, "tok")) } var wg sync.WaitGroup + called := make(chan struct{}, 2) for range 2 { wg.Go(func() { + called <- struct{}{} srv.closeBoxes() if got := releases.Load(); got != 3 { t.Errorf("releases = %d after closeBoxes returned, want 3", got) } }) } + <-called + <-called + arrived.Wait() + close(gate) wg.Wait() } @@ -240,14 +249,23 @@ func TestCappedOutputMarksTruncation(t *testing.T) { } } +func as[T any](t *testing.T, v any) T { + t.Helper() + got, ok := v.(T) + if !ok { + t.Fatalf("%v is %T, want %T", v, v, got) + } + return got +} + func toolText(t *testing.T, resp map[string]any) string { t.Helper() res, ok := resp["result"].(map[string]any) if !ok { t.Fatalf("reply without result: %+v", resp) } - content := res["content"].([]any) - return fmt.Sprintf("%v", content[0].(map[string]any)["text"]) + content := as[[]any](t, res["content"]) + return fmt.Sprintf("%v", as[map[string]any](t, content[0])["text"]) } func newTestServer(t *testing.T, extra func(http.ResponseWriter, *http.Request) bool) *server { @@ -313,7 +331,7 @@ func serveLines(t *testing.T, srv *server, lines ...string) map[int]map[string]a if err := json.Unmarshal(line, &resp); err != nil { t.Fatalf("decode reply %q: %v", line, err) } - replies[int(resp["id"].(float64))] = resp + replies[int(as[float64](t, resp["id"]))] = resp } return replies } From d17342b60f9cff176eb8e30bcaed856ebb770f80 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:39:02 +0800 Subject: [PATCH 103/127] tunnet: bind the test router to the test context and replace the sleep with a sentinel ARP --- sandboxd/tunnet/router_test.go | 63 ++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/sandboxd/tunnet/router_test.go b/sandboxd/tunnet/router_test.go index be92f65c9..0c9fd77bf 100644 --- a/sandboxd/tunnet/router_test.go +++ b/sandboxd/tunnet/router_test.go @@ -2,7 +2,6 @@ package tunnet import ( "bytes" - "context" "encoding/binary" "errors" "io" @@ -21,7 +20,7 @@ var ( ) func TestRouterAnnouncesRewritesAndPassesOnlyReplies(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) var records []FlowRecord var mu sync.Mutex r.Arm(Claim{ @@ -66,7 +65,7 @@ func TestRouterAnnouncesRewritesAndPassesOnlyReplies(t *testing.T) { } func TestRouterAnswersARPOnlyForArmedClaims(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr}) u.sent(t) asker := netip.MustParseAddr("100.80.0.1") @@ -84,14 +83,21 @@ func TestRouterAnswersARPOnlyForArmedClaims(t *testing.T) { r.Disarm(claimAddr) b = appendHeader(nil, 7, broadcast, exitMAC, etherARP) u.inject(t, appendARP(b, arpReq, exitMAC, asker, [6]byte{}, claimAddr)) - time.Sleep(50 * time.Millisecond) + sentinel := netip.MustParseAddr("100.80.0.8") + r.Arm(Claim{ID: "c2", Addr: sentinel}) + u.sent(t) + b = appendHeader(nil, 7, broadcast, exitMAC, etherARP) + u.inject(t, appendARP(b, arpReq, exitMAC, asker, [6]byte{}, sentinel)) + if inner, _, _ := u.sent(t); netip.AddrFrom4([4]byte(inner[14:18])) != sentinel { + t.Fatalf("first answer after the disarm announces %x, want only the sentinel's", inner[14:18]) + } if len(u.out) != 0 { t.Error("a disarmed claim, or a sender that is no exit, got an ARP answer") } } func TestRouterSegmentsGSOIntoOneBatch(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) guest := serve(t, r, claimAddr) @@ -109,7 +115,7 @@ func TestRouterSegmentsGSOIntoOneBatch(t *testing.T) { } func TestRouterRefusesUnarmedAndDisabledAndDisarmEndsStream(t *testing.T) { - _, r := newRouter() + _, r := newRouter(t) if h := guestHello(t, serve(t, r, claimAddr)); h.Flags&FlagOn != 0 { t.Error("an unarmed address got on") } @@ -132,7 +138,7 @@ func TestRouterRefusesUnarmedAndDisabledAndDisarmEndsStream(t *testing.T) { } func TestRouterQuotaStopsTraffic(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) used := new(atomic.Int64) r.Arm(Claim{ID: "c1", Addr: claimAddr, Used: used, Limits: Limits{QuotaBytes: 100}, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) @@ -150,7 +156,7 @@ func TestRouterQuotaStopsTraffic(t *testing.T) { } func TestRouterChargesNothingForAFrameWithNoExit(t *testing.T) { - _, r := newRouter() + _, r := newRouter(t) used := new(atomic.Int64) r.Arm(Claim{ID: "c1", Addr: claimAddr, Profile: "nowhere", Used: used, Allow: func(netip.Addr, uint16) bool { return true }}) guest := serve(t, r, claimAddr) @@ -169,7 +175,7 @@ func TestRouterChargesNothingForAFrameWithNoExit(t *testing.T) { } func TestDeliverCoalescesInOrderSegmentsIntoOneGSOFrame(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) guest := serve(t, r, claimAddr) @@ -196,12 +202,11 @@ func TestDeliverCoalescesInOrderSegmentsIntoOneGSOFrame(t *testing.T) { } func TestRouterChargesSegmentsAndRefusesOversizedPackets(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Limits: Limits{PacketsPerSecond: 10}, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) go func() { - for s := range u.out { - _ = s + for range u.out { } }() guest := serve(t, r, claimAddr) @@ -220,7 +225,7 @@ func TestRouterChargesSegmentsAndRefusesOversizedPackets(t *testing.T) { } func TestRouterLimitsOutliveAReconnect(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Limits: Limits{NewFlowsPerSecond: 0.01}, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) first := serve(t, r, claimAddr) @@ -237,7 +242,7 @@ func TestRouterLimitsOutliveAReconnect(t *testing.T) { } func TestRouterHandsAnICMPErrorBackWithTheQuotedGuestAddress(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) guest := serve(t, r, claimAddr) @@ -253,7 +258,7 @@ func TestRouterHandsAnICMPErrorBackWithTheQuotedGuestAddress(t *testing.T) { } func TestRouterResumesAStreamOpenedBeforeTheArmButRefusesAnOpener(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(_ netip.Addr, port uint16) bool { return port == 443 }}) u.sent(t) guest := serve(t, r, claimAddr) @@ -270,7 +275,7 @@ func TestRouterResumesAStreamOpenedBeforeTheArmButRefusesAnOpener(t *testing.T) } func TestRouterKeepsTheMTUAStreamWasToldUntilItsNextHello(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) guest := serve(t, r, claimAddr) @@ -286,7 +291,7 @@ func TestRouterKeepsTheMTUAStreamWasToldUntilItsNextHello(t *testing.T) { } func TestEndFlowsRecordsLiveFlowsWithIPBytes(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) var mu sync.Mutex var records []FlowRecord r.Arm(Claim{ @@ -311,7 +316,7 @@ func TestEndFlowsRecordsLiveFlowsWithIPBytes(t *testing.T) { } func TestDeliverMergesOnlyForAGuestThatOfferedTSO(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) host, guest := net.Pipe() @@ -333,7 +338,7 @@ func TestDeliverMergesOnlyForAGuestThatOfferedTSO(t *testing.T) { } func TestRouterPointsAGuestICMPErrorAtTheClaimAddress(t *testing.T) { - u, r := newRouter() + u, r := newRouter(t) r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) guest := serve(t, r, claimAddr) @@ -346,10 +351,12 @@ func TestRouterPointsAGuestICMPErrorAtTheClaimAddress(t *testing.T) { } } -func newRouter() (*fakeUnder, *Router) { - u := &fakeUnder{out: make(chan sent, 64), in: make(chan []sent, 16)} +func newRouter(t *testing.T) (*fakeUnder, *Router) { + t.Helper() + u := &fakeUnder{out: make(chan sent, 64), in: make(chan []sent, 16), done: make(chan struct{})} + t.Cleanup(func() { close(u.done) }) r := New(u, &Overlay{Subnet: subnet, VNI: 7, Port: 4789, MTU: 1450, Exits: map[string]Exit{"default": {VTEP: vtep, MAC: exitMAC}}}) - go func() { _ = r.Run(context.Background()) }() + go func() { _ = r.Run(t.Context()) }() return u, r } @@ -461,6 +468,7 @@ type sent struct { type fakeUnder struct { out chan sent in chan []sent + done chan struct{} batches atomic.Int64 } @@ -475,10 +483,15 @@ func (u *fakeUnder) Send(b []byte, seg int, dst netip.AddrPort, _ uint64) error } func (u *fakeUnder) Recv(each func(d []byte, from netip.AddrPort)) error { - for _, s := range <-u.in { - each(s.d, s.to) + select { + case batch := <-u.in: + for _, s := range batch { + each(s.d, s.to) + } + return nil + case <-u.done: + return net.ErrClosed } - return nil } func (u *fakeUnder) inject(t *testing.T, d []byte) { From 945952fd64f5a955873af1318a4401632e162d81 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:44:48 +0800 Subject: [PATCH 104/127] store: hold the probe handlers on channels, share FileSum, AllocatedBytes and WaitStacks through storetest, fold peer round-trip tests --- sandboxd/store/peer/broadcast_test.go | 39 +++++--------- sandboxd/store/peer/move_test.go | 7 ++- sandboxd/store/peer/probe_test.go | 75 +++++++++++---------------- sandboxd/store/peer/transport_test.go | 67 ++++++------------------ sandboxd/store/s3/s3_test.go | 67 +++++------------------- sandboxd/store/storetest/storetest.go | 50 ++++++++++++++++++ 6 files changed, 123 insertions(+), 182 deletions(-) diff --git a/sandboxd/store/peer/broadcast_test.go b/sandboxd/store/peer/broadcast_test.go index 8f5d0fa32..88d0e89d1 100644 --- a/sandboxd/store/peer/broadcast_test.go +++ b/sandboxd/store/peer/broadcast_test.go @@ -28,32 +28,27 @@ func TestBroadcastDeleteSendsNoForward(t *testing.T) { } func TestBroadcastDeleteFansOutToEveryPeer(t *testing.T) { - var hits atomic.Int32 - newSrv := func() *httptest.Server { - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - hits.Add(1) - w.WriteHeader(http.StatusNoContent) - })) - t.Cleanup(srv.Close) - return srv + var addrs []string + var hits []*atomic.Int32 + for range 3 { + srv, n := countingServer(t, http.StatusNoContent) + addrs, hits = append(addrs, srv.URL), append(hits, n) } - addrs := []string{newSrv().URL, newSrv().URL, newSrv().URL} b := &Broadcaster{Peers: func() []string { return addrs }} b.Delete(t.Context(), testID) - if got := hits.Load(); got != 3 { + got := 0 + for _, n := range hits { + got += int(n.Load()) + } + if got != 3 { t.Errorf("peers hit %d times, want 3 (one per distinct peer)", got) } } func TestBroadcastDeleteDedupsRepeatedAddrs(t *testing.T) { - var hits atomic.Int32 - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - hits.Add(1) - w.WriteHeader(http.StatusNoContent) - })) - defer srv.Close() + srv, hits := countingServer(t, http.StatusNoContent) b := &Broadcaster{Peers: func() []string { return []string{srv.URL, srv.URL, srv.URL} }} b.Delete(t.Context(), testID) @@ -68,12 +63,7 @@ func TestBroadcastDeleteSwallowsFailures(t *testing.T) { w.WriteHeader(http.StatusInternalServerError) })) defer fail.Close() - var reached atomic.Int32 - ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - reached.Add(1) - w.WriteHeader(http.StatusNoContent) - })) - defer ok.Close() + ok, reached := countingServer(t, http.StatusNoContent) b := &Broadcaster{Peers: func() []string { return []string{fail.URL, ok.URL, "127.0.0.1:1"} }} b.Delete(t.Context(), testID) @@ -81,8 +71,3 @@ func TestBroadcastDeleteSwallowsFailures(t *testing.T) { t.Errorf("healthy peer hit %d times, want 1 (fan-out continues past a failure)", got) } } - -func TestBroadcastDeleteNoPeersNoOp(t *testing.T) { - b := &Broadcaster{Peers: func() []string { return nil }} - b.Delete(t.Context(), testID) -} diff --git a/sandboxd/store/peer/move_test.go b/sandboxd/store/peer/move_test.go index aa069829d..26b6d335a 100644 --- a/sandboxd/store/peer/move_test.go +++ b/sandboxd/store/peer/move_test.go @@ -12,7 +12,6 @@ import ( "path/filepath" "slices" "strings" - "syscall" "testing" "time" @@ -62,7 +61,7 @@ func TestSendRoundShipsTheListedFilesInOrderAndKeepsSparseFilesSparse(t *testing t.Fatalf("ReceiveRound: %v", err) } for _, name := range names { - if sha256OfFile(t, filepath.Join(dst, name)) != sha256OfFile(t, filepath.Join(src, name)) { + if storetest.FileSum(t, filepath.Join(dst, name)) != storetest.FileSum(t, filepath.Join(src, name)) { t.Errorf("%s differs from the source", name) } } @@ -70,7 +69,7 @@ func TestSendRoundShipsTheListedFilesInOrderAndKeepsSparseFilesSparse(t *testing if err != nil || info.Size() != size { t.Fatalf("received cow.raw = %v, %v, want %d bytes", info, err, size) } - if allocated := info.Sys().(*syscall.Stat_t).Blocks * 512; allocated > sent { + if allocated := storetest.AllocatedBytes(t, filepath.Join(dst, "cow.raw")); allocated > sent { t.Errorf("received cow.raw allocates %d bytes, more than the %d sent", allocated, sent) } entries, _ := os.ReadDir(dst) @@ -99,7 +98,7 @@ func TestReceiveRoundReplacesFilesOfTheSameName(t *testing.T) { if body, err := os.ReadFile(filepath.Join(dst, "state.json")); err != nil || string(body) != "new" { t.Errorf("state.json = %q, %v, want the new round's bytes", body, err) } - if sha256OfFile(t, filepath.Join(dst, "cow.raw")) != sha256OfFile(t, filepath.Join(src, "cow.raw")) { + if storetest.FileSum(t, filepath.Join(dst, "cow.raw")) != storetest.FileSum(t, filepath.Join(src, "cow.raw")) { t.Error("cow.raw keeps bytes of the replaced file in its holes") } } diff --git a/sandboxd/store/peer/probe_test.go b/sandboxd/store/peer/probe_test.go index 588e8ac76..687da6556 100644 --- a/sandboxd/store/peer/probe_test.go +++ b/sandboxd/store/peer/probe_test.go @@ -3,10 +3,13 @@ package peer import ( "net/http" "net/http/httptest" + "strings" "sync" "sync/atomic" "testing" "time" + + "github.com/cocoonstack/sandbox/sandboxd/store/storetest" ) func TestOwnersReturnsOnly200(t *testing.T) { @@ -29,21 +32,6 @@ func TestOwnersAllMissIsEmpty(t *testing.T) { } } -func TestOwnersHungPeerExcludedWithoutBlockingOthers(t *testing.T) { - hung := hungServer(t) - ok := statusServer(t, http.StatusOK) - - start := time.Now() - p := &HTTPProber{Peers: func() []string { return []string{hung.URL, ok.URL} }} - owners := p.Owners(t.Context(), testID) - if elapsed := time.Since(start); elapsed > 2*time.Second { - t.Errorf("Owners took %v, want bounded by the per-probe timeout", elapsed) - } - if len(owners) != 1 || owners[0] != ok.URL { - t.Errorf("owners = %v, want [%s]", owners, ok.URL) - } -} - func TestOwnersReturnsPromptlyWithOneOwner(t *testing.T) { hung := hungServer(t) ok := statusServer(t, http.StatusOK) @@ -60,12 +48,7 @@ func TestOwnersReturnsPromptlyWithOneOwner(t *testing.T) { } func TestOwnersDedupsAddrs(t *testing.T) { - var hits atomic.Int32 - ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - hits.Add(1) - w.WriteHeader(http.StatusOK) - })) - defer ok.Close() + ok, hits := countingServer(t, http.StatusOK) p := &HTTPProber{Peers: func() []string { return []string{ok.URL, ok.URL, ok.URL} }} owners := p.Owners(t.Context(), testID) @@ -77,15 +60,15 @@ func TestOwnersDedupsAddrs(t *testing.T) { } } -func TestOwnersCapsAtThree(t *testing.T) { +func TestOwnersCapsAtTheRedirectLimit(t *testing.T) { var addrs []string - for range 4 { + for range maxRedirectOwners + 1 { addrs = append(addrs, statusServer(t, http.StatusOK).URL) } p := &HTTPProber{Peers: func() []string { return addrs }} - if owners := p.Owners(t.Context(), testID); len(owners) != 3 { - t.Errorf("owners = %v, want capped at 3", owners) + if owners := p.Owners(t.Context(), testID); len(owners) != maxRedirectOwners { + t.Errorf("owners = %v, want capped at %d", owners, maxRedirectOwners) } } @@ -136,18 +119,23 @@ func TestHealOwnersIsExhaustiveDespiteAFastOwner(t *testing.T) { func TestOwnersCoalescesConcurrentProbes(t *testing.T) { var hits atomic.Int32 + hold := make(chan struct{}) + release := sync.OnceFunc(func() { close(hold) }) ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { hits.Add(1) - time.Sleep(50 * time.Millisecond) + <-hold w.WriteHeader(http.StatusOK) })) defer ok.Close() + defer release() p := &HTTPProber{Peers: func() []string { return []string{ok.URL} }} var wg sync.WaitGroup for range 10 { wg.Go(func() { p.Owners(t.Context(), testID) }) } + storetest.WaitStacks(t, 10, func(stack string) bool { return strings.Contains(stack, "singleflight.(*Group).Do(") }) + release() wg.Wait() if got := hits.Load(); got != 1 { t.Errorf("peer hit %d times by 10 concurrent callers, want 1 (coalesced)", got) @@ -182,16 +170,10 @@ func TestOwnersCacheCollapsesRepeatedRedirects(t *testing.T) { } func TestOwnersCacheExpires(t *testing.T) { - var hits atomic.Int32 - ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - hits.Add(1) - w.WriteHeader(http.StatusOK) - })) - defer ok.Close() + ok, hits := countingServer(t, http.StatusOK) - p := &HTTPProber{Peers: func() []string { return []string{ok.URL} }, cacheTTL: 10 * time.Millisecond} + p := &HTTPProber{Peers: func() []string { return []string{ok.URL} }, cacheTTL: -time.Second} p.Owners(t.Context(), testID) - time.Sleep(20 * time.Millisecond) p.Owners(t.Context(), testID) if got := hits.Load(); got != 2 { t.Errorf("peer hit %d times across two Owners calls spanning the TTL, want 2 (expired)", got) @@ -199,12 +181,7 @@ func TestOwnersCacheExpires(t *testing.T) { } func TestForgetEvictsCachedEntry(t *testing.T) { - var hits atomic.Int32 - ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - hits.Add(1) - w.WriteHeader(http.StatusOK) - })) - defer ok.Close() + ok, hits := countingServer(t, http.StatusOK) p := &HTTPProber{Peers: func() []string { return []string{ok.URL} }} p.Owners(t.Context(), testID) @@ -216,12 +193,7 @@ func TestForgetEvictsCachedEntry(t *testing.T) { } func TestForgetOtherIDLeavesCacheAlone(t *testing.T) { - var hits atomic.Int32 - ok := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - hits.Add(1) - w.WriteHeader(http.StatusOK) - })) - defer ok.Close() + ok, hits := countingServer(t, http.StatusOK) p := &HTTPProber{Peers: func() []string { return []string{ok.URL} }} p.Owners(t.Context(), testID) @@ -281,6 +253,17 @@ func statusServer(t *testing.T, code int) *httptest.Server { return srv } +func countingServer(t *testing.T, code int) (*httptest.Server, *atomic.Int32) { + t.Helper() + var hits atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hits.Add(1) + w.WriteHeader(code) + })) + t.Cleanup(srv.Close) + return srv, &hits +} + func hungServer(t *testing.T) *httptest.Server { t.Helper() block := make(chan struct{}) diff --git a/sandboxd/store/peer/transport_test.go b/sandboxd/store/peer/transport_test.go index 45f1ffbc2..18fbe752b 100644 --- a/sandboxd/store/peer/transport_test.go +++ b/sandboxd/store/peer/transport_test.go @@ -3,7 +3,6 @@ package peer import ( "archive/tar" "bytes" - "crypto/sha256" "errors" "io" "net/http" @@ -12,7 +11,6 @@ import ( "path/filepath" "slices" "strings" - "syscall" "testing" "time" @@ -236,12 +234,9 @@ func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { } meta := []byte(`{"id":"ck_0000000000000001","key":{"template":"t"}}`) - var buf bytes.Buffer - if err := TarRecord(exportDir, meta, &buf); err != nil { - t.Fatalf("TarRecord: %v", err) - } + buf := tarOf(t, exportDir, meta) dst := t.TempDir() - if err := Untar(&buf, dst); err != nil { + if err := Untar(buf, dst); err != nil { t.Fatalf("Untar: %v", err) } @@ -269,6 +264,9 @@ func TestTarRecordRoundTripsAWholeRecord(t *testing.T) { if _, err := os.Stat(filepath.Join(dst, "memory-ranges")); err == nil { t.Error("export contents landed at the record root; they must be under export/") } + if _, err := os.Stat(filepath.Join(dst, recordTrailer)); !errors.Is(err, os.ErrNotExist) { + t.Errorf("completion marker was written to disk (%v); it must be consumed, not stored", err) + } } func TestUntarRejectsRecordWithoutCompletionMarker(t *testing.T) { @@ -286,27 +284,6 @@ func TestUntarRejectsRecordWithoutCompletionMarker(t *testing.T) { } } -func TestTarRecordUntarRoundTripComplete(t *testing.T) { - exportDir := t.TempDir() - if err := os.WriteFile(filepath.Join(exportDir, "mem"), []byte("state"), 0o600); err != nil { - t.Fatalf("setup: %v", err) - } - var buf bytes.Buffer - if err := TarRecord(exportDir, []byte(`{"id":"ck_1"}`), &buf); err != nil { - t.Fatalf("TarRecord: %v", err) - } - dst := t.TempDir() - if err := Untar(&buf, dst); err != nil { - t.Fatalf("Untar: %v", err) - } - if _, err := os.Stat(filepath.Join(dst, recordTrailer)); !errors.Is(err, os.ErrNotExist) { - t.Errorf("completion marker was written to disk (%v); it must be consumed, not stored", err) - } - if _, err := os.Stat(filepath.Join(dst, "export", "mem")); err != nil { - t.Errorf("export content missing after round trip: %v", err) - } -} - func TestTarRecordKeepsSparseFilesSparse(t *testing.T) { size := 10*store.DigestChunkSize + 4096 exportDir := t.TempDir() @@ -318,18 +295,15 @@ func TestTarRecordKeepsSparseFilesSparse(t *testing.T) { if err := os.WriteFile(filepath.Join(exportDir, "snapshot.json"), []byte("{}"), 0o600); err != nil { t.Fatalf("write snapshot: %v", err) } - want := sha256OfFile(t, src) + want := storetest.FileSum(t, src) - var buf bytes.Buffer - if err := TarRecord(exportDir, []byte(`{"id":"ck_0000000000000001"}`), &buf); err != nil { - t.Fatalf("TarRecord: %v", err) - } + buf := tarOf(t, exportDir, []byte(`{"id":"ck_0000000000000001"}`)) sent := int64(buf.Len()) if limit := 3*store.DigestChunkSize + 1<<20; sent > limit { t.Errorf("stream carries %d bytes for a %d-byte file with 3 data chunks, want at most %d", sent, size, limit) } dst := t.TempDir() - if err := Untar(&buf, dst); err != nil { + if err := Untar(buf, dst); err != nil { t.Fatalf("Untar: %v", err) } got := filepath.Join(dst, "export", "cow.raw") @@ -337,10 +311,10 @@ func TestTarRecordKeepsSparseFilesSparse(t *testing.T) { if err != nil || info.Size() != size { t.Fatalf("pulled size = %v, %v, want %d", info, err, size) } - if sha256OfFile(t, got) != want { + if storetest.FileSum(t, got) != want { t.Error("pulled content differs from the source") } - if allocated := info.Sys().(*syscall.Stat_t).Blocks * 512; allocated > sent { + if allocated := storetest.AllocatedBytes(t, got); allocated > sent { t.Errorf("pulled file allocates %d bytes, more than the %d sent", allocated, sent) } if body, err := os.ReadFile(filepath.Join(dst, "export", "snapshot.json")); err != nil || string(body) != "{}" { @@ -353,11 +327,7 @@ func TestUntarRejectsDamagedChunk(t *testing.T) { if err := os.WriteFile(filepath.Join(exportDir, "mem"), []byte("guest-pages-payload"), 0o600); err != nil { t.Fatalf("setup: %v", err) } - var buf bytes.Buffer - if err := TarRecord(exportDir, []byte(`{"id":"ck_1"}`), &buf); err != nil { - t.Fatalf("TarRecord: %v", err) - } - raw := buf.Bytes() + raw := tarOf(t, exportDir, []byte(`{"id":"ck_1"}`)).Bytes() raw[bytes.Index(raw, []byte("guest-pages-payload"))] ^= 0xff err := Untar(bytes.NewReader(raw), t.TempDir()) if err == nil || !strings.Contains(err.Error(), "checksum") { @@ -377,16 +347,11 @@ func tarDir(src string, w io.Writer) error { return tw.Close() } -func sha256OfFile(t *testing.T, path string) [sha256.Size]byte { +func tarOf(t *testing.T, exportDir string, meta []byte) *bytes.Buffer { t.Helper() - f, err := os.Open(path) - if err != nil { - t.Fatalf("open %s: %v", path, err) - } - defer func() { _ = f.Close() }() - h := sha256.New() - if _, err = io.Copy(h, f); err != nil { - t.Fatalf("hash %s: %v", path, err) + var buf bytes.Buffer + if err := TarRecord(exportDir, meta, &buf); err != nil { + t.Fatalf("TarRecord: %v", err) } - return [sha256.Size]byte(h.Sum(nil)) + return &buf } diff --git a/sandboxd/store/s3/s3_test.go b/sandboxd/store/s3/s3_test.go index 784774ab4..57cdf01bd 100644 --- a/sandboxd/store/s3/s3_test.go +++ b/sandboxd/store/s3/s3_test.go @@ -15,12 +15,10 @@ import ( "net/http/httptest" "os" "path/filepath" - "runtime" "slices" "strconv" "strings" "sync" - "syscall" "testing" "time" @@ -158,7 +156,9 @@ func TestFetchOutlivesTheCallerThatStartedIt(t *testing.T) { _, _, _, err := st.Fetch(t.Context(), id) joined <- err }() - waitParkedFetchers(t, 2) + storetest.WaitStacks(t, 2, func(stack string) bool { + return strings.Contains(stack, "s3.(*Store).Fetch(") && !strings.Contains(stack, "readMeta") + }) hangUp() select { case err := <-started: @@ -436,7 +436,7 @@ func TestSparseExportStaysSparseThroughS3(t *testing.T) { size := 12*store.DigestChunkSize + 4096 meta := metaJSON(id) staging, src := stageSparseRecord(t, st, id, meta, size, map[int64][]byte{0: []byte("head"), 3*store.DigestChunkSize + 100: []byte("mid"), 11*store.DigestChunkSize + 10: []byte("tail")}) - srcSum := fileSum(t, src) + srcSum := storetest.FileSum(t, src) if err := st.Publish(t.Context(), staging, id, ""); err != nil { t.Fatalf("Publish: %v", err) } @@ -454,10 +454,10 @@ func TestSparseExportStaysSparseThroughS3(t *testing.T) { if err != nil || info.Size() != size { t.Fatalf("fetched size = %v, %v, want %d", info, err, size) } - if fileSum(t, got) != srcSum { + if storetest.FileSum(t, got) != srcSum { t.Error("fetched content differs from the source") } - if blocks := allocatedBytes(t, got); blocks > uploaded+(1<<20) { + if blocks := storetest.AllocatedBytes(t, got); blocks > uploaded+(1<<20) { t.Errorf("fetched file allocates %d bytes, want at most the %d uploaded", blocks, uploaded) } } @@ -569,7 +569,7 @@ func TestLineageDedupUploadsOnlyChangedChunks(t *testing.T) { t.Helper() before, _ := fake.chunkPuts("ck/chunks/") staging, src := stageSparseRecord(t, st, id, fmt.Appendf(nil, `{"id":"%s","gen":%d}`, id, gen), 8*cs, writes) - srcSum = fileSum(t, src) + srcSum = storetest.FileSum(t, src) if err := st.Publish(t.Context(), staging, id, scope); err != nil { t.Fatalf("Publish %s: %v", id, err) } @@ -595,7 +595,7 @@ func TestLineageDedupUploadsOnlyChangedChunks(t *testing.T) { if err != nil { t.Fatalf("Fetch: %v", err) } - if fileSum(t, filepath.Join(dir, "disk.img")) != srcSum { + if storetest.FileSum(t, filepath.Join(dir, "disk.img")) != srcSum { t.Error("fetched content differs from the source") } } @@ -863,7 +863,7 @@ func TestFetchFileKeepsHoles(t *testing.T) { _, st := newFakeStore(t) size := 12*store.DigestChunkSize + 4096 staging, src := stageSparseRecord(t, st, id, metaJSON(id), size, map[int64][]byte{0: []byte("head"), 11*store.DigestChunkSize + 10: []byte("tail")}) - srcSum := fileSum(t, src) + srcSum := storetest.FileSum(t, src) if err := st.Publish(t.Context(), staging, id, ""); err != nil { t.Fatalf("Publish: %v", err) } @@ -875,10 +875,10 @@ func TestFetchFileKeepsHoles(t *testing.T) { if info, err := os.Stat(dst); err != nil || info.Size() != size { t.Fatalf("fetched size = %v, %v, want %d", info, err, size) } - if fileSum(t, dst) != srcSum { + if storetest.FileSum(t, dst) != srcSum { t.Error("fetched content differs from the source") } - if blocks, limit := allocatedBytes(t, dst), 2*store.DigestChunkSize+1<<20; blocks > limit { + if blocks, limit := storetest.AllocatedBytes(t, dst), 2*store.DigestChunkSize+1<<20; blocks > limit { t.Errorf("fetched file allocates %d bytes, want at most %d for two data chunks", blocks, limit) } } @@ -918,7 +918,7 @@ func TestRaisedConcurrencyReachesItsLimitAndKeepsItsConnections(t *testing.T) { writes[i*store.DigestChunkSize] = []byte{byte(i + 1)} } staging, src := stageSparseRecord(t, st, id, metaJSON(id), chunks*store.DigestChunkSize, writes) - srcSum := fileSum(t, src) + srcSum := storetest.FileSum(t, src) gate.arm(http.MethodPut, chunks) if err := st.Publish(t.Context(), staging, id, ""); err != nil { @@ -933,7 +933,7 @@ func TestRaisedConcurrencyReachesItsLimitAndKeepsItsConnections(t *testing.T) { if err := st.FetchFile(t.Context(), id, "disk.img", dst); err != nil { t.Fatalf("FetchFile round %d: %v", round, err) } - if fileSum(t, dst) != srcSum { + if storetest.FileSum(t, dst) != srcSum { t.Fatalf("FetchFile round %d: content differs from the source", round) } peak, opened := gate.result() @@ -1238,29 +1238,6 @@ func stageSparseRecord(t *testing.T, st *Store, id string, meta []byte, size int return staging, storetest.WriteSparse(t, filepath.Join(staging, store.ExportDir, "disk.img"), size, writes) } -func fileSum(t *testing.T, path string) [sha256.Size]byte { - t.Helper() - f, err := os.Open(path) - if err != nil { - t.Fatalf("open %s: %v", path, err) - } - defer func() { _ = f.Close() }() - h := sha256.New() - if _, err = io.Copy(h, f); err != nil { - t.Fatalf("hash %s: %v", path, err) - } - return [sha256.Size]byte(h.Sum(nil)) -} - -func allocatedBytes(t *testing.T, path string) int64 { - t.Helper() - info, err := os.Stat(path) - if err != nil { - t.Fatalf("stat %s: %v", path, err) - } - return info.Sys().(*syscall.Stat_t).Blocks * 512 -} - func denseChunkSums(t *testing.T, path string) [][sha256.Size]byte { t.Helper() f, err := os.Open(path) @@ -1322,24 +1299,6 @@ func newTestStoreAt(t *testing.T, endpoint http.Handler, cfg store.S3Config) *St return st } -func waitParkedFetchers(t *testing.T, want int) { - t.Helper() - buf := make([]byte, 1<<20) - for range 5000 { - parked := 0 - for _, g := range strings.Split(string(buf[:runtime.Stack(buf, true)]), "\n\n") { - if strings.Contains(g, "s3.(*Store).Fetch(") && !strings.Contains(g, "readMeta") { - parked++ - } - } - if parked >= want { - return - } - time.Sleep(time.Millisecond) - } - t.Fatalf("fewer than %d Fetch callers joined the shared download", want) -} - func assertSingleMetaGet(t *testing.T, fake *fakeS3, metaKey string) { t.Helper() if got := fake.requestCount(http.MethodGet, metaKey); got != 1 { diff --git a/sandboxd/store/storetest/storetest.go b/sandboxd/store/storetest/storetest.go index 7e700f52c..ff5ccb0b2 100644 --- a/sandboxd/store/storetest/storetest.go +++ b/sandboxd/store/storetest/storetest.go @@ -3,11 +3,17 @@ package storetest import ( "context" + "crypto/sha256" "errors" + "io" "os" "path/filepath" + "runtime" "slices" + "strings" + "syscall" "testing" + "time" "github.com/cocoonstack/sandbox/sandboxd/store" ) @@ -147,6 +153,50 @@ func WriteSparse(t testing.TB, path string, size int64, writes map[int64][]byte) return path } +// FileSum hashes the file at path. +func FileSum(t testing.TB, path string) [sha256.Size]byte { + t.Helper() + f, err := os.Open(path) //nolint:gosec // test path + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + defer func() { _ = f.Close() }() + h := sha256.New() + if _, err = io.Copy(h, f); err != nil { + t.Fatalf("hash %s: %v", path, err) + } + return [sha256.Size]byte(h.Sum(nil)) +} + +// AllocatedBytes reports the disk blocks path occupies. +func AllocatedBytes(t testing.TB, path string) int64 { + t.Helper() + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + return info.Sys().(*syscall.Stat_t).Blocks * 512 +} + +// WaitStacks polls until at least want goroutine stacks satisfy match. +func WaitStacks(t testing.TB, want int, match func(stack string) bool) { + t.Helper() + buf := make([]byte, 1<<20) + for range 5000 { + matched := 0 + for _, stack := range strings.Split(string(buf[:runtime.Stack(buf, true)]), "\n\n") { + if match(stack) { + matched++ + } + } + if matched >= want { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("fewer than %d goroutines reached the expected state", want) +} + func runFetchFileContract(t *testing.T, st store.Store, id string) { t.Helper() ctx := t.Context() From 7524203dcc7f69108e11e1544a433f9e92706664 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:45:01 +0800 Subject: [PATCH 105/127] engine: drop the sleep-ordered probe test, the retry loop is covered by the failure-retry test --- sandboxd/engine/engine_test.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/sandboxd/engine/engine_test.go b/sandboxd/engine/engine_test.go index a9b39d713..8a81707b9 100644 --- a/sandboxd/engine/engine_test.go +++ b/sandboxd/engine/engine_test.go @@ -105,20 +105,6 @@ func TestProbeRetriesPastFailures(t *testing.T) { } } -func TestProbeRetriesUntilListenerAppears(t *testing.T) { - path := sockPath(t) - done := make(chan error, 1) - go func() { - done <- newTestEngine().Probe(t.Context(), path, 2*time.Second) - }() - - time.Sleep(60 * time.Millisecond) - listenMuxer(t, path, "OK 2048\n", infoFrame) - if err := <-done; err != nil { - t.Errorf("Probe: %v", err) - } -} - func TestDialGuestPortCtxCancel(t *testing.T) { path := sockPath(t) ln, err := net.Listen("unix", path) From 2c37638c7f093555c48304daaae2c7f047ebbd15 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:45:32 +0800 Subject: [PATCH 106/127] server: run the dead-client splice test under synctest --- sandboxd/server/port_test.go | 57 +++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/sandboxd/server/port_test.go b/sandboxd/server/port_test.go index 440dc2384..694d262cd 100644 --- a/sandboxd/server/port_test.go +++ b/sandboxd/server/port_test.go @@ -148,37 +148,40 @@ func TestPortRelayEndsOnGuestClose(t *testing.T) { } func TestSpliceTearsDownWhenTheClientDies(t *testing.T) { - srv := New("", "node:7777", &fakeManager{}, &fakeDialer{}, nil, nil, nil, nil) - t.Cleanup(srv.CloseRelays) - client, clientPeer := net.Pipe() - t.Cleanup(func() { _ = client.Close(); _ = clientPeer.Close() }) - guest, guestPeer := net.Pipe() - t.Cleanup(func() { _ = guestPeer.Close() }) - go func() { _, _ = io.Copy(io.Discard, clientPeer) }() + synctest.Test(t, func(t *testing.T) { + srv := New("", "node:7777", &fakeManager{}, &fakeDialer{}, nil, nil, nil, nil) + t.Cleanup(srv.CloseRelays) + client, clientPeer := net.Pipe() + t.Cleanup(func() { _ = client.Close(); _ = clientPeer.Close() }) + guest, guestPeer := net.Pipe() + t.Cleanup(func() { _ = guestPeer.Close() }) + go func() { _, _ = io.Copy(io.Discard, clientPeer) }() - guestClosed := make(chan error, 1) - go func() { - _, err := guestPeer.Read(make([]byte, 1)) - guestClosed <- err - }() + guestClosed := make(chan error, 1) + go func() { + _, err := guestPeer.Read(make([]byte, 1)) + guestClosed <- err + }() - ended := make(chan struct{}) - go func() { - defer close(ended) - srv.splice(client, iotest.ErrReader(errors.New("client vanished")), guest, - switchingProtocolsTCP, tunnelEnds) - }() + ended := make(chan struct{}) + go func() { + defer close(ended) + srv.splice(client, iotest.ErrReader(errors.New("client vanished")), guest, + switchingProtocolsTCP, tunnelEnds) + }() - select { - case err := <-guestClosed: - if err == nil { - t.Error("guest read succeeded; the relay should have closed it") + synctest.Wait() + select { + case err := <-guestClosed: + if err == nil { + t.Error("guest read succeeded; the relay should have closed it") + } + default: + t.Fatal("a dead client left the guest connection open; the relay holds the sandbox") } - case <-time.After(3 * time.Second): - t.Fatal("a dead client left the guest connection open; the relay holds the sandbox") - } - _ = client.Close() - <-ended + _ = client.Close() + <-ended + }) } func TestSpliceTearsDownWhenTheGuestFails(t *testing.T) { From 1bd3f18564485099cbddb38f7e99f3f985a4ef99 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:50:03 +0800 Subject: [PATCH 107/127] silkd tests: sync on marker files, shared helpers in common, tempdir clone path, manifest env for the missing-manifest test --- silkd/tests/common/mod.rs | 67 +++++++++++++++++++++++++++++++++ silkd/tests/exec_e2e.rs | 48 ++++++------------------ silkd/tests/find_e2e.rs | 12 +++--- silkd/tests/forward_e2e.rs | 71 +++++++++++++---------------------- silkd/tests/git_e2e.rs | 3 +- silkd/tests/keepalive_e2e.rs | 64 ++++++++++++------------------- silkd/tests/lsp_e2e.rs | 73 ++++++++++++++++++------------------ silkd/tests/pty_e2e.rs | 46 ++++++++++------------- silkd/tests/session_e2e.rs | 33 +++++++++++----- silkd/tests/tree_e2e.rs | 14 ++----- silkd/tests/tun_e2e.rs | 3 +- silkd/tests/watch_e2e.rs | 9 ++--- 12 files changed, 227 insertions(+), 216 deletions(-) diff --git a/silkd/tests/common/mod.rs b/silkd/tests/common/mod.rs index 3d5b2f892..f566be0f7 100644 --- a/silkd/tests/common/mod.rs +++ b/silkd/tests/common/mod.rs @@ -2,6 +2,7 @@ #![allow(clippy::unwrap_used, clippy::expect_used)] #![allow(dead_code)] +use std::path::Path; use std::sync::Arc; use std::time::Duration; @@ -11,6 +12,7 @@ use silkd::server::State; use tokio::io::{ AsyncBufReadExt, AsyncWriteExt, BufReader, DuplexStream, Lines, ReadHalf, WriteHalf, }; +use tokio::net::TcpListener; use tokio::task::JoinHandle; use tokio::time::timeout; @@ -28,6 +30,19 @@ pub fn connect(state: &Arc) -> (FrameWriter, FrameLines, JoinHandle, + first: Value, +) -> (FrameWriter, FrameLines, JoinHandle>) { + let (mut cw, lines, handle) = connect(state); + send(&mut cw, first).await; + (cw, lines, handle) +} + +pub async fn under_deadline(fut: impl Future) -> T { + timeout(DEADLINE, fut).await.expect("deadline") +} + pub async fn send(cw: &mut FrameWriter, frame: Value) { let mut line = serde_json::to_vec(&frame).expect("encode frame"); line.push(b'\n'); @@ -43,6 +58,12 @@ pub async fn next_frame(lines: &mut FrameLines) -> Value { serde_json::from_str(&line).unwrap_or_else(|e| panic!("bad frame {line:?}: {e}")) } +pub async fn expect_frame(lines: &mut FrameLines, kind: &str) -> Value { + let frame = next_frame(lines).await; + assert_type(&frame, kind); + frame +} + pub async fn frames_until(lines: &mut FrameLines, pred: impl Fn(&Value) -> bool) -> Vec { let mut frames = Vec::new(); loop { @@ -69,6 +90,11 @@ pub fn assert_error(frame: &Value, kind: &str) { assert_eq!(frame["kind"], kind, "{frame}"); } +#[track_caller] +pub fn assert_type(frame: &Value, kind: &str) { + assert_eq!(type_of(frame), kind, "{frame}"); +} + pub async fn one(state: &Arc, request_line: &str) -> Vec { request_on(state, &[request_line.to_string()]).await } @@ -77,6 +103,47 @@ pub async fn exchange(lines: &[String]) -> Vec { request_on(&Arc::new(State::new()), lines).await } +pub fn proc_entry(ps: &[Value], pid: u64) -> Option<&Value> { + ps[0]["procs"] + .as_array() + .unwrap() + .iter() + .find(|p| p["pid"].as_u64() == Some(pid)) +} + +pub async fn bind_local() -> (TcpListener, u16) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let port = listener.local_addr().expect("addr").port(); + (listener, port) +} + +pub async fn fs_push(dest: &Path, archive: &[u8]) -> Vec { + let mut lines = vec![json!({"op":"fs_push","dest":dest.to_str().unwrap()}).to_string()]; + lines.extend(data_frames(archive)); + exchange(&lines).await +} + +pub async fn wait_for_path(path: &Path) { + under_deadline(async { + while !path.exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await; +} + +pub fn alive(pid: u32) -> bool { + // SAFETY: kill(2) with signal 0 only probes whether the pid exists; it takes no pointers. + if unsafe { libc::kill(pid as libc::pid_t, 0) } != 0 { + return false; + } + !std::fs::read_to_string(format!("/proc/{pid}/stat")).is_ok_and(|stat| { + stat.rsplit(')') + .next() + .is_some_and(|rest| rest.trim_start().starts_with('Z')) + }) +} + pub fn type_of(frame: &Value) -> &str { frame["type"].as_str().unwrap_or("") } diff --git a/silkd/tests/exec_e2e.rs b/silkd/tests/exec_e2e.rs index 5a0f81875..e7027e9aa 100644 --- a/silkd/tests/exec_e2e.rs +++ b/silkd/tests/exec_e2e.rs @@ -11,8 +11,8 @@ use silkd::server::State; use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader}; use common::{ - assert_error, b64, connect, exchange, frames_until, next_frame, one, roundtrip, send, - stdout_body, type_of, + alive, assert_error, b64, connect_with, exchange, frames_until, next_frame, one, proc_entry, + roundtrip, send, stdout_body, type_of, wait_for_path, }; async fn detached_pid(state: &Arc, script: &str) -> u64 { @@ -27,8 +27,7 @@ async fn detached_pid(state: &Arc, script: &str) -> u64 { } async fn attach_frames(state: &Arc, pid: u64) -> Vec { - let (mut cw, mut lines, _) = connect(state); - send(&mut cw, json!({"op":"attach","pid":pid})).await; + let (_cw, mut lines, _) = connect_with(state, json!({"op":"attach","pid":pid})).await; frames_until(&mut lines, |f| { matches!(type_of(f), "exit" | "done" | "error") }) @@ -49,12 +48,7 @@ async fn wait_for_exit(state: &Arc, pid: u64) -> Vec { async fn wait_listed(state: &Arc, pid: u64) -> Value { for _ in 0..250 { let ps = one(state, r#"{"op":"ps"}"#).await; - if let Some(p) = ps[0]["procs"] - .as_array() - .unwrap() - .iter() - .find(|p| p["pid"].as_u64() == Some(pid)) - { + if let Some(p) = proc_entry(&ps, pid) { return p.clone(); } tokio::time::sleep(Duration::from_millis(20)).await; @@ -217,8 +211,7 @@ async fn attach_keeps_a_tail_emitted_during_the_replay() { async fn attach_ends_when_its_client_hangs_up() { let state = Arc::new(State::new()); let pid = detached_pid(&state, "sleep 30").await; - let (mut cw, lines, handle) = connect(&state); - send(&mut cw, json!({"op":"attach","pid":pid})).await; + let (cw, lines, handle) = connect_with(&state, json!({"op":"attach","pid":pid})).await; drop(lines); drop(cw); @@ -349,9 +342,8 @@ async fn detached_exec_is_listed_then_logs_replay_output_and_exit() { #[tokio::test] async fn disconnect_during_drain_publishes_real_exit_code() { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - send( - &mut cw, + let (cw, mut lines, _) = connect_with( + &state, json!({ "op": "exec", "argv": ["/bin/sh", "-c", "{ sleep 0.5; echo late; sleep 30; } & exit 7"] @@ -385,9 +377,8 @@ async fn disconnect_during_drain_publishes_real_exit_code() { #[tokio::test] async fn stdin_held_by_a_forked_child_does_not_wedge_the_next_rpc() { let state = Arc::new(State::new()); - let (mut cw, mut lines, handle) = connect(&state); - send( - &mut cw, + let (mut cw, mut lines, handle) = connect_with( + &state, json!({ "op": "exec", "argv": ["/bin/sh", "-c", "exec 3<&0; sleep 8 <&3 3<&- & exit 0"] @@ -415,9 +406,8 @@ async fn a_client_gone_mid_output_releases_the_stdin_a_forked_child_holds() { let dir = tempfile::tempdir().unwrap(); let out = dir.path().join("count"); let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - send( - &mut cw, + let (mut cw, mut lines, _) = connect_with( + &state, json!({ "op": "exec", "argv": ["/bin/sh", "-c", "exec 3<&0; perl -e 'setpgrp; exec @ARGV' /bin/sh -c 'sleep 1; wc -c <&3 >\"$OUT.tmp\"; mv \"$OUT.tmp\" \"$OUT\"' >/dev/null 2>&1 & while :; do echo x; sleep 0.05; done"], @@ -433,14 +423,7 @@ async fn a_client_gone_mid_output_releases_the_stdin_a_forked_child_holds() { drop(lines); drop(cw); - let deadline = tokio::time::Instant::now() + Duration::from_secs(8); - while !out.exists() { - assert!( - tokio::time::Instant::now() < deadline, - "the forked child never saw stdin end" - ); - tokio::time::sleep(Duration::from_millis(50)).await; - } + wait_for_path(&out).await; let read: usize = std::fs::read_to_string(&out) .unwrap() .trim() @@ -451,10 +434,3 @@ async fn a_client_gone_mid_output_releases_the_stdin_a_forked_child_holds() { "the forked child read {read} bytes: stdin kept flowing after its client left" ); } - -fn alive(pid: u32) -> bool { - std::process::Command::new("/bin/sh") - .args(["-c", &format!("kill -0 {pid} 2>/dev/null")]) - .status() - .is_ok_and(|s| s.success()) -} diff --git a/silkd/tests/find_e2e.rs b/silkd/tests/find_e2e.rs index 66edc3bb3..1679322a3 100644 --- a/silkd/tests/find_e2e.rs +++ b/silkd/tests/find_e2e.rs @@ -10,11 +10,10 @@ use serde_json::{Value, json}; use silkd::server::State; use tokio::io::AsyncWriteExt; -use common::{assert_error, connect, frames_until, request, send, type_of}; +use common::{assert_error, connect_with, expect_frame, frames_until, request, send, type_of}; async fn find_frames(req: Value) -> Vec { - let (mut cw, mut out, handle) = connect(&Arc::new(State::new())); - send(&mut cw, req).await; + let (mut cw, mut out, handle) = connect_with(&Arc::new(State::new()), req).await; let frames = frames_until(&mut out, |f| matches!(type_of(f), "done" | "error")).await; cw.shutdown().await.unwrap(); handle.await.unwrap().unwrap(); @@ -27,9 +26,8 @@ async fn find_serves_a_request_pipelined_behind_it_after_done() { tokio::fs::write(dir.path().join("a.txt"), "needle\n") .await .unwrap(); - let (mut cw, mut out, handle) = connect(&Arc::new(State::new())); - send( - &mut cw, + let (mut cw, mut out, handle) = connect_with( + &Arc::new(State::new()), json!({"op":"fs_find","path":dir.path().to_str().unwrap(),"pattern":"needle"}), ) .await; @@ -37,7 +35,7 @@ async fn find_serves_a_request_pipelined_behind_it_after_done() { let frames = frames_until(&mut out, |f| matches!(type_of(f), "done" | "error")).await; assert_eq!(type_of(frames.last().unwrap()), "done", "{frames:?}"); assert!(frames.iter().any(|f| type_of(f) == "match"), "{frames:?}"); - assert_eq!(type_of(&common::next_frame(&mut out).await), "info"); + expect_frame(&mut out, "info").await; cw.shutdown().await.unwrap(); handle.await.unwrap().unwrap(); } diff --git a/silkd/tests/forward_e2e.rs b/silkd/tests/forward_e2e.rs index b6b3fd716..c8a0cd51a 100644 --- a/silkd/tests/forward_e2e.rs +++ b/silkd/tests/forward_e2e.rs @@ -9,18 +9,15 @@ use std::time::{Duration, Instant}; use serde_json::json; use silkd::server::State; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::TcpListener; use tokio::sync::oneshot; -use tokio::time::timeout; use common::{ - DEADLINE, FrameLines, FrameWriter, assert_error, b64, connect, decode, next_frame, send, - type_of, + FrameLines, FrameWriter, assert_error, b64, bind_local, connect, connect_with, decode, + expect_frame, next_frame, send, type_of, under_deadline, }; async fn echo_listener() -> u16 { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let port = listener.local_addr().expect("addr").port(); + let (listener, port) = bind_local().await; tokio::spawn(async move { while let Ok((mut conn, _)) = listener.accept().await { tokio::spawn(async move { @@ -44,14 +41,12 @@ async fn echo_listener() -> u16 { async fn forwarded(state: &Arc, port: u16) -> (FrameWriter, FrameLines) { let (mut cw, mut lines, _) = connect(state); send(&mut cw, json!({"v":1,"op":"port_forward","port":port})).await; - let ready = next_frame(&mut lines).await; - assert_eq!(type_of(&ready), "ready", "got {ready}"); + expect_frame(&mut lines, "ready").await; (cw, lines) } async fn request_listener(request: usize) -> u16 { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let port = listener.local_addr().expect("addr").port(); + let (listener, port) = bind_local().await; tokio::spawn(async move { let (mut conn, _) = listener.accept().await.expect("accept"); let mut buf = vec![0u8; request]; @@ -65,8 +60,7 @@ async fn request_listener(request: usize) -> u16 { } async fn half_close_listener() -> (u16, oneshot::Receiver>) { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let port = listener.local_addr().expect("addr").port(); + let (listener, port) = bind_local().await; let (tx, rx) = oneshot::channel(); tokio::spawn(async move { let (conn, _) = listener.accept().await.expect("accept"); @@ -82,37 +76,32 @@ async fn half_close_listener() -> (u16, oneshot::Receiver>) { #[tokio::test] async fn forward_round_trips_and_done_on_server_close() { - timeout(DEADLINE, async { + under_deadline(async { let port = echo_listener().await; let state = Arc::new(State::new()); let (mut cw, mut lines) = forwarded(&state, port).await; send(&mut cw, json!({"v":1,"op":"data","data":b64(b"hi")})).await; - let echoed = next_frame(&mut lines).await; - assert_eq!(type_of(&echoed), "data", "got {echoed}"); + let echoed = expect_frame(&mut lines, "data").await; assert_eq!(decode(&echoed), b"hi"); send(&mut cw, json!({"v":1,"op":"data_end"})).await; - let done = next_frame(&mut lines).await; - assert_eq!(type_of(&done), "done", "got {done}"); + expect_frame(&mut lines, "done").await; }) - .await - .expect("test deadline"); + .await; } #[tokio::test] async fn forward_accepts_client_writes_after_the_guest_half_closes() { - timeout(DEADLINE, async { + under_deadline(async { let (port, got) = half_close_listener().await; let state = Arc::new(State::new()); let (mut cw, mut lines) = forwarded(&state, port).await; - let greeting = next_frame(&mut lines).await; - assert_eq!(type_of(&greeting), "data", "got {greeting}"); + let greeting = expect_frame(&mut lines, "data").await; assert_eq!(decode(&greeting), b"greeting"); - let done = next_frame(&mut lines).await; - assert_eq!(type_of(&done), "done", "got {done}"); + expect_frame(&mut lines, "done").await; send( &mut cw, @@ -127,8 +116,7 @@ async fn forward_accepts_client_writes_after_the_guest_half_closes() { "guest lost the client's post-done bytes" ); }) - .await - .expect("test deadline"); + .await; } #[tokio::test] @@ -136,7 +124,7 @@ async fn forward_bidirectional_bulk_no_deadlock() { const CHUNK: usize = 3072; const FRAMES: usize = 400; - timeout(DEADLINE, async { + under_deadline(async { let port = echo_listener().await; let state = Arc::new(State::new()); let (mut cw, mut lines) = forwarded(&state, port).await; @@ -159,44 +147,38 @@ async fn forward_bidirectional_bulk_no_deadlock() { writer.await.expect("writer"); assert_eq!(got, CHUNK * FRAMES); }) - .await - .expect("test deadline"); + .await; } #[tokio::test] async fn forward_refused_port_is_not_found() { - timeout(DEADLINE, async { + under_deadline(async { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - - send(&mut cw, json!({"v":1,"op":"port_forward","port":1})).await; + let (_cw, mut lines, _) = + connect_with(&state, json!({"v":1,"op":"port_forward","port":1})).await; let err = next_frame(&mut lines).await; assert_error(&err, "not_found"); }) - .await - .expect("test deadline"); + .await; } #[tokio::test] async fn a_request_during_forward_ends_its_input_and_runs_after_it() { - timeout(DEADLINE, async { + under_deadline(async { let port = echo_listener().await; let state = Arc::new(State::new()); let (mut cw, mut lines) = forwarded(&state, port).await; send(&mut cw, json!({"v":1,"op":"ps"})).await; - let done = next_frame(&mut lines).await; - assert_eq!(type_of(&done), "done", "got {done}"); - let procs = next_frame(&mut lines).await; - assert_eq!(type_of(&procs), "procs", "got {procs}"); + expect_frame(&mut lines, "done").await; + expect_frame(&mut lines, "procs").await; }) - .await - .expect("test deadline"); + .await; } #[tokio::test] async fn forward_split_requests_do_not_wait_on_delayed_acks() { - timeout(DEADLINE, async { + under_deadline(async { let port = request_listener(8192 + 1024).await; let state = Arc::new(State::new()); let (mut cw, mut lines) = forwarded(&state, port).await; @@ -221,6 +203,5 @@ async fn forward_split_requests_do_not_wait_on_delayed_acks() { "20 split requests took {elapsed:?}: the guest-side write waits on delayed ACKs" ); }) - .await - .expect("test deadline"); + .await; } diff --git a/silkd/tests/git_e2e.rs b/silkd/tests/git_e2e.rs index 7fc03ee13..de844ed38 100644 --- a/silkd/tests/git_e2e.rs +++ b/silkd/tests/git_e2e.rs @@ -136,8 +136,9 @@ async fn commit_failure_surfaces_git_error() { #[tokio::test] async fn clone_lane_behavior() { silkd::net::override_egress_for_tests(Some(false)); + let refused = tempfile::tempdir().unwrap(); let none = request(json!({ - "op":"git_clone","url":"https://example.com/x.git","path":"/tmp/silkd-x" + "op":"git_clone","url":"https://example.com/x.git","path":refused.path().join("x").to_str().unwrap() })) .await; assert_error(&none[0], "unimplemented"); diff --git a/silkd/tests/keepalive_e2e.rs b/silkd/tests/keepalive_e2e.rs index 345548a33..2f928ddff 100644 --- a/silkd/tests/keepalive_e2e.rs +++ b/silkd/tests/keepalive_e2e.rs @@ -11,7 +11,8 @@ use tokio::io::AsyncWriteExt; use tokio::time::timeout; use common::{ - DEADLINE, FrameLines, assert_error, b64, connect, frames_until, next_frame, send, type_of, + DEADLINE, FrameLines, assert_error, b64, connect_with, expect_frame, frames_until, next_frame, + send, type_of, }; async fn types_until(lines: &mut FrameLines, terminal: &str) -> Vec { @@ -25,14 +26,12 @@ async fn types_until(lines: &mut FrameLines, terminal: &str) -> Vec { #[tokio::test] async fn rpcs_run_back_to_back_on_one_connection() { let state = Arc::new(State::new()); - let (mut cw, mut lines, handle) = connect(&state); - - send(&mut cw, json!({"v":1,"op":"info"})).await; - assert_eq!(type_of(&next_frame(&mut lines).await), "info"); + let (mut cw, mut lines, handle) = connect_with(&state, json!({"v":1,"op":"info"})).await; + expect_frame(&mut lines, "info").await; send(&mut cw, json!({"v":1,"op":"fs_stat","path":"/"})).await; - assert_eq!(type_of(&next_frame(&mut lines).await), "stat"); + expect_frame(&mut lines, "stat").await; send(&mut cw, json!({"v":1,"op":"ps"})).await; - assert_eq!(type_of(&next_frame(&mut lines).await), "procs"); + expect_frame(&mut lines, "procs").await; cw.shutdown().await.unwrap(); timeout(DEADLINE, handle) @@ -45,10 +44,8 @@ async fn rpcs_run_back_to_back_on_one_connection() { #[tokio::test] async fn exec_hands_the_connection_back_after_exit() { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - - send( - &mut cw, + let (mut cw, mut lines, _) = connect_with( + &state, json!({"v":1,"op":"exec","argv":["echo","hi"],"detach":false}), ) .await; @@ -62,16 +59,14 @@ async fn exec_hands_the_connection_back_after_exit() { assert!(types.iter().any(|t| t == "stdout"), "got {types:?}"); send(&mut cw, json!({"v":1,"op":"info"})).await; - assert_eq!(type_of(&next_frame(&mut lines).await), "info"); + expect_frame(&mut lines, "info").await; } #[tokio::test] async fn late_stdin_after_exit_is_dropped() { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - - send( - &mut cw, + let (mut cw, mut lines, _) = connect_with( + &state, json!({"v":1,"op":"exec","argv":["true"],"detach":false}), ) .await; @@ -81,16 +76,14 @@ async fn late_stdin_after_exit_is_dropped() { send(&mut cw, json!({"v":1,"op":"stdin","data":b64(b"late")})).await; send(&mut cw, json!({"v":1,"op":"stdin_close"})).await; send(&mut cw, json!({"v":1,"op":"info"})).await; - assert_eq!(type_of(&next_frame(&mut lines).await), "info"); + expect_frame(&mut lines, "info").await; } #[tokio::test] async fn a_malformed_request_after_exec_is_bad_request() { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - - send( - &mut cw, + let (mut cw, mut lines, _) = connect_with( + &state, json!({"v":1,"op":"exec","argv":["true"],"detach":false}), ) .await; @@ -115,46 +108,39 @@ async fn upload_hands_the_connection_back_after_done() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("f").to_string_lossy().into_owned(); let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - - send(&mut cw, json!({"v":1,"op":"fs_write","path":path})).await; + let (mut cw, mut lines, _) = + connect_with(&state, json!({"v":1,"op":"fs_write","path":path})).await; send(&mut cw, json!({"v":1,"op":"data","data":b64(b"hello")})).await; send(&mut cw, json!({"v":1,"op":"data_end"})).await; - assert_eq!(type_of(&next_frame(&mut lines).await), "done"); + expect_frame(&mut lines, "done").await; send(&mut cw, json!({"v":1,"op":"fs_stat","path":path})).await; - let stat = next_frame(&mut lines).await; - assert_eq!(type_of(&stat), "stat", "got {stat}"); + let stat = expect_frame(&mut lines, "stat").await; assert_eq!(stat["info"]["size"], 5, "got {stat}"); } #[tokio::test] async fn a_request_during_exec_ends_its_stdin_and_runs_after_it() { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - - send( - &mut cw, + let (mut cw, mut lines, _) = connect_with( + &state, json!({"v":1,"op":"exec","argv":["cat"],"detach":false}), ) .await; - assert_eq!(type_of(&next_frame(&mut lines).await), "started"); + expect_frame(&mut lines, "started").await; send(&mut cw, json!({"v":1,"op":"info"})).await; - let exit = next_frame(&mut lines).await; - assert_eq!(type_of(&exit), "exit", "got {exit}"); + let exit = expect_frame(&mut lines, "exit").await; assert_eq!(exit["code"], 0, "got {exit}"); - assert_eq!(type_of(&next_frame(&mut lines).await), "info"); + expect_frame(&mut lines, "info").await; } #[tokio::test] async fn a_leading_input_frame_is_bad_request_and_the_connection_stays() { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - - send(&mut cw, json!({"v":1,"op":"stdin_close"})).await; + let (mut cw, mut lines, _) = connect_with(&state, json!({"v":1,"op":"stdin_close"})).await; let err = next_frame(&mut lines).await; assert_error(&err, "bad_request"); send(&mut cw, json!({"v":1,"op":"info"})).await; - assert_eq!(type_of(&next_frame(&mut lines).await), "info"); + expect_frame(&mut lines, "info").await; } diff --git a/silkd/tests/lsp_e2e.rs b/silkd/tests/lsp_e2e.rs index 4fa754105..590f5b882 100644 --- a/silkd/tests/lsp_e2e.rs +++ b/silkd/tests/lsp_e2e.rs @@ -12,7 +12,10 @@ use tempfile::TempDir; use tokio::io::AsyncWriteExt; use tokio::time::timeout; -use common::{DEADLINE, assert_error, b64, connect, decode, next_frame, one, send, type_of}; +use common::{ + DEADLINE, FrameLines, FrameWriter, alive, assert_error, b64, connect_with, decode, + expect_frame, one, send, type_of, under_deadline, +}; const FAKE_SERVER: &str = r#"#!/bin/sh IFS= read -r line @@ -51,6 +54,20 @@ async fn started_server(lang: &str, server_body: &str) -> (TempDir, Arc, (env, state, server_id) } +async fn lsp_stream( + state: &Arc, + server_id: &str, +) -> ( + FrameWriter, + FrameLines, + tokio::task::JoinHandle>, +) { + let (cw, mut out, handle) = + connect_with(state, json!({"op":"lsp_request","server_id":server_id})).await; + expect_frame(&mut out, "ready").await; + (cw, out, handle) +} + async fn assert_stopped(state: &Arc, server_id: &str) { let gone = lsp_stop(state, server_id).await; assert_eq!(gone.last().unwrap()["kind"], "not_found", "{gone:?}"); @@ -67,15 +84,12 @@ async fn lsp_stop(state: &Arc, server_id: &str) -> Vec { #[tokio::test] async fn lsp_start_missing_manifest_is_not_found() { let _env_lock = ENV_LOCK.lock().await; - let frames = timeout( - DEADLINE, - one( - &Arc::new(State::new()), - &json!({"op":"lsp_start","language":"nope"}).to_string(), - ), - ) - .await - .expect("deadline"); + let _env = manifest_env("#!/bin/sh\n"); + let frames = under_deadline(one( + &Arc::new(State::new()), + &json!({"op":"lsp_start","language":"nope"}).to_string(), + )) + .await; let last = frames.last().unwrap(); assert_error(last, "not_found"); } @@ -105,15 +119,12 @@ async fn lsp_broker_relays_to_the_server() { let _env_lock = ENV_LOCK.lock().await; let (_env, state, server_id) = started_server("faketest", FAKE_SERVER).await; - let (mut cw, mut out, handle) = connect(&state); - send(&mut cw, json!({"op":"lsp_request","server_id":server_id})).await; - assert_eq!(type_of(&next_frame(&mut out).await), "ready"); + let (mut cw, mut out, handle) = lsp_stream(&state, &server_id).await; send(&mut cw, json!({"op":"data","data":b64(b"ping\n")})).await; - let frame = next_frame(&mut out).await; - assert_eq!(type_of(&frame), "data"); + let frame = expect_frame(&mut out, "data").await; assert_eq!(decode(&frame), b"reply:ping\n"); - assert_eq!(type_of(&next_frame(&mut out).await), "done"); + expect_frame(&mut out, "done").await; cw.shutdown().await.unwrap(); handle.await.unwrap().unwrap(); @@ -139,7 +150,7 @@ async fn lsp_stop_kills_the_servers_children_too() { .await; let pid_file = env.path().join("child"); let deadline = tokio::time::Instant::now() + DEADLINE; - let pid: libc::pid_t = loop { + let pid: u32 = loop { if let Some(pid) = std::fs::read_to_string(&pid_file) .ok() .and_then(|s| s.trim().parse().ok()) @@ -153,8 +164,7 @@ async fn lsp_stop_kills_the_servers_children_too() { tokio::time::sleep(std::time::Duration::from_millis(20)).await; }; lsp_stop(&state, &server_id).await; - // SAFETY: kill(2) with signal 0 only probes whether the pid exists; it takes no pointers. - while unsafe { libc::kill(pid, 0) } == 0 { + while alive(pid) { assert!( tokio::time::Instant::now() < deadline, "server child {pid} outlived lsp_stop" @@ -174,9 +184,7 @@ async fn lsp_request_reaps_when_the_client_vanishes_before_ready() { client.write_all(b"\n").await.unwrap(); drop(client); let (sr, sw) = tokio::io::split(server); - let served = timeout(DEADLINE, state.serve(tokio::io::BufReader::new(sr), sw)) - .await - .expect("deadline"); + let served = under_deadline(state.serve(tokio::io::BufReader::new(sr), sw)).await; assert!(served.is_err(), "the ready write must fail: {served:?}"); assert_stopped(&state, &server_id).await; @@ -188,16 +196,13 @@ async fn lsp_data_end_half_closes_stdin() { let (_env, state, server_id) = started_server("eoftest", "#!/bin/sh\nprintf 'ate:%s\\n' $(cat | wc -c)\n").await; - let (mut cw, mut out, handle) = connect(&state); - send(&mut cw, json!({"op":"lsp_request","server_id":server_id})).await; - assert_eq!(type_of(&next_frame(&mut out).await), "ready"); + let (mut cw, mut out, handle) = lsp_stream(&state, &server_id).await; send(&mut cw, json!({"op":"data","data":b64(b"12345")})).await; send(&mut cw, json!({"op":"data_end"})).await; - let frame = next_frame(&mut out).await; - assert_eq!(type_of(&frame), "data"); + let frame = expect_frame(&mut out, "data").await; assert_eq!(decode(&frame), b"ate:5\n"); - assert_eq!(type_of(&next_frame(&mut out).await), "done"); + expect_frame(&mut out, "done").await; cw.shutdown().await.unwrap(); handle.await.unwrap().unwrap(); } @@ -209,9 +214,7 @@ async fn lsp_client_hang_up_reaps_a_silent_server() { let (_env, state, server_id) = started_server("silenttest", "#!/bin/sh\nexec sleep 60\n").await; - let (mut cw, mut out, handle) = connect(&state); - send(&mut cw, json!({"op":"lsp_request","server_id":server_id})).await; - assert_eq!(type_of(&next_frame(&mut out).await), "ready"); + let (mut cw, out, handle) = lsp_stream(&state, &server_id).await; if half_closed_first { send(&mut cw, json!({"op":"data_end"})).await; } @@ -232,11 +235,9 @@ async fn lsp_request_during_a_stream_ends_it_and_runs_after_it() { let (_env, state, server_id) = started_server("pipelinetest", "#!/bin/sh\nexec sleep 60\n").await; - let (mut cw, mut out, _) = connect(&state); - send(&mut cw, json!({"op":"lsp_request","server_id":server_id})).await; - assert_eq!(type_of(&next_frame(&mut out).await), "ready"); + let (mut cw, mut out, _) = lsp_stream(&state, &server_id).await; send(&mut cw, json!({"op":"info"})).await; - assert_eq!(type_of(&next_frame(&mut out).await), "done"); - assert_eq!(type_of(&next_frame(&mut out).await), "info"); + expect_frame(&mut out, "done").await; + expect_frame(&mut out, "info").await; assert_stopped(&state, &server_id).await; } diff --git a/silkd/tests/pty_e2e.rs b/silkd/tests/pty_e2e.rs index 723a66f96..df6e9d3d9 100644 --- a/silkd/tests/pty_e2e.rs +++ b/silkd/tests/pty_e2e.rs @@ -18,7 +18,8 @@ use tokio::io::{AsyncWrite, AsyncWriteExt}; use tokio::sync::mpsc; use common::{ - FrameLines, assert_error, b64, connect, decode, frames_until, one, request, send, type_of, + FrameLines, FrameWriter, alive, assert_error, b64, connect_with, decode, frames_until, one, + proc_entry, request, send, type_of, }; struct FailingSink(Vec); @@ -46,10 +47,19 @@ async fn read_until(lines: &mut FrameLines, pred: impl Fn(&Value) -> bool) -> Va frames_until(lines, pred).await.pop().unwrap() } +async fn open_pty( + state: &Arc, +) -> ( + FrameWriter, + FrameLines, + tokio::task::JoinHandle>, +) { + connect_with(state, json!({"op":"pty_open","cols":80,"rows":24})).await +} + async fn wait_gone(pid: u32) { let deadline = tokio::time::Instant::now() + Duration::from_secs(5); - // SAFETY: kill(2) with signal 0 only probes whether the pid exists; it takes no pointers. - while unsafe { libc::kill(pid as libc::pid_t, 0) } == 0 { + while alive(pid) { assert!( tokio::time::Instant::now() < deadline, "shell {pid} still alive 5 s after the started write failed" @@ -61,8 +71,7 @@ async fn wait_gone(pid: u32) { #[tokio::test] async fn pty_runs_a_shell_and_echoes() { let state = Arc::new(State::new()); - let (mut cw, mut lines, handle) = connect(&state); - send(&mut cw, json!({"op":"pty_open","cols":80,"rows":24})).await; + let (mut cw, mut lines, handle) = open_pty(&state).await; let started = read_until(&mut lines, |v| v["type"] == "started").await; let pid = started["pid"].as_u64().unwrap(); @@ -90,8 +99,7 @@ async fn pty_runs_a_shell_and_echoes() { #[tokio::test] async fn the_pty_shell_holds_no_master_fd() { let state = Arc::new(State::new()); - let (mut cw, mut lines, handle) = connect(&state); - send(&mut cw, json!({"op":"pty_open","cols":80,"rows":24})).await; + let (mut cw, mut lines, handle) = open_pty(&state).await; read_until(&mut lines, |v| v["type"] == "started").await; send( &mut cw, @@ -113,21 +121,13 @@ async fn the_pty_shell_holds_no_master_fd() { #[tokio::test] async fn pty_appears_in_ps_and_resizes() { let state = Arc::new(State::new()); - let (mut cw, mut lines, _) = connect(&state); - send(&mut cw, json!({"op":"pty_open","cols":80,"rows":24})).await; + let (cw, mut lines, _) = open_pty(&state).await; let pid = read_until(&mut lines, |v| v["type"] == "started").await["pid"] .as_u64() .unwrap(); let ps = one(&state, r#"{"op":"ps"}"#).await; - assert!( - ps[0]["procs"] - .as_array() - .unwrap() - .iter() - .any(|p| p["pid"].as_u64() == Some(pid)), - "pty not in ps: {ps:?}" - ); + assert!(proc_entry(&ps, pid).is_some(), "pty not in ps: {ps:?}"); let resized = one( &state, &json!({"op":"pty_resize","pid":pid,"cols":120,"rows":40}).to_string(), @@ -147,8 +147,7 @@ async fn resize_unknown_pid_is_not_found() { #[tokio::test] async fn stdin_nobody_reads_after_the_shell_exits_does_not_wedge_the_next_request() { let state = Arc::new(State::new()); - let (mut cw, mut lines, handle) = connect(&state); - send(&mut cw, json!({"op":"pty_open","cols":80,"rows":24})).await; + let (mut cw, mut lines, handle) = open_pty(&state).await; read_until(&mut lines, |v| v["type"] == "started").await; send( &mut cw, @@ -178,8 +177,7 @@ async fn stdin_nobody_reads_after_the_shell_exits_does_not_wedge_the_next_reques #[tokio::test] async fn pty_disconnect_tears_down_without_spin() { let state = Arc::new(State::new()); - let (mut cw, mut lines, handle) = connect(&state); - send(&mut cw, json!({"op":"pty_open","cols":80,"rows":24})).await; + let (cw, mut lines, handle) = open_pty(&state).await; let pid = read_until(&mut lines, |v| v["type"] == "started").await["pid"] .as_u64() .unwrap(); @@ -193,11 +191,7 @@ async fn pty_disconnect_tears_down_without_spin() { .expect("pty did not tear down on disconnect"); let ps = one(&state, r#"{"op":"ps"}"#).await; assert!( - !ps[0]["procs"] - .as_array() - .unwrap() - .iter() - .any(|p| p["pid"].as_u64() == Some(pid)), + proc_entry(&ps, pid).is_none(), "pty still in table after disconnect: {ps:?}" ); } diff --git a/silkd/tests/session_e2e.rs b/silkd/tests/session_e2e.rs index 5235e0c43..798a67c90 100644 --- a/silkd/tests/session_e2e.rs +++ b/silkd/tests/session_e2e.rs @@ -10,7 +10,7 @@ use serde_json::{Value, json}; use silkd::server::State; use tokio::task::JoinHandle; -use common::{assert_error, one, stdout_body, type_of}; +use common::{assert_error, one, stdout_body, type_of, wait_for_path}; async fn create(state: &Arc) -> String { create_with(state, json!({"op":"session_create"})).await @@ -31,10 +31,28 @@ async fn sh(state: &Arc, id: &str, cmd: &[&str]) -> Vec { .await } -fn spawn_sh(state: &Arc, id: &str, cmd: &'static [&'static str]) -> JoinHandle> { +async fn spawn_busy(state: &Arc, id: &str, seconds: &'static str) -> JoinHandle> { + let dir = tempfile::tempdir().unwrap(); + let marker = dir.path().join("started"); + let marker_arg = marker.to_str().unwrap().to_string(); let s = Arc::clone(state); let sid = id.to_string(); - tokio::spawn(async move { sh(&s, &sid, cmd).await }) + let busy = tokio::spawn(async move { + sh( + &s, + &sid, + &[ + "sh", + "-c", + "touch \"$0\"; exec sleep \"$1\"", + &marker_arg, + seconds, + ], + ) + .await + }); + wait_for_path(&marker).await; + busy } #[tokio::test] @@ -216,8 +234,7 @@ async fn reap_idle_removes_idle_sessions() { async fn reap_skips_a_session_running_a_command() { let state = Arc::new(State::new()); let id = create(&state).await; - let busy = spawn_sh(&state, &id, &["sleep", "1"]); - tokio::time::sleep(Duration::from_millis(150)).await; + let busy = spawn_busy(&state, &id, "1").await; let reaped = state.sessions.reap_idle(Duration::ZERO); assert_eq!(reaped, 0, "a running session must not be reaped"); let done = busy.await.unwrap(); @@ -242,8 +259,7 @@ async fn session_argv_with_metacharacters_is_one_literal_token() { async fn session_rm_unwedges_a_running_external_command() { let state = Arc::new(State::new()); let id = create(&state).await; - let busy = spawn_sh(&state, &id, &["sleep", "3600"]); - tokio::time::sleep(Duration::from_millis(200)).await; + let busy = spawn_busy(&state, &id, "3600").await; let rm = one(&state, &json!({"op":"session_rm","id":id}).to_string()).await; assert_eq!(type_of(&rm[0]), "done"); let done = tokio::time::timeout(Duration::from_secs(5), busy) @@ -308,8 +324,7 @@ async fn a_dead_session_command_leaves_a_recreated_session_alone() { ) .await; assert_eq!(type_of(bg.last().unwrap()), "exit", "{bg:?}"); - let busy = spawn_sh(&state, &id, &["sleep", "3600"]); - tokio::time::sleep(Duration::from_millis(200)).await; + let busy = spawn_busy(&state, &id, "3600").await; let rm = one(&state, &json!({"op":"session_rm","id":id}).to_string()).await; assert_eq!(type_of(&rm[0]), "done"); let again = create_with(&state, json!({"op":"session_create","id":"s"})).await; diff --git a/silkd/tests/tree_e2e.rs b/silkd/tests/tree_e2e.rs index 41ae0067d..b1a168e27 100644 --- a/silkd/tests/tree_e2e.rs +++ b/silkd/tests/tree_e2e.rs @@ -10,7 +10,7 @@ use std::process::Command; use serde_json::json; -use common::{assert_error, b64, data_frames, exchange, payload, request, type_of}; +use common::{assert_error, b64, exchange, fs_push, payload, request, type_of}; fn sys_tar_create(dir: &Path) -> Vec { let out = Command::new("tar") @@ -46,9 +46,7 @@ async fn push_extracts_a_tar_stream() { let dest = tempfile::tempdir().unwrap(); let dest_path = dest.path().join("proj"); - let mut lines = vec![json!({"op":"fs_push","dest":dest_path.to_str().unwrap()}).to_string()]; - lines.extend(data_frames(&archive)); - let frames = exchange(&lines).await; + let frames = fs_push(&dest_path, &archive).await; assert_eq!( type_of(&frames[0]), @@ -92,9 +90,7 @@ async fn push_then_pull_roundtrips_a_tree() { let work = tempfile::tempdir().unwrap(); let dest = work.path().join("t"); - let mut lines = vec![json!({"op":"fs_push","dest":dest.to_str().unwrap()}).to_string()]; - lines.extend(data_frames(&archive)); - assert_eq!(type_of(&exchange(&lines).await[0]), "done"); + assert_eq!(type_of(&fs_push(&dest, &archive).await[0]), "done"); let pulled = request(json!({"op":"fs_pull","path":dest.to_str().unwrap()})).await; let out_archive = payload(&pulled, "data"); @@ -238,9 +234,7 @@ async fn push_overlays_existing_tree_and_cleans_staging() { std::fs::create_dir(dest.path().join("sub")).unwrap(); std::fs::write(dest.path().join("sub/old.txt"), b"OLD").unwrap(); - let mut lines = vec![json!({"op":"fs_push","dest":dest.path().to_str().unwrap()}).to_string()]; - lines.extend(data_frames(&archive)); - let frames = exchange(&lines).await; + let frames = fs_push(dest.path(), &archive).await; assert_eq!(type_of(&frames[0]), "done", "push failed: {frames:?}"); assert_eq!( diff --git a/silkd/tests/tun_e2e.rs b/silkd/tests/tun_e2e.rs index b1296a216..71d2b45ec 100644 --- a/silkd/tests/tun_e2e.rs +++ b/silkd/tests/tun_e2e.rs @@ -2,6 +2,7 @@ #![cfg(target_os = "linux")] #![allow(clippy::unwrap_used, clippy::expect_used)] +use std::io::{Read, Write}; use std::net::{Ipv4Addr, SocketAddr}; use std::os::fd::{AsFd, AsRawFd}; use std::os::unix::net::UnixStream; @@ -562,8 +563,6 @@ fn forward( to: Ipv4Addr, ) -> JoinHandle<()> { std::thread::spawn(move || { - use std::io::{Read, Write}; - let mut r = std::io::BufReader::with_capacity(256 << 10, r); let mut hello = [0u8; HELLO_LEN]; if r.read_exact(&mut hello).is_err() || w.write_all(&hello).is_err() { diff --git a/silkd/tests/watch_e2e.rs b/silkd/tests/watch_e2e.rs index 9bd5d5585..bbb77d0fd 100644 --- a/silkd/tests/watch_e2e.rs +++ b/silkd/tests/watch_e2e.rs @@ -8,19 +8,18 @@ use std::time::Duration; use serde_json::json; use silkd::server::State; -use common::{assert_error, connect, next_frame, request, send, type_of}; +use common::{assert_error, connect_with, expect_frame, next_frame, request, type_of}; #[tokio::test] async fn watch_streams_events_until_disconnect() { let dir = tempfile::tempdir().unwrap(); let state = Arc::new(State::new()); - let (mut cw, mut lines, handle) = connect(&state); - send( - &mut cw, + let (cw, mut lines, handle) = connect_with( + &state, json!({"op": "fs_watch", "path": dir.path().to_str().unwrap(), "recursive": true}), ) .await; - assert_eq!(type_of(&next_frame(&mut lines).await), "ready"); + expect_frame(&mut lines, "ready").await; tokio::fs::write(dir.path().join("new.txt"), b"hi") .await .unwrap(); From ee3e012590274a1b48724f6f7962d071ef601034 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:50:14 +0800 Subject: [PATCH 108/127] ci: fail the Caddy TLS step when its test skips --- .github/workflows/sandboxd.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sandboxd.yml b/.github/workflows/sandboxd.yml index 5e4c17792..78063b1e4 100644 --- a/.github/workflows/sandboxd.yml +++ b/.github/workflows/sandboxd.yml @@ -104,10 +104,12 @@ jobs: - name: Test TLS edge with Caddy working-directory: e2e + shell: bash env: CADDY_BIN: ${{ runner.temp }}/caddy run: | curl -fsSL --retry 3 https://github.com/caddyserver/caddy/releases/download/v2.11.4/caddy_2.11.4_linux_amd64.tar.gz -o "$RUNNER_TEMP/caddy.tar.gz" printf '%s %s\n' '8220d1f013b6f27510247b2360c9e0ca9f018feebd82515f07635318b34ff9777ccc8fd0b6e6f2486ce3a33fe389fbb7db12d05baa474f4587509fb4f5ebf1c9' "$RUNNER_TEMP/caddy.tar.gz" | sha512sum -c - tar -xzf "$RUNNER_TEMP/caddy.tar.gz" -C "$RUNNER_TEMP" caddy - GOWORK=off go test -race -count=1 -run TestCaddyTLSCluster -timeout 60s -v . + GOWORK=off go test -race -count=1 -run TestCaddyTLSCluster -timeout 60s -v . | tee "$RUNNER_TEMP/caddy-test.log" + ! grep -- '--- SKIP' "$RUNNER_TEMP/caddy-test.log" From dd19c239d48fcc8378535d88c743ef36855cc19a Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:51:07 +0800 Subject: [PATCH 109/127] e2e: fold the volume end-to-end and wire-shape pairs into tables, accessors for fake engine state --- e2e/e2e_test.go | 185 +++++++++++++++-------------------------- e2e/fakeengine_test.go | 18 ++++ e2e/reader_test.go | 8 +- 3 files changed, 88 insertions(+), 123 deletions(-) diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 0c987ca10..c130dd906 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -429,39 +429,67 @@ func TestVolumesEndToEnd(t *testing.T) { } } -func TestWritableVolumeEndToEnd(t *testing.T) { - scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") - st := startTenantStack(t, nil, - []config.VolumeSpec{{Name: "scratch", Path: scratch, Writable: true}}) +func TestWriterVolumeEndToEnd(t *testing.T) { + for _, tt := range []struct { + name string + volume sandbox.Volume + applied []string + released []string + marker bool + }{ + { + name: "writable", + volume: sandbox.Volume{Name: "scratch", Mount: "/datasets/rw", Mode: "rw"}, + applied: []string{"attach:scratch:rw", "mount:scratch:/datasets/rw:rw"}, + released: []string{"umount:/datasets/rw", "remove"}, + }, + { + name: "attach-only", + volume: sandbox.Volume{Name: "scratch", Mode: "rw", AttachOnly: true}, + applied: []string{"attach:scratch:rw"}, + released: []string{"remove"}, + marker: true, + }, + } { + t.Run(tt.name, func(t *testing.T) { + scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") + st := startTenantStack(t, nil, + []config.VolumeSpec{{Name: "scratch", Path: scratch, Writable: true}}) - sb, err := st.client.New(t.Context(), "rt:24.04", - sandbox.WithVolumes(sandbox.Volume{Name: "scratch", Mount: "/datasets/rw", Mode: "rw"})) - if err != nil { - t.Fatalf("writable claim: %v", err) - } - if want := []sandbox.Volume{{Name: "scratch", Mount: "/datasets/rw", Mode: "rw"}}; !slices.Equal(sb.Volumes, want) { - t.Errorf("claim volumes %+v, want %+v", sb.Volumes, want) - } - applied := []string{"attach:scratch:rw", "mount:scratch:/datasets/rw:rw"} - if got := st.eng.volumeOpsLog(); !slices.Equal(got, applied) { - t.Errorf("engine ops %v, want %v", got, applied) - } - for _, requested := range []string{`{"name":"scratch"}`, `{"name":"scratch","mode":"rw"}`} { - if status, _ := rawClaim(t, st, requested); status != http.StatusConflict { - t.Errorf("claim %s under a live writer: %d, want 409", requested, status) - } - } + sb, err := st.client.New(t.Context(), "rt:24.04", sandbox.WithVolumes(tt.volume)) + if err != nil { + t.Fatalf("%s claim: %v", tt.name, err) + } + if want := []sandbox.Volume{tt.volume}; !slices.Equal(sb.Volumes, want) { + t.Errorf("claim volumes %+v, want %+v", sb.Volumes, want) + } + if got := st.eng.volumeOpsLog(); !slices.Equal(got, tt.applied) { + t.Errorf("engine ops %v, want %v", got, tt.applied) + } + if tt.marker { + assertNoDirtyMarker(t, scratch, "apply") + } + for _, requested := range []string{`{"name":"scratch"}`, `{"name":"scratch","mode":"rw"}`} { + if status, _ := rawClaim(t, st, requested); status != http.StatusConflict { + t.Errorf("claim %s under a live %s writer: %d, want 409", requested, tt.name, status) + } + } - if err := sb.Close(); err != nil { - t.Fatalf("Close: %v", err) - } - want := slices.Concat(applied, []string{"umount:/datasets/rw", "remove"}) - if got := st.eng.volumeOpsLog(); !slices.Equal(got, want) { - t.Errorf("engine ops after release %v, want %v", got, want) + if err := sb.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + want := slices.Concat(tt.applied, tt.released) + if got := st.eng.volumeOpsLog(); !slices.Equal(got, want) { + t.Errorf("engine ops after release %v, want %v", got, want) + } + if tt.marker { + assertNoDirtyMarker(t, scratch, "release") + } + }) } } -func TestVolumeModeWireShape(t *testing.T) { +func TestVolumeWireShape(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") st := startTenantStack(t, nil, []config.VolumeSpec{{Name: "scratch", Path: scratch, Writable: true}}) @@ -480,6 +508,16 @@ func TestVolumeModeWireShape(t *testing.T) { `{"name":"scratch","mount":"/datasets/x"}`, map[string]any{"name": "scratch", "mount": "/datasets/x"}, }, + { + "attach-only omits the mount", + `{"name":"scratch","mode":"rw","attach_only":true}`, + map[string]any{"name": "scratch", "mode": "rw", "attach_only": true}, + }, + { + "eager claim is unchanged", + `{"name":"scratch","mode":"rw"}`, + map[string]any{"name": "scratch", "mount": "/volumes/scratch", "mode": "rw"}, + }, } { t.Run(tt.name, func(t *testing.T) { status, claimed := rawClaim(t, st, tt.requested) @@ -602,10 +640,7 @@ func TestClaimEnvEndToEnd(t *testing.T) { if gotA, gotB := seen(a), seen(b); gotA != "Bearer tenant-a" || gotB != "Bearer tenant-b" { t.Fatalf("injected %q / %q, want each claim's own value", gotA, gotB) } - st.eng.mu.Lock() - guestEnv := st.eng.guestEnvs[sockA] - st.eng.mu.Unlock() - if guestEnv != "AGENT_MODE=\"on\"\n" { + if guestEnv := st.eng.guestEnv(sockA); guestEnv != "AGENT_MODE=\"on\"\n" { t.Errorf("guest env file %q, want only the guest entry", guestEnv) } if status, body := rawJSON(t, st, http.MethodPut, "/v1/sandboxes/"+idA+"/env", `{"env":{"GW_KEY":{"value":"Bearer rotated","guest":false}}}`); status != http.StatusNoContent { @@ -636,9 +671,7 @@ func TestClaimEnvThroughTheSDK(t *testing.T) { if err != nil { t.Fatalf("socket: %v", err) } - st.eng.mu.Lock() - defer st.eng.mu.Unlock() - return st.eng.guestEnvs[sock] + return st.eng.guestEnv(sock) } readEnv := func(sb *sandbox.Sandbox, want map[string]sandbox.EnvVar) { t.Helper() @@ -753,88 +786,6 @@ func TestForkClaimRefPrefixWireShape(t *testing.T) { } } -func TestAttachOnlyVolumeEndToEnd(t *testing.T) { - scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") - st := startTenantStack(t, nil, - []config.VolumeSpec{{Name: "scratch", Path: scratch, Writable: true}}) - - sb, err := st.client.New(t.Context(), "rt:24.04", - sandbox.WithVolumes(sandbox.Volume{Name: "scratch", Mode: "rw", AttachOnly: true})) - if err != nil { - t.Fatalf("attach-only claim: %v", err) - } - if want := []sandbox.Volume{{Name: "scratch", Mode: "rw", AttachOnly: true}}; !slices.Equal(sb.Volumes, want) { - t.Errorf("claim volumes %+v, want %+v", sb.Volumes, want) - } - applied := []string{"attach:scratch:rw"} - if got := st.eng.volumeOpsLog(); !slices.Equal(got, applied) { - t.Errorf("engine ops %v, want %v", got, applied) - } - assertNoDirtyMarker(t, scratch, "apply") - for _, requested := range []string{`{"name":"scratch"}`, `{"name":"scratch","mode":"rw"}`} { - if status, _ := rawClaim(t, st, requested); status != http.StatusConflict { - t.Errorf("claim %s under a live attach-only writer: %d, want 409", requested, status) - } - } - - if err := sb.Close(); err != nil { - t.Fatalf("Close: %v", err) - } - want := slices.Concat(applied, []string{"remove"}) - if got := st.eng.volumeOpsLog(); !slices.Equal(got, want) { - t.Errorf("engine ops after release %v, want %v", got, want) - } - assertNoDirtyMarker(t, scratch, "release") -} - -func TestAttachOnlyVolumeWireShape(t *testing.T) { - scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") - st := startTenantStack(t, nil, - []config.VolumeSpec{{Name: "scratch", Path: scratch, Writable: true}}) - for _, tt := range []struct { - name string - request string - want map[string]any - }{ - { - "attach-only omits the mount", - `{"template":"rt:24.04","volumes":[{"name":"scratch","mode":"rw","attach_only":true}]}`, - map[string]any{"name": "scratch", "mode": "rw", "attach_only": true}, - }, - { - "eager claim is unchanged", - `{"template":"rt:24.04","volumes":[{"name":"scratch","mode":"rw"}]}`, - map[string]any{"name": "scratch", "mount": "/volumes/scratch", "mode": "rw"}, - }, - } { - t.Run(tt.name, func(t *testing.T) { - status, body := rawJSON(t, st, http.MethodPost, "/v1/claim", tt.request) - if status != http.StatusOK { - t.Fatalf("claim: %d %s, want 200", status, body) - } - var reply map[string]any - if err := json.Unmarshal(body, &reply); err != nil { - t.Fatalf("decode claim %s: %v", body, err) - } - entries, _ := reply["volumes"].([]any) - if len(entries) != 1 { - t.Fatalf("reply volumes %v, want one entry", reply["volumes"]) - } - entry, _ := entries[0].(map[string]any) - if !maps.Equal(entry, tt.want) { - t.Errorf("reply volume %v, want %v", entry, tt.want) - } - var claimed rawClaimResponse - if err := json.Unmarshal(body, &claimed); err != nil { - t.Fatalf("decode claim %s: %v", body, err) - } - if err := st.client.Attach(st.addr, claimed.ID, claimed.Token).Close(); err != nil { - t.Fatalf("release: %v", err) - } - }) - } -} - func TestDirtyVolumeRefusesReader(t *testing.T) { scratch := writeVolumeImage(t, "scratch.img", "scratch-bytes") if err := os.WriteFile(scratch+".dirty", nil, 0o600); err != nil { diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index 2e7d57908..f8ac3c6f7 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -254,6 +254,24 @@ func (f *fakeEngine) volumeOpsLog() []string { return slices.Clone(f.volumeOps) } +func (f *fakeEngine) guestEnv(sock string) string { + f.mu.Lock() + defer f.mu.Unlock() + return f.guestEnvs[sock] +} + +func (f *fakeEngine) setReaderRoot(root string) { + f.mu.Lock() + defer f.mu.Unlock() + f.readerRoot = root +} + +func (f *fakeEngine) readerCounts() (started, attached int) { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.readerStarts), len(f.readerAttaches) +} + func (f *fakeEngine) create(name string) (string, error) { f.mu.Lock() defer f.mu.Unlock() diff --git a/e2e/reader_test.go b/e2e/reader_test.go index 05d29b0a1..df447fc98 100644 --- a/e2e/reader_test.go +++ b/e2e/reader_test.go @@ -21,9 +21,7 @@ func TestReaderServesAHibernatedClaimUntilAWriteWakesIt(t *testing.T) { if err := os.WriteFile(filepath.Join(root, "work", "out.txt"), []byte("result"), 0o600); err != nil { t.Fatalf("setup: %v", err) } - st.eng.mu.Lock() - st.eng.readerRoot = root - st.eng.mu.Unlock() + st.eng.setReaderRoot(root) sb, err := st.client.New(t.Context(), testKey.Template) if err != nil { t.Fatalf("New: %v", err) @@ -44,9 +42,7 @@ func TestReaderServesAHibernatedClaimUntilAWriteWakesIt(t *testing.T) { if !hibernated(t, st, sb.ID) { t.Fatal("reading a file woke the claim") } - st.eng.mu.Lock() - started, attached := len(st.eng.readerStarts), len(st.eng.readerAttaches) - st.eng.mu.Unlock() + started, attached := st.eng.readerCounts() if started != 1 || attached != 1 { t.Errorf("%d readers started over %d snapshot copies, want one of each", started, attached) } From fd5b086a1ce40f036ae04b78245a90167c3d8235 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:52:06 +0800 Subject: [PATCH 110/127] sandboxd: table the pool flag tests, shared CA fixtures and guest roots in the egress tests --- sandboxd/config/config_test.go | 25 +++++----- sandboxd/egress/bench_test.go | 14 ++---- sandboxd/egress/ca_test.go | 81 ++++++++++++------------------- sandboxd/egress/intercept_test.go | 19 +++++--- 4 files changed, 59 insertions(+), 80 deletions(-) diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 80ca3c552..37c2e3f5c 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -41,10 +41,20 @@ func TestLoadAppliesDefaults(t *testing.T) { } } -func TestLoadReadsCaptureTrim(t *testing.T) { - cfg := mustLoad(t, writeConfig(t, `{"pools":[{"template":"rt:24.04","capture_trim":true},{"template":"py:3.12"}]}`)) - if !cfg.Pools[0].CaptureTrim || cfg.Pools[1].CaptureTrim { - t.Errorf("capture_trim %v/%v, want on for the pool that names it and off by default", cfg.Pools[0].CaptureTrim, cfg.Pools[1].CaptureTrim) +func TestLoadReadsPoolFlags(t *testing.T) { + for _, tt := range []struct { + key string + get func(PoolSpec) bool + }{ + {"capture_trim", func(p PoolSpec) bool { return p.CaptureTrim }}, + {"reader", func(p PoolSpec) bool { return p.Reader }}, + } { + t.Run(tt.key, func(t *testing.T) { + cfg := mustLoad(t, writeConfig(t, `{"pools":[{"template":"rt:24.04","`+tt.key+`":true},{"template":"py:3.12"}]}`)) + if got, other := tt.get(cfg.Pools[0]), tt.get(cfg.Pools[1]); !got || other { + t.Errorf("%s %v/%v, want on for the pool that names it and off by default", tt.key, got, other) + } + }) } } @@ -378,13 +388,6 @@ func TestLoadAcceptsPreviewAdvertiseShapes(t *testing.T) { } } -func TestLoadReadsReader(t *testing.T) { - cfg := mustLoad(t, writeConfig(t, `{"pools":[{"template":"rt:24.04","reader":true},{"template":"py:3.12"}]}`)) - if !cfg.Pools[0].Reader || cfg.Pools[1].Reader { - t.Errorf("reader %v/%v, want on for the pool that names it and off by default", cfg.Pools[0].Reader, cfg.Pools[1].Reader) - } -} - func TestLoadAcceptsVolumes(t *testing.T) { path := writeConfig(t, `{"pools":[],"volume_roots":["/srv/models/"],"volumes":[ {"name":"imagenet","path":"/srv/datasets/imagenet.img"}, diff --git a/sandboxd/egress/bench_test.go b/sandboxd/egress/bench_test.go index 5f6d1ee75..0354fe8ae 100644 --- a/sandboxd/egress/bench_test.go +++ b/sandboxd/egress/bench_test.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/http/httptest" + "slices" "strings" "testing" ) @@ -146,11 +147,7 @@ func BenchmarkScrubBody(b *testing.B) { }{ {"64KiB-page", func() io.Reader { return bytes.NewReader(page) }}, {"sse-256-events", func() io.Reader { - events := make([]string, 256) - for i := range events { - events[i] = string(event) - } - return chunked(events...) + return chunked(slices.Repeat([]string{string(event)}, 256)...) }}, } { b.Run(arm.name+"/plain", func(b *testing.B) { @@ -178,14 +175,11 @@ func benchFront(b *testing.B, mode InterceptMode) (proxyAddr string, roots *x509 b.Cleanup(upstream.Close) rule := Rule{Host: "example.com", Intercept: mode} var ca *CA - roots = x509.NewCertPool() if mode == InterceptAlways { ca = testCAOnly(b) - if !roots.AppendCertsFromPEM(ca.CertPEM()) { - b.Fatal("append cluster root") - } + roots = rootsFor(b, ca) } else { - roots.AddCert(upstream.Certificate()) + roots = trustUpstream(upstream) } p := New(Policy{Allow: []Rule{rule}}, nil, ca, fixedDial(upstream.Listener.Addr().String()), nil, nil) if ca != nil { diff --git a/sandboxd/egress/ca_test.go b/sandboxd/egress/ca_test.go index af78f34cb..096a0873a 100644 --- a/sandboxd/egress/ca_test.go +++ b/sandboxd/egress/ca_test.go @@ -13,28 +13,16 @@ import ( ) func TestLoadCARejectsForeignIntermediate(t *testing.T) { - rootA, keyA, err := GenerateRoot("A") - if err != nil { - t.Fatalf("root A: %v", err) - } - rootB, _, err := GenerateRoot("B") - if err != nil { - t.Fatalf("root B: %v", err) - } - interCert, interKey, err := IssueIntermediate(rootA, keyA, "n") - if err != nil { - t.Fatalf("intermediate: %v", err) - } + rootA, keyA := testRoot(t, "A") + rootB, _ := testRoot(t, "B") + interCert, interKey := testIntermediate(t, rootA, keyA, "n") if _, err := LoadCA(rootB, interCert, interKey); err == nil { t.Error("LoadCA accepted an intermediate not signed by the given root") } } func TestLoadCARejectsRootAsIntermediate(t *testing.T) { - rootCert, rootKey, err := GenerateRoot("root") - if err != nil { - t.Fatalf("root: %v", err) - } + rootCert, rootKey := testRoot(t, "root") if _, err := LoadCA(rootCert, rootCert, rootKey); err == nil { t.Error("LoadCA accepted the root as its own intermediate; want rejection (would put the root key on the node)") } @@ -77,10 +65,7 @@ func TestSignLeafChainsToRoot(t *testing.T) { } func TestLoadCARejectsExpiredIntermediate(t *testing.T) { - rootCert, rootKey, err := GenerateRoot("root") - if err != nil { - t.Fatalf("root: %v", err) - } + rootCert, rootKey := testRoot(t, "root") root, err := parseCert(rootCert, "root") if err != nil { t.Fatalf("parse root: %v", err) @@ -123,18 +108,9 @@ func TestLoadCARejectsExpiredIntermediate(t *testing.T) { } func TestLoadCARootBundle(t *testing.T) { - rootA, keyA, err := GenerateRoot("A") - if err != nil { - t.Fatalf("root A: %v", err) - } - rootB, _, err := GenerateRoot("B") - if err != nil { - t.Fatalf("root B: %v", err) - } - interCert, interKey, err := IssueIntermediate(rootA, keyA, "n") - if err != nil { - t.Fatalf("intermediate: %v", err) - } + rootA, keyA := testRoot(t, "A") + rootB, _ := testRoot(t, "B") + interCert, interKey := testIntermediate(t, rootA, keyA, "n") bundle := slices.Concat(rootA, rootB) ca, err := LoadCA(bundle, interCert, interKey) if err != nil { @@ -153,14 +129,8 @@ func TestLoadCARootBundle(t *testing.T) { } func TestLoadCARejectsNonCertBlockInRoot(t *testing.T) { - rootCert, rootKey, err := GenerateRoot("root") - if err != nil { - t.Fatalf("root: %v", err) - } - interCert, interKey, err := IssueIntermediate(rootCert, rootKey, "n") - if err != nil { - t.Fatalf("intermediate: %v", err) - } + rootCert, rootKey := testRoot(t, "root") + interCert, interKey := testIntermediate(t, rootCert, rootKey, "n") polluted := slices.Concat(rootCert, rootKey) if _, err := LoadCA(polluted, interCert, interKey); err == nil { t.Error("LoadCA accepted a non-certificate pem block in the root bundle") @@ -168,12 +138,9 @@ func TestLoadCARejectsNonCertBlockInRoot(t *testing.T) { } func TestFingerprintIsRootAndStable(t *testing.T) { - rootCert, rootKey, err := GenerateRoot("root") - if err != nil { - t.Fatalf("root: %v", err) - } - c1, k1, _ := IssueIntermediate(rootCert, rootKey, "n1") - c2, k2, _ := IssueIntermediate(rootCert, rootKey, "n2") + rootCert, rootKey := testRoot(t, "root") + c1, k1 := testIntermediate(t, rootCert, rootKey, "n1") + c2, k2 := testIntermediate(t, rootCert, rootKey, "n2") ca1, err := LoadCA(rootCert, c1, k1) if err != nil { t.Fatalf("load ca1: %v", err) @@ -187,16 +154,28 @@ func TestFingerprintIsRootAndStable(t *testing.T) { } } -func testCA(tb testing.TB) (*CA, []byte) { +func testRoot(tb testing.TB, name string) (cert, key []byte) { tb.Helper() - rootCert, rootKey, err := GenerateRoot("test root") + cert, key, err := GenerateRoot(name) if err != nil { - tb.Fatalf("generate root: %v", err) + tb.Fatalf("generate root %s: %v", name, err) } - interCert, interKey, err := IssueIntermediate(rootCert, rootKey, "node1") + return cert, key +} + +func testIntermediate(tb testing.TB, rootCert, rootKey []byte, name string) (cert, key []byte) { + tb.Helper() + cert, key, err := IssueIntermediate(rootCert, rootKey, name) if err != nil { - tb.Fatalf("issue intermediate: %v", err) + tb.Fatalf("issue intermediate %s: %v", name, err) } + return cert, key +} + +func testCA(tb testing.TB) (*CA, []byte) { + tb.Helper() + rootCert, rootKey := testRoot(tb, "test root") + interCert, interKey := testIntermediate(tb, rootCert, rootKey, "node1") ca, err := LoadCA(rootCert, interCert, interKey) if err != nil { tb.Fatalf("load ca: %v", err) diff --git a/sandboxd/egress/intercept_test.go b/sandboxd/egress/intercept_test.go index 547825d73..0b50fcce1 100644 --- a/sandboxd/egress/intercept_test.go +++ b/sandboxd/egress/intercept_test.go @@ -540,8 +540,7 @@ func TestInterceptStreamsAnEventStreamAsItArrives(t *testing.T) { defer close(release) ca, _ := testCA(t) p := New(Policy{Allow: []Rule{{Host: "example.com", Intercept: InterceptAlways}}}, nil, ca, fixedDial(upstream.Listener.Addr().String()), nil, nil) - guestRoots := x509.NewCertPool() - guestRoots.AppendCertsFromPEM(ca.CertPEM()) + guestRoots := rootsFor(t, ca) front := serveTrusting(t, p, upstream) tc := connectTLS(t, front.Listener.Addr().String(), "example.com:443", "example.com", guestRoots) @@ -568,8 +567,7 @@ func TestAnUpstreamDyingMidBodyTruncatesTheInterceptedResponse(t *testing.T) { defer upstream.Close() ca, _ := testCA(t) p := New(Policy{Allow: []Rule{{Host: "example.com", Intercept: InterceptAlways}}}, tt.secrets, ca, fixedDial(upstream.Listener.Addr().String()), nil, nil) - guestRoots := x509.NewCertPool() - guestRoots.AppendCertsFromPEM(ca.CertPEM()) + guestRoots := rootsFor(t, ca) tc := connectTLS(t, serveTrusting(t, p, upstream).Listener.Addr().String(), "example.com:443", "example.com", guestRoots) defer func() { _ = tc.Close() }() req := newRequest(t, http.MethodGet, tt.target, "") @@ -703,11 +701,16 @@ func interceptProxySecrets(t *testing.T, policy Policy, secrets Secrets, events } p := New(policy, secrets, ca, fixedDial(upstream.Listener.Addr().String()), audit, nil) - guestRoots := x509.NewCertPool() - if !guestRoots.AppendCertsFromPEM(ca.CertPEM()) { - t.Fatal("guest trust: append ca cert") + return p, rootsFor(t, ca), upstream +} + +func rootsFor(tb testing.TB, ca *CA) *x509.CertPool { + tb.Helper() + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(ca.CertPEM()) { + tb.Fatal("guest trust: append ca cert") } - return p, guestRoots, upstream + return roots } func serveTrusting(t *testing.T, p *Proxy, upstream *httptest.Server) *httptest.Server { From 4ef275814848ec29bbcd6a37c3e5181b0d2b1ea6 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:53:36 +0800 Subject: [PATCH 111/127] engine: shared port conn fixture size, countOf helper, drop the determinism test of a pure function --- sandboxd/engine/cloneargs_test.go | 10 ---------- sandboxd/engine/installca_test.go | 4 ++-- sandboxd/engine/portconn_test.go | 28 ++++++++++++---------------- sandboxd/engine/volume_test.go | 14 ++++++++++++-- 4 files changed, 26 insertions(+), 30 deletions(-) diff --git a/sandboxd/engine/cloneargs_test.go b/sandboxd/engine/cloneargs_test.go index 7ea2a8078..24a790b35 100644 --- a/sandboxd/engine/cloneargs_test.go +++ b/sandboxd/engine/cloneargs_test.go @@ -93,16 +93,6 @@ func TestEgressVMsSpreadOverEveryConfiguredShard(t *testing.T) { } } -func TestNetworkChoiceIsStableForAName(t *testing.T) { - shards := []string{"a", "b", "c"} - first := shardOf(shards, "sbx-pool1-42") - for range 100 { - if got := shardOf(shards, "sbx-pool1-42"); got != first { - t.Fatalf("shardOf returned %q then %q for one name", first, got) - } - } -} - func TestNetArgsHonorsTheLaneAndTheAttachment(t *testing.T) { none := types.PoolKey{Template: "rt:24.04", Net: types.NetNone, Size: types.SizeMedium} egress := types.PoolKey{Template: "rt:24.04", Net: types.NetEgress, Size: types.SizeMedium} diff --git a/sandboxd/engine/installca_test.go b/sandboxd/engine/installca_test.go index 84e6f945d..d5a1a18f5 100644 --- a/sandboxd/engine/installca_test.go +++ b/sandboxd/engine/installca_test.go @@ -40,7 +40,7 @@ func TestInstallCACertWritesCertAndUpdates(t *testing.T) { func TestInstallCACertNonzeroExitFails(t *testing.T) { path := sockPath(t) fake := serveFakeSilkd(t, path) - fake.execCode = 3 + fake.set(func(f *fakeSilkd) { f.execCode = 3 }) err := newTestEngine().InstallCACert(t.Context(), path, []byte("x")) if err == nil || !strings.Contains(err.Error(), "exit code 3") { t.Errorf("got %v, want exit code 3 failure", err) @@ -50,7 +50,7 @@ func TestInstallCACertNonzeroExitFails(t *testing.T) { func TestInstallCACertWriteErrorFrameFails(t *testing.T) { path := sockPath(t) fake := serveFakeSilkd(t, path) - fake.writeErr = "disk full" + fake.set(func(f *fakeSilkd) { f.writeErr = "disk full" }) err := newTestEngine().InstallCACert(t.Context(), path, []byte("x")) if err == nil || !strings.Contains(err.Error(), "disk full") { t.Errorf("got %v, want fs_write error-frame failure", err) diff --git a/sandboxd/engine/portconn_test.go b/sandboxd/engine/portconn_test.go index 5801621ed..4d7b13707 100644 --- a/sandboxd/engine/portconn_test.go +++ b/sandboxd/engine/portconn_test.go @@ -17,7 +17,7 @@ import ( ) func TestGuestPortConnReadsDataUntilDone(t *testing.T) { - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) go func() { frame, _ := json.Marshal(map[string]string{"type": "data", "data": base64.StdEncoding.EncodeToString([]byte("HTTP/1.1 200 OK"))}, json.Deterministic(true)) @@ -37,7 +37,7 @@ func TestGuestPortConnReadsDataUntilDone(t *testing.T) { func TestGuestPortConnCutBeforeDoneIsUnexpectedEOF(t *testing.T) { for name, trailer := range map[string]string{"between frames": "", "mid frame": `{"type":"da`} { t.Run(name, func(t *testing.T) { - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) go func() { frame, _ := json.Marshal(map[string]string{"type": "data", "data": base64.StdEncoding.EncodeToString([]byte("partial"))}, json.Deterministic(true)) @@ -57,7 +57,7 @@ func TestGuestPortConnCutBeforeDoneIsUnexpectedEOF(t *testing.T) { } func TestGuestPortConnWriteWrapsInDataFrame(t *testing.T) { - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) go func() { _, _ = gc.Write([]byte("GET / HTTP/1.1\r\n")) }() @@ -77,7 +77,7 @@ func TestGuestPortConnWriteWrapsInDataFrame(t *testing.T) { } func TestGuestPortConnCloseWriteSendsDataEnd(t *testing.T) { - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) go func() { _ = gc.CloseWrite() }() @@ -105,7 +105,7 @@ func TestGuestPortConnRoundTripsFixtures(t *testing.T) { return append(bytes.TrimSpace(raw), '\n') } - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) respData, respDone := fixture("resp_data.json"), fixture("resp_done.json") go func() { @@ -121,18 +121,14 @@ func TestGuestPortConnRoundTripsFixtures(t *testing.T) { t.Errorf("fixture done frame: %v, want EOF", err) } - client2, silk2 := net.Pipe() - t.Cleanup(func() { _ = client2.Close(); _ = silk2.Close() }) - gc2 := &guestPortConn{Conn: client2, r: bufio.NewReaderSize(client2, portReadBuf)} + silk2, gc2 := newTestGuestPortConn(t, portReadBuf) respError := fixture("resp_error.json") go func() { _, _ = silk2.Write(respError) }() if _, err := gc2.Read(buf); err == nil || errors.Is(err, io.EOF) { t.Errorf("fixture error frame: %v, want a typed error", err) } - client3, silk3 := net.Pipe() - t.Cleanup(func() { _ = client3.Close(); _ = silk3.Close() }) - gc3 := &guestPortConn{Conn: client3, r: bufio.NewReaderSize(client3, portReadBuf)} + silk3, gc3 := newTestGuestPortConn(t, portReadBuf) go func() { _, _ = gc3.Write([]byte("hello")) }() line := make([]byte, 256) n, _ = silk3.Read(line) @@ -152,7 +148,7 @@ func TestPortReadBufHoldsOneBulkFrame(t *testing.T) { } func TestGuestPortConnReadsAFirstFrameThatIsAFullBulkFrame(t *testing.T) { - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) payload := patterned(wire.BulkChunk) go writeDataFrames(t, silk, payload) @@ -163,7 +159,7 @@ func TestGuestPortConnReadsAFirstFrameThatIsAFullBulkFrame(t *testing.T) { } func TestGuestPortConnReadsSmallFramesThenBulkFrames(t *testing.T) { - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) chunks := [][]byte{[]byte("HTTP/1.1 200 OK\r\n"), patterned(100), patterned(3000), patterned(wire.BulkChunk), patterned(wire.BulkChunk - 1), []byte("tail")} go writeDataFrames(t, silk, chunks...) @@ -179,7 +175,7 @@ func TestGuestPortConnReadsSmallFramesThenBulkFrames(t *testing.T) { func TestGuestPortConnRefusesAFrameOverMaxFrame(t *testing.T) { for name, trailer := range map[string]string{"unterminated": "", "terminated": "\n"} { t.Run(name, func(t *testing.T) { - silk, gc := newTestGuestPortConn(t) + silk, gc := newTestGuestPortConn(t, portForwardMax) go func() { _, _ = silk.Write(append(bytes.Repeat([]byte("a"), wire.MaxFrame+1), trailer...)) @@ -192,11 +188,11 @@ func TestGuestPortConnRefusesAFrameOverMaxFrame(t *testing.T) { } } -func newTestGuestPortConn(t *testing.T) (net.Conn, *guestPortConn) { +func newTestGuestPortConn(t *testing.T, size int) (net.Conn, *guestPortConn) { t.Helper() client, silk := net.Pipe() t.Cleanup(func() { _ = client.Close(); _ = silk.Close() }) - return silk, &guestPortConn{Conn: client, r: bufio.NewReaderSize(client, portForwardMax)} + return silk, &guestPortConn{Conn: client, r: bufio.NewReaderSize(client, size)} } func writeDataFrames(t *testing.T, silk net.Conn, chunks ...[]byte) { diff --git a/sandboxd/engine/volume_test.go b/sandboxd/engine/volume_test.go index e3783a5b6..ee40b25d2 100644 --- a/sandboxd/engine/volume_test.go +++ b/sandboxd/engine/volume_test.go @@ -286,10 +286,10 @@ func TestMountVolumeReadsEachOtherSerialOnce(t *testing.T) { if fake.listCalls != 4 { t.Fatalf("list calls = %d, want 4 (3 blank serials + the successful poll)", fake.listCalls) } - if got := len(slices.DeleteFunc(slices.Clone(fake.readCalls), func(p string) bool { return p != "/sys/block/vda/serial" })); got != 1 { + if got := countOf(fake.readCalls, "/sys/block/vda/serial"); got != 1 { t.Errorf("read the root disk's serial %d times, want once: %v", got, fake.readCalls) } - if got := len(slices.DeleteFunc(slices.Clone(fake.readCalls), func(p string) bool { return p != "/sys/block/vdc/serial" })); got != 4 { + if got := countOf(fake.readCalls, "/sys/block/vdc/serial"); got != 4 { t.Errorf("read the awaited disk's serial %d times, want 4: %v", got, fake.readCalls) } } @@ -399,6 +399,16 @@ func TestSilkdListAndReadAreBounded(t *testing.T) { } } +func countOf(items []string, want string) int { + n := 0 + for _, item := range items { + if item == want { + n++ + } + } + return n +} + func configureVolumeDevices(f *fakeSilkd) { f.mu.Lock() defer f.mu.Unlock() From 19b3b4ddb6955526b1f8439d5836bc2ea21880f2 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:56:52 +0800 Subject: [PATCH 112/127] outbound: one dial assertion helper, nameserver address as a string, drop the host-dependent self-address test --- sandboxd/mesh/mesh_test.go | 32 +++++++++--------- sandboxd/outbound/dial_test.go | 60 +++++++++++----------------------- sandboxd/outbound/dns_test.go | 33 +++++++++---------- 3 files changed, 51 insertions(+), 74 deletions(-) diff --git a/sandboxd/mesh/mesh_test.go b/sandboxd/mesh/mesh_test.go index fcbeac466..8511e2bbb 100644 --- a/sandboxd/mesh/mesh_test.go +++ b/sandboxd/mesh/mesh_test.go @@ -177,19 +177,17 @@ func TestTwoNodeClusterGossipsPools(t *testing.T) { } a.mesh.UpdateSelf(map[string]int{"kk": 4}, []string{"tpl-hash"}, []string{"dataset"}) - deadline := time.Now().Add(10 * time.Second) - for time.Now().Before(deadline) { + learned := eventually(func() bool { cands := b.mesh.Candidates("kk") owners := b.mesh.TemplateOwners("tpl-hash") volumeOwners := b.mesh.VolumeOwners([]string{"dataset"}) - if len(cands) == 1 && cands[0] == "node-a:7777" && + return len(cands) == 1 && cands[0] == "node-a:7777" && len(owners) == 1 && owners[0] == "node-a:7777" && - len(volumeOwners) == 1 && volumeOwners[0] == "node-a:7777" { - return - } - time.Sleep(100 * time.Millisecond) + len(volumeOwners) == 1 && volumeOwners[0] == "node-a:7777" + }) + if !learned { + t.Fatalf("node-b never learned node-a's state: view=%+v", b.mesh.Members()) } - t.Fatalf("node-b never learned node-a's state: view=%+v", b.mesh.Members()) } func TestARuntimeDigestChangeReachesThePeer(t *testing.T) { @@ -199,14 +197,9 @@ func TestARuntimeDigestChangeReachesThePeer(t *testing.T) { t.Fatalf("join: %v", err) } a.mesh.UpdateDigest("rotated") - deadline := time.Now().Add(10 * time.Second) - for time.Now().Before(deadline) { - if b.mesh.ConfigMismatches() == 1 { - return - } - time.Sleep(100 * time.Millisecond) + if !eventually(func() bool { return b.mesh.ConfigMismatches() == 1 }) { + t.Fatalf("node-b never saw node-a's new digest: view=%+v", b.mesh.Members()) } - t.Fatalf("node-b never saw node-a's new digest: view=%+v", b.mesh.Members()) } func TestVolumeOwnersRequireEveryNameAndExcludeSelf(t *testing.T) { @@ -275,6 +268,15 @@ func TestVolumeHoldersCountSelfAndPeers(t *testing.T) { } } +func eventually(cond func() bool) bool { + for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(100 * time.Millisecond) { + if cond() { + return true + } + } + return false +} + func mergeStates(t *testing.T, m *Mesh, states []NodeState) { t.Helper() for _, st := range states { diff --git a/sandboxd/outbound/dial_test.go b/sandboxd/outbound/dial_test.go index df8abf9c1..3bca90661 100644 --- a/sandboxd/outbound/dial_test.go +++ b/sandboxd/outbound/dial_test.go @@ -1,6 +1,7 @@ package outbound import ( + "net" "net/netip" "strings" "testing" @@ -28,24 +29,18 @@ func TestEgressDialerBlocksInternal(t *testing.T) { "[64:ff9b::a9fe:a9fe]:80", "[64:ff9b::a00:1]:80", } + d := newDialer(verdictOf(nil)) for _, addr := range blocked { - if err := newDialer(verdictOf(nil)).Control("tcp", addr, nil); err == nil { - t.Errorf("dial to internal %s allowed; SSRF not blocked", addr) - } + assertControl(t, d, addr, false) } for _, addr := range []string{"93.184.216.34:443", "[2606:4700:4700::1111]:443", "[64:ff9b::5db8:d822]:443"} { - if err := newDialer(verdictOf(nil)).Control("tcp", addr, nil); err != nil { - t.Errorf("dial to public %s blocked: %v", addr, err) - } + assertControl(t, d, addr, true) } } func TestEgressDialerAdmitsOnlyNamedInternalPrefixes(t *testing.T) { d := newDialer(verdictOf([]string{"fdc8::/16", "10.8.0.0/16"})) - check := func(addr string) error { - return d.Control("tcp", addr, nil) - } for name, tt := range map[string]struct { addr string allowed bool @@ -63,13 +58,7 @@ func TestEgressDialerAdmitsOnlyNamedInternalPrefixes(t *testing.T) { "CGNAT metadata": {"100.100.100.200:80", false}, } { t.Run(name, func(t *testing.T) { - err := check(tt.addr) - if tt.allowed && err != nil { - t.Errorf("%s was blocked: %v", tt.addr, err) - } - if !tt.allowed && err == nil { - t.Errorf("%s was allowed; it must not be reachable from a guest", tt.addr) - } + assertControl(t, d, tt.addr, tt.allowed) }) } } @@ -84,9 +73,7 @@ func TestEgressDialerWildcardAllowsEverything(t *testing.T) { "127.0.0.1:7777", "169.254.169.254:80", } { - if err := d.Control("tcp", addr, nil); err != nil { - t.Errorf("%s blocked under a wildcard allow-list: %v", addr, err) - } + assertControl(t, d, addr, true) } } @@ -118,13 +105,7 @@ func TestEgressDialerScopesInternalAllowToPorts(t *testing.T) { "outside the prefix": {"10.8.0.2:18090", false}, } { t.Run(name, func(t *testing.T) { - err := d.Control("tcp", tt.addr, nil) - if tt.allowed && err != nil { - t.Errorf("%s was blocked: %v", tt.addr, err) - } - if !tt.allowed && (err == nil || !strings.Contains(err.Error(), "blocked internal address")) { - t.Errorf("%s: %v, want blocked internal address", tt.addr, err) - } + assertControl(t, d, tt.addr, tt.allowed) }) } } @@ -145,25 +126,11 @@ func TestEgressDialerBlocksTheNodesOwnAddresses(t *testing.T) { "another public host": {"93.184.216.35:22", true}, } { t.Run(name, func(t *testing.T) { - err := d.Control("tcp", tt.addr, nil) - if tt.allowed && err != nil { - t.Errorf("%s was blocked: %v", tt.addr, err) - } - if !tt.allowed && (err == nil || !strings.Contains(err.Error(), "blocked internal address")) { - t.Errorf("%s: %v, want blocked internal address", tt.addr, err) - } + assertControl(t, d, tt.addr, tt.allowed) }) } } -func TestSelfAddrsAreOnlyPublic(t *testing.T) { - for _, ip := range selfAddrs() { - if !ip.IsGlobalUnicast() || ip.IsPrivate() || ip.Is4In6() { - t.Errorf("self address %s is not a public unicast address", ip) - } - } -} - func TestTheHostGuardFollowsARefresh(t *testing.T) { h := newHost(t, &fakeEngine{}, &config.Config{}, nil) stale := netip.MustParseAddr("93.184.216.34") @@ -191,3 +158,14 @@ func verdictOf(allow []string, self ...netip.Addr) VerdictFunc { a := parseInternalAllow(allow) return func(ip netip.Addr, port uint16) (bool, error) { return destVerdict(a, self, ip, port) } } + +func assertControl(t *testing.T, d *net.Dialer, addr string, allowed bool) { + t.Helper() + err := d.Control("tcp", addr, nil) + if allowed && err != nil { + t.Errorf("%s was blocked: %v", addr, err) + } + if !allowed && (err == nil || !strings.Contains(err.Error(), "blocked internal address")) { + t.Errorf("%s: %v, want blocked internal address; it must not be reachable from a guest", addr, err) + } +} diff --git a/sandboxd/outbound/dns_test.go b/sandboxd/outbound/dns_test.go index 6446c032d..751aa99e7 100644 --- a/sandboxd/outbound/dns_test.go +++ b/sandboxd/outbound/dns_test.go @@ -3,6 +3,7 @@ package outbound import ( "bytes" "context" + "encoding/binary" "net" "net/netip" "os" @@ -68,7 +69,7 @@ func TestDNSDoorDropsInternalAnswersAndRemembersTheRest(t *testing.T) { cfg.EgressInternalAllow = []string{"10.9.0.0/16:5432"} events := &dnsEvents{} h := newHost(t, &fakeEngine{}, cfg, func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } o.RecordDNS = events.record }) h.self.Store(&[]netip.Addr{netip.MustParseAddr("93.184.216.99")}) @@ -103,7 +104,7 @@ func TestDNSDoorDropsInternalAnswersAndRemembersTheRest(t *testing.T) { func TestDNSDoorPassesAnUnfilteredAnswerByteForByte(t *testing.T) { upstream := newFakeNameserver(t) h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } }) conn := dialDNS(t, armDNS(t, h)) query := packQuery(t, dnsmessage.Header{ID: 0x5a5a, RecursionDesired: true}, "raw.test.") @@ -123,7 +124,7 @@ func TestDNSDoorDropsAnInternalGlueRecord(t *testing.T) { upstream := newFakeNameserver(t) events := &dnsEvents{} h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } o.RecordDNS = events.record }) msg := queryDoor(t, armDNS(t, h), "glue.test.") @@ -141,7 +142,7 @@ func TestDNSDoorDropsAnInternalGlueRecord(t *testing.T) { func TestDNSDoorDisarmEndsAnOpenConnection(t *testing.T) { upstream := newFakeNameserver(t) h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } }) sb := armDNS(t, h) conn := dialDNS(t, sb) @@ -159,7 +160,7 @@ func TestDNSDoorDisarmEndsAnOpenConnection(t *testing.T) { func TestDNSDoorsHandATruncatedAnswerToEachTransport(t *testing.T) { upstream := newFakeNameserver(t) h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } }) sb := armDNS(t, h) gram, err := net.Dial("unix", engine.DNSDatagramSocketPath(sb.VsockSocket)) @@ -179,7 +180,7 @@ func TestADNSDialBeforeTheClaimForcesARebind(t *testing.T) { upstream := newFakeNameserver(t) events := &dnsEvents{} h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } o.RecordDNS = events.record }) sb := outboundtest.VsockSandbox(t, "sb_dns", testKey) @@ -217,7 +218,7 @@ func TestDNSDoorMovesPastANameserverThatAnswersServfail(t *testing.T) { upstream := newFakeNameserver(t) failing := newServfailNameserver(t) h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{failing, upstream.udp} } + o.Nameservers = func() []string { return []string{failing, upstream} } }) if got := addresses(queryDoor(t, armDNS(t, h), "public.test.")); len(got) != 2 { t.Errorf("answers %v, want the second nameserver's two addresses", got) @@ -252,7 +253,7 @@ func TestDNSDoorRefusesAQueryFloodPastItsBucket(t *testing.T) { upstream := newFakeNameserver(t) events := &dnsEvents{} h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } o.RecordDNS = events.record }) sb := armDNS(t, h) @@ -279,7 +280,7 @@ func TestDNSDoorRefusesAQueryFloodPastItsBucket(t *testing.T) { func TestDNSDoorDropsAndCountsMalformedQueries(t *testing.T) { upstream := newFakeNameserver(t) h := newHost(t, &fakeEngine{}, poolConfig(testKey, dnsPolicy), func(o *Options) { - o.Nameservers = func() []string { return []string{upstream.udp} } + o.Nameservers = func() []string { return []string{upstream} } }) sb := armDNS(t, h) for _, tt := range []struct { @@ -324,7 +325,7 @@ func FuzzDNSDoor(f *testing.F) { var forwarded atomic.Int64 h := New(Options{View: func() *View { return v }, Nameservers: func() []string { forwarded.Add(1) - return []string{upstream.udp} + return []string{upstream} }}) valid := packQuery(f, dnsmessage.Header{RecursionDesired: true}, "public.test.") for _, seed := range [][]byte{ @@ -380,11 +381,7 @@ func TestReadNameserversReadsResolvConf(t *testing.T) { } } -type fakeNameserver struct { - udp string -} - -func newFakeNameserver(t testing.TB) *fakeNameserver { +func newFakeNameserver(t testing.TB) string { t.Helper() pc, ln := listenUDPAndTCP(t) t.Cleanup(func() { _ = pc.Close(); _ = ln.Close() }) @@ -412,7 +409,7 @@ func newFakeNameserver(t testing.TB) *fakeNameserver { }() } }() - return &fakeNameserver{udp: pc.LocalAddr().String()} + return pc.LocalAddr().String() } func listenUDPAndTCP(t testing.TB) (net.PacketConn, net.Listener) { @@ -616,7 +613,7 @@ func rawAnswer(q dnsmessage.Message) []byte { } func frame(msg []byte) []byte { - return append([]byte{byte(len(msg) >> 8), byte(len(msg))}, msg...) + return append(binary.BigEndian.AppendUint16(nil, uint16(len(msg))), msg...) } func packQuery(t testing.TB, hdr dnsmessage.Header, names ...string) []byte { @@ -651,7 +648,7 @@ func standardQuery(query []byte) bool { func selfPointerAdditional(query []byte) []byte { query[11] = 1 off := len(query) - return append(query, 0xc0|byte(off>>8), byte(off), 0, 1, 0, 1, 0, 0, 0, 0, 0, 4, 1, 2, 3, 4) + return append(binary.BigEndian.AppendUint16(query, 0xc000|uint16(off)), 0, 1, 0, 1, 0, 0, 0, 0, 0, 4, 1, 2, 3, 4) } type fuzzConn struct { From fab533d7060d1323a7fcc3cde8c7fcdc9517d8f9 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:58:33 +0800 Subject: [PATCH 113/127] server: shared guest and frame fixtures, await helper for channel waits, subtests per table row, one table for the preview failure statuses --- sandboxd/server/exec_test.go | 97 ++++++++---------- sandboxd/server/preview_test.go | 81 ++++----------- sandboxd/server/reader_test.go | 30 +----- sandboxd/server/relay_test.go | 6 +- sandboxd/server/server_test.go | 176 +++++++++++++++++++------------- 5 files changed, 174 insertions(+), 216 deletions(-) diff --git a/sandboxd/server/exec_test.go b/sandboxd/server/exec_test.go index 8965ca275..4581004e6 100644 --- a/sandboxd/server/exec_test.go +++ b/sandboxd/server/exec_test.go @@ -10,6 +10,7 @@ import ( "net" "net/http" "net/http/httptest" + "strconv" "strings" "sync/atomic" "testing" @@ -18,6 +19,8 @@ import ( "github.com/cocoonstack/sandbox/protocol/wire" ) +const startedFrame = `{"type":"started","pid":7}` + "\n" + func TestExecReturnsBufferedOutput(t *testing.T) { var gotExec, gotStdinClose wire.Request ts, _ := newRelayServer(t, func(conn net.Conn) { @@ -33,10 +36,10 @@ func TestExecReturnsBufferedOutput(t *testing.T) { return } gotStdinClose, _ = wire.DecodeRequest(bytes.TrimSpace(frame)) - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(conn, startedFrame) _, _ = io.WriteString(conn, stdoutFrame("v22\n")) _, _ = io.WriteString(conn, `{"type":"stderr","data":"`+base64.StdEncoding.EncodeToString([]byte("warn\n"))+`"}`+"\n") - _, _ = io.WriteString(conn, `{"type":"exit","code":3}`+"\n") + _, _ = io.WriteString(conn, exitFrame(3)) }) status, body := postExec(t, ts, `{"argv":["node","-v"],"cwd":"/work","env":{"A":"1"},"timeout_seconds":5}`) if status != http.StatusOK { @@ -64,9 +67,9 @@ func TestExecKeepsInvalidUTF8OutputAsReplacementCharacters(t *testing.T) { r := bufio.NewReader(conn) _, _ = r.ReadBytes('\n') _, _ = r.ReadBytes('\n') - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(conn, startedFrame) _, _ = io.WriteString(conn, stdoutFrame("ok\xff\n")) - _, _ = io.WriteString(conn, `{"type":"exit","code":0}`+"\n") + _, _ = io.WriteString(conn, exitFrame(0)) }) status, body := postExec(t, ts, `{"argv":["cat","blob"],"timeout_seconds":5}`) if status != http.StatusOK { @@ -82,16 +85,7 @@ func TestExecKeepsInvalidUTF8OutputAsReplacementCharacters(t *testing.T) { } func TestExecAuditsTheFrameWithoutItsDelimiter(t *testing.T) { - ts, srv := newRelayServer(t, func(conn net.Conn) { - defer conn.Close() - r := bufio.NewReader(conn) - for range 2 { - if _, err := r.ReadBytes('\n'); err != nil { - return - } - } - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n"+`{"type":"exit","code":0}`+"\n") - }) + ts, srv := newRelayServer(t, okGuest) audited := make(chan []byte, 1) srv.mgr.(*fakeManager).audited = func(_ string, frame []byte) { audited <- bytes.Clone(frame) } if status, body := postExec(t, ts, `{"argv":["true"]}`); status != http.StatusOK { @@ -104,16 +98,7 @@ func TestExecAuditsTheFrameWithoutItsDelimiter(t *testing.T) { } func TestExecRunsUnderAnOverflowingTimeout(t *testing.T) { - ts, _ := newRelayServer(t, func(conn net.Conn) { - defer conn.Close() - r := bufio.NewReader(conn) - for range 2 { - if _, err := r.ReadBytes('\n'); err != nil { - return - } - } - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n"+`{"type":"exit","code":0}`+"\n") - }) + ts, _ := newRelayServer(t, okGuest) status, body := postExec(t, ts, `{"argv":["true"],"timeout_seconds":10000000000}`) if status != http.StatusOK { t.Fatalf("status %d, want 200: a huge timeout_seconds must not wrap into an expired deadline: %s", status, body) @@ -147,32 +132,28 @@ func TestExecTimeoutKillsTheCommand(t *testing.T) { ts, _ := newRelayServer(t, killWatchGuest(killed, func(conn net.Conn, r *bufio.Reader) { defer close(closed) _, _ = r.ReadBytes('\n') - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(conn, startedFrame) _, _ = r.ReadBytes('\n') _, _ = r.ReadBytes('\n') })) if status, _ := postExec(t, ts, `{"argv":["sleep","60"],"timeout_seconds":1}`); status != http.StatusGatewayTimeout { t.Fatalf("status %d, want 504", status) } - select { - case <-closed: - case <-time.After(3 * time.Second): - t.Fatal("guest conn still open after the timeout") - } + await(t, closed, "guest conn still open after the timeout") expectKill(t, killed, "timeout") } -func TestExecRejectsUnknownFields(t *testing.T) { - ts, _ := newRelayServer(t, func(conn net.Conn) { _ = conn.Close() }) - if status, _ := postExec(t, ts, `{"argv":["whoami"],"user":"nobody"}`); status != http.StatusBadRequest { - t.Errorf("status %d, want 400", status) - } -} - -func TestExecRejectsEmptyArgv(t *testing.T) { - ts, _ := newRelayServer(t, func(conn net.Conn) { _ = conn.Close() }) - if status, _ := postExec(t, ts, `{"argv":[]}`); status != http.StatusBadRequest { - t.Errorf("status %d, want 400", status) +func TestExecRejectsBadRequests(t *testing.T) { + for _, tt := range []struct{ name, body string }{ + {"unknown fields", `{"argv":["whoami"],"user":"nobody"}`}, + {"empty argv", `{"argv":[]}`}, + } { + t.Run(tt.name, func(t *testing.T) { + ts, _ := newRelayServer(t, func(conn net.Conn) { _ = conn.Close() }) + if status, _ := postExec(t, ts, tt.body); status != http.StatusBadRequest { + t.Errorf("status %d, want 400", status) + } + }) } } @@ -200,7 +181,7 @@ func TestExecGuestClosingAfterTheStartWakesOnce(t *testing.T) { r := bufio.NewReader(guestEnd) _, _ = r.ReadBytes('\n') _, _ = r.ReadBytes('\n') - _, _ = io.WriteString(guestEnd, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(guestEnd, startedFrame) })) if status, body := postExec(t, ts, `{"argv":["sleep","60"]}`); status != http.StatusBadGateway { @@ -226,7 +207,7 @@ func TestExecTimeoutStartsAfterWake(t *testing.T) { r := bufio.NewReader(guestEnd) _, _ = r.ReadBytes('\n') _, _ = r.ReadBytes('\n') - _, _ = io.WriteString(guestEnd, `{"type":"exit","code":0}`+"\n") + _, _ = io.WriteString(guestEnd, exitFrame(0)) })) if status, body := postExec(t, ts, `{"argv":["true"],"timeout_seconds":1}`); status != http.StatusOK { @@ -241,7 +222,7 @@ func TestExecOverTheNodeBudgetIs503AndKillsTheCommand(t *testing.T) { killed := make(chan uint32, 1) ts, srv := newRelayServer(t, killWatchGuest(killed, func(conn net.Conn, r *bufio.Reader) { _, _ = r.ReadBytes('\n') - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(conn, startedFrame) _, _ = io.WriteString(conn, stdoutFrame("x")) _, _ = io.Copy(io.Discard, r) })) @@ -261,7 +242,7 @@ func TestExecOverTheOutputCapIs413AndKillsTheCommand(t *testing.T) { chunk := stdoutFrame(strings.Repeat("x", 1<<20)) ts, _ := newRelayServer(t, killWatchGuest(killed, func(conn net.Conn, r *bufio.Reader) { _, _ = r.ReadBytes('\n') - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(conn, startedFrame) for range execOutputCap>>20 + 1 { if _, err := io.WriteString(conn, chunk); err != nil { return @@ -280,9 +261,9 @@ func TestExecReturnsItsBudgetAfterAnswering(t *testing.T) { defer conn.Close() r := bufio.NewReader(conn) _, _ = r.ReadBytes('\n') - _, _ = io.WriteString(conn, `{"type":"started","pid":7}`+"\n") + _, _ = io.WriteString(conn, startedFrame) _, _ = io.WriteString(conn, stdoutFrame("v22\n")) - _, _ = io.WriteString(conn, `{"type":"exit","code":0}`+"\n") + _, _ = io.WriteString(conn, exitFrame(0)) }) if status, body := postExec(t, ts, `{"argv":["node","-v"]}`); status != http.StatusOK { t.Fatalf("status %d, want 200: %s", status, body) @@ -328,14 +309,24 @@ func killWatchGuest(killed chan<- uint32, first func(net.Conn, *bufio.Reader)) g func expectKill(t *testing.T, killed <-chan uint32, after string) { t.Helper() - select { - case pid := <-killed: - if pid != 7 { - t.Errorf("killed pid %d, want 7", pid) + if pid := await(t, killed, "no kill reached the guest after the "+after); pid != 7 { + t.Errorf("killed pid %d, want 7", pid) + } +} + +func okGuest(conn net.Conn) { + defer conn.Close() + r := bufio.NewReader(conn) + for range 2 { + if _, err := r.ReadBytes('\n'); err != nil { + return } - case <-time.After(3 * time.Second): - t.Fatalf("no kill reached the guest after the %s", after) } + _, _ = io.WriteString(conn, startedFrame+exitFrame(0)) +} + +func exitFrame(code int) string { + return `{"type":"exit","code":` + strconv.Itoa(code) + `}` + "\n" } func stdoutFrame(data string) string { diff --git a/sandboxd/server/preview_test.go b/sandboxd/server/preview_test.go index 06d349c6b..68e2a93b4 100644 --- a/sandboxd/server/preview_test.go +++ b/sandboxd/server/preview_test.go @@ -79,46 +79,22 @@ func TestPreviewProxiesToGuest(t *testing.T) { } } -func TestPreviewRevokedWhenDialFails(t *testing.T) { - ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ - dial: func(string, uint16) (net.Conn, error) { return nil, net.ErrClosed }, - }) - ts := httptest.NewServer(ps.Handler()) - t.Cleanup(ts.Close) - token := mintToken(ps, "sb_gone", 3000, time.Hour) - - resp, err := http.Get(ts.URL + "/p/" + token + "/") - if err != nil { - t.Fatalf("get: %v", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusBadGateway { - t.Errorf("status %d, want 502 for a gone sandbox", resp.StatusCode) - } -} - -func TestPreviewAnswers404ForAReleasedSandbox(t *testing.T) { - ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ - dial: func(string, uint16) (net.Conn, error) { return nil, pool.ErrUnknownSandbox }, - }) - ts := httptest.NewServer(ps.Handler()) - t.Cleanup(ts.Close) - - resp, err := http.Get(ts.URL + "/p/" + mintToken(ps, "sb_released", 3000, time.Hour) + "/") - if err != nil { - t.Fatalf("get: %v", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusNotFound { - t.Errorf("status %d, want 404 for a released sandbox", resp.StatusCode) - } -} - -func TestPreviewAnswers503WhileAClaimCannotServe(t *testing.T) { - for _, dialErr := range []error{pool.ErrLeaseLost, pool.ErrEvacuating} { - t.Run(dialErr.Error(), func(t *testing.T) { +func TestPreviewAnswersTheStatusOfAFailedDial(t *testing.T) { + for _, tt := range []struct { + name string + dialErr error + status int + wantRetry bool + }{ + {"gone sandbox", net.ErrClosed, http.StatusBadGateway, false}, + {"released sandbox", pool.ErrUnknownSandbox, http.StatusNotFound, false}, + {pool.ErrLeaseLost.Error(), pool.ErrLeaseLost, http.StatusServiceUnavailable, true}, + {pool.ErrEvacuating.Error(), pool.ErrEvacuating, http.StatusServiceUnavailable, true}, + {"quota", fmt.Errorf("wake sb_1: %w", pool.ErrQuota), http.StatusTooManyRequests, false}, + } { + t.Run(tt.name, func(t *testing.T) { ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ - dial: func(string, uint16) (net.Conn, error) { return nil, dialErr }, + dial: func(string, uint16) (net.Conn, error) { return nil, tt.dialErr }, }) ts := httptest.NewServer(ps.Handler()) t.Cleanup(ts.Close) @@ -129,33 +105,16 @@ func TestPreviewAnswers503WhileAClaimCannotServe(t *testing.T) { t.Fatalf("get: %v", err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("Retry-After") == "" { - t.Errorf("status %d retry-after %q, want 503 with a retry hint", resp.StatusCode, resp.Header.Get("Retry-After")) + if resp.StatusCode != tt.status { + t.Errorf("status %d, want %d", resp.StatusCode, tt.status) + } + if tt.wantRetry && resp.Header.Get("Retry-After") == "" { + t.Error("no Retry-After on a 503, want a retry hint") } }) } } -func TestPreviewAnswers429AtACap(t *testing.T) { - ps := NewPreviewServer("secret", "node:9000", "node:7777", &fakePreviewMgr{ - dial: func(string, uint16) (net.Conn, error) { - return nil, fmt.Errorf("wake sb_1: %w", pool.ErrQuota) - }, - }) - ts := httptest.NewServer(ps.Handler()) - t.Cleanup(ts.Close) - token := mintToken(ps, "sb_1", 3000, time.Hour) - - resp, err := http.Get(ts.URL + "/p/" + token + "/") - if err != nil { - t.Fatalf("get: %v", err) - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusTooManyRequests { - t.Errorf("status %d, want 429", resp.StatusCode) - } -} - func TestPreviewRechecksClaimForEveryRequest(t *testing.T) { guestAddr := newGuestServer(t, func(*http.Request) string { return "guest" }) diff --git a/sandboxd/server/reader_test.go b/sandboxd/server/reader_test.go index ec18eb22c..5244d0201 100644 --- a/sandboxd/server/reader_test.go +++ b/sandboxd/server/reader_test.go @@ -97,11 +97,7 @@ func TestSleepingLaterWriteMovesTheConnectionToTheGuest(t *testing.T) { send(t, conn, execFrame) expectLine(t, r, "guest") - select { - case <-env.readerClosed: - case <-time.After(3 * time.Second): - t.Fatal("the reader connection stayed open after the claim woke") - } + await(t, env.readerClosed, "the reader connection stayed open after the claim woke") send(t, conn, statFrame) expectLine(t, r, "guest") if wakes, dials := env.reader.counts(); wakes != 1 || dials != 1 { @@ -122,11 +118,7 @@ func TestSleepingReaderStopsServingOnceTheClaimWoke(t *testing.T) { env.reader.setSource("") send(t, conn, statFrame) expectLine(t, r, "guest") - select { - case <-env.readerClosed: - case <-time.After(3 * time.Second): - t.Fatal("the stale reader connection stayed open") - } + await(t, env.readerClosed, "the stale reader connection stayed open") } func TestSleepingConnectionMovesToAFreshReaderWhenTheClaimSleepsAgain(t *testing.T) { @@ -138,11 +130,7 @@ func TestSleepingConnectionMovesToAFreshReaderWhenTheClaimSleepsAgain(t *testing env.reader.setSource("snap-2") send(t, conn, readFrame) expectLine(t, r, "reader") - select { - case <-env.readerClosed: - case <-time.After(3 * time.Second): - t.Fatal("the stale reader connection stayed open") - } + await(t, env.readerClosed, "the stale reader connection stayed open") if wakes, dials := env.reader.counts(); wakes != 0 || dials != 2 { t.Errorf("wakes=%d reader dials=%d, want a fresh reader for the new sleep and no wake", wakes, dials) } @@ -311,11 +299,7 @@ func TestSleepingReaderClosesWhenItsClientLeavesMidReply(t *testing.T) { send(t, conn, readFrame) send(t, conn, statFrame) _ = conn.Close() - select { - case <-env.readerClosed: - case <-time.After(5 * time.Second): - t.Fatal("the reader connection outlived its client") - } + await(t, env.readerClosed, "the reader connection outlived its client") } func TestSleepingCloseRelaysEndsReaderConnections(t *testing.T) { @@ -329,11 +313,7 @@ func TestSleepingCloseRelaysEndsReaderConnections(t *testing.T) { env.srv.CloseRelays() close(done) }() - select { - case <-done: - case <-time.After(3 * time.Second): - t.Fatal("CloseRelays hung on a reader connection") - } + await(t, done, "CloseRelays hung on a reader connection") if _, err := r.ReadString('\n'); !errors.Is(err, io.EOF) { t.Errorf("client after CloseRelays: %v, want EOF", err) } diff --git a/sandboxd/server/relay_test.go b/sandboxd/server/relay_test.go index 3ff0ca4b9..5d66735de 100644 --- a/sandboxd/server/relay_test.go +++ b/sandboxd/server/relay_test.go @@ -125,11 +125,7 @@ func TestRelayClientDisconnectClosesGuest(t *testing.T) { conn, _ := upgradeConn(t, ts) _ = conn.Close() - select { - case <-guestClosed: - case <-time.After(3 * time.Second): - t.Fatal("guest side not closed after client disconnect") - } + await(t, guestClosed, "guest side not closed after client disconnect") } func TestRelayEndsWhenItsClaimIsFenced(t *testing.T) { diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index c3664982b..72df20400 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -495,15 +495,17 @@ func TestSandboxesNarrowToAClaimRef(t *testing.T) { {"/v1/sandboxes?claim_ref=team-a%2Fdemo", "team-a/demo"}, {"/v1/sandboxes", ""}, } { - got := "unset" - mgr := &fakeManager{sandboxIndex: func(_, claimRef string) []pool.SandboxSummary { - got = claimRef - return nil - }} - ts := newTenantTestServer(t, "root", nil, mgr, nil) - if code := statusOf(t, http.MethodGet, ts.URL+tt.path, "root", ""); code != http.StatusOK || got != tt.want { - t.Errorf("%s: status %d, manager saw claim_ref %q, want %q", tt.path, code, got, tt.want) - } + t.Run(tt.path, func(t *testing.T) { + got := "unset" + mgr := &fakeManager{sandboxIndex: func(_, claimRef string) []pool.SandboxSummary { + got = claimRef + return nil + }} + ts := newTenantTestServer(t, "root", nil, mgr, nil) + if code := statusOf(t, http.MethodGet, ts.URL+tt.path, "root", ""); code != http.StatusOK || got != tt.want { + t.Errorf("%s: status %d, manager saw claim_ref %q, want %q", tt.path, code, got, tt.want) + } + }) } } @@ -689,9 +691,11 @@ func TestInstanceMetadataVerb(t *testing.T) { {"over 4 KiB", "sekret", "sb_1", `{"a":"` + strings.Repeat("x", 4<<10) + `"}`, http.StatusBadRequest}, {"unknown sandbox", "sekret", "sb_2", valid, http.StatusNotFound}, } { - if code := put(tt.token, tt.id, tt.body); code != tt.want { - t.Errorf("%s: status %d, want %d", tt.name, code, tt.want) - } + t.Run(tt.name, func(t *testing.T) { + if code := put(tt.token, tt.id, tt.body); code != tt.want { + t.Errorf("%s: status %d, want %d", tt.name, code, tt.want) + } + }) } for _, err := range []error{pool.ErrPaused, pool.ErrArchived, pool.ErrNoInstanceMetadata} { fail = err @@ -711,12 +715,14 @@ func TestDrainEndpoints(t *testing.T) { {http.MethodPost, true}, {http.MethodDelete, false}, } { - resp := doReq(t, tt.method, ts.URL+"/v1/drain", "sekret", "") - info := decodeJSON[InfoResponse](t, resp.Body) - resp.Body.Close() - if resp.StatusCode != http.StatusOK || info.Draining != tt.draining || mgr.draining != tt.draining { - t.Fatalf("%s: status=%d draining=%v mgr=%v, want 200/%v", tt.method, resp.StatusCode, info.Draining, mgr.draining, tt.draining) - } + t.Run(tt.method, func(t *testing.T) { + resp := doReq(t, tt.method, ts.URL+"/v1/drain", "sekret", "") + info := decodeJSON[InfoResponse](t, resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK || info.Draining != tt.draining || mgr.draining != tt.draining { + t.Fatalf("%s: status=%d draining=%v mgr=%v, want 200/%v", tt.method, resp.StatusCode, info.Draining, mgr.draining, tt.draining) + } + }) } } @@ -731,15 +737,17 @@ func TestCordonEndpoints(t *testing.T) { {http.MethodPost, true, "sandboxd_cordoned 1\n"}, {http.MethodDelete, false, "sandboxd_cordoned 0\n"}, } { - resp := doReq(t, tt.method, ts.URL+"/v1/cordon", "sekret", "") - info := decodeJSON[InfoResponse](t, resp.Body) - resp.Body.Close() - if resp.StatusCode != http.StatusOK || info.Cordoned != tt.cordoned || mgr.cordoned != tt.cordoned || info.Draining { - t.Fatalf("%s: status=%d info=%+v mgr=%v, want 200/%v and no drain", tt.method, resp.StatusCode, info, mgr.cordoned, tt.cordoned) - } - if _, body := bodyOf(t, http.MethodGet, ts.URL+"/metrics", "sekret", ""); !strings.Contains(string(body), tt.metric) { - t.Errorf("%s: metrics miss %q", tt.method, tt.metric) - } + t.Run(tt.method, func(t *testing.T) { + resp := doReq(t, tt.method, ts.URL+"/v1/cordon", "sekret", "") + info := decodeJSON[InfoResponse](t, resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK || info.Cordoned != tt.cordoned || mgr.cordoned != tt.cordoned || info.Draining { + t.Fatalf("%s: status=%d info=%+v mgr=%v, want 200/%v and no drain", tt.method, resp.StatusCode, info, mgr.cordoned, tt.cordoned) + } + if _, body := bodyOf(t, http.MethodGet, ts.URL+"/metrics", "sekret", ""); !strings.Contains(string(body), tt.metric) { + t.Errorf("%s: metrics miss %q", tt.method, tt.metric) + } + }) } } @@ -751,11 +759,13 @@ func TestCordonPersistFailureAnswers500AndKeepsState(t *testing.T) { {http.MethodPost, false}, {http.MethodDelete, true}, } { - mgr := &fakeManager{cordoned: tt.cordoned, cordonErr: errors.New("disk gone")} - ts := newTestServer(t, "sekret", mgr, nil) - if code := statusOf(t, tt.method, ts.URL+"/v1/cordon", "sekret", ""); code != http.StatusInternalServerError || mgr.cordoned != tt.cordoned { - t.Errorf("%s: status %d cordoned=%v, want 500 and cordoned=%v", tt.method, code, mgr.cordoned, tt.cordoned) - } + t.Run(tt.method, func(t *testing.T) { + mgr := &fakeManager{cordoned: tt.cordoned, cordonErr: errors.New("disk gone")} + ts := newTestServer(t, "sekret", mgr, nil) + if code := statusOf(t, tt.method, ts.URL+"/v1/cordon", "sekret", ""); code != http.StatusInternalServerError || mgr.cordoned != tt.cordoned { + t.Errorf("%s: status %d cordoned=%v, want 500 and cordoned=%v", tt.method, code, mgr.cordoned, tt.cordoned) + } + }) } } @@ -2105,17 +2115,19 @@ func TestClaimRejectsMetadataOrClaimRefOverABound(t *testing.T) { {"/v1/claim", `{"template":"rt:24.04","volumes":[{"name":"data"}],"claim_ref":"` + strings.Repeat("r", 513) + `"}`, "claim_ref must"}, {"/v1/sandboxes/sb_1/fork", `{"token":"tok","count":1,"claim_ref_prefix":"ns/\u0007"}`, "claim_ref_prefix must"}, } { - mgr := &fakeManager{} - ts := newTestServer(t, "sekret", mgr, nil) - resp := postJSON(t, ts.URL+tt.path, "sekret", tt.body) - got := decodeJSON[errorReply](t, resp.Body) - resp.Body.Close() - if resp.StatusCode != http.StatusBadRequest || !strings.Contains(got.Error, tt.want) { - t.Errorf("%s: status %d error %q, want 400 naming %q", tt.path, resp.StatusCode, got.Error, tt.want) - } - if mgr.warmCalls+mgr.provisionCalls+mgr.healCalls != 0 || mgr.gotMetadata != nil { - t.Errorf("%s: the manager ran on a rejected claim", tt.path) - } + t.Run(tt.want+" "+tt.path, func(t *testing.T) { + mgr := &fakeManager{} + ts := newTestServer(t, "sekret", mgr, nil) + resp := postJSON(t, ts.URL+tt.path, "sekret", tt.body) + got := decodeJSON[errorReply](t, resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest || !strings.Contains(got.Error, tt.want) { + t.Errorf("%s: status %d error %q, want 400 naming %q", tt.path, resp.StatusCode, got.Error, tt.want) + } + if mgr.warmCalls+mgr.provisionCalls+mgr.healCalls != 0 || mgr.gotMetadata != nil { + t.Errorf("%s: the manager ran on a rejected claim", tt.path) + } + }) } } @@ -2140,12 +2152,14 @@ func TestClaimThreadsMetadataAndEnv(t *testing.T) { {"checkpoint", "/v1/checkpoints/ck_00000000000000aa/claim", `{"metadata":{"team":"a"},"env":{"GW":{"value":"Bearer k","guest":false}}}`, func(f *fakeManager) { f.claimCheckpoint = ckpt }}, {"checkpoint heal", "/v1/checkpoints/ck_00000000000000aa/claim", `{"metadata":{"team":"a"},"env":{"GW":{"value":"Bearer k","guest":false}}}`, func(f *fakeManager) { f.healCheckpoint = ckpt }}, } { - mgr := &fakeManager{} - tt.setup(mgr) - ts := newTestServer(t, "sekret", mgr, nil) - if code := statusOf(t, http.MethodPost, ts.URL+tt.path, "sekret", tt.body); code != http.StatusOK || !maps.Equal(mgr.gotMetadata, want) || !mgr.gotEnv.Equal(wantEnv) { - t.Errorf("%s: status %d, manager saw metadata %v env %v, want 200 and %v %v", tt.name, code, mgr.gotMetadata, mgr.gotEnv, want, wantEnv) - } + t.Run(tt.name, func(t *testing.T) { + mgr := &fakeManager{} + tt.setup(mgr) + ts := newTestServer(t, "sekret", mgr, nil) + if code := statusOf(t, http.MethodPost, ts.URL+tt.path, "sekret", tt.body); code != http.StatusOK || !maps.Equal(mgr.gotMetadata, want) || !mgr.gotEnv.Equal(wantEnv) { + t.Errorf("%s: status %d, manager saw metadata %v env %v, want 200 and %v %v", tt.name, code, mgr.gotMetadata, mgr.gotEnv, want, wantEnv) + } + }) } } @@ -2164,11 +2178,13 @@ func TestSandboxesFilterByMetadata(t *testing.T) { {"?metadata=%3Dv", http.StatusBadRequest, nil}, {"?metadata=a%3D1&metadata=a%3D2", http.StatusBadRequest, nil}, } { - mgr := &fakeManager{} - ts := newTestServer(t, "root", mgr, nil) - if code := statusOf(t, http.MethodGet, ts.URL+"/v1/sandboxes"+tt.query, "root", ""); code != tt.code || !maps.Equal(mgr.gotMetadata, tt.want) { - t.Errorf("%q: status %d, manager saw %v, want %d and %v", tt.query, code, mgr.gotMetadata, tt.code, tt.want) - } + t.Run(tt.query, func(t *testing.T) { + mgr := &fakeManager{} + ts := newTestServer(t, "root", mgr, nil) + if code := statusOf(t, http.MethodGet, ts.URL+"/v1/sandboxes"+tt.query, "root", ""); code != tt.code || !maps.Equal(mgr.gotMetadata, tt.want) { + t.Errorf("%q: status %d, manager saw %v, want %d and %v", tt.query, code, mgr.gotMetadata, tt.code, tt.want) + } + }) } } @@ -2191,22 +2207,24 @@ func TestExpireActionThreadsThroughEveryDecoder(t *testing.T) { f.renew = func(string, string, time.Duration) (time.Time, error) { return time.Now(), nil } }}, } { - mgr := &fakeManager{} - tt.setup(mgr) - ts := newTestServer(t, "sekret", mgr, nil) - if code := statusOf(t, http.MethodPost, ts.URL+tt.path, "sekret", tt.body); code != http.StatusOK || mgr.gotOnExpire != types.ExpireArchive { - t.Errorf("%s: status %d, manager saw on_expire %q, want 200 and archive", tt.name, code, mgr.gotOnExpire) - } + t.Run(tt.name, func(t *testing.T) { + mgr := &fakeManager{} + tt.setup(mgr) + ts := newTestServer(t, "sekret", mgr, nil) + if code := statusOf(t, http.MethodPost, ts.URL+tt.path, "sekret", tt.body); code != http.StatusOK || mgr.gotOnExpire != types.ExpireArchive { + t.Errorf("%s: status %d, manager saw on_expire %q, want 200 and archive", tt.name, code, mgr.gotOnExpire) + } - mgr = &fakeManager{} - tt.setup(mgr) - ts = newTestServer(t, "sekret", mgr, nil) - resp := postJSON(t, ts.URL+tt.path, "sekret", strings.Replace(tt.body, `"archive"`, `"pause"`, 1)) - got := decodeJSON[errorReply](t, resp.Body) - resp.Body.Close() - if resp.StatusCode != http.StatusBadRequest || !strings.Contains(got.Error, "on_expire") { - t.Errorf("%s with on_expire pause: status %d error %q, want 400 naming on_expire", tt.name, resp.StatusCode, got.Error) - } + mgr = &fakeManager{} + tt.setup(mgr) + ts = newTestServer(t, "sekret", mgr, nil) + resp := postJSON(t, ts.URL+tt.path, "sekret", strings.Replace(tt.body, `"archive"`, `"pause"`, 1)) + got := decodeJSON[errorReply](t, resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest || !strings.Contains(got.Error, "on_expire") { + t.Errorf("%s with on_expire pause: status %d error %q, want 400 naming on_expire", tt.name, resp.StatusCode, got.Error) + } + }) } } @@ -2244,9 +2262,11 @@ func TestTunVerbIsRootOnlyAndPassesOnlyTheNamedFields(t *testing.T) { {"unknown field", "sekret", "sb_1", `{"quota": 1}`, http.StatusBadRequest}, {"unknown sandbox", "sekret", "sb_2", `{"enabled": true}`, http.StatusNotFound}, } { - if code := put(tt.token, tt.id, tt.body); code != tt.want { - t.Errorf("%s: status %d, want %d", tt.name, code, tt.want) - } + t.Run(tt.name, func(t *testing.T) { + if code := put(tt.token, tt.id, tt.body); code != tt.want { + t.Errorf("%s: status %d, want %d", tt.name, code, tt.want) + } + }) } } @@ -2933,6 +2953,18 @@ func postJSON(t *testing.T, url, token, body string) *http.Response { return doReq(t, http.MethodPost, url, token, body) } +func await[T any](t *testing.T, ch <-chan T, msg string) T { + t.Helper() + select { + case v := <-ch: + return v + case <-time.After(5 * time.Second): + t.Fatal(msg) + var zero T + return zero + } +} + func scrapeMetrics(t *testing.T, base string) string { t.Helper() _, raw := bodyOf(t, http.MethodGet, base+"/metrics", "", "") From ba159491ccd00526306f70ea3d7187574a5c08a1 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:58:33 +0800 Subject: [PATCH 114/127] review: move guestFunc beside the other function vocabulary types --- sandboxd/server/server_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index 72df20400..238b09eb3 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -2286,6 +2286,8 @@ type checkpointClaimFunc func(string) (*types.Sandbox, error) type idActionFunc func(string) error +type guestFunc func(net.Conn) + type fakeManager struct { counters pool.Counters registerVolume func(context.Context, string, types.VolumeRegistration) error @@ -2789,8 +2791,6 @@ func (f *fakeManager) record(o pool.ClaimOptions) { f.gotNoEgress = o.NoEgress } -type guestFunc func(net.Conn) - type fakeDialer struct { dial func(ctx context.Context, sock string) (net.Conn, error) } From de0047bb246094142cb074485c8a3a267b2e0725 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:59:07 +0800 Subject: [PATCH 115/127] store/dir: assertGone and assertPresent for the on-disk checks --- sandboxd/store/dir/dir_test.go | 90 +++++++++++++--------------------- 1 file changed, 33 insertions(+), 57 deletions(-) diff --git a/sandboxd/store/dir/dir_test.go b/sandboxd/store/dir/dir_test.go index 80200f02e..4ef31abcf 100644 --- a/sandboxd/store/dir/dir_test.go +++ b/sandboxd/store/dir/dir_test.go @@ -46,12 +46,8 @@ func TestFetchPinsGenerationAcrossStoreInstances(t *testing.T) { t.Fatalf("fetch digest %q, want first generation %q", digest, firstDigest) } secondDigest := mustPublishDigested(t, writer, testID, "second") - if _, statErr := os.Stat(filepath.Join(dir, "disk.img")); statErr != nil { - t.Fatalf("first generation disturbed by re-publish: %v", statErr) - } - if _, statErr := os.Stat(firstDigestPath); statErr != nil { - t.Fatalf("first generation digest disturbed by re-publish: %v", statErr) - } + assertPresent(t, filepath.Join(dir, "disk.img"), "first generation disturbed by re-publish") + assertPresent(t, firstDigestPath, "first generation digest disturbed by re-publish") _, meta, digest = mustFetch(t, reader, testID) if string(meta) != storetest.MetaJSON("second") { @@ -77,9 +73,7 @@ func TestPlainPublishReplacesDigestWithEmpty(t *testing.T) { if firstDigest == "" { t.Fatal("digested generation returned an empty digest") } - if _, statErr := os.Stat(firstDigestPath); statErr != nil { - t.Fatalf("superseded digest removed inside the grace: %v", statErr) - } + assertPresent(t, firstDigestPath, "superseded digest removed inside the grace") mustPublish(t, st, testID, "digested") if _, _, digest = mustFetch(t, st, testID); digest != "" { t.Errorf("a plain publish of the digested meta kept digest %q, want empty", digest) @@ -270,14 +264,10 @@ func TestSweepGenerationsPairsCurrentAndSupersededDigests(t *testing.T) { t.Fatalf("SweepGenerations: %v", err) } for _, path := range []string{firstGen, firstSidecar} { - if _, statErr := os.Stat(path); !errors.Is(statErr, fs.ErrNotExist) { - t.Errorf("superseded entry %s survived sweep: %v", filepath.Base(path), statErr) - } + assertGone(t, path, "superseded entry %s survived sweep", filepath.Base(path)) } for _, path := range []string{secondGen, secondSidecar, labels} { - if _, statErr := os.Stat(path); statErr != nil { - t.Errorf("current entry %s was swept: %v", filepath.Base(path), statErr) - } + assertPresent(t, path, "current entry %s was swept", filepath.Base(path)) } _, meta, digest := mustFetch(t, st, testID) if string(meta) != storetest.MetaJSON("second") || digest != secondDigest { @@ -304,17 +294,13 @@ func TestPublishRetriesAfterExpiredInstall(t *testing.T) { if sweepErr := st.SweepGenerations(t.Context()); sweepErr != nil { t.Fatalf("sweep expired generation: %v", sweepErr) } - if _, statErr := os.Stat(gen); !errors.Is(statErr, fs.ErrNotExist) { - t.Fatalf("expired generation survived sweep: %v", statErr) - } + assertGone(t, gen, "expired generation survived sweep") mustPublish(t, st, testID, "same") backdate(t, gen) mustPublish(t, st, testID, "same") dir, _, _ := mustFetch(t, st, testID) - if _, err := os.Stat(filepath.Join(dir, "disk.img")); err != nil { - t.Fatalf("export after retried publish: %v", err) - } + assertPresent(t, filepath.Join(dir, "disk.img"), "export after retried publish") } func TestPublishSweepsGenerationsBySupersessionAge(t *testing.T) { @@ -325,26 +311,18 @@ func TestPublishSweepsGenerationsBySupersessionAge(t *testing.T) { mustPublish(t, st, testID, "first") mustPublish(t, st, testID, "second") mustPublish(t, st, testID, "third") - if _, statErr := os.Stat(gen1); statErr != nil { - t.Fatalf("superseded generation reclaimed inside the grace: %v", statErr) - } + assertPresent(t, gen1, "superseded generation reclaimed inside the grace") setAge(t, gen1, store.GenerationGrace+time.Minute) setAge(t, gen2, store.GenerationGrace/2) mustPublish(t, st, testID, "fourth") - if _, statErr := os.Stat(gen1); !errors.Is(statErr, fs.ErrNotExist) { - t.Fatalf("generation past its supersession grace survived publish: %v", statErr) - } - if _, statErr := os.Stat(gen2); statErr != nil { - t.Fatalf("generation inside its supersession grace was reclaimed: %v", statErr) - } + assertGone(t, gen1, "generation past its supersession grace survived publish") + assertPresent(t, gen2, "generation inside its supersession grace was reclaimed") dir, meta, _ := mustFetch(t, st, testID) if string(meta) != storetest.MetaJSON("fourth") { t.Fatalf("fetch meta %q, want the current generation", meta) } - if _, err := os.Stat(filepath.Join(dir, "disk.img")); err != nil { - t.Fatalf("current export after sweep: %v", err) - } + assertPresent(t, filepath.Join(dir, "disk.img"), "current export after sweep") } func TestSweepSparesPublishingGeneration(t *testing.T) { @@ -358,9 +336,7 @@ func TestSweepSparesPublishingGeneration(t *testing.T) { if sweepErr := sweeper.SweepStaging(); sweepErr != nil { t.Fatalf("sweep during publish: %v", sweepErr) } - if _, statErr := os.Stat(gen); statErr != nil { - t.Fatalf("publishing generation was swept: %v", statErr) - } + assertPresent(t, gen, "publishing generation was swept") tmp := filepath.Join(root, testID, store.MetaFile+".tmp") if writeErr := os.WriteFile(tmp, meta, 0o600); writeErr != nil { @@ -413,15 +389,9 @@ func TestSweepReclaimsOnlyAgedUncommittedGenerations(t *testing.T) { if sweepErr := sweeper.SweepStaging(); sweepErr != nil { t.Fatalf("sweep old: %v", sweepErr) } - if _, statErr := os.Stat(orphan); !errors.Is(statErr, fs.ErrNotExist) { - t.Fatalf("aged uncommitted generation survived: %v", statErr) - } - if _, statErr := os.Stat(filepath.Join(root, orphanID)); statErr != nil { - t.Fatalf("orphan record shell was removed: %v", statErr) - } - if _, statErr := os.Stat(fresh); statErr != nil { - t.Fatalf("publishing generation was swept: %v", statErr) - } + assertGone(t, orphan, "aged uncommitted generation survived") + assertPresent(t, filepath.Join(root, orphanID), "orphan record shell was removed") + assertPresent(t, fresh, "publishing generation was swept") if removeErr := os.RemoveAll(commitBlocker); removeErr != nil { t.Fatalf("remove commit blocker: %v", removeErr) @@ -448,17 +418,13 @@ func TestSweepStagingSparesAPeersFreshStaging(t *testing.T) { if sweepErr := restarted.SweepStaging(); sweepErr != nil { t.Fatalf("sweep: %v", sweepErr) } - if _, statErr := os.Stat(filepath.Join(staging, store.MetaFile)); statErr != nil { - t.Fatalf("a peer's in-progress staging was swept: %v", statErr) - } + assertPresent(t, filepath.Join(staging, store.MetaFile), "a peer's in-progress staging was swept") backdate(t, staging) if sweepErr := restarted.SweepStaging(); sweepErr != nil { t.Fatalf("sweep aged: %v", sweepErr) } - if _, statErr := os.Stat(staging); !errors.Is(statErr, fs.ErrNotExist) { - t.Errorf("aged staging survived: %v", statErr) - } + assertGone(t, staging, "aged staging survived") } func TestSweepGenerationsRemovesOnlyAgedStaging(t *testing.T) { @@ -474,12 +440,8 @@ func TestSweepGenerationsRemovesOnlyAgedStaging(t *testing.T) { t.Fatalf("SweepGenerations: %v", err) } - if _, err := os.Stat(aged); !errors.Is(err, fs.ErrNotExist) { - t.Errorf("staging aged past the grace survived the periodic sweep: %v", err) - } - if _, err := os.Stat(fresh); err != nil { - t.Errorf("fresh staging was swept: %v", err) - } + assertGone(t, aged, "staging aged past the grace survived the periodic sweep") + assertPresent(t, fresh, "fresh staging was swept") } func TestSweepGenerationsToleratesConcurrentDelete(t *testing.T) { @@ -713,3 +675,17 @@ func setAge(t *testing.T, path string, age time.Duration) { t.Fatalf("backdate %s: %v", path, err) } } + +func assertGone(t *testing.T, path, format string, args ...any) { + t.Helper() + if _, err := os.Stat(path); !errors.Is(err, fs.ErrNotExist) { + t.Errorf(format+": %v", append(args, err)...) + } +} + +func assertPresent(t *testing.T, path, format string, args ...any) { + t.Helper() + if _, err := os.Stat(path); err != nil { + t.Errorf(format+": %v", append(args, err)...) + } +} From 6e15957c249d5a7c0ae7a65d3065bd13dcca9144 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:59:21 +0800 Subject: [PATCH 116/127] tunnet: one overlay fixture --- sandboxd/tunnet/activity_test.go | 2 +- sandboxd/tunnet/bench_test.go | 2 +- sandboxd/tunnet/router_test.go | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sandboxd/tunnet/activity_test.go b/sandboxd/tunnet/activity_test.go index b88faef28..f7bbe2c72 100644 --- a/sandboxd/tunnet/activity_test.go +++ b/sandboxd/tunnet/activity_test.go @@ -8,7 +8,7 @@ import ( func TestOnlyAdmittedTrafficTouchesClaim(t *testing.T) { u := &fakeUnder{out: make(chan sent, 64)} - r := New(u, &Overlay{Subnet: subnet, VNI: 7, Port: 4789, MTU: 1450, Exits: map[string]Exit{"default": {VTEP: vtep, MAC: exitMAC}}}) + r := New(u, testOverlay(1450)) touches := 0 r.Arm(Claim{ ID: "c1", Addr: claimAddr, diff --git a/sandboxd/tunnet/bench_test.go b/sandboxd/tunnet/bench_test.go index d727c2b8e..3f304348c 100644 --- a/sandboxd/tunnet/bench_test.go +++ b/sandboxd/tunnet/bench_test.go @@ -53,7 +53,7 @@ func BenchmarkDeliver(b *testing.B) { } func benchLink() *link { - r := New(nopUnder{}, &Overlay{Subnet: subnet, VNI: 7, Port: 4789, MTU: 1450, Exits: map[string]Exit{"default": {VTEP: vtep, MAC: exitMAC}}}) + r := New(nopUnder{}, testOverlay(1450)) c := Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }} r.Arm(c) a := r.claims[claimAddr] diff --git a/sandboxd/tunnet/router_test.go b/sandboxd/tunnet/router_test.go index 0c9fd77bf..722895b2e 100644 --- a/sandboxd/tunnet/router_test.go +++ b/sandboxd/tunnet/router_test.go @@ -280,7 +280,7 @@ func TestRouterKeepsTheMTUAStreamWasToldUntilItsNextHello(t *testing.T) { u.sent(t) guest := serve(t, r, claimAddr) guestHello(t, guest) - r.Reload(&Overlay{Subnet: subnet, VNI: 7, Port: 4789, MTU: 1410, Exits: map[string]Exit{"default": {VTEP: vtep, MAC: exitMAC}}}) + r.Reload(testOverlay(1410)) u.sent(t) full := gsoFrame(ProtoUDP, 1412, 1412) full[0], full[1] = 0, gsoNone @@ -355,11 +355,15 @@ func newRouter(t *testing.T) (*fakeUnder, *Router) { t.Helper() u := &fakeUnder{out: make(chan sent, 64), in: make(chan []sent, 16), done: make(chan struct{})} t.Cleanup(func() { close(u.done) }) - r := New(u, &Overlay{Subnet: subnet, VNI: 7, Port: 4789, MTU: 1450, Exits: map[string]Exit{"default": {VTEP: vtep, MAC: exitMAC}}}) + r := New(u, testOverlay(1450)) go func() { _ = r.Run(t.Context()) }() return u, r } +func testOverlay(mtu uint16) *Overlay { + return &Overlay{Subnet: subnet, VNI: 7, Port: 4789, MTU: mtu, Exits: map[string]Exit{"default": {VTEP: vtep, MAC: exitMAC}}} +} + func serve(t *testing.T, r *Router, addr netip.Addr) net.Conn { t.Helper() host, guest := net.Pipe() From 915ae33c8095e7fcbac4c2a125e1afd47a5475ce Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 16:59:34 +0800 Subject: [PATCH 117/127] types: build the volume fixtures from the limit, drop the bare mutex benchmark --- sandboxd/types/bench_test.go | 21 +-------------------- sandboxd/types/volume_test.go | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 31 deletions(-) diff --git a/sandboxd/types/bench_test.go b/sandboxd/types/bench_test.go index 279e571e4..5f81488b8 100644 --- a/sandboxd/types/bench_test.go +++ b/sandboxd/types/bench_test.go @@ -1,10 +1,6 @@ package types -import ( - "sync" - "testing" - "time" -) +import "testing" func BenchmarkSandboxTouch(b *testing.B) { sb := &Sandbox{} @@ -14,18 +10,3 @@ func BenchmarkSandboxTouch(b *testing.B) { } }) } - -func BenchmarkSandboxTouchMutex(b *testing.B) { - var ( - mu sync.Mutex - t time.Time - ) - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - mu.Lock() - t = time.Now() - mu.Unlock() - } - }) - _ = t -} diff --git a/sandboxd/types/volume_test.go b/sandboxd/types/volume_test.go index 79947d597..6fde38cc7 100644 --- a/sandboxd/types/volume_test.go +++ b/sandboxd/types/volume_test.go @@ -45,16 +45,7 @@ func TestValidateVolumes(t *testing.T) { if !slices.Equal(volumes, wantInput) { t.Errorf("input mutated to %v, want %v", volumes, wantInput) } - if got, err := ValidateVolumes([]Volume{ - {Name: "a"}, - {Name: "b"}, - {Name: "c"}, - {Name: "d"}, - {Name: "e"}, - {Name: "f"}, - {Name: "g"}, - {Name: "h"}, - }); err != nil || len(got) != MaxClaimVolumes { + if got, err := ValidateVolumes(namedVolumes(MaxClaimVolumes)); err != nil || len(got) != MaxClaimVolumes { t.Errorf("ValidateVolumes at limit: got=%v err=%v", got, err) } @@ -62,7 +53,7 @@ func TestValidateVolumes(t *testing.T) { name string volumes []Volume }{ - {"too-many", []Volume{{Name: "a"}, {Name: "b"}, {Name: "c"}, {Name: "d"}, {Name: "e"}, {Name: "f"}, {Name: "g"}, {Name: "h"}, {Name: "i"}}}, + {"too-many", namedVolumes(MaxClaimVolumes + 1)}, {"invalid-name", []Volume{{Name: "bad/name"}}}, {"duplicate-name", []Volume{{Name: "dataset"}, {Name: "dataset", Mount: "/other"}}}, {"relative-mount", []Volume{{Name: "dataset", Mount: "data"}}}, @@ -167,3 +158,11 @@ func TestValidateMixedVolumesPreservesPerEntryIntent(t *testing.T) { t.Fatalf("second validation changed mixed intent: %+v, %v", again, err) } } + +func namedVolumes(n int) []Volume { + volumes := make([]Volume, n) + for i := range volumes { + volumes[i].Name = string(rune('a' + i)) + } + return volumes +} From 9fcb7f8201e254ff188be19203fa7c60680123fb Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 17:01:19 +0800 Subject: [PATCH 118/127] sdk/go: shared server builders, table the redirect fallback tests, counting sandbox for keep-alive, git plumbing asserts the guest request --- sdk/go/checkpoint_test.go | 10 +- sdk/go/client_test.go | 306 +++++++++++++++----------------------- sdk/go/files_test.go | 7 - sdk/go/git_test.go | 26 +++- sdk/go/keepalive_test.go | 42 ++---- sdk/go/pools_test.go | 3 +- sdk/go/tenants_test.go | 3 +- sdk/go/volumes_test.go | 4 +- 8 files changed, 174 insertions(+), 227 deletions(-) diff --git a/sdk/go/checkpoint_test.go b/sdk/go/checkpoint_test.go index 18ec77867..225e543be 100644 --- a/sdk/go/checkpoint_test.go +++ b/sdk/go/checkpoint_test.go @@ -23,7 +23,7 @@ func TestCheckpointNewFollowsRedirect(t *testing.T) { nodeA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { nodeACalls++ _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(nodeB.URL, "http://")}, + Redirect: []string{addrOf(nodeB)}, }) })) t.Cleanup(nodeA.Close) @@ -37,7 +37,7 @@ func TestCheckpointNewFollowsRedirect(t *testing.T) { if sb.ID != "sb_ck1" { t.Errorf("id %q, want sb_ck1 (claimed at the redirect target)", sb.ID) } - if sb.owner != strings.TrimPrefix(nodeB.URL, "http://") { + if sb.owner != addrOf(nodeB) { t.Errorf("owner %q, want node B", sb.owner) } if nodeACalls != 1 { @@ -78,7 +78,7 @@ func TestCheckpointNewRedirectAllCandidatesFail(t *testing.T) { entryCalls++ if entryCalls == 1 { _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(broken.URL, "http://")}, + Redirect: []string{addrOf(broken)}, }) return } @@ -114,7 +114,7 @@ func TestCheckpointNewRedirectFallbackHeals(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { entryCalls++ if entryCalls == 1 { - _ = json.MarshalWrite(w, claimResponse{Redirect: []string{strings.TrimPrefix(busy.URL, "http://")}}) + _ = json.MarshalWrite(w, claimResponse{Redirect: []string{addrOf(busy)}}) return } var req checkpointClaimRequest @@ -165,7 +165,7 @@ func TestCheckpointNewSecondLevelRedirectFails(t *testing.T) { nodeA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(nodeB.URL, "http://")}, + Redirect: []string{addrOf(nodeB)}, }) })) t.Cleanup(nodeA.Close) diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 05d462eeb..59d82885d 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -137,7 +137,7 @@ func TestCloseReleasesASandboxThatMoved(t *testing.T) { adopter = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/v1/sandboxes/sb_9/owner": - _ = json.MarshalWrite(w, map[string]string{"owner_addr": strings.TrimPrefix(adopter.URL, "http://")}) + _ = json.MarshalWrite(w, map[string]string{"owner_addr": addrOf(adopter)}) case "/v1/sandboxes/sb_9/release": released.Store(true) w.WriteHeader(http.StatusNoContent) @@ -146,7 +146,7 @@ func TestCloseReleasesASandboxThatMoved(t *testing.T) { t.Cleanup(adopter.Close) old := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/peers" { - _ = json.MarshalWrite(w, map[string]any{"peers": []string{strings.TrimPrefix(adopter.URL, "http://")}}) + _ = json.MarshalWrite(w, map[string]any{"peers": []string{addrOf(adopter)}}) return } w.WriteHeader(http.StatusNotFound) @@ -218,10 +218,10 @@ func TestAWriterErrorTaggedForThisSandboxIsNotReplayed(t *testing.T) { }) owner := newAgentServer(t, fake.ServeConn) entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, map[string]string{"owner_addr": strings.TrimPrefix(other.URL, "http://")}) + _ = json.MarshalWrite(w, map[string]string{"owner_addr": addrOf(other)}) })) t.Cleanup(entry.Close) - sb := &Sandbox{ID: "sb_1", c: testClient(t, entry), token: "tok", owner: strings.TrimPrefix(owner.URL, "http://")} + sb := &Sandbox{ID: "sb_1", c: testClient(t, entry), token: "tok", owner: addrOf(owner)} t.Cleanup(sb.pool.drain) tagged := writerFunc(func([]byte) (int, error) { return 0, &ownerGoneError{err: syscall.ECONNREFUSED, id: sb.ID, addr: sb.Owner()} @@ -247,11 +247,11 @@ func TestAnotherSandboxsErrorInAWriterIsNotReplayed(t *testing.T) { w.WriteHeader(http.StatusNotFound) return } - _ = json.MarshalWrite(w, map[string]string{"owner_addr": strings.TrimPrefix(other.URL, "http://")}) + _ = json.MarshalWrite(w, map[string]string{"owner_addr": addrOf(other)}) })) t.Cleanup(entry.Close) c := testClient(t, entry) - sb := &Sandbox{ID: "sb_1", c: c, token: "tok", owner: strings.TrimPrefix(owner.URL, "http://")} + sb := &Sandbox{ID: "sb_1", c: c, token: "tok", owner: addrOf(owner)} gone := &Sandbox{ID: "sb_2", c: c, token: "tok", owner: retiredAddr(t)} t.Cleanup(sb.pool.drain) writeElsewhere := writerFunc(func([]byte) (int, error) { @@ -284,7 +284,7 @@ func TestCloseFollowsAMoveOffARetiredNode(t *testing.T) { adopter = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/v1/sandboxes/sb_9/owner": - _ = json.MarshalWrite(w, map[string]string{"owner_addr": strings.TrimPrefix(adopter.URL, "http://")}) + _ = json.MarshalWrite(w, map[string]string{"owner_addr": addrOf(adopter)}) case "/v1/sandboxes/sb_9/release": released.Store(true) w.WriteHeader(http.StatusNoContent) @@ -317,17 +317,12 @@ func TestNewFollowsRedirect(t *testing.T) { var nodeB *httptest.Server nodeB = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.MarshalWrite(w, claimResponse{ - ID: "sb_2", Token: "tok", OwnerAddr: strings.TrimPrefix(nodeB.URL, "http://"), + ID: "sb_2", Token: "tok", OwnerAddr: addrOf(nodeB), }) })) t.Cleanup(nodeB.Close) - nodeA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(nodeB.URL, "http://")}, - }) - })) - t.Cleanup(nodeA.Close) + nodeA := redirectEntry(t, addrOf(nodeB)) sb, err := testClient(t, nodeA).New(t.Context(), "rt:24.04") if err != nil { @@ -336,7 +331,7 @@ func TestNewFollowsRedirect(t *testing.T) { if sb.ID != "sb_2" { t.Errorf("id %q, want sb_2 (claimed at the redirect target)", sb.ID) } - if sb.owner != strings.TrimPrefix(nodeB.URL, "http://") { + if sb.owner != addrOf(nodeB) { t.Errorf("owner %q, want node B", sb.owner) } } @@ -354,12 +349,7 @@ func TestDeleteTemplateFollowsRedirect(t *testing.T) { })) t.Cleanup(nodeB.Close) - nodeA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(nodeB.URL, "http://")}, - }) - })) - t.Cleanup(nodeA.Close) + nodeA := redirectEntry(t, addrOf(nodeB)) if err := testClient(t, nodeA).DeleteTemplate(t.Context(), "tpl"); err != nil { t.Fatalf("DeleteTemplate: %v", err) @@ -373,31 +363,11 @@ func TestDeleteTemplateFollowsRedirect(t *testing.T) { } func TestDeleteTemplateRetryPolicy(t *testing.T) { - status := func(code int, hit *bool) *httptest.Server { - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - *hit = true - w.WriteHeader(code) - })) - t.Cleanup(s.Close) - return s - } - entry := func(candidates ...*httptest.Server) *httptest.Server { - addrs := make([]string, len(candidates)) - for i, c := range candidates { - addrs[i] = strings.TrimPrefix(c.URL, "http://") - } - s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, claimResponse{Redirect: addrs}) - })) - t.Cleanup(s.Close) - return s - } - t.Run("miss walks to the next owner", func(t *testing.T) { var missHit, ownerHit bool - miss := status(http.StatusNotFound, &missHit) - owner := status(http.StatusNoContent, &ownerHit) - if err := testClient(t, entry(miss, owner)).DeleteTemplate(t.Context(), "tpl"); err != nil { + miss := statusServer(t, http.StatusNotFound, &missHit) + owner := statusServer(t, http.StatusNoContent, &ownerHit) + if err := testClient(t, redirectEntry(t, addrOf(miss), addrOf(owner))).DeleteTemplate(t.Context(), "tpl"); err != nil { t.Fatalf("DeleteTemplate: %v", err) } if !missHit || !ownerHit { @@ -406,9 +376,9 @@ func TestDeleteTemplateRetryPolicy(t *testing.T) { }) t.Run("served error stops the walk", func(t *testing.T) { var brokenHit, ownerHit bool - broken := status(http.StatusInternalServerError, &brokenHit) - owner := status(http.StatusNoContent, &ownerHit) - err := testClient(t, entry(broken, owner)).DeleteTemplate(t.Context(), "tpl") + broken := statusServer(t, http.StatusInternalServerError, &brokenHit) + owner := statusServer(t, http.StatusNoContent, &ownerHit) + err := testClient(t, redirectEntry(t, addrOf(broken), addrOf(owner))).DeleteTemplate(t.Context(), "tpl") if err == nil || !strings.Contains(err.Error(), "http 500") { t.Fatalf("err %v, want the 500 surfaced", err) } @@ -428,12 +398,7 @@ func TestRedirectSetsNoRedirect(t *testing.T) { })) t.Cleanup(target.Close) - entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(target.URL, "http://")}, - }) - })) - t.Cleanup(entry.Close) + entry := redirectEntry(t, addrOf(target)) sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") if err != nil { @@ -447,25 +412,28 @@ func TestRedirectSetsNoRedirect(t *testing.T) { } } -func TestRedirectTriesAllCandidates(t *testing.T) { - good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, claimResponse{ID: "sb_4", Token: "t"}) - })) - t.Cleanup(good.Close) - - entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{"127.0.0.1:1", strings.TrimPrefix(good.URL, "http://")}, +func TestRedirectWalksPastACandidateThatRefusedTheClaim(t *testing.T) { + for _, tt := range []struct { + name string + first func(*testing.T) string + }{ + {"unreachable", func(*testing.T) string { return "127.0.0.1:1" }}, + {"forbidden", func(t *testing.T) string { return addrOf(statusServer(t, http.StatusForbidden, nil)) }}, + } { + t.Run(tt.name, func(t *testing.T) { + good := serveFunc(t, func(w http.ResponseWriter, _ *http.Request) { + _ = json.MarshalWrite(w, claimResponse{ID: "sb_5", Token: "t"}) + }) + entry := redirectEntry(t, tt.first(t), addrOf(good)) + + sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") + if err != nil { + t.Fatalf("New: %v", err) + } + if sb.ID != "sb_5" { + t.Errorf("id %q, want sb_5 (second candidate, reached despite the first's refusal)", sb.ID) + } }) - })) - t.Cleanup(entry.Close) - - sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") - if err != nil { - t.Fatalf("New: %v", err) - } - if sb.ID != "sb_4" { - t.Errorf("id %q, want sb_4 (second candidate)", sb.ID) } } @@ -488,7 +456,7 @@ func TestRedirectStopsWhenACandidateMayHaveClaimed(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { entryCalls.Add(1) _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(lost.URL, "http://"), strings.TrimPrefix(good.URL, "http://")}, + Redirect: []string{addrOf(lost), addrOf(good)}, }) })) t.Cleanup(entry.Close) @@ -501,72 +469,55 @@ func TestRedirectStopsWhenACandidateMayHaveClaimed(t *testing.T) { } } -func TestRedirectCandidateDefinitiveErrorStillTriesNextCandidate(t *testing.T) { - forbidden := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusForbidden) - })) - t.Cleanup(forbidden.Close) - - good := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, claimResponse{ID: "sb_5", Token: "t"}) - })) - t.Cleanup(good.Close) +func TestRedirectFallbackProvisionsAtTheOrigin(t *testing.T) { + for _, tt := range []struct { + name string + code int + body string + }{ + {"candidate full", http.StatusTooManyRequests, ""}, + {"rotating token", http.StatusUnauthorized, "invalid api token"}, + } { + t.Run(tt.name, func(t *testing.T) { + var candidateCalls, entryCalls int + candidate := serveFunc(t, func(w http.ResponseWriter, _ *http.Request) { + candidateCalls++ + w.WriteHeader(tt.code) + if tt.body != "" { + _ = json.MarshalWrite(w, errorResponse{Error: tt.body}) + } + }) + entry := serveFunc(t, func(w http.ResponseWriter, r *http.Request) { + entryCalls++ + if entryCalls == 1 { + _ = json.MarshalWrite(w, claimResponse{Redirect: []string{addrOf(candidate)}}) + return + } + var req claimRequest + _ = json.UnmarshalRead(r.Body, &req) + if !req.NoRedirect { + t.Error("origin fallback did not carry no_redirect") + } + _ = json.MarshalWrite(w, claimResponse{ID: "sb_origin", Token: "t"}) + }) - entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(forbidden.URL, "http://"), strings.TrimPrefix(good.URL, "http://")}, + sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") + if err != nil { + t.Fatalf("New: %v", err) + } + if sb.ID != "sb_origin" { + t.Errorf("id %q, want sb_origin (provisioned at the origin)", sb.ID) + } + if sb.owner != addrOf(entry) { + t.Errorf("owner %q, want the origin", sb.owner) + } + if entryCalls != 2 { + t.Errorf("origin called %d times, want 2 (initial + fallback)", entryCalls) + } + if candidateCalls != 1 { + t.Errorf("candidate called %d times, want 1", candidateCalls) + } }) - })) - t.Cleanup(entry.Close) - - sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") - if err != nil { - t.Fatalf("New: %v", err) - } - if sb.ID != "sb_5" { - t.Errorf("id %q, want sb_5 (second candidate, reached despite the first's 403)", sb.ID) - } -} - -func TestRedirectFallbackHeals(t *testing.T) { - var fullCalls int - full := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - fullCalls++ - w.WriteHeader(http.StatusTooManyRequests) - })) - t.Cleanup(full.Close) - - var entryCalls int - entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - entryCalls++ - if entryCalls == 1 { - _ = json.MarshalWrite(w, claimResponse{Redirect: []string{strings.TrimPrefix(full.URL, "http://")}}) - return - } - var req claimRequest - _ = json.UnmarshalRead(r.Body, &req) - if !req.NoRedirect { - t.Error("origin fallback did not carry no_redirect") - } - _ = json.MarshalWrite(w, claimResponse{ID: "sb_healed", Token: "t"}) - })) - t.Cleanup(entry.Close) - - sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") - if err != nil { - t.Fatalf("New: %v", err) - } - if sb.ID != "sb_healed" { - t.Errorf("id %q, want sb_healed", sb.ID) - } - if sb.owner != strings.TrimPrefix(entry.URL, "http://") { - t.Errorf("owner %q, want the origin", sb.owner) - } - if entryCalls != 2 { - t.Errorf("origin called %d times, want 2 (initial + fallback)", entryCalls) - } - if fullCalls != 1 { - t.Errorf("candidate called %d times, want 1", fullCalls) } } @@ -583,7 +534,7 @@ func TestRedirectFallbackAlsoFails(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { entryCalls++ if entryCalls == 1 { - _ = json.MarshalWrite(w, claimResponse{Redirect: []string{strings.TrimPrefix(full.URL, "http://")}}) + _ = json.MarshalWrite(w, claimResponse{Redirect: []string{addrOf(full)}}) return } w.WriteHeader(http.StatusConflict) @@ -633,7 +584,7 @@ func TestRedirectDefinitiveErrorSkipsFallback(t *testing.T) { var entryCalls int entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { entryCalls++ - _ = json.MarshalWrite(w, claimResponse{Redirect: []string{strings.TrimPrefix(bad.URL, "http://")}}) + _ = json.MarshalWrite(w, claimResponse{Redirect: []string{addrOf(bad)}}) })) t.Cleanup(entry.Close) @@ -648,48 +599,11 @@ func TestRedirectDefinitiveErrorSkipsFallback(t *testing.T) { } } -func TestRedirectRotating401CandidateFallsBackToOrigin(t *testing.T) { - var staleCalls int - stale := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - staleCalls++ - w.WriteHeader(http.StatusUnauthorized) - _ = json.MarshalWrite(w, errorResponse{Error: "invalid api token"}) - })) - t.Cleanup(stale.Close) - - var entryCalls int - entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - entryCalls++ - if entryCalls == 1 { - _ = json.MarshalWrite(w, claimResponse{Redirect: []string{strings.TrimPrefix(stale.URL, "http://")}}) - return - } - var req claimRequest - _ = json.UnmarshalRead(r.Body, &req) - if !req.NoRedirect { - t.Error("origin fallback did not carry no_redirect") - } - _ = json.MarshalWrite(w, claimResponse{ID: "sb_local", Token: "t"}) - })) - t.Cleanup(entry.Close) - - sb, err := testClient(t, entry).New(t.Context(), "rt:24.04") - if err != nil { - t.Fatalf("New: %v", err) - } - if sb.ID != "sb_local" { - t.Errorf("id %q, want sb_local (provisioned at the already-authorized origin)", sb.ID) - } - if staleCalls != 1 || entryCalls != 2 { - t.Errorf("stale=%d entry=%d calls, want 1 and 2 (redirect + fallback)", staleCalls, entryCalls) - } -} - func TestLookupScatter(t *testing.T) { var owner *httptest.Server owner = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/sandboxes/sb_1/owner" && r.Header.Get("Authorization") == "Bearer tok" { - _ = json.MarshalWrite(w, map[string]string{"owner_addr": strings.TrimPrefix(owner.URL, "http://")}) + _ = json.MarshalWrite(w, map[string]string{"owner_addr": addrOf(owner)}) return } w.WriteHeader(http.StatusNotFound) @@ -701,7 +615,7 @@ func TestLookupScatter(t *testing.T) { case "/v1/sandboxes/sb_1/owner": w.WriteHeader(http.StatusNotFound) case "/v1/peers": - _ = json.MarshalWrite(w, map[string]any{"peers": []string{strings.TrimPrefix(owner.URL, "http://")}}) + _ = json.MarshalWrite(w, map[string]any{"peers": []string{addrOf(owner)}}) } })) t.Cleanup(entry.Close) @@ -710,7 +624,7 @@ func TestLookupScatter(t *testing.T) { if err != nil { t.Fatalf("Lookup: %v", err) } - if sb.ID != "sb_1" || sb.owner != strings.TrimPrefix(owner.URL, "http://") { + if sb.ID != "sb_1" || sb.owner != addrOf(owner) { t.Errorf("handle %+v, want owner=B", sb) } } @@ -764,7 +678,7 @@ func movedSandbox(t *testing.T) (*Sandbox, string) { fake.ServeConn(conn) }) mux.HandleFunc("GET /v1/sandboxes/sb_1/owner", func(w http.ResponseWriter, _ *http.Request) { - _ = json.MarshalWrite(w, map[string]string{"owner_addr": strings.TrimPrefix(adopter.URL, "http://")}) + _ = json.MarshalWrite(w, map[string]string{"owner_addr": addrOf(adopter)}) }) mux.HandleFunc("POST /v1/sandboxes/sb_1/renew", func(w http.ResponseWriter, _ *http.Request) { _ = json.MarshalWrite(w, map[string]time.Time{"deadline": time.Now().Add(time.Minute)}) @@ -773,14 +687,14 @@ func movedSandbox(t *testing.T) (*Sandbox, string) { t.Cleanup(adopter.Close) old := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/peers" { - _ = json.MarshalWrite(w, map[string]any{"peers": []string{strings.TrimPrefix(adopter.URL, "http://")}}) + _ = json.MarshalWrite(w, map[string]any{"peers": []string{addrOf(adopter)}}) return } w.WriteHeader(http.StatusNotFound) })) t.Cleanup(old.Close) sb := testSandbox(t, old) - return sb, strings.TrimPrefix(adopter.URL, "http://") + return sb, addrOf(adopter) } func retiredAddr(t *testing.T) string { @@ -794,9 +708,37 @@ func retiredAddr(t *testing.T) string { return addr } +func serveFunc(t testing.TB, h http.HandlerFunc) *httptest.Server { + t.Helper() + ts := httptest.NewServer(h) + t.Cleanup(ts.Close) + return ts +} + +func statusServer(t testing.TB, code int, hit *bool) *httptest.Server { + t.Helper() + return serveFunc(t, func(w http.ResponseWriter, _ *http.Request) { + if hit != nil { + *hit = true + } + w.WriteHeader(code) + }) +} + +func redirectEntry(t testing.TB, candidates ...string) *httptest.Server { + t.Helper() + return serveFunc(t, func(w http.ResponseWriter, _ *http.Request) { + _ = json.MarshalWrite(w, claimResponse{Redirect: candidates}) + }) +} + +func addrOf(ts *httptest.Server) string { + return strings.TrimPrefix(ts.URL, "http://") +} + func testClient(t testing.TB, ts *httptest.Server, opts ...ClientOption) *Client { t.Helper() - c, err := Connect(strings.TrimPrefix(ts.URL, "http://"), opts...) + c, err := Connect(addrOf(ts), opts...) if err != nil { t.Fatalf("connect: %v", err) } diff --git a/sdk/go/files_test.go b/sdk/go/files_test.go index eb9dc7040..85da86c2c 100644 --- a/sdk/go/files_test.go +++ b/sdk/go/files_test.go @@ -98,13 +98,6 @@ func TestReadFileToStopsWhenTheWriterRefuses(t *testing.T) { } } -func TestReadMissingFileErrors(t *testing.T) { - sb := fakeSandbox(t) - if _, err := sb.ReadFile(t.Context(), "/nope"); err == nil { - t.Error("read of missing file succeeded") - } -} - func TestSessionLifecycle(t *testing.T) { sb := fakeSandbox(t) ctx := t.Context() diff --git a/sdk/go/git_test.go b/sdk/go/git_test.go index 37f16f720..b67090eb7 100644 --- a/sdk/go/git_test.go +++ b/sdk/go/git_test.go @@ -1,8 +1,11 @@ package sandbox import ( + "net" "slices" "testing" + + "github.com/cocoonstack/sandbox/protocol/wire" ) func TestGitBranchLifecycle(t *testing.T) { @@ -39,11 +42,32 @@ func TestGitBranchLifecycle(t *testing.T) { } func TestGitPushPullPlumbing(t *testing.T) { - sb := fakeSandbox(t) + reqs := make(chan wire.Request, 2) + sb := testSandbox(t, newAgentServer(t, func(conn net.Conn) { + defer func() { _ = conn.Close() }() + sc := wire.NewFrameScanner(conn) + for sc.Scan() { + req, err := wire.DecodeRequest(sc.Bytes()) + if err != nil { + return + } + reqs <- req + line, _ := wire.EncodeResponse(wire.Done{}) + if _, err = conn.Write(append(line, '\n')); err != nil { + return + } + } + })) if err := sb.GitPush(t.Context(), "/repo", ""); err != nil { t.Errorf("push: %v", err) } if err := sb.GitPull(t.Context(), "/repo", "tok"); err != nil { t.Errorf("pull: %v", err) } + if push, ok := (<-reqs).(*wire.GitPush); !ok || push.Path != "/repo" || push.Auth != "" { + t.Errorf("guest saw %+v, want a git push of /repo", push) + } + if pull, ok := (<-reqs).(*wire.GitPull); !ok || pull.Path != "/repo" || pull.Auth != "tok" { + t.Errorf("guest saw %+v, want a git pull of /repo with the token", pull) + } } diff --git a/sdk/go/keepalive_test.go b/sdk/go/keepalive_test.go index ef89a96a7..85f001986 100644 --- a/sdk/go/keepalive_test.go +++ b/sdk/go/keepalive_test.go @@ -14,12 +14,7 @@ import ( ) func TestCallsShareOneConnection(t *testing.T) { - var upgrades atomic.Int32 - fake := silkdtest.NewFake(t.TempDir()) - sb := testSandbox(t, newAgentServer(t, func(c net.Conn) { - upgrades.Add(1) - fake.ServeConn(c) - })) + sb, upgrades := countingSandbox(t, silkdtest.NewFake(t.TempDir()).ServeConn) for range 3 { if _, err := sb.Stat(t.Context(), "/"); err != nil { t.Fatalf("stat: %v", err) @@ -40,12 +35,7 @@ func TestCallsShareOneConnection(t *testing.T) { } func TestRefusedStreamOpenKeepsItsConnection(t *testing.T) { - var upgrades atomic.Int32 - fake := silkdtest.NewFake(t.TempDir()) - sb := testSandbox(t, newAgentServer(t, func(c net.Conn) { - upgrades.Add(1) - fake.ServeConn(c) - })) + sb, upgrades := countingSandbox(t, silkdtest.NewFake(t.TempDir()).ServeConn) for range 3 { if _, err := sb.Watch(t.Context(), "/missing", false); err == nil { t.Fatal("watch on a missing path succeeded") @@ -72,11 +62,7 @@ func TestCanceledCallSendsNothingOnAParkedConnection(t *testing.T) { } func TestKeepAliveOffDialsPerCall(t *testing.T) { - var upgrades atomic.Int32 - sb := testSandbox(t, newAgentServer(t, func(c net.Conn) { - upgrades.Add(1) - silkdtest.ServeConn(c) - }), WithKeepAlive(0)) + sb, upgrades := countingSandbox(t, silkdtest.ServeConn, WithKeepAlive(0)) for range 3 { if _, err := sb.Exec(t.Context(), "echo", "42"); err != nil { t.Fatalf("exec: %v", err) @@ -105,11 +91,7 @@ func TestIdleConnectionCloses(t *testing.T) { } func TestStalledStdinKeepsItsConnectionOut(t *testing.T) { - var upgrades atomic.Int32 - sb := testSandbox(t, newAgentServer(t, func(c net.Conn) { - upgrades.Add(1) - silkdtest.ServeConn(c) - })) + sb, upgrades := countingSandbox(t, silkdtest.ServeConn) pr, pw := io.Pipe() t.Cleanup(func() { _ = pw.Close() }) if code, err := sb.Run(t.Context(), Cmd{Argv: []string{"echo", "hi"}, Stdin: pr}); err != nil || code != 0 { @@ -127,14 +109,12 @@ func TestPeerHangUpIsNoticedBeforeReuse(t *testing.T) { if !canProbe { t.Skip("no parked-connection probe on this platform") } - var upgrades atomic.Int32 gone := make(chan struct{}, 2) fake := silkdtest.NewFake(t.TempDir()) - sb := testSandbox(t, newAgentServer(t, func(c net.Conn) { - upgrades.Add(1) + sb, upgrades := countingSandbox(t, func(c net.Conn) { fake.ServeConn(&hangUpAfterReply{Conn: c}) gone <- struct{}{} - })) + }) if _, err := sb.Stat(t.Context(), "/"); err != nil { t.Fatalf("stat: %v", err) } @@ -184,6 +164,16 @@ func TestPeerQuiet(t *testing.T) { waitUntil(t, func() bool { return !probe.quiet() }, "hang-up not noticed") } +func countingSandbox(t *testing.T, serve func(net.Conn), opts ...ClientOption) (*Sandbox, *atomic.Int32) { + t.Helper() + var upgrades atomic.Int32 + sb := testSandbox(t, newAgentServer(t, func(c net.Conn) { + upgrades.Add(1) + serve(c) + }), opts...) + return sb, &upgrades +} + func waitUntil(t *testing.T, cond func() bool, msg string) { t.Helper() for start := time.Now(); time.Since(start) < 3*time.Second; time.Sleep(time.Millisecond) { diff --git a/sdk/go/pools_test.go b/sdk/go/pools_test.go index 1046da775..93beb763d 100644 --- a/sdk/go/pools_test.go +++ b/sdk/go/pools_test.go @@ -4,7 +4,6 @@ import ( "encoding/json/v2" "net/http" "net/http/httptest" - "strings" "sync" "testing" ) @@ -49,7 +48,7 @@ func TestSetPoolsClusterFansOut(t *testing.T) { } peer := httptest.NewServer(http.HandlerFunc(pools)) defer peer.Close() - peerAddr := strings.TrimPrefix(peer.URL, "http://") + peerAddr := addrOf(peer) entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/peers" { diff --git a/sdk/go/tenants_test.go b/sdk/go/tenants_test.go index d55bedf0f..f37cab5e2 100644 --- a/sdk/go/tenants_test.go +++ b/sdk/go/tenants_test.go @@ -7,7 +7,6 @@ import ( "maps" "net/http" "net/http/httptest" - "strings" "sync" "testing" ) @@ -81,7 +80,7 @@ func TestDeleteTenantClusterCountsAMissingTenantAsDone(t *testing.T) { defer broken.Close() old := httptest.NewServer(http.NotFoundHandler()) defer old.Close() - peers := []string{strings.TrimPrefix(gone.URL, "http://"), strings.TrimPrefix(broken.URL, "http://"), strings.TrimPrefix(old.URL, "http://")} + peers := []string{addrOf(gone), addrOf(broken), addrOf(old)} entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/v1/peers" { _ = json.MarshalWrite(w, map[string][]string{"peers": peers}) diff --git a/sdk/go/volumes_test.go b/sdk/go/volumes_test.go index e2d46cb75..efd554d9e 100644 --- a/sdk/go/volumes_test.go +++ b/sdk/go/volumes_test.go @@ -94,7 +94,7 @@ func TestTemplateNewVolumeClaimFollowsRedirect(t *testing.T) { entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { originAuth = r.Header.Get("Authorization") _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(target.URL, "http://")}, RequirePromoted: true, + Redirect: []string{addrOf(target)}, RequirePromoted: true, }) })) t.Cleanup(entry.Close) @@ -162,7 +162,7 @@ func TestVolumeClaimRedirectPreservesEntries(t *testing.T) { t.Cleanup(target.Close) entry := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _ = json.MarshalWrite(w, claimResponse{ - Redirect: []string{strings.TrimPrefix(target.URL, "http://")}, + Redirect: []string{addrOf(target)}, }) })) t.Cleanup(entry.Close) From 1e52432d728e3575c90739e66926f39896c746fc Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 17:02:46 +0800 Subject: [PATCH 119/127] store/storetest: drop the helper doc comments --- sandboxd/store/storetest/storetest.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/sandboxd/store/storetest/storetest.go b/sandboxd/store/storetest/storetest.go index ff5ccb0b2..019637ae2 100644 --- a/sandboxd/store/storetest/storetest.go +++ b/sandboxd/store/storetest/storetest.go @@ -153,7 +153,6 @@ func WriteSparse(t testing.TB, path string, size int64, writes map[int64][]byte) return path } -// FileSum hashes the file at path. func FileSum(t testing.TB, path string) [sha256.Size]byte { t.Helper() f, err := os.Open(path) //nolint:gosec // test path @@ -168,7 +167,6 @@ func FileSum(t testing.TB, path string) [sha256.Size]byte { return [sha256.Size]byte(h.Sum(nil)) } -// AllocatedBytes reports the disk blocks path occupies. func AllocatedBytes(t testing.TB, path string) int64 { t.Helper() info, err := os.Stat(path) @@ -178,7 +176,6 @@ func AllocatedBytes(t testing.TB, path string) int64 { return info.Sys().(*syscall.Stat_t).Blocks * 512 } -// WaitStacks polls until at least want goroutine stacks satisfy match. func WaitStacks(t testing.TB, want int, match func(stack string) bool) { t.Helper() buf := make([]byte, 1<<20) From e515cd6a4248cc67cea4ffb8ac4a9ccd693573bd Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 17:05:09 +0800 Subject: [PATCH 120/127] sdk/go, storetest: match the server's VolumePatch tags and range the stack dump lazily --- sandboxd/store/storetest/storetest.go | 2 +- sdk/go/volumes.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sandboxd/store/storetest/storetest.go b/sandboxd/store/storetest/storetest.go index 019637ae2..9bf6781ec 100644 --- a/sandboxd/store/storetest/storetest.go +++ b/sandboxd/store/storetest/storetest.go @@ -181,7 +181,7 @@ func WaitStacks(t testing.TB, want int, match func(stack string) bool) { buf := make([]byte, 1<<20) for range 5000 { matched := 0 - for _, stack := range strings.Split(string(buf[:runtime.Stack(buf, true)]), "\n\n") { + for stack := range strings.SplitSeq(string(buf[:runtime.Stack(buf, true)]), "\n\n") { if match(stack) { matched++ } diff --git a/sdk/go/volumes.go b/sdk/go/volumes.go index bd9f90706..f2e00df9e 100644 --- a/sdk/go/volumes.go +++ b/sdk/go/volumes.go @@ -36,8 +36,8 @@ type SandboxVolumesResponse struct { // VolumePatch adds and removes volumes by name; a nil slice changes nothing. type VolumePatch struct { - Add []Volume `json:"add,omitzero"` - Remove []string `json:"remove,omitzero"` + Add []Volume `json:"add,omitempty"` + Remove []string `json:"remove,omitempty"` } // VolumeError reports a failed volume request; Response is nil unless the reply carried the full view. From e3faff88430dd2bb69bfe937e234a5b6238c4580 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 17:05:43 +0800 Subject: [PATCH 121/127] types, sdk/go: keep omitzero on VolumePatch so an explicit empty list stays on the wire The SDK test pins that an explicit empty add/remove list is sent; omitempty dropped it. --- sandboxd/types/api.go | 4 ++-- sdk/go/volumes.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sandboxd/types/api.go b/sandboxd/types/api.go index 218e0f546..0d81b5407 100644 --- a/sandboxd/types/api.go +++ b/sandboxd/types/api.go @@ -32,8 +32,8 @@ type VolumeRegistration struct { // VolumePatch is the wire body of PATCH /v1/sandboxes/{id}/volumes. type VolumePatch struct { - Add []Volume `json:"add,omitempty"` - Remove []string `json:"remove,omitempty"` + Add []Volume `json:"add,omitzero"` + Remove []string `json:"remove,omitzero"` } // VolumePhase is how far one held volume has converged. diff --git a/sdk/go/volumes.go b/sdk/go/volumes.go index f2e00df9e..bd9f90706 100644 --- a/sdk/go/volumes.go +++ b/sdk/go/volumes.go @@ -36,8 +36,8 @@ type SandboxVolumesResponse struct { // VolumePatch adds and removes volumes by name; a nil slice changes nothing. type VolumePatch struct { - Add []Volume `json:"add,omitempty"` - Remove []string `json:"remove,omitempty"` + Add []Volume `json:"add,omitzero"` + Remove []string `json:"remove,omitzero"` } // VolumeError reports a failed volume request; Response is nil unless the reply carried the full view. From 87829fee211d9535890cc42db291dacf6ef5dec8 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 17:07:10 +0800 Subject: [PATCH 122/127] sdk/go: re-send stream calls whose request is their whole input after an owner moves DialPort, Lsp.Request, OpenPty, Watch, Logs, Attach and WriteFile only rebound and failed once, while the Python SDK re-sends them; the move tag is raised at the dial or upgrade reply, before any frame, so a re-send is safe. Run with Stdin and Push still fail once: they stream the caller's reader. --- docs/sdk.md | 7 ++++--- sdk/go/client_test.go | 21 +++++++++++++++++++++ sdk/go/files.go | 4 +++- sdk/go/port.go | 28 +++++++++++++++------------- sdk/go/proc.go | 28 +++++++++++++++------------- sdk/go/pty.go | 31 ++++++++++++++++--------------- sdk/go/watch.go | 34 ++++++++++++++++++---------------- 7 files changed, 92 insertions(+), 61 deletions(-) diff --git a/docs/sdk.md b/docs/sdk.md index 60e42dc69..ad13c1d42 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -216,9 +216,10 @@ one already gone is not an error, and `Close` is bounded internally so it stays other call follows a move too: a 404 or a refused connection from the owner runs `Lookup` once, and a different owner replaces the old one. A call whose request is its whole input is sent again there: the control calls, `Run` -without `Stdin`, and the file, git, process and session calls that send no -data. `Run` with `Stdin`, `WriteFile`, `Push`, `OpenPty`, `Watch`, `DialPort`, `Lsp.Request`, -`Logs` and `Attach` fail once and the next call reaches the new owner. An +without `Stdin`, `WriteFile`, `OpenPty`, `Watch`, `DialPort`, `Lsp.Request`, +`Logs`, `Attach`, and the file, git, process and session calls. `Run` with +`Stdin` and `Push` stream the caller's reader, so they fail once and the next +call reaches the new owner. An error from the caller's own writer is returned as it is and never retried, and a released sandbox still answers 404. Volume sandboxes and egress-lane sandboxes cannot hibernate, fork, checkpoint, diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 59d82885d..94f4b9dd6 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -183,6 +183,27 @@ func TestAMovedSandboxRetriesAControlCallAtItsNewOwner(t *testing.T) { } } +func TestAMovedSandboxRetriesAStreamCallAtItsNewOwner(t *testing.T) { + for name, call := range map[string]func(*Sandbox) error{ + "write file": func(sb *Sandbox) error { return sb.WriteFile(t.Context(), "/f", []byte("x"), nil) }, + "watch": func(sb *Sandbox) error { + w, err := sb.Watch(t.Context(), "/", false) + if err == nil { + err = w.Close() + } + return err + }, + } { + sb, adopter := movedSandbox(t) + if err := call(sb); err != nil { + t.Errorf("%s after the move: %v", name, err) + } + if got := sb.Owner(); got != adopter { + t.Errorf("%s: owner %s, want the adopter %s", name, got, adopter) + } + } +} + func TestAMovedSandboxDoesNotReplayARunThatStreamedStdin(t *testing.T) { sb, adopter := movedSandbox(t) if _, err := sb.Run(t.Context(), Cmd{Argv: []string{"cat"}, Stdin: strings.NewReader("once")}); err == nil { diff --git a/sdk/go/files.go b/sdk/go/files.go index bd98ac4d4..16f4aea01 100644 --- a/sdk/go/files.go +++ b/sdk/go/files.go @@ -10,7 +10,9 @@ import ( // WriteFile atomically writes data to path in the sandbox; a non-nil mode sets the file's permission bits. func (s *Sandbox) WriteFile(ctx context.Context, path string, data []byte, mode *uint32) error { - return s.uploadRPC(ctx, &wire.FsWrite{Path: path, Mode: mode}, bytes.NewReader(data)) + return s.retryMoved(ctx, func() error { + return s.uploadRPC(ctx, &wire.FsWrite{Path: path, Mode: mode}, bytes.NewReader(data)) + }) } // ReadFile returns the contents of path. diff --git a/sdk/go/port.go b/sdk/go/port.go index 3dd33d7b3..49881e3d0 100644 --- a/sdk/go/port.go +++ b/sdk/go/port.go @@ -111,19 +111,21 @@ func (s *Sandbox) ProxyPort(ctx context.Context, localAddr string, port uint16) } // openStream turns a call into a raw byte stream. -func (s *Sandbox) openStream(ctx context.Context, req wire.Request) (_ *PortConn, err error) { - defer func() { s.rebind(ctx, err) }() - conn, l, err := s.call(ctx, req) - if err != nil { - return nil, err - } - if _, err = expect[wire.Ready](ctx, conn); err != nil { - return nil, l.done(err) - } - pr, pw := io.Pipe() - p := &PortConn{conn: conn, stop: l.close, out: pr} - go p.drain(ctx, pw) - return p, nil +func (s *Sandbox) openStream(ctx context.Context, req wire.Request) (p *PortConn, err error) { + err = s.retryMoved(ctx, func() error { + conn, l, err := s.call(ctx, req) + if err != nil { + return err + } + if _, err = expect[wire.Ready](ctx, conn); err != nil { + return l.done(err) + } + pr, pw := io.Pipe() + p = &PortConn{conn: conn, stop: l.close, out: pr} + go p.drain(ctx, pw) + return nil + }) + return p, err } func (s *Sandbox) proxyConn(ctx context.Context, local net.Conn, port uint16) { diff --git a/sdk/go/proc.go b/sdk/go/proc.go index 17b4c7a02..11f73ca56 100644 --- a/sdk/go/proc.go +++ b/sdk/go/proc.go @@ -52,17 +52,19 @@ func (s *Sandbox) Attach(ctx context.Context, pid uint32, stdout, stderr io.Writ return s.drainProc(ctx, &wire.Attach{PID: pid}, stdout, stderr) } -func (s *Sandbox) drainProc(ctx context.Context, req wire.Request, stdout, stderr io.Writer) (_ int32, _ bool, err error) { - defer func() { s.rebind(ctx, err) }() - conn, l, err := s.call(ctx, req) - if err != nil { - return 0, false, err - } - defer l.close() - _, code, exited, err := pumpStdio(ctx, conn, stdout, stderr) - // Logs closes with done after the exit frame - if _, replay := req.(*wire.Logs); replay && exited && err == nil { - err = terminalErr(ctx, conn) - } - return code, exited, l.done(err) +func (s *Sandbox) drainProc(ctx context.Context, req wire.Request, stdout, stderr io.Writer) (code int32, exited bool, err error) { + err = s.retryMoved(ctx, func() error { + conn, l, err := s.call(ctx, req) + if err != nil { + return err + } + defer l.close() + _, code, exited, err = pumpStdio(ctx, conn, stdout, stderr) + // Logs closes with done after the exit frame + if _, replay := req.(*wire.Logs); replay && exited && err == nil { + err = terminalErr(ctx, conn) + } + return l.done(err) + }) + return code, exited, err } diff --git a/sdk/go/pty.go b/sdk/go/pty.go index f037bd276..ac29a62a9 100644 --- a/sdk/go/pty.go +++ b/sdk/go/pty.go @@ -87,19 +87,20 @@ func (p *Pty) drain(ctx context.Context, pw *io.PipeWriter, stop func()) { } // OpenPty starts a shell whose lifetime is governed by ctx or Close. -func (s *Sandbox) OpenPty(ctx context.Context, opts wire.PtyOpen) (_ *Pty, err error) { - defer func() { s.rebind(ctx, err) }() - conn, l, err := s.call(ctx, &opts) - if err != nil { - return nil, err - } - started, err := expect[wire.Started](ctx, conn) - if err != nil { - return nil, l.done(err) - } - - pr, pw := io.Pipe() - p := &Pty{PID: started.PID, sb: s, conn: conn, stop: l.close, out: pr} - go p.drain(ctx, pw, l.close) - return p, nil +func (s *Sandbox) OpenPty(ctx context.Context, opts wire.PtyOpen) (p *Pty, err error) { + err = s.retryMoved(ctx, func() error { + conn, l, err := s.call(ctx, &opts) + if err != nil { + return err + } + started, err := expect[wire.Started](ctx, conn) + if err != nil { + return l.done(err) + } + pr, pw := io.Pipe() + p = &Pty{PID: started.PID, sb: s, conn: conn, stop: l.close, out: pr} + go p.drain(ctx, pw, l.close) + return nil + }) + return p, err } diff --git a/sdk/go/watch.go b/sdk/go/watch.go index 4f5495ffe..a2e055279 100644 --- a/sdk/go/watch.go +++ b/sdk/go/watch.go @@ -76,20 +76,22 @@ func (w *Watcher) setErr(err error) { } // Watch streams filesystem events under path; events caused after it returns are guaranteed captured. -func (s *Sandbox) Watch(ctx context.Context, path string, recursive bool) (_ *Watcher, err error) { - defer func() { s.rebind(ctx, err) }() - conn, l, err := s.call(ctx, &wire.FsWatch{Path: path, Recursive: recursive}) - if err != nil { - return nil, err - } - if _, err = expect[wire.Ready](ctx, conn); err != nil { - return nil, l.done(err) - } - w := &Watcher{ - events: make(chan wire.Event, 16), - stop: l.close, - closed: make(chan struct{}), - } - go w.drain(ctx, conn) - return w, nil +func (s *Sandbox) Watch(ctx context.Context, path string, recursive bool) (w *Watcher, err error) { + err = s.retryMoved(ctx, func() error { + conn, l, err := s.call(ctx, &wire.FsWatch{Path: path, Recursive: recursive}) + if err != nil { + return err + } + if _, err = expect[wire.Ready](ctx, conn); err != nil { + return l.done(err) + } + w = &Watcher{ + events: make(chan wire.Event, 16), + stop: l.close, + closed: make(chan struct{}), + } + go w.drain(ctx, conn) + return nil + }) + return w, err } From 41e7d5260f45a2a28415328c5147fbafb8575293 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 17:07:53 +0800 Subject: [PATCH 123/127] sdk: refuse a comma-separated seed list instead of silently keeping its first address --- docs/sdk.md | 3 +-- sdk/go/client.go | 9 ++++----- sdk/go/client_test.go | 4 ++-- sdk/python/cocoonsandbox/client.py | 2 +- sdk/python/tests/test_endpoint.py | 2 ++ 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/sdk.md b/docs/sdk.md index ad13c1d42..7ada4ff17 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -103,8 +103,7 @@ client, err := sandbox.Connect("10.0.0.5:7777", sandbox.WithAPIToken(os.Getenv("SANDBOXD_TOKEN"))) ``` -- `Connect(addr, opts...)` — `addr` accepts a comma-separated seed list for - forward compatibility; the current version uses the first entry. +- `Connect(addr, opts...)` — `addr` is one node's address or origin. - `WithAPIToken(token)` — the node token: a root `api_token` (full access) or a tenant token (resource-creating verbs only; operator surfaces like `Info` answer it 403). On a cluster every node shares the same root token diff --git a/sdk/go/client.go b/sdk/go/client.go index 22c18be73..7636f6a91 100644 --- a/sdk/go/client.go +++ b/sdk/go/client.go @@ -99,14 +99,13 @@ type Client struct { keepAlive time.Duration } -// Connect returns a client for a sandboxd node; a comma-separated seed list keeps only its first address. +// Connect returns a client for a sandboxd node. func Connect(addr string, opts ...ClientOption) (*Client, error) { - first, _, _ := strings.Cut(addr, ",") - first = strings.TrimSpace(first) - if first == "" { + addr = strings.TrimSpace(addr) + if addr == "" { return nil, errors.New("empty sandboxd address") } - u, err := endpointURL(first, "http") + u, err := endpointURL(addr, "http") if err != nil { return nil, err } diff --git a/sdk/go/client_test.go b/sdk/go/client_test.go index 94f4b9dd6..4a44bf5e2 100644 --- a/sdk/go/client_test.go +++ b/sdk/go/client_test.go @@ -25,9 +25,9 @@ func TestConnectAddress(t *testing.T) { ok bool }{ {"single", "n1:7777", "n1:7777", true}, - {"seed list uses first", "n1:7777, n2:7777", "n1:7777", true}, + {"seed list refused", "n1:7777, n2:7777", "", false}, {"empty", "", "", false}, - {"only commas", " , ", "", false}, + {"blank", " ", "", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/sdk/python/cocoonsandbox/client.py b/sdk/python/cocoonsandbox/client.py index c98672496..0790dc76d 100644 --- a/sdk/python/cocoonsandbox/client.py +++ b/sdk/python/cocoonsandbox/client.py @@ -43,7 +43,7 @@ def __init__( ssl_context is set to offer ALPN http/1.1, which the relay upgrade needs; pass a dedicated context. """ - endpoint = _endpoint_url(addr.split(",")[0].strip()) + endpoint = _endpoint_url(addr.strip()) self.addr = endpoint.geturl().removeprefix("http://") self._scheme = endpoint.scheme self._ssl_context = ssl_context diff --git a/sdk/python/tests/test_endpoint.py b/sdk/python/tests/test_endpoint.py index 6d6cfdb2e..f7a6c5a96 100644 --- a/sdk/python/tests/test_endpoint.py +++ b/sdk/python/tests/test_endpoint.py @@ -38,6 +38,8 @@ def test_endpoint_url(addr: str, want: str) -> None: "https://node:65536", "https://node:bad", "https://node\r\nX: bad", + "n1:7777,n2:7777", + "n1:7777, n2:7777", ], ) def test_endpoint_rejects_invalid_origins(addr: str) -> None: From afc37d2b068a5d5a036860012ba1a3562b453dea Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 17:17:38 +0800 Subject: [PATCH 124/127] silkd: hang up a pty shell whose client disconnects instead of killing it A SIGKILLed shell cannot pass SIGHUP to its jobs, so a background job started in an abandoned web terminal kept running; ssh delivers the hang-up. The shell now gets SIGHUP and SIGKILL only after a 2 s grace, and the master keeps draining after the client is gone because a session leader exits only once its tty output drains. --- docs/silkd.md | 2 +- silkd/src/pty.rs | 28 +++++++++++++++++++++++----- silkd/tests/pty_e2e.rs | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/docs/silkd.md b/docs/silkd.md index 1bedac49e..6478feffd 100644 --- a/docs/silkd.md +++ b/docs/silkd.md @@ -58,7 +58,7 @@ cannot parse fails CI. | tree | `fs_push {dest}` (+tar as `data` frames) / `fs_pull {path}` | whole trees as tar streams through the guest tar: `fs_pull` streams `data{data}`… → `done`, `fs_push` terminates with `done` | | search | `fs_find {path, pattern, glob?}` → `match{file, line, content}`… → `done` / `fs_replace {files, pattern, replacement}` → `replaced{file, replacements}`… → `done` | regex as data, no shell quoting; `glob` is anchored `*`/`?` wildcards over file names; find skips binary and >8 MiB files, and replace skips the same 8 MiB bound with a zero count. One match whose frame would pass the 8 MiB cap ends the find with an `error`. Replace is atomic per file, not per list: it stops with an `error` at the first missing, unreadable or non-UTF-8 file, and the files before it stay rewritten | | watch | `fs_watch {path, recursive?}` | `ready` once armed (events after it are guaranteed captured), then `event{kind, path}` until the client disconnects or sends another request, which ends the watch with no terminal frame; watcher or delivery-queue overflow errors arrive as a terminal `error` instead of silently losing events | -| pty | `pty_open {cols, rows, cwd?, env?, user?}` / `pty_resize {pid, cols, rows}` | `pty_open` answers `started{pid}`, then a shell under a pseudo-terminal: output as `stdout` frames, input as `stdin` frames, `exit{code}` terminal; `pty_resize` answers `done`. PTYs register in the proc table like any exec | +| pty | `pty_open {cols, rows, cwd?, env?, user?}` / `pty_resize {pid, cols, rows}` | `pty_open` answers `started{pid}`, then a shell under a pseudo-terminal: output as `stdout` frames, input as `stdin` frames, `exit{code}` terminal; `pty_resize` answers `done`. PTYs register in the proc table like any exec. A client that disconnects hangs the shell up (SIGHUP, so it hangs up its jobs, as a closed ssh session does), and a shell still alive 2 s later is killed | | git | `git_clone {url, path, branch?, depth?, auth?}` / `git_status {path}` / `git_add {path, files}` / `git_commit {path, message, author}` / `git_push {path, auth?}` / `git_pull {path, auth?}` / `git_branch {path, action, name?}` | `git_status_result` (porcelain v2), `git_commit_result{hash}`, `git_branches{current, branches}` for `git_branch {action: "list"}`; every other verb terminates with `done`. A status past about 1 MiB of entries carries `truncated: true` with the head of the list, so one frame never exceeds the cap. `auth` is injected as an in-memory header, never written to guest disk | | port | `port_forward {port}` | relays guest TCP 127.0.0.1:port over this connection: `ready` once connected, then `data` both ways. Each direction ends on its own: `data_end` half-closes the guest socket, `done` reports the guest closing its write side, and the RPC ends once both have. A client that only drops the connection leaves the relay open until the guest side closes or writes. Works on both lanes — the no-network lane's only way in | | lsp | `lsp_start {language, root?}` / `lsp_request {server_id}` / `lsp_stop {server_id}` | a broker for the language server a flavor image ships: `lsp_start` spawns the argv named in `/etc/silkd/lsp.d/` (absent on the base image → `not_found` naming the flavor) and answers `lsp_started{server_id}`; `lsp_request` attaches the JSON-RPC byte stream (`ready`, then `data` both ways — silkd pipes bytes, it never parses LSP; `data_end` half-closes the server's stdin) and the stream ending reaps the server (v1 single-shot); `lsp_stop` kills it early, and a server nothing attaches within 5 minutes is reaped | diff --git a/silkd/src/pty.rs b/silkd/src/pty.rs index feca62efc..3c6afd975 100644 --- a/silkd/src/pty.rs +++ b/silkd/src/pty.rs @@ -9,7 +9,7 @@ use tokio::io::AsyncWrite; use tokio::io::unix::AsyncFd; use tokio::process::Command; use tokio::sync::{mpsc, oneshot}; -use tokio::time::timeout; +use tokio::time::{Instant, sleep_until, timeout}; use crate::proc::{Proc, Table}; use crate::proto::{self, ErrorKind, PtyReq, READ_CHUNK, Request, Response}; @@ -17,6 +17,7 @@ use crate::sysutil; /// Bounds the post-exit drain of the master's buffered tail so a stuck fd cannot wedge teardown. const POST_EXIT_DRAIN: Duration = Duration::from_secs(1); +const HUP_GRACE: Duration = Duration::from_secs(2); type Master = Arc>; @@ -129,6 +130,8 @@ async fn pump( let mut frame = Vec::new(); let mut eof = false; let mut disc = false; + let mut gone = false; + let mut kill_at = None; loop { tokio::select! { status = child.wait() => { @@ -137,9 +140,13 @@ async fn pump( stdin.abort(); return code; } - // kill the shell so child.wait publishes its real signalled code. + // hang up like a closed terminal so the shell hangs up its jobs; SIGKILL follows the grace. _ = &mut disc_rx, if !disc => { disc = true; + kill_at = Some(hang_up(child)); + } + _ = sleep_until(kill_at.unwrap_or_else(Instant::now)), if kill_at.is_some() => { + kill_at = None; let _ = child.start_kill(); } readable = master.readable(), if !eof => { @@ -152,10 +159,14 @@ async fn pump( Ok(Ok(0)) | Ok(Err(_)) => eof = true, Ok(Ok(n)) => { proc.emit_bytes(false, &buf[..n]); - if proto::write_chunk_frame(out, &mut frame, "stdout", &buf[..n]).await.is_err() { - let _ = child.start_kill(); + // keep reading after the client is gone: a shell exits only once its tty output drains. + if !gone && proto::write_chunk_frame(out, &mut frame, "stdout", &buf[..n]).await.is_err() { + gone = true; stdin.abort(); - return sysutil::wait_code(child).await; + if !disc { + disc = true; + kill_at = Some(hang_up(child)); + } } } Err(_would_block) => {} @@ -165,6 +176,13 @@ async fn pump( } } +fn hang_up(child: &tokio::process::Child) -> Instant { + if let Some(pid) = child.id() { + sysutil::signal_group(pid, libc::SIGHUP); + } + Instant::now() + HUP_GRACE +} + /// Writes client stdin frames into the master; StdinClose is ignored because a pty outlives input EOF. async fn pump_stdin( master: Master, diff --git a/silkd/tests/pty_e2e.rs b/silkd/tests/pty_e2e.rs index df6e9d3d9..92f3d0155 100644 --- a/silkd/tests/pty_e2e.rs +++ b/silkd/tests/pty_e2e.rs @@ -19,7 +19,7 @@ use tokio::sync::mpsc; use common::{ FrameLines, FrameWriter, alive, assert_error, b64, connect_with, decode, frames_until, one, - proc_entry, request, send, type_of, + proc_entry, request, send, type_of, wait_for_path, }; struct FailingSink(Vec); @@ -196,6 +196,37 @@ async fn pty_disconnect_tears_down_without_spin() { ); } +#[tokio::test] +async fn pty_disconnect_hangs_up_background_jobs() { + let dir = tempfile::tempdir().unwrap(); + let pid_file = dir.path().join("job"); + let state = Arc::new(State::new()); + let (mut cw, mut lines, handle) = open_pty(&state).await; + read_until(&mut lines, |v| v["type"] == "started").await; + let cmd = format!("sleep 300 & echo $! > {}\n", pid_file.display()); + send(&mut cw, json!({"op":"stdin","data":b64(cmd.as_bytes())})).await; + wait_for_path(&pid_file).await; + let job: u32 = loop { + if let Ok(pid) = std::fs::read_to_string(&pid_file).unwrap().trim().parse() { + break pid; + } + tokio::time::sleep(Duration::from_millis(10)).await; + }; + drop(lines); + drop(cw); + let _ = tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("pty did not tear down on disconnect"); + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + while alive(job) { + assert!( + tokio::time::Instant::now() < deadline, + "background job {job} outlived the hang-up" + ); + tokio::time::sleep(Duration::from_millis(20)).await; + } +} + #[tokio::test] async fn pty_started_write_failure_kills_the_shell_and_unregisters_it() { let table = Table::default(); From 5618c0b270974ed95b8783433eb8b46cb0582046 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 17:19:44 +0800 Subject: [PATCH 125/127] silkd: open a pty with close-on-exec set from birth openpty opened /dev/ptmx without O_CLOEXEC and set it with fcntl afterwards, so an exec forked on another connection in between inherited the pair and kept the pty alive after the shell exited. posix_openpt and the slave open now pass O_CLOEXEC; a unit test asserts both fds on Linux and macOS. --- silkd/src/sysutil.rs | 53 ++++++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/silkd/src/sysutil.rs b/silkd/src/sysutil.rs index 0d39517b3..9174ce853 100644 --- a/silkd/src/sysutil.rs +++ b/silkd/src/sysutil.rs @@ -12,7 +12,7 @@ use std::process::ExitStatus; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{LazyLock, Mutex}; -use nix::fcntl::{FcntlArg, FdFlag, OFlag, fcntl}; +use nix::fcntl::{FcntlArg, OFlag, fcntl}; use tokio::io::unix::AsyncFd; use tokio::process::Command; @@ -135,20 +135,33 @@ pub fn start_in(cmd: &mut Command, dir: &OwnedFd) { }; } -/// Opens a pty, returning the (master, slave) fds; the master is non-blocking for async I/O. +/// Opens a pty, returning the (master, slave) fds; both are close-on-exec from birth and the master is non-blocking. pub fn openpty(cols: u16, rows: u16) -> std::io::Result<(OwnedFd, OwnedFd)> { - let ws = nix::pty::Winsize { - ws_row: rows, - ws_col: cols, - ws_xpixel: 0, - ws_ypixel: 0, - }; - let pty = nix::pty::openpty(&ws, None)?; - fcntl(&pty.master, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC))?; - fcntl(&pty.slave, FcntlArg::F_SETFD(FdFlag::FD_CLOEXEC))?; - let flags = OFlag::from_bits_retain(fcntl(&pty.master, FcntlArg::F_GETFL)?); - fcntl(&pty.master, FcntlArg::F_SETFL(flags | OFlag::O_NONBLOCK))?; - Ok((pty.master, pty.slave)) + let master = nix::pty::posix_openpt(OFlag::O_RDWR | OFlag::O_NOCTTY | OFlag::O_CLOEXEC)?; + nix::pty::grantpt(&master)?; + nix::pty::unlockpt(&master)?; + let slave = nix::fcntl::open( + pty_name(&master)?.as_str(), + OFlag::O_RDWR | OFlag::O_NOCTTY | OFlag::O_CLOEXEC, + nix::sys::stat::Mode::empty(), + )?; + let master = OwnedFd::from(master); + set_winsize(master.as_raw_fd(), cols, rows)?; + set_nonblocking(master.as_fd(), true)?; + Ok((master, slave)) +} + +#[cfg(target_os = "linux")] +fn pty_name(master: &nix::pty::PtyMaster) -> nix::Result { + nix::pty::ptsname_r(master) +} + +#[cfg(not(target_os = "linux"))] +fn pty_name(master: &nix::pty::PtyMaster) -> nix::Result { + static PTSNAME: Mutex<()> = Mutex::new(()); + let _serial = lock(&PTSNAME); + // SAFETY: PTSNAME serializes every ptsname call in silkd, so its static buffer is copied before another call reuses it. + unsafe { nix::pty::ptsname(master) } } pub fn watch_fd(fd: T) -> std::io::Result> { @@ -500,6 +513,18 @@ fn check(rc: libc::c_int) -> std::io::Result<()> { mod tests { use super::*; + #[test] + fn openpty_fds_are_close_on_exec_and_the_master_non_blocking() { + let (master, slave) = openpty(80, 24).unwrap(); + for fd in [&master, &slave] { + let fd_flags = + nix::fcntl::FdFlag::from_bits_retain(fcntl(fd, FcntlArg::F_GETFD).unwrap()); + assert!(fd_flags.contains(nix::fcntl::FdFlag::FD_CLOEXEC)); + } + let fl = OFlag::from_bits_retain(fcntl(&master, FcntlArg::F_GETFL).unwrap()); + assert!(fl.contains(OFlag::O_NONBLOCK)); + } + #[test] fn base_env_has_path_and_term() { let env = base_env(); From 0bc30cb3784811bc09a620ee863434468d0c5212 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 17:24:17 +0800 Subject: [PATCH 126/127] sdk/go: drop the named results the move retry wrappers shadowed --- sdk/go/port.go | 5 +++-- sdk/go/proc.go | 6 ++++-- sdk/go/pty.go | 5 +++-- sdk/go/watch.go | 5 +++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/sdk/go/port.go b/sdk/go/port.go index 49881e3d0..4365d6468 100644 --- a/sdk/go/port.go +++ b/sdk/go/port.go @@ -111,8 +111,9 @@ func (s *Sandbox) ProxyPort(ctx context.Context, localAddr string, port uint16) } // openStream turns a call into a raw byte stream. -func (s *Sandbox) openStream(ctx context.Context, req wire.Request) (p *PortConn, err error) { - err = s.retryMoved(ctx, func() error { +func (s *Sandbox) openStream(ctx context.Context, req wire.Request) (*PortConn, error) { + var p *PortConn + err := s.retryMoved(ctx, func() error { conn, l, err := s.call(ctx, req) if err != nil { return err diff --git a/sdk/go/proc.go b/sdk/go/proc.go index 11f73ca56..7675eed00 100644 --- a/sdk/go/proc.go +++ b/sdk/go/proc.go @@ -52,8 +52,10 @@ func (s *Sandbox) Attach(ctx context.Context, pid uint32, stdout, stderr io.Writ return s.drainProc(ctx, &wire.Attach{PID: pid}, stdout, stderr) } -func (s *Sandbox) drainProc(ctx context.Context, req wire.Request, stdout, stderr io.Writer) (code int32, exited bool, err error) { - err = s.retryMoved(ctx, func() error { +func (s *Sandbox) drainProc(ctx context.Context, req wire.Request, stdout, stderr io.Writer) (int32, bool, error) { + var code int32 + var exited bool + err := s.retryMoved(ctx, func() error { conn, l, err := s.call(ctx, req) if err != nil { return err diff --git a/sdk/go/pty.go b/sdk/go/pty.go index ac29a62a9..93b82db51 100644 --- a/sdk/go/pty.go +++ b/sdk/go/pty.go @@ -87,8 +87,9 @@ func (p *Pty) drain(ctx context.Context, pw *io.PipeWriter, stop func()) { } // OpenPty starts a shell whose lifetime is governed by ctx or Close. -func (s *Sandbox) OpenPty(ctx context.Context, opts wire.PtyOpen) (p *Pty, err error) { - err = s.retryMoved(ctx, func() error { +func (s *Sandbox) OpenPty(ctx context.Context, opts wire.PtyOpen) (*Pty, error) { + var p *Pty + err := s.retryMoved(ctx, func() error { conn, l, err := s.call(ctx, &opts) if err != nil { return err diff --git a/sdk/go/watch.go b/sdk/go/watch.go index a2e055279..61ce0984f 100644 --- a/sdk/go/watch.go +++ b/sdk/go/watch.go @@ -76,8 +76,9 @@ func (w *Watcher) setErr(err error) { } // Watch streams filesystem events under path; events caused after it returns are guaranteed captured. -func (s *Sandbox) Watch(ctx context.Context, path string, recursive bool) (w *Watcher, err error) { - err = s.retryMoved(ctx, func() error { +func (s *Sandbox) Watch(ctx context.Context, path string, recursive bool) (*Watcher, error) { + var w *Watcher + err := s.retryMoved(ctx, func() error { conn, l, err := s.call(ctx, &wire.FsWatch{Path: path, Recursive: recursive}) if err != nil { return err From 82607e54ef0117847fd2576e3ba05010bc74a79c Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 17:24:17 +0800 Subject: [PATCH 127/127] sandboxd: name the printf-style dir test helpers and fill the router test's drain loop --- sandboxd/store/dir/dir_test.go | 42 +++++++++++++++++----------------- sandboxd/tunnet/router_test.go | 3 ++- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/sandboxd/store/dir/dir_test.go b/sandboxd/store/dir/dir_test.go index 4ef31abcf..42bf7eb5c 100644 --- a/sandboxd/store/dir/dir_test.go +++ b/sandboxd/store/dir/dir_test.go @@ -46,8 +46,8 @@ func TestFetchPinsGenerationAcrossStoreInstances(t *testing.T) { t.Fatalf("fetch digest %q, want first generation %q", digest, firstDigest) } secondDigest := mustPublishDigested(t, writer, testID, "second") - assertPresent(t, filepath.Join(dir, "disk.img"), "first generation disturbed by re-publish") - assertPresent(t, firstDigestPath, "first generation digest disturbed by re-publish") + assertPresentf(t, filepath.Join(dir, "disk.img"), "first generation disturbed by re-publish") + assertPresentf(t, firstDigestPath, "first generation digest disturbed by re-publish") _, meta, digest = mustFetch(t, reader, testID) if string(meta) != storetest.MetaJSON("second") { @@ -73,7 +73,7 @@ func TestPlainPublishReplacesDigestWithEmpty(t *testing.T) { if firstDigest == "" { t.Fatal("digested generation returned an empty digest") } - assertPresent(t, firstDigestPath, "superseded digest removed inside the grace") + assertPresentf(t, firstDigestPath, "superseded digest removed inside the grace") mustPublish(t, st, testID, "digested") if _, _, digest = mustFetch(t, st, testID); digest != "" { t.Errorf("a plain publish of the digested meta kept digest %q, want empty", digest) @@ -264,10 +264,10 @@ func TestSweepGenerationsPairsCurrentAndSupersededDigests(t *testing.T) { t.Fatalf("SweepGenerations: %v", err) } for _, path := range []string{firstGen, firstSidecar} { - assertGone(t, path, "superseded entry %s survived sweep", filepath.Base(path)) + assertGonef(t, path, "superseded entry %s survived sweep", filepath.Base(path)) } for _, path := range []string{secondGen, secondSidecar, labels} { - assertPresent(t, path, "current entry %s was swept", filepath.Base(path)) + assertPresentf(t, path, "current entry %s was swept", filepath.Base(path)) } _, meta, digest := mustFetch(t, st, testID) if string(meta) != storetest.MetaJSON("second") || digest != secondDigest { @@ -294,13 +294,13 @@ func TestPublishRetriesAfterExpiredInstall(t *testing.T) { if sweepErr := st.SweepGenerations(t.Context()); sweepErr != nil { t.Fatalf("sweep expired generation: %v", sweepErr) } - assertGone(t, gen, "expired generation survived sweep") + assertGonef(t, gen, "expired generation survived sweep") mustPublish(t, st, testID, "same") backdate(t, gen) mustPublish(t, st, testID, "same") dir, _, _ := mustFetch(t, st, testID) - assertPresent(t, filepath.Join(dir, "disk.img"), "export after retried publish") + assertPresentf(t, filepath.Join(dir, "disk.img"), "export after retried publish") } func TestPublishSweepsGenerationsBySupersessionAge(t *testing.T) { @@ -311,18 +311,18 @@ func TestPublishSweepsGenerationsBySupersessionAge(t *testing.T) { mustPublish(t, st, testID, "first") mustPublish(t, st, testID, "second") mustPublish(t, st, testID, "third") - assertPresent(t, gen1, "superseded generation reclaimed inside the grace") + assertPresentf(t, gen1, "superseded generation reclaimed inside the grace") setAge(t, gen1, store.GenerationGrace+time.Minute) setAge(t, gen2, store.GenerationGrace/2) mustPublish(t, st, testID, "fourth") - assertGone(t, gen1, "generation past its supersession grace survived publish") - assertPresent(t, gen2, "generation inside its supersession grace was reclaimed") + assertGonef(t, gen1, "generation past its supersession grace survived publish") + assertPresentf(t, gen2, "generation inside its supersession grace was reclaimed") dir, meta, _ := mustFetch(t, st, testID) if string(meta) != storetest.MetaJSON("fourth") { t.Fatalf("fetch meta %q, want the current generation", meta) } - assertPresent(t, filepath.Join(dir, "disk.img"), "current export after sweep") + assertPresentf(t, filepath.Join(dir, "disk.img"), "current export after sweep") } func TestSweepSparesPublishingGeneration(t *testing.T) { @@ -336,7 +336,7 @@ func TestSweepSparesPublishingGeneration(t *testing.T) { if sweepErr := sweeper.SweepStaging(); sweepErr != nil { t.Fatalf("sweep during publish: %v", sweepErr) } - assertPresent(t, gen, "publishing generation was swept") + assertPresentf(t, gen, "publishing generation was swept") tmp := filepath.Join(root, testID, store.MetaFile+".tmp") if writeErr := os.WriteFile(tmp, meta, 0o600); writeErr != nil { @@ -389,9 +389,9 @@ func TestSweepReclaimsOnlyAgedUncommittedGenerations(t *testing.T) { if sweepErr := sweeper.SweepStaging(); sweepErr != nil { t.Fatalf("sweep old: %v", sweepErr) } - assertGone(t, orphan, "aged uncommitted generation survived") - assertPresent(t, filepath.Join(root, orphanID), "orphan record shell was removed") - assertPresent(t, fresh, "publishing generation was swept") + assertGonef(t, orphan, "aged uncommitted generation survived") + assertPresentf(t, filepath.Join(root, orphanID), "orphan record shell was removed") + assertPresentf(t, fresh, "publishing generation was swept") if removeErr := os.RemoveAll(commitBlocker); removeErr != nil { t.Fatalf("remove commit blocker: %v", removeErr) @@ -418,13 +418,13 @@ func TestSweepStagingSparesAPeersFreshStaging(t *testing.T) { if sweepErr := restarted.SweepStaging(); sweepErr != nil { t.Fatalf("sweep: %v", sweepErr) } - assertPresent(t, filepath.Join(staging, store.MetaFile), "a peer's in-progress staging was swept") + assertPresentf(t, filepath.Join(staging, store.MetaFile), "a peer's in-progress staging was swept") backdate(t, staging) if sweepErr := restarted.SweepStaging(); sweepErr != nil { t.Fatalf("sweep aged: %v", sweepErr) } - assertGone(t, staging, "aged staging survived") + assertGonef(t, staging, "aged staging survived") } func TestSweepGenerationsRemovesOnlyAgedStaging(t *testing.T) { @@ -440,8 +440,8 @@ func TestSweepGenerationsRemovesOnlyAgedStaging(t *testing.T) { t.Fatalf("SweepGenerations: %v", err) } - assertGone(t, aged, "staging aged past the grace survived the periodic sweep") - assertPresent(t, fresh, "fresh staging was swept") + assertGonef(t, aged, "staging aged past the grace survived the periodic sweep") + assertPresentf(t, fresh, "fresh staging was swept") } func TestSweepGenerationsToleratesConcurrentDelete(t *testing.T) { @@ -676,14 +676,14 @@ func setAge(t *testing.T, path string, age time.Duration) { } } -func assertGone(t *testing.T, path, format string, args ...any) { +func assertGonef(t *testing.T, path, format string, args ...any) { t.Helper() if _, err := os.Stat(path); !errors.Is(err, fs.ErrNotExist) { t.Errorf(format+": %v", append(args, err)...) } } -func assertPresent(t *testing.T, path, format string, args ...any) { +func assertPresentf(t *testing.T, path, format string, args ...any) { t.Helper() if _, err := os.Stat(path); err != nil { t.Errorf(format+": %v", append(args, err)...) diff --git a/sandboxd/tunnet/router_test.go b/sandboxd/tunnet/router_test.go index 722895b2e..6b10bd5da 100644 --- a/sandboxd/tunnet/router_test.go +++ b/sandboxd/tunnet/router_test.go @@ -206,7 +206,8 @@ func TestRouterChargesSegmentsAndRefusesOversizedPackets(t *testing.T) { r.Arm(Claim{ID: "c1", Addr: claimAddr, Limits: Limits{PacketsPerSecond: 10}, Allow: func(netip.Addr, uint16) bool { return true }}) u.sent(t) go func() { - for range u.out { + for s := range u.out { + _ = s } }() guest := serve(t, r, claimAddr)