From 46c4227caa8deaaa33db23b840d2e0d533a211e5 Mon Sep 17 00:00:00 2001 From: CMGS Date: Tue, 10 Nov 2026 21:21:29 +0800 Subject: [PATCH 1/4] review: v0.1.31 whole-range round follow-ups - deploy.md Upgrading: from v0.1.31 a config with a balloon block fails to load; claims v0.1.30 inflated stay inflated (deflate running ones with a v0.1.30 reload that removes the block, hibernated ones with a POST after the wake); POST /balloon now needs a target body. - A move aborted after it consumed the claim's row marks the claim due, so the next replica sweep puts the row back instead of leaving the claim without failover for up to an interval. - A never-replicated claim claimed within the last interval anchors its first round on its claim time, as before v0.1.31; only older claims without replica_at get the id offset, the restart case. - settleReplicaIntent stamps replica_at when it promotes a landed round. - The balloon verb check caches only a pass, so a failed or timed-out first check no longer refuses every later call until a restart. - Docs: an adoption carries the round before the last; the idle clock across a clean restart and a crash; a warm VM retires an old one only once the warm set is back at target. - Tests: a move and an adoption carrying replica_at, a downtime abort replicating at the next sweep, a new claim's first round, the settle stamp, the verb check retried, and an outgoing move refused a balloon. --- docs/cluster.md | 13 +++++--- docs/deploy.md | 16 +++++++-- sandboxd/pool/balloon.go | 8 +++-- sandboxd/pool/balloon_test.go | 20 +++++++++++ sandboxd/pool/move_out.go | 3 ++ sandboxd/pool/move_test.go | 24 ++++++++++++- sandboxd/pool/pool.go | 5 ++- sandboxd/pool/replica.go | 9 ++++- sandboxd/pool/replica_test.go | 63 ++++++++++++++++++++++++++++++++++- sandboxd/types/types.go | 1 + 10 files changed, 147 insertions(+), 15 deletions(-) diff --git a/docs/cluster.md b/docs/cluster.md index 35a533ed..a2c54e7d 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -511,7 +511,7 @@ fields and deletes the row, so no third node can adopt a stale replica while the target runs the claim. It keeps the replica until the move ends. The target's first replica round puts a new row when the claim's carried interval comes due, from its `replica_at`, at once when the last round is older than the interval, -and an aborted move lets the source's next round put it again. A source cut off from the +and an aborted move makes the claim due at the source's next replica sweep, which puts it again. A source cut off from the registry changes nothing and its move fails: it keeps the row's epoch, so the fence still finds a peer that adopted the claim during the outage, and the source drops it. A consume whose reply was lost leaves the source's own tombstone, which @@ -564,10 +564,13 @@ checkpoint store and keeps each claim's registry row on the latest snapshot: write that never landed otherwise. A refused write reads the row: one this node still owns only had a stale epoch and is refreshed, and an absent or foreign row drops the claim and its VM. A claim's interval runs from its last round, which `claims.json` keeps - as `replica_at` and a restart, a move or an adoption carries, so a restart does - not reset it and a claim whose last round is older than the interval is due at - the first sweep. A claim that has never replicated gets its first round within - one interval of the sweep first seeing it, at an offset taken from its id, so + as `replica_at` and a restart or a move carries, so a restart does not reset + it and a claim whose last round is older than the interval is due at the first + sweep. An adoption carries the round before the last, since a round writes the + row before it stamps its own time. A claim that has never replicated gets its + first round one interval after it was claimed; one claimed more than an + interval ago (a restart onto claims that never replicated) gets it within one + interval of the sweep first seeing it, at an offset taken from its id, so those first rounds spread instead of landing together. Until a claim woken from an archive or an adoption commits its first replica it has no row, and it is lost with its node. A hibernated, archived or failed claim, diff --git a/docs/deploy.md b/docs/deploy.md index 72f419d6..1be99ed1 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -38,6 +38,18 @@ Roll the guest images with the daemon: a guest image from another release can read a different lane file. Each note names the release it starts from; a note for a release you already run does not apply. +From v0.1.31, the `balloon` config block is gone: a config that still has it +fails to load, so the node does not start and a reload fails. Remove the block +on every node before the roll. Claims that v0.1.30 inflated stay inflated, +both running idle claims and idle-hibernated ones, whose snapshots carry the +target. To deflate the running ones, remove the block on v0.1.30 with a config +[reload](#reloading-the-config), not a restart: the next sweep then deflates +them. Then roll. A hibernated claim that v0.1.30 inflated keeps that target +when it wakes; send it [`POST /v1/sandboxes/{id}/balloon`](sandboxd-api.md#post-v1sandboxesidballoon) +after the wake, with `target` set to the boot target, the `default` that any +balloon reply reports. That call now needs a +`target` body: an empty `POST`, which v0.1.30 answered 202, answers 400. + An S3 `checkpoint_store` written before v0.1.18 used another object layout (whole export files, now chunk manifests). Those records are not readable and there is no fallback reader: before starting the new binary, wake or release @@ -175,12 +187,12 @@ the intersection of the two. | `audit_log` | false | append every relayed frame that opens an RPC — its op + addressing fields, never payloads — to `/audit.jsonl`; continuation `data`, `data_end`, `stdin`, and `stdin_close` frames pass through unrecorded. The file is size-rotated with one `.1` backup. Records are `{t, id, op}` plus whichever addressing fields the op carries (`argv`, `path`, `dest`, `from`, `to`, `url`, `session`, `port`), plus `method` (`GET`, `CONNECT`, `SOCKS5`, …), `decision` (`allow`, `deny`, `rate_limited`), `secret` (the ref name, never its value), `upstream` (the [upstream proxy](egress.md#upstream-proxies)'s `host:port`, when one carried it), `tenant`, and on an allowed connection `src_addr` and `dst_addr` (the node-side and destination `ip:port` of the upstream TCP connection, the tunnel target through an upstream proxy) on `egress` records, which a relayed frame cannot set; preview accesses record as op `preview`, one per request; a tenant change records as op `tenants` with the `added`, `changed` and `removed` names, never a token; a config reload records as op `config_reload` with `changed`, a runtime volume change as op `volumes` with `added` or `removed`, and each ended flow of the [advanced network mode](advanced-network.md) as op `tun_flow` with `path` (`tun`), `tenant`, `proto` (`tcp`, `udp`, `icmp`), `dst_addr`, `decision` (`allow`, `deny`), `sent` and `received` bytes, `start`, `end` and the exit `profile`; a guest DNS-relay query records as op `dns` ([fields](egress.md#name-resolution-dns)), a cordon and its lift as ops `cordon` and `uncordon`, and a guest-port relay as op `port` with `port`. An opening frame whose first line exceeds 4 KiB records as op `oversized` with no addressing fields. A record a root, operator or sandbox-token request caused carries `caller` (`root`, `operator` or `claim`), which a relayed frame cannot set; an operator's read of a claim token by id records as op `token` | | `disk_reserve` | unset | `{"free": "100G", "dirs": ["/srv/sbx/run"]}`: while the filesystem of any listed directory (default `data_dir`) has less than `free` available, checked every 2 s, the node refuses new claims, forks, checkpoint claims, wakes of hibernated or archived sandboxes, hibernates, checkpoint captures and promotes with 429, and parks refill; the idle sweep and a lease-end archive leave their sandboxes running and retry on the next tick; `/v1/info` reports it as at capacity and the mesh sees no pools on it. Running sandboxes keep running. On an XFS project quota the free space is the project's. List cocoon's `run_dir` when it sits on another filesystem, since the copy-on-write disks grow there. Size `free` to at least two sandboxes' worst-case growth (copy-on-write disk plus memory file each) and keep the refusal above your disk alert's warning level, so the warning comes first: on a 640G project with 40G disks and 8G memory files, `100G` refuses at 84% used | | `memory_limit` | unset | `{"anon": "300G", "psi_some_avg10": 40}`, either key alone: while anon, `AnonPages` plus used swap (`SwapTotal` minus `SwapFree`) less `SwapCached`, the swapped pages already back in memory, in `/proc/meminfo`, is above `anon`, or the `some avg10` of `/proc/pressure/memory` is above `psi_some_avg10`, checked every 2 s, the node refuses new claims, forks and checkpoint claims with 429 and parks refill; `/v1/info` reports it as at capacity and the mesh sees no pools on it, so a warm or pool redirect skips it; a template or volume held only there still redirects to it and answers 429. Hibernates, checkpoints and archives go on, and running sandboxes keep running. Wakes of its own claims go on too, except an archived claim's on a node with `archive_failover`: that wake is [handed to a peer with room](cluster.md#waking-an-archived-claim-on-a-full-node), and one no peer adopts wakes here anyway; a squeezed peer does not adopt it. Guest memory is a private file mapping, so page cache and clean guest pages are reclaimable and only anon is what a hibernate has to free. Both signals are node-wide; anon leaves out explicit hugepages and counts swapped shmem as anon. A signal the node cannot read, such as PSI on a kernel without it, logs an error and is ignored. PSI also counts a task stalled by its own cgroup limit and weighs by busy CPUs, so on a mostly idle node a few throttled tasks in one small cgroup can cross it: 16 tasks under a 64M `MemoryHigh` reached 75 on an idle 384-CPU host with memory to spare. Treat `anon` as the main limit, and set `psi_some_avg10` only where no workload runs under a tight memory cgroup. Size `anon` so the running guests' further growth still fits under physical memory | -| `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a none-lane claim is hibernated once it has had no open data-plane connection (relay, buffered exec, preview), no egress request in flight and no [advanced-network](advanced-network.md) packet carried for this long; the clock restarts when the last connection closes, so a long command is never cut short. The next call that reaches the guest wakes it transparently. Per-pool `idle_hibernate_seconds` does the same for that pool's claims; pooled keys ignore the node-wide value, and egress pools reject it because they cannot resume safely. Opt in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice | +| `idle_hibernate_seconds` | 0 (off) | node-wide idle policy for unpooled claims (template/checkpoint claims): a none-lane claim is hibernated once it has had no open data-plane connection (relay, buffered exec, preview), no egress request in flight and no [advanced-network](advanced-network.md) packet carried for this long; the clock restarts when the last connection closes, so a long command is never cut short. The next call that reaches the guest wakes it transparently. Per-pool `idle_hibernate_seconds` does the same for that pool's claims; pooled keys ignore the node-wide value, and egress pools reject it because they cannot resume safely. Opt in deliberately: a wake costs latency and the snapshot, so callers with their own idle logic must not pay twice. A clean restart keeps each claim's idle clock from the shutdown flush, so claims already idle past the limit hibernate at the first sweep; after a crash the clock starts again at the restart | | `archive_after_seconds` | 0 (off) | tier below hibernation: a hibernated claim idle this long is checkpointed to the store and its local VM dropped, freeing the node entirely; the next call that reaches the guest restores it transparently (a checkpoint restore's latency) with a fresh lease of the length the claim was granted (the server default of 5m when it asked for none); a lease moves only when a client calls [`/renew`](sandboxd-api.md#post-v1sandboxesidrenew). Keep `idle_hibernate_seconds` under the leases your claims use, or a woken sandbox that idles again is destroyed by the reap before it can hibernate, its archive already consumed by the wake; one still in use when its lease ends is destroyed the same way, unless it claimed `on_expire: archive`, which archives at the lease end whatever this setting. A call that wakes a hibernated claim past its lease, before the export, starts that fresh lease too. Requires `idle_hibernate_seconds > 0` and must exceed it. Node-wide for unpooled keys; per-pool overrides for that pool | | `archive_delete_after_seconds` | 0 (keep) | purge an archived claim's store checkpoint this long after it was archived; the claim is then gone for good. On S3, chunks and manifests remain until [offline reclamation](#reclaiming-s3-storage). On archive this retention window replaces the live claim deadline; 0 clears the deadline so the archive is kept forever. An [evacuation](cluster.md#evacuating-a-node) keeps a longer live deadline, and a claim of a pool without an archive tier, whose `on_expire` is not `archive`, keeps its deadline until its first wake. Same node-wide/per-pool split | | `archive_failover` | false | let another node adopt this node's archived claims once its node lease expires, and the running claims of pools with `replicate_seconds`: each archive and replication round also writes a row (token SHA-256, owner, epoch, checkpoint id, retention deadline, and the claim record with its env, `guest: false` values included, in plaintext, unless `host_env_memory_only` is on) to the shared PostgreSQL, and a `Lookup` that reaches any node takes the claim over there (see [archived claims across node loss](cluster.md#archived-claims-across-node-loss)). Requires `meta_store` with a `cell` (rows are scoped by it, so meshes sharing one database never adopt from each other), a `checkpoint_store` of kind `s3` and a `mesh` with a `cluster_key` (a hand-off presents the claim's token, or the root `api_token` for a claim whose token this node has not seen, to an address learned from gossip), all checked at config load. The node also renews a lease row every 5 s (see [node lease](cluster.md#node-lease)). With the meta store down, archives, wakes, releases and env writes of archived claims, and env writes and renews of hibernated replicated claims, fail and retry, and replicated claims stop being served once the lease is overdue (see [running claims](cluster.md#running-claims-across-node-loss)); other running claims never touch it. Restart only | | `mesh` | unset | join a cluster ([Clusters](cluster.md)); unset = single node | -| `pools[]` | — | warm pools, keyed by `(template, net, size)`. `warm` defaults to 4 only when both `warm` and `warm_max` are omitted; an explicit `warm: 0` is kept, so `warm: 0` under a `warm_max` starts the pool empty and grows it on demand, and `warm: 0` alone registers the key with no warm VMs (`PUT /v1/pools` never defaults `warm`); `net` is `none` (the default) or `egress`; `size` is a tier, below (default `small`). Retune online without a restart via [`PUT /v1/pools`](sandboxd-api.md#put-v1pools) — omitted pools drain, and their goldens are deleted once they are idle (`warm: 0` keeps the golden). This is the **first-boot seed**: once a node takes a `PUT /v1/pools`, the applied set persists to `/pools.json` and overrides this section on every later boot (a startup log notes it); delete `pools.json` to return to config-owned pools. With a `meta_store` `cell`, the cell's set overrides it instead (see [shared pool set](#shared-pool-set)). Egress stays config-owned either way. A template re-pulled or re-imported under the same ref is picked up within a minute: the pool rebuilds its golden while claims keep getting the old image's warm VMs, and once the new golden lands each new warm VM retires one old VM. See [state ownership](cluster.md#state-ownership) | +| `pools[]` | — | warm pools, keyed by `(template, net, size)`. `warm` defaults to 4 only when both `warm` and `warm_max` are omitted; an explicit `warm: 0` is kept, so `warm: 0` under a `warm_max` starts the pool empty and grows it on demand, and `warm: 0` alone registers the key with no warm VMs (`PUT /v1/pools` never defaults `warm`); `net` is `none` (the default) or `egress`; `size` is a tier, below (default `small`). Retune online without a restart via [`PUT /v1/pools`](sandboxd-api.md#put-v1pools) — omitted pools drain, and their goldens are deleted once they are idle (`warm: 0` keeps the golden). This is the **first-boot seed**: once a node takes a `PUT /v1/pools`, the applied set persists to `/pools.json` and overrides this section on every later boot (a startup log notes it); delete `pools.json` to return to config-owned pools. With a `meta_store` `cell`, the cell's set overrides it instead (see [shared pool set](#shared-pool-set)). Egress stays config-owned either way. A template re-pulled or re-imported under the same ref is picked up within a minute: the pool rebuilds its golden while claims keep getting the old image's warm VMs, and once the new golden lands and the warm set is back at target, each new warm VM retires one old VM. See [state ownership](cluster.md#state-ownership) | Size tiers (free-form CPU/memory is deliberately not accepted — it would fragment the warm pools): diff --git a/sandboxd/pool/balloon.go b/sandboxd/pool/balloon.go index 7f3ac6b2..3a46d309 100644 --- a/sandboxd/pool/balloon.go +++ b/sandboxd/pool/balloon.go @@ -45,11 +45,15 @@ func (m *Manager) SetBalloon(ctx context.Context, id, target string) (types.VMBa return b, err } -func (m *Manager) checkBalloon() bool { +func (m *Manager) balloonReady() bool { + if m.balloonOK.Load() { + return true + } ctx := context.Background() if err := m.eng.CheckBalloonSupport(ctx); err != nil { - log.WithFunc("pool.checkBalloon").Error(ctx, err, "the balloon verb is off") + log.WithFunc("pool.balloonReady").Error(ctx, err, "the balloon verb is off") return false } + m.balloonOK.Store(true) return true } diff --git a/sandboxd/pool/balloon_test.go b/sandboxd/pool/balloon_test.go index a4e19fd4..c16661a4 100644 --- a/sandboxd/pool/balloon_test.go +++ b/sandboxd/pool/balloon_test.go @@ -33,9 +33,11 @@ func TestSetBalloonRefusesWhatItCannotReach(t *testing.T) { eng := newFakeEngine() m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey, Warm: 6}) asleep, archived, failed, waking, capturing, arriving := mustClaim(t, m, testKey), mustClaim(t, m, testKey), mustClaim(t, m, testKey), mustClaim(t, m, testKey), mustClaim(t, m, testKey), mustClaim(t, m, testKey) + leaving := mustClaim(t, m, testKey) m.mu.Lock() asleep.HibernateSnap, archived.ArchiveCk, failed.Failed = "snap", "ck_00000000000000aa", "vmm exited" waking.Waking, capturing.PendingSnap, arriving.MoveIn = true, "pending", true + m.moving[leaving.ID] = &outMove{} m.mu.Unlock() for _, tt := range []struct { id string @@ -48,6 +50,7 @@ func TestSetBalloonRefusesWhatItCannotReach(t *testing.T) { {waking.ID, ErrPaused}, {capturing.ID, ErrPaused}, {arriving.ID, ErrEvacuating}, + {leaving.ID, ErrEvacuating}, } { if _, err := m.SetBalloon(t.Context(), tt.id, "50%"); !errors.Is(err, tt.want) { t.Errorf("set balloon of %s: %v, want %v", tt.id, err, tt.want) @@ -56,6 +59,7 @@ func TestSetBalloonRefusesWhatItCannotReach(t *testing.T) { m.mu.Lock() asleep.HibernateSnap, archived.ArchiveCk, failed.Failed = "", "", "" waking.Waking, capturing.PendingSnap, arriving.MoveIn = false, "", false + delete(m.moving, leaving.ID) m.mu.Unlock() if calls := balloonCalls(eng); len(calls) != 0 { t.Errorf("balloon calls %q for refused claims, want none", calls) @@ -90,6 +94,22 @@ func TestSetBalloonRefusesOnACocoonWithoutTheVerb(t *testing.T) { } } +func TestSetBalloonChecksTheVerbAgainAfterAFailedCheck(t *testing.T) { + eng := newFakeEngine() + eng.balloonSupportErr = errors.New("cocoon vm balloon --help: context deadline exceeded") + m := newTestManager(t, eng, config.PoolSpec{PoolKey: testKey}) + sb := mustClaim(t, m, testKey) + if _, err := m.SetBalloon(t.Context(), sb.ID, "50%"); !errors.Is(err, ErrBalloonOff) { + t.Fatalf("set balloon while the check fails: %v, want ErrBalloonOff", err) + } + eng.mu.Lock() + eng.balloonSupportErr = nil + eng.mu.Unlock() + if _, err := m.SetBalloon(t.Context(), sb.ID, "50%"); err != nil { + t.Errorf("set balloon once the check passes: %v, want it set", err) + } +} + func balloonCalls(eng *fakeEngine) []string { eng.mu.Lock() defer eng.mu.Unlock() diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index 50b9d464..b1573183 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -349,6 +349,9 @@ func (m *Manager) abortMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMo logger := log.WithFunc("pool.abortMoveOut") m.mu.Lock() vm, live := sb.VMName, m.claimed[sb.ID] == sb + if live && mv.replica != "" { + sb.ReplicaDue = true + } m.mu.Unlock() if live && mv.Series != "" { if err := m.eng.MigrateAbort(ctx, vm, mv.Series); err != nil { diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 3643cc8e..34e3031b 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -517,7 +517,7 @@ func TestMoveOfAReplicatedClaimDropsItsRowBeforeTheFinalRound(t *testing.T) { sb := mustClaim(t, mc.s, testKey) replicateNow(t, mc.s, sb) mc.s.mu.Lock() - replica := sb.ReplicaCk + replica, replicaAt := sb.ReplicaCk, sb.ReplicaAt mc.s.mu.Unlock() if _, ok, _ := mc.reg.Get(t.Context(), sb.ID); !ok || replica == "" { t.Fatal("setup: the claim has no replica row") @@ -539,6 +539,9 @@ func TestMoveOfAReplicatedClaimDropsItsRowBeforeTheFinalRound(t *testing.T) { if ckExists(t, mc.s, replica) { t.Error("the stale replica outlived the move") } + if moved, ok := mc.t.byID(sb.ID); !ok || !moved.ReplicaAt.Equal(replicaAt) { + t.Errorf("target claim %v replica_at %v, want the source's last round %v", ok, moved.ReplicaAt, replicaAt) + } } func TestMoveAbortedAfterDroppingTheRowPutsItBackOnTheNextRound(t *testing.T) { @@ -566,6 +569,25 @@ func TestMoveAbortedAfterDroppingTheRowPutsItBackOnTheNextRound(t *testing.T) { } } +func TestMoveAbortedByTheDowntimeCheckReplicatesAtTheNextSweep(t *testing.T) { + mc := newMoveClusterWith(t, func(c *failoverCluster) { c.replicate = replicateEvery }) + sb := mustClaim(t, mc.s, testKey) + replicateNow(t, mc.s, sb) + fakeOf(mc.s).finalCaptureTime = 200 * time.Millisecond + if _, err := mc.s.Move(t.Context(), sb.ID, "b", MoveStartAllowance+100*time.Millisecond); !errors.Is(err, ErrMoveDowntime) { + t.Fatalf("move: %v, want ErrMoveDowntime", err) + } + assertStillOnSource(t, mc, sb) + if _, ok, _ := mc.reg.Get(t.Context(), sb.ID); ok { + t.Fatal("the row survived the final round") + } + mc.s.replicateOnce(t.Context()) + waitFor(t, func() bool { return !mc.s.replicating.Load() }) + if _, ok, _ := mc.reg.Get(t.Context(), sb.ID); !ok { + t.Error("the first sweep after the abort did not put the row back") + } +} + func TestMoveNeedsNoArchiveRegistry(t *testing.T) { mc := newMoveClusterWith(t, func(c *failoverCluster) { c.noArchives = true }) sb := claimTenant(t, mc.s, "acme") diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 607bc6c1..52585f8f 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -304,8 +304,8 @@ type Manager struct { poolVersion int64 configSeedHash string // config pools' hash, to warn when a file edit is overridden - sweeps struct{ idle, liveness, archive, archiveDelete, orphan, held, ckpt, gen atomic.Bool } - balloonReady func() bool + sweeps struct{ idle, liveness, archive, archiveDelete, orphan, held, ckpt, gen atomic.Bool } + balloonOK atomic.Bool cocoondSocket string // vmmEvents is set while the cocoon daemon stream is live; the poll is then only a backstop. @@ -454,7 +454,6 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg healSem: make(chan struct{}, maxConcurrentHeals), } m.store.dropHostEnv = cfg.HostEnvMemoryOnly - m.balloonReady = sync.OnceValue(m.checkBalloon) m.seedVolumes(cfg.Volumes) if err := m.loadCordon(); err != nil { return nil, err diff --git a/sandboxd/pool/replica.go b/sandboxd/pool/replica.go index 78a505b1..284ba050 100644 --- a/sandboxd/pool/replica.go +++ b/sandboxd/pool/replica.go @@ -37,6 +37,10 @@ func (m *Manager) replicateOnce(ctx context.Context) { if sb.HasReplica() { retire = append(retire, sb) } + case sb.ReplicaDue: + sb.ReplicaDue = false + next[id] = now + due = append(due, sb) case tried.IsZero(): next[id] = firstTry(sb, every, now) case now.Sub(tried) < every: @@ -181,7 +185,7 @@ func (m *Manager) settleReplicaIntent(ctx context.Context, sb *types.Sandbox) er case ok && !rec.Consumed && rec.Checkpoint == pending: var prev string live, commitErr := m.commitIfLive(ctx, sb, func() { - prev, sb.ReplicaCk, sb.ReplicaPending, sb.ArchiveEpoch = sb.ReplicaCk, pending, "", rec.Epoch + prev, sb.ReplicaCk, sb.ReplicaPending, sb.ArchiveEpoch, sb.ReplicaAt = sb.ReplicaCk, pending, "", rec.Epoch, time.Now() }) if !live { return ErrUnknownSandbox @@ -281,6 +285,9 @@ func firstTry(sb *types.Sandbox, every time.Duration, now time.Time) time.Time { if at := sb.ReplicaAt; !at.IsZero() && at.Before(now) { return at } + if at := sb.ClaimedAt; at.After(now.Add(-every)) && !at.After(now) { + return at + } h := fnv.New32a() _, _ = h.Write([]byte(sb.ID)) return now.Add(-time.Duration(float64(every) * float64(h.Sum32()) / (1 << 32))) diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 1a707cfd..ce470a45 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -120,7 +120,11 @@ func TestClaimsNeverReplicatedSpreadTheirFirstRoundOverAnInterval(t *testing.T) a := c.node(t, "a", t.TempDir(), keepAnHour) ids := make([]string, 6) for i := range ids { - ids[i] = mustClaim(t, a, testKey).ID + sb := mustClaim(t, a, testKey) + a.mu.Lock() + sb.ClaimedAt = time.Now().Add(-2 * replicateEvery * time.Second) + a.mu.Unlock() + ids[i] = sb.ID } now := time.Now() a.replicateOnce(t.Context()) @@ -139,6 +143,41 @@ func TestClaimsNeverReplicatedSpreadTheirFirstRoundOverAnInterval(t *testing.T) } } +func TestANewClaimFirstReplicatesOneIntervalAfterItsClaim(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + a.replicateOnce(t.Context()) + waitFor(t, func() bool { return !a.replicating.Load() }) + a.mu.Lock() + defer a.mu.Unlock() + if got := a.replicas[sb.ID]; !got.Equal(sb.ClaimedAt) { + t.Errorf("first try at %v, want the claim's start %v so its first round comes one interval later", got, sb.ClaimedAt) + } +} + +func TestReplicaAdoptionCarriesTheRoundBeforeItsLast(t *testing.T) { + c := newReplicaCluster(t) + a, b := c.node(t, "a", t.TempDir(), keepAnHour), c.node(t, "b", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + replicateNow(t, a, sb) + a.mu.Lock() + first := sb.ReplicaAt + a.mu.Unlock() + replicateNow(t, a, sb) + c.expire("a") + mustAdopt(t, b, sb, "adopt") + held, ok := b.byID(sb.ID) + if !ok { + t.Fatal("the adopter holds no claim") + } + b.mu.Lock() + defer b.mu.Unlock() + if !held.ReplicaAt.Equal(first) { + t.Errorf("adopted replica_at %v, want the first round's %v, which the row of the second round carries", held.ReplicaAt, first) + } +} + func TestOverdueClaimsKeepTheirPhasesAfterTheCatchUpRound(t *testing.T) { c := newReplicaCluster(t) a := c.node(t, "a", t.TempDir(), keepAnHour) @@ -264,6 +303,28 @@ func TestReplicaUnavailablePutThatNeverLandedIsSettledAtTheNextRound(t *testing. } } +func TestReplicaSettledLostPutReplyStampsItsRound(t *testing.T) { + c := newReplicaCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + a.archives = &hookRegistry{Registry: c.reg, losePutReply: true} + if err := a.replicate(t.Context(), sb); !errors.Is(err, ErrArchiveStoreDown) { + t.Fatalf("round whose put reply was lost: %v", err) + } + a.archives = c.reg + sb.Transition.Lock() + err := a.settleReplicaIntent(t.Context(), sb) + sb.Transition.Unlock() + if err != nil { + t.Fatalf("settle: %v", err) + } + a.mu.Lock() + defer a.mu.Unlock() + if sb.ReplicaPending != "" || sb.ReplicaCk == "" || sb.ReplicaAt.IsZero() { + t.Errorf("settled claim pending %q replica %q replica_at %v, want the landed round promoted and stamped", sb.ReplicaPending, sb.ReplicaCk, sb.ReplicaAt) + } +} + func TestReplicaRestartSettlesALostPutReply(t *testing.T) { c := newReplicaCluster(t) dirA := t.TempDir() diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index af94e04a..a01d13a8 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -361,6 +361,7 @@ type Sandbox struct { Replicated bool `json:"-"` ArchiveRetryAt time.Time `json:"-"` + ReplicaDue bool `json:"-"` // lastActivity is unix-nanos of the last data-plane connection, stamped lock-free. lastActivity atomic.Int64 From 36d116b58f5c40ed7ba61c8113797d62783682c5 Mon Sep 17 00:00:00 2001 From: CMGS Date: Tue, 10 Nov 2026 21:27:16 +0800 Subject: [PATCH 2/4] review: simplify the round follow-ups balloonReady takes the request's context, so a cancelled POST stops its verb check. The abort-after-unrow move test now ends on a replicate sweep instead of a direct round, which is the path the due mark fixes, so the separate downtime-abort test goes; a no-op wait goes too. --- sandboxd/pool/balloon.go | 5 ++--- sandboxd/pool/move_test.go | 18 ------------------ sandboxd/pool/replica_test.go | 1 - 3 files changed, 2 insertions(+), 22 deletions(-) diff --git a/sandboxd/pool/balloon.go b/sandboxd/pool/balloon.go index 3a46d309..c0c7388d 100644 --- a/sandboxd/pool/balloon.go +++ b/sandboxd/pool/balloon.go @@ -12,7 +12,7 @@ import ( // SetBalloon sets a running claim's balloon target through cocoon and returns cocoon's report; the caller restores the reply's default. func (m *Manager) SetBalloon(ctx context.Context, id, target string) (types.VMBalloon, error) { - if !m.balloonReady() { + if !m.balloonReady(ctx) { return types.VMBalloon{}, ErrBalloonOff } m.mu.Lock() @@ -45,11 +45,10 @@ func (m *Manager) SetBalloon(ctx context.Context, id, target string) (types.VMBa return b, err } -func (m *Manager) balloonReady() bool { +func (m *Manager) balloonReady(ctx context.Context) bool { if m.balloonOK.Load() { return true } - ctx := context.Background() if err := m.eng.CheckBalloonSupport(ctx); err != nil { log.WithFunc("pool.balloonReady").Error(ctx, err, "the balloon verb is off") return false diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 34e3031b..fcc26c1e 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -563,24 +563,6 @@ func TestMoveAbortedAfterDroppingTheRowPutsItBackOnTheNextRound(t *testing.T) { t.Fatal("the row survived the final round") } mc.p.commitHook = nil - replicateNow(t, mc.s, sb) - if _, ok, _ := mc.reg.Get(t.Context(), sb.ID); !ok { - t.Error("the next round did not put the row back") - } -} - -func TestMoveAbortedByTheDowntimeCheckReplicatesAtTheNextSweep(t *testing.T) { - mc := newMoveClusterWith(t, func(c *failoverCluster) { c.replicate = replicateEvery }) - sb := mustClaim(t, mc.s, testKey) - replicateNow(t, mc.s, sb) - fakeOf(mc.s).finalCaptureTime = 200 * time.Millisecond - if _, err := mc.s.Move(t.Context(), sb.ID, "b", MoveStartAllowance+100*time.Millisecond); !errors.Is(err, ErrMoveDowntime) { - t.Fatalf("move: %v, want ErrMoveDowntime", err) - } - assertStillOnSource(t, mc, sb) - if _, ok, _ := mc.reg.Get(t.Context(), sb.ID); ok { - t.Fatal("the row survived the final round") - } mc.s.replicateOnce(t.Context()) waitFor(t, func() bool { return !mc.s.replicating.Load() }) if _, ok, _ := mc.reg.Get(t.Context(), sb.ID); !ok { diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index ce470a45..0c4286ba 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -148,7 +148,6 @@ func TestANewClaimFirstReplicatesOneIntervalAfterItsClaim(t *testing.T) { a := c.node(t, "a", t.TempDir(), keepAnHour) sb := mustClaim(t, a, testKey) a.replicateOnce(t.Context()) - waitFor(t, func() bool { return !a.replicating.Load() }) a.mu.Lock() defer a.mu.Unlock() if got := a.replicas[sb.ID]; !got.Equal(sb.ClaimedAt) { From a523bad3a0642f5d7e610681e9e88bb2bd11a6e9 Mon Sep 17 00:00:00 2001 From: CMGS Date: Tue, 10 Nov 2026 21:32:49 +0800 Subject: [PATCH 3/4] review: a restarted move abort marks the unrowed claim due, and the upgrade note says which claims to restore The due mark came from mv.replica, which the journal does not keep, so an abort resumed after a restart never set it. It now reads the state unrowForMove leaves: no replica checkpoint and a replica_at kept. The upgrade note restores every claim whose inspected balloon target is above its default, including running claims the reload did not deflate. --- docs/deploy.md | 11 +++++++---- sandboxd/pool/move_out.go | 2 +- sandboxd/pool/move_test.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/docs/deploy.md b/docs/deploy.md index 1be99ed1..0ec9a001 100644 --- a/docs/deploy.md +++ b/docs/deploy.md @@ -44,10 +44,13 @@ on every node before the roll. Claims that v0.1.30 inflated stay inflated, both running idle claims and idle-hibernated ones, whose snapshots carry the target. To deflate the running ones, remove the block on v0.1.30 with a config [reload](#reloading-the-config), not a restart: the next sweep then deflates -them. Then roll. A hibernated claim that v0.1.30 inflated keeps that target -when it wakes; send it [`POST /v1/sandboxes/{id}/balloon`](sandboxd-api.md#post-v1sandboxesidballoon) -after the wake, with `target` set to the boot target, the `default` that any -balloon reply reports. That call now needs a +them. Then roll. After the roll, restore every claim whose balloon target is +still above its default; `cocoon vm inspect` reports both as `balloon.target` +and `balloon.default`. That covers claims that were hibernated when the block +came off, which keep the inflated target when they wake, and running claims +the reload did not deflate, such as one moved onto a v0.1.30 node without the +block. Send each [`POST /v1/sandboxes/{id}/balloon`](sandboxd-api.md#post-v1sandboxesidballoon) +with its `default` as `target`, waking a hibernated one first. That call now needs a `target` body: an empty `POST`, which v0.1.30 answered 202, answers 400. An S3 `checkpoint_store` written before v0.1.18 used another object layout diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index b1573183..c28b27ed 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -349,7 +349,7 @@ func (m *Manager) abortMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMo logger := log.WithFunc("pool.abortMoveOut") m.mu.Lock() vm, live := sb.VMName, m.claimed[sb.ID] == sb - if live && mv.replica != "" { + if live && sb.ReplicaCk == "" && !sb.ReplicaAt.IsZero() { sb.ReplicaDue = true } m.mu.Unlock() diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index fcc26c1e..6b6d2cb5 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -1162,6 +1162,35 @@ func TestMoveRestartResolvesAJournaledMoveThroughItsRecord(t *testing.T) { } } +func TestMoveRestartAbortOfAnUnrowedClaimMakesItDue(t *testing.T) { + const vm, series = "sbx-unrowed", "s1" + c := newFailoverCluster(t) + reg := movestest.NewMem() + c.moves = reg + epoch := seedMoveRecord(t, reg, "sb_unrowed", "a", "b", moves.Final) + sb := &types.Sandbox{ID: "sb_unrowed", VMName: vm, Key: testKey, Token: "tok", Deadline: time.Now().Add(time.Hour), ReplicaAt: time.Now().Add(-time.Minute)} + sb.MoveOut = &types.MoveOut{Epoch: epoch, Series: series} + dataDir := t.TempDir() + seedClaims(t, dataDir, map[string]*types.Sandbox{sb.ID: sb}) + eng := newFakeEngine() + eng.vms[vm], eng.migrateSeries[vm] = "/vsock/"+vm, series + m := c.nodeOn(t, eng, "a", dataDir, keepAnHour) + m.liftFence() + if err := m.Reconcile(t.Context()); err != nil { + t.Fatalf("reconcile: %v", err) + } + waitFor(t, func() bool { return !m.moveHeld(sb.ID) }) + held, ok := m.byID(sb.ID) + if !ok { + t.Fatal("the source lost the claim whose move it aborted") + } + m.mu.Lock() + defer m.mu.Unlock() + if !held.ReplicaDue { + t.Error("the claim whose row the move dropped is not due at the next replica sweep after a restarted abort") + } +} + func TestStaleMoveRecordsAreDeletedOnlyByTheirSource(t *testing.T) { mc := newMoveCluster(t) ctx, reg := t.Context(), mc.moveReg From 87789ad4447ea00b9517e7cd9a3afc4381f4c6b6 Mon Sep 17 00:00:00 2001 From: CMGS Date: Tue, 10 Nov 2026 21:47:41 +0800 Subject: [PATCH 4/4] review: a move aborted after its pause, or a restart-promoted first replica, keeps the claim due for failover An abort after the guest paused marks the claim due whatever the row writes answered, so a consume whose reply was lost (the row a tombstone, ReplicaCk kept) is repaired at the next sweep instead of the hourly one. A restart that promotes a landed pending round stamps replica_at, so a later unrowed abort of that claim still reads as due. --- sandboxd/pool/failover.go | 3 +++ sandboxd/pool/move_out.go | 2 +- sandboxd/pool/move_test.go | 6 ++++++ sandboxd/pool/replica_test.go | 4 ++-- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index 6cf48e85..7a86356a 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -669,6 +669,9 @@ func (m *Manager) fenceArchived(ctx context.Context, claims map[string]*types.Sa drop = true default: if ok && !rec.Consumed { + if rec.Checkpoint == sb.ReplicaPending { + sb.ReplicaAt = time.Now() + } sb.ReplicaCk, sb.ArchiveEpoch = rec.Checkpoint, rec.Epoch } sb.ReplicaPending = "" diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index c28b27ed..2a06b3b6 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -349,7 +349,7 @@ func (m *Manager) abortMoveOut(ctx context.Context, sb *types.Sandbox, mv *outMo logger := log.WithFunc("pool.abortMoveOut") m.mu.Lock() vm, live := sb.VMName, m.claimed[sb.ID] == sb - if live && sb.ReplicaCk == "" && !sb.ReplicaAt.IsZero() { + if live && (mv.final || sb.ReplicaCk == "" && !sb.ReplicaAt.IsZero()) { sb.ReplicaDue = true } m.mu.Unlock() diff --git a/sandboxd/pool/move_test.go b/sandboxd/pool/move_test.go index 6b6d2cb5..68643fbd 100644 --- a/sandboxd/pool/move_test.go +++ b/sandboxd/pool/move_test.go @@ -626,6 +626,12 @@ func TestMoveWhoseRowConsumeReplyIsLostKeepsTheGuestAndItsReplica(t *testing.T) if !ckExists(t, mc.s, replica) { t.Fatal("the failed move deleted the replica the claim still names") } + mc.s.archives = mc.reg + mc.s.replicateOnce(t.Context()) + waitFor(t, func() bool { return !mc.s.replicating.Load() }) + if rec, ok, _ := mc.reg.Get(t.Context(), sb.ID); !ok || rec.Consumed { + t.Fatalf("row %+v %v after the first sweep, want a live row again", rec, ok) + } mc.s.sweepOrphanArchives(t.Context()) if rec, ok, _ := mc.reg.Get(t.Context(), sb.ID); !ok || rec.Consumed || rec.Checkpoint != replica { t.Errorf("row %+v %v after the sweep, want the replica back on it", rec, ok) diff --git a/sandboxd/pool/replica_test.go b/sandboxd/pool/replica_test.go index 0c4286ba..6bfaeb39 100644 --- a/sandboxd/pool/replica_test.go +++ b/sandboxd/pool/replica_test.go @@ -343,8 +343,8 @@ func TestReplicaRestartSettlesALostPutReply(t *testing.T) { if !ok || restarted.replicaOf(held) != pending || !ckExists(t, restarted, pending) { t.Fatalf("claim held %v, replica %q, ck kept %v, want the row's checkpoint %s adopted", ok, restarted.replicaOf(held), ckExists(t, restarted, pending), pending) } - if held.ReplicaPending != "" { - t.Errorf("intent %q left after the restart settled it", held.ReplicaPending) + if held.ReplicaPending != "" || held.ReplicaAt.IsZero() { + t.Errorf("intent %q left, replica_at %v after the restart settled it, want no intent and the landed round stamped", held.ReplicaPending, held.ReplicaAt) } }