pool: settle a capture interrupted by sandboxd's death

docs/deploy.md

@@ -680,7 +680,11 @@ dying with the service's cgroup. sandboxd starts no new `cocoon` command once
 stopped and waits for the running ones, since cocoon dies on its next log line
 once sandboxd's end of its stderr closes, and a hibernate killed that way leaves
 the guest paused. A save or hibernate can take 26 minutes, hence
-`TimeoutStopSec=27min`; stops normally take seconds. Claimed
+`TimeoutStopSec=27min`; stops normally take seconds. If sandboxd dies abruptly during
+a capture instead, the next start holds that claim until the `cocoon` command has
+exited, then settles it: a hibernate whose snapshot landed stays hibernated and wakes
+from it, and any other capture resumes the guest, or marks the claim failed if it
+does not answer within 30 minutes. Claimed
 sandboxes are reaped when their TTL expires (default 5m, capped at 24h).
 
 A snapshot save, hibernate, export, restore or clone has 15 minutes. Once canceled, a

e2e/fakeengine_test.go

@@ -180,6 +180,8 @@ func (f *fakeEngine) VMEvents(ctx context.Context, _ string, _ engine.VMSyncFunc
 	return ctx.Err()
 }
 
+func (f *fakeEngine) WaitCapture(context.Context, string) error { return nil }
+
 func (f *fakeEngine) Probe(ctx context.Context, vsockSocket string, timeout time.Duration) error {
 	return f.real.Probe(ctx, vsockSocket, timeout)
 }

sandboxd/engine/capture.go

@@ -0,0 +1,32 @@
+package engine
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+	"slices"
+	"time"
+)
+
+const captureExitPoll = time.Second
+
+func (e *Engine) WaitCapture(ctx context.Context, snap string) error {
+	for captureRunning(snap) {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(captureExitPoll):
+		}
+	}
+	return nil
+}
+
+func captureRunning(snap string) bool {
+	procs, _ := os.ReadDir("/proc")
+	arg := []byte("\x00" + snap + "\x00")
+	return slices.ContainsFunc(procs, func(proc os.DirEntry) bool {
+		cmdline, err := os.ReadFile(filepath.Join("/proc", proc.Name(), "cmdline"))
+		return err == nil && bytes.Contains(cmdline, arg)
+	})
+}

sandboxd/engine/engine_test.go

@@ -7,7 +7,9 @@ import (
 	"io"
 	"net"
 	"os"
+	"os/exec"
 	"path/filepath"
+	"runtime"
 	"slices"
 	"strconv"
 	"strings"
@@ -328,6 +330,33 @@ func TestCloseWaitsForARunningCommandAndRefusesNewOnes(t *testing.T) {
 	}
 }
 
+func TestWaitCaptureBlocksOnAnExactArgument(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip("reads /proc")
+	}
+	snap := "sbx-hib-" + strings.ReplaceAll(t.Name(), "/", "-")
+	cmd := exec.Command("sh", "-c", "sleep 30; :", "--name", snap)
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() })
+	done := make(chan error, 1)
+	go func() { done <- newTestEngine().WaitCapture(t.Context(), snap) }()
+	select {
+	case err := <-done:
+		t.Fatalf("WaitCapture returned %v with the process alive", err)
+	case <-time.After(2 * captureExitPoll):
+	}
+	if err := newTestEngine().WaitCapture(t.Context(), snap[:len(snap)-1]); err != nil {
+		t.Errorf("a prefix of the name: %v", err)
+	}
+	_ = cmd.Process.Kill()
+	_ = cmd.Wait()
+	if err := <-done; err != nil {
+		t.Errorf("WaitCapture after exit: %v", err)
+	}
+}
+
 func TestADialOfAGuestThatNeverAnswersFails(t *testing.T) {
 	t.Parallel()
 	path := sockPath(t)

sandboxd/pool/capture.go

@@ -2,6 +2,8 @@ package pool
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"strings"
 
 	"github.com/projecteru2/core/log"
@@ -22,8 +24,20 @@ func (m *Manager) sourceSnap(ctx context.Context, sb *types.Sandbox) (string, fu
 		return sb.HibernateSnap, func() {}, nil
 	}
 	snap := forkPrefix + strings.TrimPrefix(sb.VMName, vmPrefix) + "-" + randHex(3)
+	intent, live := m.setIfLive(sb, func() { sb.SaveSnap = snap })
+	if !live {
+		return "", nil, ErrUnknownSandbox
+	}
+	if err := m.store.commit(intent); err != nil {
+		m.setIfLive(sb, func() { sb.SaveSnap = "" })
+		return "", nil, fmt.Errorf("save %s: persist intent: %w", sb.ID, err)
+	}
 	if err := m.eng.SnapshotSave(ctx, sb.VMName, snap); err != nil {
-		return "", nil, err
+		_, settleErr := m.settleFailedCapture(ctx, sb)
+		return "", nil, errors.Join(err, settleErr)
+	}
+	if _, err := m.commitIfLive(ctx, sb, func() { sb.SaveSnap = "" }); err != nil {
+		log.WithFunc("pool.sourceSnap").Warnf(ctx, "clear save intent of %s: %v", sb.ID, err)
 	}
 	return snap, func() { m.dropSnap(ctx, snap) }, nil
 }

sandboxd/pool/claims.go

@@ -54,6 +54,7 @@ type claimDTO struct {
 	TAP            string             `json:"tap,omitempty"`
 	HibernateSnap  string             `json:"hibernate_snap,omitempty"`
 	PendingSnap    string             `json:"pending_snap,omitempty"`
+	SaveSnap       string             `json:"save_snap,omitempty"`
 	ReaderSafe     bool               `json:"reader_safe,omitzero"`
 	ArchiveCk      string             `json:"archive_ck,omitempty"`
 	ArchiveEpoch   int64              `json:"archive_epoch,omitzero"`
@@ -394,7 +395,7 @@ func dtoOf(sb *types.Sandbox) claimDTO {
 		PolicySource: sb.PolicySource, NoEgress: sb.NoEgress,
 		Tenant: sb.Tenant, EgressClass: sb.EgressClass, ClaimRef: sb.ClaimRef, Metadata: sb.Metadata, OnExpire: sb.OnExpire,
 		Volumes: slices.Clone(sb.Volumes), VolumesWant: volumesWantDTO(sb), Env: sb.Env, VsockSocket: sb.VsockSocket,
-		TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, ReaderSafe: sb.ReaderSafe,
+		TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, SaveSnap: sb.SaveSnap, ReaderSafe: sb.ReaderSafe,
 		ArchiveCk: sb.ArchiveCk, ArchiveEpoch: sb.ArchiveEpoch, ArchivePending: sb.ArchivePending, ReplicaCk: sb.ReplicaCk, ReplicaPending: sb.ReplicaPending, FromCheckpoint: sb.FromCheckpoint,
 		Restarts: sb.Restarts, RestartedAt: sb.RestartedAt, Failed: sb.Failed, MoveOut: sb.MoveOut, MoveIn: sb.MoveIn,
 	}

sandboxd/pool/hibernate.go

@@ -83,7 +83,7 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error
 		// cocoon resumes the guest before its fresh tap can be re-locked
 		return ErrNoEgressHibernate
 	}
-	if _, err := m.settlePendingSnap(ctx, sb); err != nil {
+	if err := m.settleCapture(ctx, sb); err != nil {
 		return err
 	}
 	if sb.HibernateSnap != "" {
@@ -122,17 +122,16 @@ func (m *Manager) hibernateLocked(ctx context.Context, sb *types.Sandbox) error
 	m.mu.Unlock()
 	if hibErr := m.eng.Hibernate(ctx, sb.VMName, snap); hibErr != nil {
 		// The engine can report failure after the snapshot landed
-		adopted, resolveErr := m.settlePendingSnap(ctx, sb)
+		adopted, settleErr := m.settleFailedCapture(ctx, sb)
 		if adopted {
-			m.out.Disarm(sb.ID, true)
-			if resolveErr != nil {
-				return fmt.Errorf("hibernate %s: persist claims: %w", sb.ID, resolveErr)
+			if settleErr != nil {
+				return fmt.Errorf("hibernate %s: persist claims: %w", sb.ID, settleErr)
 			}
 			m.dropStale(ctx, sb)
 			return nil
 		}
-		if resolveErr != nil {
-			return errors.Join(fmt.Errorf("hibernate %s: %w", sb.ID, hibErr), resolveErr)
+		if settleErr != nil {
+			return errors.Join(fmt.Errorf("hibernate %s: %w", sb.ID, hibErr), settleErr)
 		}
 		return fmt.Errorf("hibernate %s: %w", sb.ID, hibErr)
 	}
@@ -170,7 +169,7 @@ func (m *Manager) wakeOrRead(ctx context.Context, sb *types.Sandbox, readerOK bo
 	if sb.Failed != "" {
 		return "", fmt.Errorf("%w: %s", ErrFailed, sb.Failed)
 	}
-	if _, err := m.settlePendingSnap(ctx, sb); err != nil {
+	if err := m.settleCapture(ctx, sb); err != nil {
 		return "", fmt.Errorf("wake %s: %w", sb.ID, err)
 	}
 	if sb.HibernateSnap == "" {

sandboxd/pool/hibernate_test.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"testing"
 	"testing/synctest"
+	"time"
 
 	"github.com/cocoonstack/sandbox/sandboxd/config"
 	"github.com/cocoonstack/sandbox/sandboxd/types"
@@ -109,10 +110,10 @@ func TestHibernateVerifiedFailureClearsIntent(t *testing.T) {
 	if _, g := m.Info(); g.Hibernated != 0 {
 		t.Errorf("hibernated %d, want 0 (still running)", g.Hibernated)
 	}
-	got := journaled(t, m)
-	if got[sb.ID] == nil || got[sb.ID].HibernateSnap != "" || got[sb.ID].PendingSnap != "" {
-		t.Fatalf("journal %+v, want running (no snaps)", got[sb.ID])
-	}
+	waitFor(t, func() bool {
+		row := journaled(t, m)[sb.ID]
+		return row != nil && row.HibernateSnap == "" && row.PendingSnap == ""
+	})
 	eng.hibernateErr = nil
 	if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil {
 		t.Fatalf("Hibernate retry: %v", err)
@@ -192,8 +193,9 @@ func TestRetryResolvesDanglingIntent(t *testing.T) {
 		t.Fatalf("intent %q, want the real snapshot %q kept", got[sb.ID].PendingSnap, realSnap)
 	}
 
-	eng.hibernateErr = nil
-	eng.snapListErr = nil
+	eng.mu.Lock()
+	eng.hibernateErr, eng.snapListErr = nil, nil
+	eng.mu.Unlock()
 	if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil {
 		t.Fatalf("Hibernate retry: %v", err)
 	}
@@ -247,24 +249,23 @@ func TestResolvePendingSnapReleaseRaceDropsOrphan(t *testing.T) {
 		leaveDanglingIntent(t, m, eng, sb)
 		realSnap := eng.hibernates[0]
 
-		eng.snapListErr = nil
-		eng.snapListStall = make(chan struct{})
 		callsBefore := eng.snapListCalls()
-		wakeErr := make(chan error, 1)
-		go func() {
-			_, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token)
-			wakeErr <- err
-		}()
-		waitFor(t, func() bool { return eng.snapListCalls() > callsBefore })
+		eng.mu.Lock()
+		eng.snapListErr, eng.snapListStall = nil, make(chan struct{})
+		eng.mu.Unlock()
+		time.Sleep(2 * captureResumeRetry)
+		synctest.Wait()
+		if eng.snapListCalls() <= callsBefore {
+			t.Fatal("the settle never re-read the snapshot list")
+		}
 		if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil {
 			t.Fatalf("release: %v", err)
 		}
 		close(eng.snapListStall)
-
-		if err := <-wakeErr; !errors.Is(err, ErrUnknownSandbox) {
+		waitFor(t, func() bool { return eng.snapRemoved(realSnap) })
+		if _, _, _, err := m.WakeAgentSocket(t.Context(), sb.ID, sb.Token); !errors.Is(err, ErrUnknownSandbox) {
 			t.Fatalf("wake: %v, want ErrUnknownSandbox", err)
 		}
-		waitFor(t, func() bool { return eng.snapRemoved(realSnap) })
 	})
 }
 
@@ -275,11 +276,12 @@ func TestResolveAdoptBillsWhenPersistFails(t *testing.T) {
 		sb := mustClaim(t, m, testKey)
 		leaveDanglingIntent(t, m, eng, sb)
 
-		eng.snapListErr = nil
 		breakStore(t, m)
-		if err := m.Hibernate(t.Context(), sb.ID, Cred{Token: sb.Token}); err == nil {
-			t.Fatal("Hibernate reported success despite a persist failure")
-		}
+		eng.mu.Lock()
+		eng.snapListErr = nil
+		eng.mu.Unlock()
+		time.Sleep(2 * captureResumeRetry)
+		synctest.Wait()
 		if n := m.Counters().Hibernates; n != 1 {
 			t.Errorf("hibernates billed=%d, want 1 (billed even while the persist lags)", n)
 		}