diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90a93c75..5a8d0364 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -336,19 +336,38 @@ jobs: echo "pgtest: checked $(printf '%s\n' "$pkgs" | wc -l) real-Postgres packages against the service database" exit "$rc" - - name: Enable KVM - # The tagged microVM suites open /dev/kvm; on a fresh ubuntu-latest runner - # the device is root:kvm 0660, so the invoking uid cannot open it. Write - # the udev group-perms rule the ecosystem standardizes on — the same rule - # the Determinate nix-installer-action installs — then reload+trigger so - # it applies to the live node. We install Nix via cachix/install-nix-action - # (above), not the Determinate action, so we write the rule ourselves. + - name: Enable KVM and unprivileged user namespaces + # Two ephemeral-runner relaxations the microVM boot leg needs, both the + # §E-D2 throwaway-runner class (a GHA runner is a single-tenant throwaway, + # so a looser device/kernel posture that the dev box would never take is + # harmless here). sudo is the runner's passwordless sudo; the test + # processes themselves run rootless as the invoking uid. # - # MODE="0666" (world-RW), deliberately looser than the dev box's kvm-group - # 0660 (record §E-D2): a GHA runner is an ephemeral single-tenant throwaway - # where a world-RW /dev/kvm is harmless, so we skip the group-membership - # dance and just make it openable. sudo is the runner's passwordless sudo; - # the test processes themselves run rootless as the invoking uid. + # 1. /dev/kvm openable. The tagged microVM suites open /dev/kvm; on a + # fresh ubuntu-latest runner the device is root:kvm 0660, so the + # invoking uid cannot open it. Write the udev group-perms rule the + # ecosystem standardizes on — the same rule the Determinate + # nix-installer-action installs — then reload+trigger so it applies to + # the live node. We install Nix via cachix/install-nix-action (above), + # not the Determinate action, so we write the rule ourselves. MODE + # "0666" (world-RW) is deliberately looser than the dev box's kvm-group + # 0660: on this throwaway runner a world-RW /dev/kvm is harmless, so we + # skip the group-membership dance and just make it openable. + # + # 2. Unprivileged user namespaces allowed. ubuntu-latest is now Ubuntu + # 24.04, which ships AppArmor's apparmor_restrict_unprivileged_userns=1 + # — an unconfined non-root process can no longer create a user + # namespace. passt AND virtiofsd both self-sandbox by detaching into + # their own userns (passt's --sandbox has no opt-out; virtiofsd's + # --sandbox=namespace is what the record §T4 specifies), so under the + # restriction they die at startup with "Failed to detach isolating + # namespaces: Operation not permitted" and the guest never boots. Set + # the sysctl to 0 to lift the restriction for this boot. This is a + # kernel-hardening knob, not a capability grant: the daemons still run + # rootless as the invoking uid with no added privilege — the record's + # "only privilege is the kvm group, no CAP_NET_ADMIN, no rootful + # helper" constraint holds. The setting reverts on the runner's + # teardown (it is a throwaway VM); nothing persists. # # Assume-KVM required-leg posture (§E-D2): the microVM step below sets # COMPASS_REQUIRE_MICROVM=1, so a run where GitHub withholds /dev/kvm reds @@ -358,6 +377,7 @@ jobs: | sudo tee /etc/udev/rules.d/99-kvm4all.rules sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm + echo 0 | sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns - name: microVM suites working-directory: go diff --git a/.moon/workspace.yml b/.moon/workspace.yml index 619e1326..b639015a 100644 --- a/.moon/workspace.yml +++ b/.moon/workspace.yml @@ -39,6 +39,12 @@ projects: # cache lane or V2a boot bring-up. Same affected-detection posture as # compass-agent-image (see guest-image/moon.yml). compass-guest-image: 'guest-image' + # The local microVM boot-test lane (RIG-2591): realises the guest image + VMM + # stack from nix and execs the KVM-gated `go test -tags microvm` suite the + # untagged compass-go:test lane never builds. Registered so its typecheck + + # unit tests ride the moon-driven CI sweep; the boot lane itself lives on + # compass-go:test-microvm (runInCI:false — it needs KVM + a nix build). + microvm-boot-test: 'tools/microvm-boot-test' # The Compass native-app release bundle: a heavy nix build (realises the # WebKitGTK cc/pkg-config closure) that stages the versioned tarball. Same # affected-detection posture as compass-agent-image — registered here so the diff --git a/bun.lock b/bun.lock index 0ffb347a..418086f4 100644 --- a/bun.lock +++ b/bun.lock @@ -148,6 +148,16 @@ "typescript": "catalog:", }, }, + "tools/microvm-boot-test": { + "name": "@compass/microvm-boot-test", + "bin": { + "microvm-boot-test": "./run.ts", + }, + "devDependencies": { + "@types/bun": "catalog:", + "typescript": "catalog:", + }, + }, "tools/orion-ref-gate": { "name": "@compass/orion-ref-gate", "bin": { @@ -400,6 +410,8 @@ "@compass/forge-linear-token": ["@compass/forge-linear-token@workspace:tools/forge-linear-token"], + "@compass/microvm-boot-test": ["@compass/microvm-boot-test@workspace:tools/microvm-boot-test"], + "@compass/orion-ref-gate": ["@compass/orion-ref-gate@workspace:tools/orion-ref-gate"], "@compass/renovate": ["@compass/renovate@workspace:tools/renovate"], diff --git a/go/internal/microvmtest/canary_microvm_test.go b/go/internal/microvmtest/canary_microvm_test.go index 65c8e1a0..07cc5c41 100644 --- a/go/internal/microvmtest/canary_microvm_test.go +++ b/go/internal/microvmtest/canary_microvm_test.go @@ -63,6 +63,9 @@ func TestCanaryMicroVMEnv(t *testing.T) { if env.KernelImage == "" { t.Error("resolved Env.KernelImage is empty") } + if env.InitrdImage == "" { + t.Error("resolved Env.InitrdImage is empty") + } if env.RootfsImage == "" { t.Error("resolved Env.RootfsImage is empty") } @@ -72,6 +75,9 @@ func TestCanaryMicroVMEnv(t *testing.T) { if env.VirtiofsdPath == "" { t.Error("resolved Env.VirtiofsdPath is empty") } + if env.PasstPath == "" { + t.Error("resolved Env.PasstPath is empty") + } // The two guest-image paths must exist on disk: this is what proves E3's // attrs were realized and exported, not merely that the env vars were set to @@ -84,6 +90,7 @@ func TestCanaryMicroVMEnv(t *testing.T) { }{ {"guest kernel", env.KernelImage}, {"guest rootfs", env.RootfsImage}, + {"guest initrd", env.InitrdImage}, } { if _, err := os.Stat(img.path); err != nil { t.Errorf("%s image %q does not exist on disk: %v", img.name, img.path, err) diff --git a/go/internal/microvmtest/microvmtest.go b/go/internal/microvmtest/microvmtest.go index 4d257cd6..f479b8a7 100644 --- a/go/internal/microvmtest/microvmtest.go +++ b/go/internal/microvmtest/microvmtest.go @@ -53,21 +53,32 @@ const KernelEnvVar = "COMPASS_TEST_GUEST_KERNEL" // export it pointing at the `nix build .#compass-guest-rootfs` result). const RootfsEnvVar = "COMPASS_TEST_GUEST_ROOTFS" +// InitrdEnvVar points the harness at the guest initramfs image to boot. Like +// KernelEnvVar it is supplied by the environment (the dev shell / CI KVM leg +// export it pointing at the `nix build .#compass-guest-initrd` result). The +// initrd is load-bearing, not optional: the pinned generic kernel ships virtio/ +// erofs/overlay as modules, so the initrd is what loads them and mounts the +// root before switch_root (record §(a)). +const InitrdEnvVar = "COMPASS_TEST_GUEST_INITRD" + // vmmBinary is the VMM the microVM suites drive; virtiofsdBinary is the // virtio-fs daemon they pair it with. Both are resolved from PATH (the dev shell // and CI KVM leg put them there) when Require builds the Env. const ( vmmBinary = "cloud-hypervisor" virtiofsdBinary = "virtiofsd" + passtBinary = "passt" ) // Env is the resolved microVM test environment Require hands back: the guest // images to boot and the host binaries to drive them with. type Env struct { KernelImage string + InitrdImage string RootfsImage string VMMPath string VirtiofsdPath string + PasstPath string } // kvmSource is which of the three KVM-availability paths Require takes. @@ -148,11 +159,20 @@ func resolveEnv(t *testing.T) Env { if err != nil { t.Fatalf("microVM test requires %s on PATH: %v", virtiofsdBinary, err) } + passtPath, err := exec.LookPath(passtBinary) + if err != nil { + t.Fatalf("microVM test requires %s on PATH: %v", passtBinary, err) + } kernelImage := os.Getenv(KernelEnvVar) if kernelImage == "" { t.Fatalf("microVM test requires %s to point at the guest kernel image "+ "(exported by the dev shell / CI KVM leg from `nix build .#compass-guest-kernel`)", KernelEnvVar) } + initrdImage := os.Getenv(InitrdEnvVar) + if initrdImage == "" { + t.Fatalf("microVM test requires %s to point at the guest initramfs image "+ + "(exported by the dev shell / CI KVM leg from `nix build .#compass-guest-initrd`)", InitrdEnvVar) + } rootfsImage := os.Getenv(RootfsEnvVar) if rootfsImage == "" { t.Fatalf("microVM test requires %s to point at the guest rootfs image "+ @@ -160,8 +180,10 @@ func resolveEnv(t *testing.T) Env { } return Env{ KernelImage: kernelImage, + InitrdImage: initrdImage, RootfsImage: rootfsImage, VMMPath: vmmPath, VirtiofsdPath: virtiofsdPath, + PasstPath: passtPath, } } diff --git a/go/internal/runtime/microvm/boot_microvm_test.go b/go/internal/runtime/microvm/boot_microvm_test.go new file mode 100644 index 00000000..4fba5b18 --- /dev/null +++ b/go/internal/runtime/microvm/boot_microvm_test.go @@ -0,0 +1,300 @@ +//go:build microvm && unix + +// boot_microvm_test.go is T4's KVM-gated boot spike: it boots a real session +// guest under cloud-hypervisor (rootless, as the invoking user), completes the +// GuestControl.Health handshake over the hybrid vsock, measures per-process +// PSS, and tears the stack down with no orphans — the deliverable of the V2a +// milestone (record §T4, §(g)). Every test calls microvmtest.Require(t) first: +// on a KVM-less box it SKIPS (unless COMPASS_REQUIRE_MICROVM=1 forces a hard +// fail), so the suite is only real where /dev/kvm is openable. +// +// The tests run in a deliberate order via subtests off one parent, but each is +// independent (its own VM, its own temp dir). The net-only smoke is first +// because it isolates OQ-G — the passt×cloud-hypervisor vhost-user-net +// negotiation, the spike's primary unknown — so a negotiation failure surfaces +// as itself (no address in the serial console) rather than as an ambiguous +// full-stack timeout. + +package microvm + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/RigelBuild/compass/go/internal/microvmtest" +) + +const ( + // testVsockCID is the guest CID (must be >= 3; 0-2 are reserved). + testVsockCID uint32 = 3 + // testVsockPort is the guest port guestd serves the handshake on. It is + // carried into the guest via the appended compass.vsock_port= cmdline + // parameter, which guestd reads from /proc/cmdline (T2). + testVsockPort uint32 = 1024 + // fullBootDeadline bounds Launch→Health-OK for the full boot (record §T4: + // 60 s). + fullBootDeadline = 60 * time.Second + // negativeBootDeadline bounds the corrupt-rootfs negative: the boot must + // fail-closed inside this window, not hang to the outer -timeout. + negativeBootDeadline = 60 * time.Second + // dhcpLeaseDeadline bounds the net-only smoke's wait for the guest to + // acquire its address (observable in the serial console). + dhcpLeaseDeadline = 60 * time.Second +) + +// bootConfig builds a BootConfig from the resolved test env and a fresh temp +// dir. The temp dir holds every AF_UNIX socket, the passt pidfile, the captured +// logs, and the virtio-fs shared dir, so a test leaves nothing outside its own +// t.TempDir(). FSSharedDir is a throwaway dir here (V2b supplies the per-session +// checkout); the spike only asserts the mount happened via Health's flag, not +// content. +func bootConfig(t *testing.T, env microvmtest.Env, cpus, memoryMB int) BootConfig { + t.Helper() + dir := t.TempDir() + return BootConfig{ + Kernel: env.KernelImage, + Initrd: env.InitrdImage, + Rootfs: env.RootfsImage, + VsockCID: testVsockCID, + VsockPort: testVsockPort, + VsockSocket: dir + "/vsock.sock", + FSTag: "workspace", + FSSocket: dir + "/virtiofsd.sock", + FSSharedDir: dir, // a throwaway shared tree; content round-trip is V2b's + CPUs: cpus, + MemoryMB: memoryMB, + Net: NetConfig{ + VhostUserSocket: dir + "/net.sock", + MAC: "12:34:56:78:9a:bc", + }, + } +} + +// TestNetOnlyBootSmoke isolates OQ-G: it boots with the vhost-user-net device +// but WITHOUT virtio-fs or vsock, and asserts only that the passt×CH vhost-user +// negotiation succeeded and the guest acquired its DHCP-delivered address — read +// from the captured serial console (guestd logs "network provisioned … addr=…"). +// The guest boot then fails-closed at the absent workspace mount; that is +// expected and irrelevant here. On failure the serial console + daemon logs are +// surfaced so the cause is visible, not an opaque timeout — the STOP-condition +// evidence the driver needs to decide the pre-authorized gvproxy fallback. +func TestNetOnlyBootSmoke(t *testing.T) { + env := microvmtest.Require(t) + cfg := bootConfig(t, env, 1, 512) + + vm, err := LaunchNetOnly(t.Context(), cfg) + if err != nil { + t.Fatalf("LaunchNetOnly failed: %v", err) + } + t.Cleanup(func() { + if shutErr := vm.Shutdown(context.WithoutCancel(t.Context())); shutErr != nil { + t.Errorf("Shutdown: %v", shutErr) + } + }) + + // Gate on the guest logging its acquired address to the serial console — the + // proof the vhost-user link negotiated and DHCP delivered the host-controlled + // lease. Poll the console (there is no in-band event to receive on: the guest + // has no vsock here), bounded by dhcpLeaseDeadline. + leased := waitForConsole(t, vm, dhcpLeaseDeadline, func(console string) bool { + return strings.Contains(console, "network provisioned") && strings.Contains(console, guestAddr) + }) + if !leased { + t.Fatalf("guest did not acquire its address within %s — passt×cloud-hypervisor "+ + "vhost-user negotiation likely failed (OQ-G STOP CONDITION: report this to the driver).\n%s", + dhcpLeaseDeadline, vm.Diagnostics()) + } + t.Logf("net-only smoke: guest acquired %s (passt×CH vhost-user negotiation OK)", guestAddr) +} + +// TestFullBoot is the milestone deliverable: boot the full stack (2 CPUs / +// 1024 MB), complete the Health handshake with net_provisioned && +// workspace_mounted inside the 60 s deadline, record boot latency and +// per-process PSS, and verify Shutdown leaves no orphan process and removes the +// unix sockets. +func TestFullBoot(t *testing.T) { + env := microvmtest.Require(t) + cfg := bootConfig(t, env, 2, 1024) + + start := time.Now() + vm, err := Launch(t.Context(), cfg) + if err != nil { + t.Fatalf("Launch failed: %v", err) + } + t.Cleanup(func() { + if shutErr := vm.Shutdown(context.WithoutCancel(t.Context())); shutErr != nil { + t.Errorf("Shutdown: %v", shutErr) + } + // No orphans: all three processes gone after Shutdown. + for _, name := range []string{"cloud-hypervisor", "virtiofsd", "passt"} { + if vm.Running(name) { + t.Errorf("%s still running after Shutdown", name) + } + } + // Sockets removed. + for _, s := range vm.sockets { + if fileExists(s) { + t.Errorf("socket %s not removed by Shutdown", s) + } + } + }) + + // Poll Health until the guest reports both flags true, bounded by the boot + // deadline. Early Health calls fail (guest not yet serving the vsock); that + // is the expected pre-ready state, so a failing call is retried, not fatal. + var latency time.Duration + ok := waitFor(t, fullBootDeadline, func() bool { + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + resp, healthErr := vm.Health(ctx) + if healthErr != nil { + return false + } + if resp.GetNetProvisioned() && resp.GetWorkspaceMounted() { + latency = time.Since(start) + return true + } + return false + }) + if !ok { + t.Fatalf("guest did not reach Health{net_provisioned && workspace_mounted} within %s.\n%s", + fullBootDeadline, vm.Diagnostics()) + } + t.Logf("full boot: Health OK (net_provisioned && workspace_mounted) in %s", latency) + + // PSS is informational spike data, not a pass/fail gate: log what is + // readable and note what is not. A rootless harness cannot read a + // sandboxed helper's smaps_rollup (passt sets PR_SET_DUMPABLE=0), so its + // entry is legitimately absent — that is reported, never failed. + pss, pssErr := vm.PSS() + if pssErr != nil { + t.Logf("reading PSS (best-effort): %v", pssErr) + } + for _, name := range []string{"cloud-hypervisor", "virtiofsd", "passt"} { + if kb, present := pss[name]; present { + t.Logf("PSS %s: %d kB", name, kb) + } else { + t.Logf("PSS %s: unavailable (sandboxed or exited)", name) + } + } +} + +// TestCorruptRootfsFailsClosed is the fail-closed negative: boot with a +// nonexistent rootfs path and assert the boot fails inside the deadline with the +// cause visible in the serial console (the initrd's erofs-mount failure), not as +// a silent hang. This exercises the fail-closed teardown path — the VM must +// still Shutdown cleanly. +func TestCorruptRootfsFailsClosed(t *testing.T) { + env := microvmtest.Require(t) + cfg := bootConfig(t, env, 1, 512) + // A raw-sized but non-erofs rootfs (16 MiB of zeros), NOT a missing or tiny + // path: CH must be able to detect a raw image and boot the kernel, so the + // initrd's `mount -t erofs /dev/vda` then fails-closed with a named cause on + // the serial console — the GUEST fail-closed teardown the design exercises. + // A missing path (or a byte-sized garbage file) instead trips CH's disk-open + // / image-type detection before the kernel ever runs, leaving no serial log. + corrupt := t.TempDir() + "/corrupt.erofs" + f, createErr := os.Create(corrupt) + if createErr != nil { + t.Fatalf("creating corrupt rootfs: %v", createErr) + } + if truncErr := f.Truncate(16 << 20); truncErr != nil { + _ = f.Close() // cleanup on an already-failing setup path + t.Fatalf("sizing corrupt rootfs: %v", truncErr) + } + if closeErr := f.Close(); closeErr != nil { + t.Fatalf("closing corrupt rootfs: %v", closeErr) + } + cfg.Rootfs = corrupt + + vm, err := Launch(t.Context(), cfg) + if err != nil { + // A pre-VM spawn failure is also a valid fail-closed outcome. + t.Logf("Launch fail-closed before VM start: %v", err) + return + } + t.Cleanup(func() { + if shutErr := vm.Shutdown(context.WithoutCancel(t.Context())); shutErr != nil { + t.Errorf("Shutdown: %v", shutErr) + } + }) + + // The guest can never reach Health (the root never mounts → no switch_root → + // no guestd), so assert the named fail-closed cause appears in the serial + // console inside the deadline: the initrd's erofs-mount failure. + failed := waitForConsole(t, vm, negativeBootDeadline, func(console string) bool { + return strings.Contains(console, "mount erofs root") || strings.Contains(console, "boot aborted") + }) + if !failed { + t.Fatalf("corrupt-rootfs boot did not surface a named failure in the serial console within %s.\n%s", + negativeBootDeadline, vm.Diagnostics()) + } + // Health must NOT succeed on a fail-closed boot. + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + if _, healthErr := vm.Health(ctx); healthErr == nil { + t.Fatalf("Health unexpectedly succeeded on a corrupt-rootfs boot") + } + t.Logf("corrupt-rootfs boot fail-closed with a named serial-console cause (as designed)") +} + +// waitFor polls cond every 250 ms until it returns true or the deadline elapses, +// returning whether it became true. It is a gated poll-until (not a blind +// time.Sleep): there is no single in-band completion event for a guest boot — +// readiness is only observable by probing Health — so a bounded poll is the +// correct wait, and it fails loudly at a real timeout. +func waitFor(t *testing.T, deadline time.Duration, cond func() bool) bool { + t.Helper() + timeout := time.NewTimer(deadline) + defer timeout.Stop() + tick := time.NewTicker(250 * time.Millisecond) + defer tick.Stop() + if cond() { + return true + } + for { + select { + case <-timeout.C: + return false + case <-tick.C: + if cond() { + return true + } + } + } +} + +// waitForConsole polls the captured serial console until match reports the +// expected content or the deadline elapses. Used where the guest has no vsock to +// probe (the net-only smoke and the corrupt-rootfs negative), so the console is +// the only observable signal. +// +// It matches against the WHOLE console file, not VM.ConsoleTail's bounded +// diagnostic tail: a fail-closed boot prints a kernel panic backtrace AFTER the +// evidence line, and a dump larger than the 8 KiB tail window would evict the +// evidence and turn a real failure into a false-negative timeout. The bounded +// tail stays where it belongs — the failure MESSAGE (Diagnostics) — not the poll +// predicate. +func waitForConsole(t *testing.T, vm *VM, deadline time.Duration, match func(string) bool) bool { + t.Helper() + return waitFor(t, deadline, func() bool { return match(readConsole(vm)) }) +} + +// readConsole returns the full captured serial console, falling back to the +// bounded tail if the file cannot be read yet (e.g. before the first byte is +// flushed). +func readConsole(vm *VM) string { + raw, err := os.ReadFile(vm.consolePath) + if err != nil { + return vm.ConsoleTail() + } + return string(raw) +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/go/internal/runtime/microvm/launch.go b/go/internal/runtime/microvm/launch.go new file mode 100644 index 00000000..b8171c9c --- /dev/null +++ b/go/internal/runtime/microvm/launch.go @@ -0,0 +1,528 @@ +//go:build unix + +package microvm + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "connectrpc.com/connect" + + compassv1 "github.com/RigelBuild/compass/go/internal/gen/compass/v1" +) + +// The host-controlled IPv4 plan passt offers the guest over its built-in DHCP +// server (§(c) lines 128-136): a fixed private /24, delivered as the lease the +// guest's in-process DHCP client (T2) consumes. The address plan is passt's +// launch flags (-a/-g/-n/-D), not a guest guess, so the host stays in control +// of addressing; DHCP is only the delivery mechanism. The 10.x block is a +// throwaway per-boot plan — nothing outside the guest routes it. +const ( + guestAddr = "10.0.2.15" // -a: the address passt leases the guest + guestGW = "10.0.2.2" // -g: the default gateway passt advertises + guestPrefix = "24" // -n: the netmask, as a prefix length + guestDNS = "10.0.2.3" // -D: the resolver passt hands out +) + +// socketReadyTimeout bounds the wait for virtiofsd's and passt's control +// sockets to appear before cloud-hypervisor is started against them. Both +// daemons create their AF_UNIX socket early in startup; a bounded poll (not an +// unbounded sleep) fails fast with a named cause if a daemon dies on launch +// instead of hanging the boot until the outer test deadline. +const socketReadyTimeout = 10 * time.Second + +// reapGrace is how long Shutdown waits for a SIGTERM'd auxiliary daemon +// (virtiofsd/passt) to exit before escalating to SIGKILL. +const reapGrace = 5 * time.Second + +// diagnosticTailBytes caps how much of a captured log (serial console, or a +// daemon's stderr) is surfaced in a failure message — enough to carry the cause +// (a kernel panic banner, a passt/CH negotiation error) without dumping a whole +// boot log into the test output. +const diagnosticTailBytes = 8 << 10 + +// launchOptions selects which virtio devices cloud-hypervisor is booted with. +// The full boot wires every device; the OQ-G net-only smoke omits virtio-fs and +// vsock so the passt×CH vhost-user-net negotiation — the spike's primary +// unknown (§(c)/OQ-G) — is exercised in isolation, and a negotiation failure +// surfaces from the serial console as itself rather than as an ambiguous +// full-stack timeout. +type launchOptions struct { + withFS bool + withVsock bool +} + +// child is one managed guest-support process: the *exec.Cmd plus the file its +// stdout+stderr are captured to, so Shutdown can reap it and a failure can +// surface its log. +type child struct { + name string + cmd *exec.Cmd + logPath string + waited bool // set once cmd.Wait has returned, so liveness probes and PSS skip a reaped process +} + +// VM is a running (or partially-started, on the Launch error path) guest and +// its two supporting host daemons. It owns their process handles, the captured +// serial console + per-daemon logs, and the AF_UNIX sockets/pidfile that must +// be removed on teardown. The zero devices (virtiofsd nil under the net-only +// smoke) are tolerated by Shutdown and PSS. +type VM struct { + vmm *child // cloud-hypervisor + virtiofsd *child // nil under the net-only smoke (no --fs) + passt *child + + consolePath string // --serial file: the guest serial console + + vsockSocket string // host end of the hybrid vsock (empty under the net-only smoke) + vsockPort uint32 + + // Cleanup targets: the AF_UNIX sockets the daemons/VMM serve and passt's + // pidfile. Removed by Shutdown after the processes are reaped. + sockets []string + pidfile string + + shutdownOnce sync.Once + shutdownErr error +} + +// Launch boots one session guest: it starts virtiofsd and passt, waits for +// their control sockets, then starts cloud-hypervisor wired to both plus the +// hybrid vsock. It is the full-stack boot (virtio-fs + vsock + vhost-user-net). +// On any spawn failure it tears down everything already started before +// returning, so no child process is orphaned and no socket is leaked. +func Launch(ctx context.Context, cfg BootConfig) (*VM, error) { + return launch(ctx, cfg, launchOptions{withFS: true, withVsock: true}) +} + +// LaunchNetOnly boots the guest with only the vhost-user-net device — no +// virtio-fs, no vsock (§(c)/OQ-G). It is the isolation smoke that exercises the +// passt×cloud-hypervisor vhost-user negotiation on its own: the guest brings its +// link up and acquires its DHCP lease (observable in the serial console) before +// the boot sequence fails-closed at the absent workspace mount. That failure is +// expected and irrelevant here — the smoke asserts only that the address landed, +// read from the console, so a negotiation failure is a named cause rather than a +// full-stack timeout. +func LaunchNetOnly(ctx context.Context, cfg BootConfig) (*VM, error) { + return launch(ctx, cfg, launchOptions{withFS: false, withVsock: false}) +} + +// launch is the shared boot body. Binaries are resolved from PATH here (not +// carried on BootConfig) so the harness stays a pure boot contract and the test +// gate (microvmtest.Require) owns proving they are present; exec.LookPath +// against the same PATH yields the identical binaries Require resolved. +func launch(ctx context.Context, cfg BootConfig, opts launchOptions) (_ *VM, err error) { + dir := filepath.Dir(cfg.Net.VhostUserSocket) + // vm is a LOCAL, not the named return. The error sites below return an + // explicit `nil, err`; were vm the named return, that nil would clobber the + // handle before the deferred cleanup runs — nil-dereferencing in Shutdown + // and orphaning any daemon already started. Keeping vm local means the defer + // always sees the real, partially-built VM. (This is the fail-closed path CI + // exercises when a daemon cannot sandbox on the runner and dies on launch.) + vm := &VM{ + consolePath: filepath.Join(dir, "console.log"), + vsockPort: cfg.VsockPort, + } + if opts.withVsock { + vm.vsockSocket = cfg.VsockSocket + } + // Tear down whatever started if a later spawn fails: no orphans, no leaked + // sockets on the error path. + defer func() { + if err != nil { + _ = vm.Shutdown(ctx) // best-effort cleanup on an already-failing launch + } + }() + + if opts.withFS { + virtiofsdPath, lookErr := exec.LookPath("virtiofsd") + if lookErr != nil { + return nil, fmt.Errorf("microvm: resolving virtiofsd on PATH: %w", lookErr) + } + vm.virtiofsd = &child{ + name: "virtiofsd", + logPath: filepath.Join(dir, "virtiofsd.log"), + //nolint:gosec // G204: the microVM harness seam — virtiofsdPath is LookPath-resolved and the argv is harness-built from BootConfig, neither user-controlled + cmd: exec.CommandContext(ctx, virtiofsdPath, + "--socket-path="+cfg.FSSocket, + "--shared-dir="+cfg.FSSharedDir, + "--sandbox=namespace"), + } + if startErr := startChild(vm.virtiofsd); startErr != nil { + return nil, fmt.Errorf("microvm: starting virtiofsd: %w", startErr) + } + vm.sockets = append(vm.sockets, cfg.FSSocket) + } + + passtPath, lookErr := exec.LookPath("passt") + if lookErr != nil { + return nil, fmt.Errorf("microvm: resolving passt on PATH: %w", lookErr) + } + vm.pidfile = filepath.Join(dir, "passt.pid") + vm.passt = &child{ + name: "passt", + logPath: filepath.Join(dir, "passt.log"), + // -f keeps passt in the foreground so this *exec.Cmd IS the passt + // process (default is to daemonize, which would orphan it and make the + // Cmd exit immediately). The -a/-g/-n/-D flags fix the host-controlled + // address plan passt serves over DHCP (§(c)). + //nolint:gosec // G204: the microVM harness seam — passtPath is LookPath-resolved and the argv is harness-built (fixed flags + BootConfig socket), neither user-controlled + cmd: exec.CommandContext(ctx, passtPath, + "--vhost-user", + "--socket", cfg.Net.VhostUserSocket, + "--pid", vm.pidfile, + "-f", + "-a", guestAddr, + "-g", guestGW, + "-n", guestPrefix, + "-D", guestDNS), + } + if startErr := startChild(vm.passt); startErr != nil { + return nil, fmt.Errorf("microvm: starting passt: %w", startErr) + } + vm.sockets = append(vm.sockets, cfg.Net.VhostUserSocket) + + // virtiofsd + passt must be serving before cloud-hypervisor connects to + // their sockets. Bounded poll, not a fixed sleep. + ready := []string{cfg.Net.VhostUserSocket} + if opts.withFS { + ready = append(ready, cfg.FSSocket) + } + if waitErr := waitForSockets(ctx, ready, socketReadyTimeout); waitErr != nil { + return nil, fmt.Errorf("microvm: waiting for daemon sockets: %w", waitErr) + } + + vmmPath, lookErr := exec.LookPath("cloud-hypervisor") + if lookErr != nil { + return nil, fmt.Errorf("microvm: resolving cloud-hypervisor on PATH: %w", lookErr) + } + vm.vmm = &child{ + name: "cloud-hypervisor", + logPath: filepath.Join(dir, "cloud-hypervisor.log"), + //nolint:gosec // G204: the microVM harness seam — vmmPath is LookPath-resolved and vmmArgs is harness-built from BootConfig, neither user-controlled + cmd: exec.CommandContext(ctx, vmmPath, vmmArgs(cfg, vm.consolePath, opts)...), + } + if startErr := startChild(vm.vmm); startErr != nil { + return nil, fmt.Errorf("microvm: starting cloud-hypervisor: %w", startErr) + } + if opts.withVsock { + vm.sockets = append(vm.sockets, cfg.VsockSocket) + } + return vm, nil +} + +// vmmArgs builds the cloud-hypervisor argv exactly per the record (lines +// 542-547), dropping --fs/--vsock under the net-only smoke. Launch appends to +// the guest cmdline (per BootConfig.Cmdline's contract): +// - console=ttyS0 so guestd's stderr reaches the captured serial console; +// - compass.vsock_port= which guestd reads from /proc/cmdline (last-wins, +// so appending is safe even if cfg.Cmdline already carries one); +// - net.ifnames=0 so the single virtio-net device is named eth0, the fixed +// name guestd's netProvisioner binds (guestd/net.go: "cloud-hypervisor +// presents the single virtio-net device as eth0"). The pinned generic +// kernel defaults to predictable naming (enpNsM); without this the guest's +// link is enp0s5 and guestd fail-closes with "Link not found". This is a +// host-side boot parameter, not a guest change — guestd's eth0 contract is +// frozen (T2), and the host is what must present that name. +func vmmArgs(cfg BootConfig, consolePath string, opts launchOptions) []string { + cmdline := strings.TrimSpace(cfg.Cmdline + + " console=ttyS0 net.ifnames=0 compass.vsock_port=" + strconv.FormatUint(uint64(cfg.VsockPort), 10)) + args := []string{ + "--kernel", cfg.Kernel, + "--initramfs", cfg.Initrd, + "--disk", "path=" + cfg.Rootfs + ",readonly=on", + "--cmdline", cmdline, + "--cpus", "boot=" + strconv.Itoa(cfg.CPUs), + "--memory", "size=" + strconv.Itoa(cfg.MemoryMB) + "M,shared=on", + "--serial", "file=" + consolePath, + "--console", "off", + } + if opts.withFS { + args = append(args, "--fs", "tag="+cfg.FSTag+",socket="+cfg.FSSocket) + } + args = append(args, "--net", "vhost_user=true,socket="+cfg.Net.VhostUserSocket+",mac="+cfg.Net.MAC) + if opts.withVsock { + args = append(args, "--vsock", "cid="+strconv.FormatUint(uint64(cfg.VsockCID), 10)+",socket="+cfg.VsockSocket) + } + return args +} + +// startChild sets the child's Pdeathsig (a best-effort orphan guard: on Linux +// PR_SET_PDEATHSIG fires on the spawning THREAD's death, so the spawn holds the +// OS thread for it to bind reliably) and captures its stdout+stderr to logPath +// so a boot failure can surface the daemon's own diagnostics. The real teardown +// guarantee is Shutdown, not Pdeathsig (record §(g) lines 300-303). +func startChild(c *child) error { + logFile, err := os.OpenFile(c.logPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) + if err != nil { + return fmt.Errorf("opening %s log %s: %w", c.name, c.logPath, err) + } + // The Cmd owns the fd for the child's lifetime; drop our copy once Start has + // dup'd it into the child. Close after Start below. + c.cmd.Stdout = logFile + c.cmd.Stderr = logFile + c.cmd.SysProcAttr = &syscall.SysProcAttr{Pdeathsig: syscall.SIGTERM} + + runtime.LockOSThread() + defer runtime.UnlockOSThread() + startErr := c.cmd.Start() + // Our handle on the log file is no longer needed: Start dup'd it into the + // child (on success) or it stays unused (on failure). Either way close it. + _ = logFile.Close() // the child holds its own dup; our copy is done with + if startErr != nil { + return startErr + } + return nil +} + +// waitForSockets polls until every path exists or the deadline elapses. A +// missing socket at the deadline is a named error naming the first path still +// absent, so a daemon that died on launch fails the boot fast. +func waitForSockets(ctx context.Context, paths []string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + missing := "" + for _, p := range paths { + if _, err := os.Stat(p); err != nil { + missing = p + break + } + } + if missing == "" { + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("socket %s did not appear within %s", missing, timeout) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(25 * time.Millisecond): + } + } +} + +// Health dials the guest over the hybrid vsock (T3's client) and returns the +// unwrapped Health response. Not valid under the net-only smoke, which boots +// without a vsock device. +func (vm *VM) Health(ctx context.Context) (*compassv1.HealthResponse, error) { + client := GuestClient(vm.vsockSocket, vm.vsockPort) + resp, err := client.Health(ctx, connect.NewRequest(&compassv1.HealthRequest{})) + if err != nil { + return nil, err + } + return resp.Msg, nil +} + +// Shutdown tears the guest and its daemons down: the VMM is killed first (a VM +// gets no graceful drain), then virtiofsd and passt are reaped (SIGTERM, a +// bounded wait, then SIGKILL), each Wait'd to avoid zombies, and finally the +// AF_UNIX sockets and passt's pidfile are removed. It runs at most once (guarded +// by sync.Once) so it is safe to call explicitly AND from t.Cleanup. The serial +// console log is deliberately NOT removed — the test reads it after teardown. +func (vm *VM) Shutdown(ctx context.Context) error { + vm.shutdownOnce.Do(func() { + var errs []error + // VMM first: kill outright, then Wait to reap. + if vm.vmm != nil && vm.vmm.cmd.Process != nil { + if killErr := vm.vmm.cmd.Process.Kill(); killErr != nil && !errors.Is(killErr, os.ErrProcessDone) { + errs = append(errs, fmt.Errorf("killing cloud-hypervisor: %w", killErr)) + } + if waitErr := waitProcess(vm.vmm); waitErr != nil { + errs = append(errs, waitErr) + } + } + // Then the auxiliary daemons: SIGTERM, bounded wait, SIGKILL. + for _, c := range []*child{vm.virtiofsd, vm.passt} { + if c == nil || c.cmd.Process == nil { + continue + } + if reapErr := reap(c); reapErr != nil { + errs = append(errs, reapErr) + } + } + // Remove the sockets and pidfile now that nothing is serving them. + for _, s := range vm.sockets { + if rmErr := os.Remove(s); rmErr != nil && !errors.Is(rmErr, os.ErrNotExist) { + errs = append(errs, fmt.Errorf("removing socket %s: %w", s, rmErr)) + } + } + if vm.pidfile != "" { + if rmErr := os.Remove(vm.pidfile); rmErr != nil && !errors.Is(rmErr, os.ErrNotExist) { + errs = append(errs, fmt.Errorf("removing pidfile %s: %w", vm.pidfile, rmErr)) + } + } + vm.shutdownErr = errors.Join(errs...) + }) + return vm.shutdownErr +} + +// reap terminates an auxiliary daemon gracefully then forcibly: SIGTERM, wait up +// to reapGrace, SIGKILL if it is still alive, then Wait to collect the exit and +// avoid a zombie. +func reap(c *child) error { + if termErr := c.cmd.Process.Signal(syscall.SIGTERM); termErr != nil && !errors.Is(termErr, os.ErrProcessDone) { + return fmt.Errorf("SIGTERM %s: %w", c.name, termErr) + } + done := make(chan error, 1) + go func() { done <- c.cmd.Wait() }() + select { + case err := <-done: + c.waited = true + return waitResult(c.name, err) + case <-time.After(reapGrace): + if killErr := c.cmd.Process.Kill(); killErr != nil && !errors.Is(killErr, os.ErrProcessDone) { + return fmt.Errorf("SIGKILL %s: %w", c.name, killErr) + } + c.waited = true + return waitResult(c.name, <-done) + } +} + +// waitProcess Wait's a process that has already been signalled to die and +// normalizes the "expected" exit (killed/exited non-zero) to nil — Shutdown +// killed it on purpose, so a non-nil ExitError is not a Shutdown failure. +func waitProcess(c *child) error { + err := c.cmd.Wait() + c.waited = true + return waitResult(c.name, err) +} + +// waitResult swallows the ExitError a deliberately-killed process yields (a +// signalled or non-zero exit is expected on teardown) but propagates a genuine +// wait failure. +func waitResult(name string, err error) error { + if err == nil { + return nil + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return nil // killed/non-zero exit is the expected teardown outcome + } + return fmt.Errorf("waiting for %s: %w", name, err) +} + +// Running reports whether the named child's process is still alive. It is used +// by the test to assert Shutdown left no orphan. A process that has been Wait'd +// is definitively gone; otherwise signal 0 probes liveness without affecting it. +func (vm *VM) Running(name string) bool { + c := vm.childByName(name) + if c == nil || c.cmd.Process == nil || c.waited { + return false + } + return c.cmd.Process.Signal(syscall.Signal(0)) == nil +} + +// PSS returns each live process's proportional set size in kB, read from +// /proc//smaps_rollup and keyed by process name. It is PSS, NOT summed +// VmHWM: under --memory shared=on the guest RAM is one shared mapping all three +// processes map, so VmHWM counts those pages in each and summing double/triple- +// counts; PSS divides shared pages among their mappers (record §(g) lines +// 292-296). The unit is kB, as smaps_rollup reports it. +func (vm *VM) PSS() (map[string]int64, error) { + out := make(map[string]int64) + var errs []error + for _, c := range []*child{vm.vmm, vm.virtiofsd, vm.passt} { + if c == nil || c.cmd.Process == nil || c.waited { + continue + } + pss, err := readPSS(c.cmd.Process.Pid) + if err != nil { + // Best-effort: a sandboxed helper makes its own smaps_rollup + // unreadable to the rootless harness — passt sets PR_SET_DUMPABLE=0, + // which reparents /proc//smaps_rollup to root and denies the + // non-root reader — and an already-exited process's proc entry is + // gone. Both are expected and leave no entry rather than failing: + // PSS is informational spike output (record §(g)), not a boot gate. + if errors.Is(err, os.ErrPermission) || errors.Is(err, os.ErrNotExist) { + continue + } + errs = append(errs, fmt.Errorf("reading PSS for %s (pid %d): %w", c.name, c.cmd.Process.Pid, err)) + continue + } + out[c.name] = pss + } + return out, errors.Join(errs...) +} + +// readPSS parses the Pss line (kB) from a process's smaps_rollup. +func readPSS(pid int) (int64, error) { + raw, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/smaps_rollup") + if err != nil { + return 0, fmt.Errorf("reading smaps_rollup: %w", err) + } + for line := range strings.Lines(string(raw)) { + rest, ok := strings.CutPrefix(line, "Pss:") + if !ok { + continue + } + fields := strings.Fields(rest) // " 1234 kB" -> ["1234", "kB"] + if len(fields) < 1 { + return 0, fmt.Errorf("malformed Pss line %q", strings.TrimSpace(line)) + } + kb, parseErr := strconv.ParseInt(fields[0], 10, 64) + if parseErr != nil { + return 0, fmt.Errorf("parsing Pss value %q: %w", fields[0], parseErr) + } + return kb, nil + } + return 0, errors.New("no Pss line in smaps_rollup") +} + +// ConsoleTail returns the tail of the captured guest serial console, so a boot +// failure surfaces the guest's own diagnostics (a guestd fail-closed line, a +// kernel panic) instead of an opaque deadline timeout. +func (vm *VM) ConsoleTail() string { + return tailFile(vm.consolePath) +} + +// Diagnostics returns the tails of the serial console and every daemon's +// captured stderr, the evidence set for an OQ-G negotiation failure (passt/CH +// vhost-user) or any other boot abort. +func (vm *VM) Diagnostics() string { + var b strings.Builder + fmt.Fprintf(&b, "=== guest serial console (%s) ===\n%s\n", vm.consolePath, tailFile(vm.consolePath)) + for _, c := range []*child{vm.vmm, vm.passt, vm.virtiofsd} { + if c == nil { + continue + } + fmt.Fprintf(&b, "=== %s (%s) ===\n%s\n", c.name, c.logPath, tailFile(c.logPath)) + } + return b.String() +} + +// tailFile returns up to diagnosticTailBytes from the end of path, or a short +// note if it cannot be read. +func tailFile(path string) string { + raw, err := os.ReadFile(path) //nolint:gosec // G304: path is a harness-owned capture-log path (console/daemon stderr under the test temp dir), not user input + if err != nil { + return fmt.Sprintf("", err) + } + if len(raw) > diagnosticTailBytes { + raw = raw[len(raw)-diagnosticTailBytes:] + } + return string(raw) +} + +func (vm *VM) childByName(name string) *child { + switch name { + case "cloud-hypervisor": + return vm.vmm + case "virtiofsd": + return vm.virtiofsd + case "passt": + return vm.passt + default: + return nil + } +} diff --git a/go/internal/runtime/microvm/launch_teardown_test.go b/go/internal/runtime/microvm/launch_teardown_test.go new file mode 100644 index 00000000..a4c0bc02 --- /dev/null +++ b/go/internal/runtime/microvm/launch_teardown_test.go @@ -0,0 +1,100 @@ +//go:build unix + +package microvm + +import ( + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +// writeFake writes an executable shell stub into dir under name. +func writeFake(t *testing.T, dir, name, body string) { + t.Helper() + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte("#!/bin/sh\n"+body+"\n"), 0o755); err != nil { + t.Fatal(err) + } +} + +// pidAlive reports whether pid names a live process (signal 0 probes without +// affecting it); a reaped process yields ESRCH. +func pidAlive(pid int) bool { + return syscall.Kill(pid, 0) == nil +} + +// TestLaunchFailClosedTeardown pins the fail-closed teardown contract that CI +// surfaced (PR #574): when a later spawn fails after virtiofsd and passt are +// already up, launch must (1) return a nil VM and the error WITHOUT panicking, +// and (2) reap the daemons it already started, leaving no orphan. The original +// bug returned `nil, err` into the NAMED vm return, clobbering the handle before +// the deferred cleanup ran — so Shutdown nil-dereferenced (SIGSEGV) and the +// started daemons were orphaned. This runs with no KVM: the aux daemons are +// shell fakes that bind their sockets and sleep, and cloud-hypervisor is absent +// from PATH so LookPath fails after both aux daemons are up. +func TestLaunchFailClosedTeardown(t *testing.T) { + bin := t.TempDir() + run := t.TempDir() + vfsPidFile := filepath.Join(run, "vfs.pid") + passtPidFile := filepath.Join(run, "passt.pid") + + // virtiofsd: record pid, touch its --socket-path=, then stay alive. + writeFake(t, bin, "virtiofsd", `echo $$ > `+vfsPidFile+` +for a in "$@"; do case "$a" in --socket-path=*) : > "${a#--socket-path=}";; esac; done +sleep 30`) + // passt: record pid, touch the path following --socket, then stay alive. + writeFake(t, bin, "passt", `echo $$ > `+passtPidFile+` +p=""; for a in "$@"; do [ "$p" = --socket ] && : > "$a"; p="$a"; done +sleep 30`) + // cloud-hypervisor deliberately absent → LookPath fails after aux are up. + t.Setenv("PATH", bin) + + dir := t.TempDir() + cfg := BootConfig{ + Kernel: "/nonexistent/kernel", Initrd: "/nonexistent/initrd", Rootfs: "/nonexistent/rootfs", + VsockCID: 3, VsockPort: 1024, VsockSocket: dir + "/vsock.sock", + FSTag: "workspace", FSSocket: dir + "/virtiofsd.sock", FSSharedDir: dir, + CPUs: 2, MemoryMB: 1024, + Net: NetConfig{VhostUserSocket: dir + "/net.sock", MAC: "12:34:56:78:9a:bc"}, + } + + vm, err := Launch(t.Context(), cfg) // must not panic + if err == nil { + t.Fatal("expected an error (cloud-hypervisor absent), got nil") + } + if vm != nil { + t.Errorf("expected a nil VM on the error path, got %v", vm) + } + + // The daemons launch already started must have been reaped by the deferred + // cleanup — no orphan left sleeping. + for name, pidFile := range map[string]string{"virtiofsd": vfsPidFile, "passt": passtPidFile} { + pid := readPidFile(t, pidFile) + if pidAlive(pid) { + t.Errorf("%s (pid %d) still alive after fail-closed launch — teardown orphaned it", name, pid) + } + } +} + +// readPidFile reads a pid a fake wrote, retrying briefly since the fake writes +// it asynchronously after exec. +func readPidFile(t *testing.T, path string) int { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for { + raw, err := os.ReadFile(path) + if err == nil { + if pid, convErr := strconv.Atoi(strings.TrimSpace(string(raw))); convErr == nil { + return pid + } + } + if time.Now().After(deadline) { + t.Fatalf("fake never wrote its pid to %s", path) + } + time.Sleep(10 * time.Millisecond) + } +} diff --git a/go/moon.yml b/go/moon.yml index 56090a30..c626cd22 100644 --- a/go/moon.yml +++ b/go/moon.yml @@ -160,6 +160,31 @@ tasks: runFromWorkspaceRoot: false inputs: *go_sources + test-microvm: + # The KVM-gated microVM boot suite (RIG-2591, record §T4). It compiles only + # under `-tags microvm`, so the untagged `test` lane above never builds it; + # this task realises the guest image + VMM stack from nix and execs + # `go test -tags microvm ./internal/runtime/microvm/...` with the env + # microvmtest.Require reads. The realisation + exec logic lives in a bun + # script (rule://scripts-ts-over-bash + the no-bash-gate CI task forbid this + # much logic in a shell `script:`); the microvm-boot-test project carries its + # unit tests. + # + # deps: compass-guest-image:build orders the guest-image realise first and + # scopes this task to the SAME affected-detection, so the nix builds the + # script runs are store cache hits rather than cold builds. + # + # runInCI:false — the CI KVM leg runs the tagged suite its own way + # (ci.yml's "microVM suites" step); this lane is the dev-box entry point and + # needs KVM + a local nix build. cache:false — nix owns the build caching and + # the subject is a live KVM boot, never a hashable input. + command: 'bun run tools/microvm-boot-test/run.ts' + deps: ['compass-guest-image:build'] + options: + runFromWorkspaceRoot: true + cache: false + runInCI: false + vuln: # govulncheck: known-vulnerability scan over the module's actual call graph # (reports only vulns in code paths the module reaches). It queries the Go diff --git a/guest-image/default.nix b/guest-image/default.nix index c45ef409..cc9642b1 100644 --- a/guest-image/default.nix +++ b/guest-image/default.nix @@ -130,30 +130,53 @@ let # (full tree, on-demand autoload) and the initrd (boot-critical subset). kernel = pkgs.linuxPackages.kernel; - # The boot-critical module set the initramfs must load to mount the root - # overlay before switch_root: the virtio transport + block device, erofs (the - # lower), and overlayfs (the writable view). Every one is `=m` in the pinned - # generic kernel (record §(a)); the derivation-time check below fails the build - # if a pin move flips one to `=y` or drops it, rather than silently producing a - # non-booting initrd. Post-switch_root module needs (virtio_net/virtiofs/vsock, - # V3's netfilter stack) autoload from the full /lib/modules tree the rootfs - # ships — they are deliberately NOT carried here. + # The module set the initramfs loads before switch_root. Two groups, one + # mechanism (kmod modprobe from the shrunk closure), because the guest has no + # udev/systemd-modules-load to autoload anything post-switch_root — guestd IS + # init (§(d)). Modules loaded here are kernel state that persists across + # switch_root, so every driver the guest needs is bound by the time guestd + # starts: + # - boot-critical (mount the root overlay before switch_root): the virtio + # transport + block device, erofs (the lower), overlayfs (the writable + # view); + # - runtime (the devices + socket families guestd binds right after + # switch_root): virtio-net (guestd's eth0), virtio-fs (the /workspace + # mount), the virtio vsock transport (the host↔guest GuestControl + # channel), and af_packet (the AF_PACKET raw socket guestd's in-process + # DHCP client opens for its broadcast DORA exchange — without it the + # lease step fails with EAFNOSUPPORT). Their AF_VSOCK/fuse dependencies + # are pulled into the closure automatically by modprobe. + # Every one is `=m` in the pinned generic kernel (record §(a)); the + # derivation-time check below fails the build if a pin move flips one to `=y` + # or drops it, rather than silently producing a guest that boots but cannot + # reach the network, its workspace, or the host. bootModules = [ "virtio_pci" "virtio_blk" "erofs" "overlay" + "virtio_net" + "virtiofs" + "vmw_vsock_virtio_transport" + "af_packet" ]; - # The .config symbol each boot-critical module is gated on, checked `=m` at - # build time. The mapping is module→Kconfig (virtio_blk⇒CONFIG_VIRTIO_BLK, - # erofs⇒CONFIG_EROFS_FS, overlay⇒CONFIG_OVERLAY_FS) — not a mechanical - # upper-casing, so it is spelled out. + # The .config symbol each named module is gated on, checked `=m` at build + # time. The mapping is module→Kconfig (virtio_blk⇒CONFIG_VIRTIO_BLK, + # erofs⇒CONFIG_EROFS_FS, overlay⇒CONFIG_OVERLAY_FS, virtiofs⇒CONFIG_VIRTIO_FS, + # vmw_vsock_virtio_transport⇒CONFIG_VIRTIO_VSOCKETS, af_packet⇒CONFIG_PACKET) + # — not a mechanical upper-casing, so it is spelled out. Only the explicitly- + # named modules are checked; their transitive deps (vsock, fuse) ride along + # via the closure. bootModuleConfigs = [ "CONFIG_VIRTIO_PCI" "CONFIG_VIRTIO_BLK" "CONFIG_EROFS_FS" "CONFIG_OVERLAY_FS" + "CONFIG_VIRTIO_NET" + "CONFIG_VIRTIO_FS" + "CONFIG_VIRTIO_VSOCKETS" + "CONFIG_PACKET" ]; # Derivation-time assertion (record §(a) test cycle): the pinned kernel's diff --git a/tools/microvm-boot-test/biome.json b/tools/microvm-boot-test/biome.json new file mode 100644 index 00000000..99b4ab8f --- /dev/null +++ b/tools/microvm-boot-test/biome.json @@ -0,0 +1,3 @@ +{ + "extends": "//" +} diff --git a/tools/microvm-boot-test/moon.yml b/tools/microvm-boot-test/moon.yml new file mode 100644 index 00000000..22664d97 --- /dev/null +++ b/tools/microvm-boot-test/moon.yml @@ -0,0 +1,36 @@ +# yaml-language-server: $schema=https://moonrepo.dev/schemas/project.json +# +# microvm-boot-test (RIG-2591): the local entry point for the KVM-gated microVM +# boot suite. The tagged suite compiles only under `-tags microvm`, which +# compass-go:test (`go test -race ./...`, no tag) never builds; this project's +# script realises the guest image + VMM stack from nix, exports the env +# microvmtest.Require reads, and execs `go test -tags microvm`. It mirrors +# ci.yml's microVM step (the reviewed source of truth for the nix invocations) +# and does NOT replace it — CI runs the microVM leg its own way. +# +# The boot lane itself (compass-go:test-microvm) needs KVM + a nix build, so it +# is never scheduled in CI (runInCI:false lives on that task, in go/moon.yml). +# The typecheck/test tasks here ARE ordinary bun gates that ride CI. +# +# A bun/TypeScript CLI, hoisted root-workspace member (`bun` tag): install is +# inherited via .moon/tasks/tag-bun.yml and lint/format are whole-repo root +# tasks, so this leaf carries no own bun.lock. +layer: 'tool' +language: 'typescript' +tags: ['bun'] + +tasks: + typecheck: + command: 'bunx tsc --noEmit' + deps: ['install'] + inputs: ['*.ts', 'tsconfig.json', 'package.json', '/bun.lock'] + test: + # The pure core (run-core.ts) over fixtures: the out-path parsing + env + # mapping, incl. the fail-closed drift edges. + command: 'bun test' + deps: ['install'] + inputs: ['*.ts', 'tsconfig.json', 'package.json', '/bun.lock'] + ci: + deps: ['typecheck', 'test'] + options: + cache: false diff --git a/tools/microvm-boot-test/package.json b/tools/microvm-boot-test/package.json new file mode 100644 index 00000000..c930c899 --- /dev/null +++ b/tools/microvm-boot-test/package.json @@ -0,0 +1,14 @@ +{ + "name": "@compass/microvm-boot-test", + "private": true, + "type": "module", + "description": "The local microVM boot-test lane (RIG-2591): realise the guest image + VMM stack from nix, export the env vars microvmtest.Require reads, and exec the KVM-gated `go test -tags microvm` suite. Mirrors ci.yml's microVM step; the dev-box entry point, not a CI replacement.", + "module": "run.ts", + "bin": { + "microvm-boot-test": "./run.ts" + }, + "devDependencies": { + "@types/bun": "catalog:", + "typescript": "catalog:" + } +} diff --git a/tools/microvm-boot-test/run-core.test.ts b/tools/microvm-boot-test/run-core.test.ts new file mode 100644 index 00000000..e39d5115 --- /dev/null +++ b/tools/microvm-boot-test/run-core.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, test } from "bun:test"; +import { guestImageEnv, parseOutPaths, prependBins } from "./run-core.ts"; + +describe("parseOutPaths", () => { + test("splits, trims, and drops blank lines", () => { + expect(parseOutPaths("/nix/store/a\n/nix/store/b\n")).toEqual([ + "/nix/store/a", + "/nix/store/b", + ]); + expect(parseOutPaths(" /nix/store/a \n\n /nix/store/b \n")).toEqual([ + "/nix/store/a", + "/nix/store/b", + ]); + }); + + test("empty stdout yields no paths (the fail-closed empty case run.ts rejects)", () => { + expect(parseOutPaths("")).toEqual([]); + expect(parseOutPaths("\n \n")).toEqual([]); + }); +}); + +describe("guestImageEnv", () => { + test("maps kernel/rootfs/initrd to the env vars Require reads, kernel gets /bzImage", () => { + expect( + guestImageEnv(["/nix/store/k", "/nix/store/r", "/nix/store/i"]), + ).toEqual({ + COMPASS_TEST_GUEST_KERNEL: "/nix/store/k/bzImage", + COMPASS_TEST_GUEST_ROOTFS: "/nix/store/r", + COMPASS_TEST_GUEST_INITRD: "/nix/store/i", + }); + }); + + test("throws on a wrong out-path count (build-drift is a named failure)", () => { + expect(() => guestImageEnv(["/a", "/b"])).toThrow(); + expect(() => guestImageEnv(["/a", "/b", "/c", "/d"])).toThrow(); + }); +}); + +describe("prependBins", () => { + test("prepends each out-path's bin/ ahead of the existing PATH", () => { + expect( + prependBins(["/nix/store/ch", "/nix/store/vf"], "/usr/bin:/bin"), + ).toBe("/nix/store/ch/bin:/nix/store/vf/bin:/usr/bin:/bin"); + }); + + test("empty current PATH yields just the bins", () => { + expect(prependBins(["/nix/store/ch"], "")).toBe("/nix/store/ch/bin"); + }); +}); diff --git a/tools/microvm-boot-test/run-core.ts b/tools/microvm-boot-test/run-core.ts new file mode 100644 index 00000000..3340fec2 --- /dev/null +++ b/tools/microvm-boot-test/run-core.ts @@ -0,0 +1,45 @@ +// The pure core of the local microVM boot-test lane (RIG-2591): parse nix +// out-paths and map them to the env the tagged Go suite (microvmtest.Require) +// reads. No I/O — every function is a total map over its inputs — so +// run-core.test.ts can drive each mapping (and its fail-closed edges) without +// nix or a subprocess. + +/** Split `nix build --print-out-paths` stdout into trimmed, non-empty store + * paths, one per line. */ +export function parseOutPaths(stdout: string): string[] { + return stdout + .split("\n") + .map((line) => line.trim()) + .filter((line) => line !== ""); +} + +/** Map the three guest-image out-paths (in the order kernel, rootfs, initrd — + * the attr order run.ts builds them in) to the env vars microvmtest.Require + * reads. The kernel path gains `/bzImage`, mirroring ci.yml:392. Throws if not + * given exactly three paths, so a build-count drift is a named failure. */ +export function guestImageEnv( + outPaths: readonly string[], +): Record { + if (outPaths.length !== 3) { + throw new Error( + `guestImageEnv expects 3 out-paths (kernel, rootfs, initrd), got ${outPaths.length}`, + ); + } + const [kernel, rootfs, initrd] = outPaths as [string, string, string]; + return { + COMPASS_TEST_GUEST_KERNEL: `${kernel}/bzImage`, + COMPASS_TEST_GUEST_ROOTFS: rootfs, + COMPASS_TEST_GUEST_INITRD: initrd, + }; +} + +/** Prepend each VMM out-path's `bin/` to an existing PATH, so the freshly + * realised cloud-hypervisor/virtiofsd/passt win over any ambient copy — the same + * ordering ci.yml:400-406 gives $GITHUB_PATH. */ +export function prependBins( + outPaths: readonly string[], + currentPath: string, +): string { + const bins = outPaths.map((p) => `${p}/bin`); + return currentPath === "" ? bins.join(":") : [...bins, currentPath].join(":"); +} diff --git a/tools/microvm-boot-test/run.ts b/tools/microvm-boot-test/run.ts new file mode 100755 index 00000000..b39035d3 --- /dev/null +++ b/tools/microvm-boot-test/run.ts @@ -0,0 +1,95 @@ +#!/usr/bin/env bun +// The LOCAL microVM boot-test lane (RIG-2591). The tagged microVM suite compiles +// only under `-tags microvm`, and the existing compass-go:test lane is `go test +// -race ./...` with NO tag, so it never builds these tests. This script is what +// the `compass-go:test-microvm` moon task runs to drive the tagged suite on a +// KVM-capable dev box: realise the guest image + the VMM stack from nix, export +// the same env vars the CI KVM leg does (ci.yml:385-406), and exec `go test +// -tags microvm`. +// +// TypeScript, not bash (rule://scripts-ts-over-bash + the no-bash-gate CI task): +// this has real logic — two nix builds whose out-paths must be parsed and mapped +// to distinct env vars, a PATH assembled from three store paths, and a fail-fast +// on any missing output — so the parsing/mapping core is pure and unit-tested +// (./run-core.test.ts) while this file is the thin I/O shell. +// +// It deliberately mirrors ci.yml's microVM step (the reviewed source of truth +// for the exact nix invocations); it does NOT replace it — CI runs the microVM +// leg its own way. This is the dev-box entry point. + +import { spawnSync } from "node:child_process"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { guestImageEnv, parseOutPaths, prependBins } from "./run-core.ts"; + +// This file is tools/microvm-boot-test/run.ts, so `../..` is the workspace root. +const workspaceRoot = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); + +/** Run `nix build --no-link --print-out-paths -f ` and return + * the realised store paths in argument order. Exits the process on failure so a + * broken build is a named, fail-closed abort, never a silent empty env. */ +function nixBuild(file: string, attrs: readonly string[]): string[] { + const result = spawnSync( + "nix", + ["build", "--no-link", "--print-out-paths", "-f", file, ...attrs], + { + cwd: workspaceRoot, + encoding: "utf8", + stdio: ["ignore", "pipe", "inherit"], + }, + ); + if (result.status !== 0) { + console.error( + `nix build -f ${file} ${attrs.join(" ")} failed (exit ${result.status})`, + ); + process.exit(result.status ?? 1); + } + const paths = parseOutPaths(result.stdout); + if (paths.length !== attrs.length) { + console.error( + `nix build -f ${file} produced ${paths.length} out-paths, expected ${attrs.length} (${attrs.join(", ")})`, + ); + process.exit(1); + } + return paths; +} + +// Realise the three guest-image artifacts and the three VMM-stack binaries, in +// the SAME attr order ci.yml relies on. +const guest = nixBuild("guest-image/default.nix", [ + "compass-guest-kernel", + "compass-guest-rootfs", + "compass-guest-initrd", +]); +const vmm = nixBuild("tools/toolchain/microvm-vmm-env.nix", [ + "cloud-hypervisor", + "virtiofsd", + "passt", +]); + +const env: NodeJS.ProcessEnv = { + ...process.env, + ...guestImageEnv(guest), + PATH: prependBins(vmm, process.env.PATH ?? ""), + // The -race lane needs cgo. + CGO_ENABLED: "1", +}; + +// Exec the tagged suite, inheriting stdio so the go test output (incl. the boot +// latency + PSS t.Logf lines and any serial-console tail) streams live. cwd=go +// so go resolves the module. +const test = spawnSync( + "go", + [ + "test", + "-tags", + "microvm", + "-race", + "-v", + "-timeout", + "15m", + "./internal/runtime/microvm/...", + ], + { cwd: join(workspaceRoot, "go"), env, stdio: "inherit" }, +); +process.exit(test.status ?? 1); diff --git a/tools/microvm-boot-test/tsconfig.json b/tools/microvm-boot-test/tsconfig.json new file mode 100644 index 00000000..47d3248b --- /dev/null +++ b/tools/microvm-boot-test/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "lib": ["ESNext"], + "target": "ESNext", + "module": "Preserve", + "moduleDetection": "force", + "allowJs": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + "strict": true, + "skipLibCheck": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "types": ["bun"] + } +}