From acf94c2a3e2793e90afa64282618c7fd38d3e0b7 Mon Sep 17 00:00:00 2001 From: tunsuy Date: Tue, 17 Nov 2026 18:17:16 +0800 Subject: [PATCH] fix(envd): give the guest a routable IPv4 address so Tailscale sees it online MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sandbox's only IPv4 address is the link-local 169.254.0.21/30 from the kernel's ip= argument, and software that requires a non-link-local address to consider the host online reads the guest as having no network at all. The reported case is Tailscale: its network monitor does not count 169.254.0.0/16 as usable, so tailscaled pauses its control client before login and auth keys never reach the control plane — while plain egress (curl to the control plane itself) works fine. Add the RFC 7600 IPv4 dummy address 192.0.0.8/32 to eth0 when envd starts in a Firecracker guest. RFC 7600 reserves that address for hosts with no other usable IPv4 address and it is never routed, so it cannot clash with user networks or VPN ranges the way RFC 1918 or Tailscale's own 100.64.0.0/10 would. It is added with IFA_F_NOPREFIXROUTE, so no prefix route is created: egress keeps selecting 169.254.0.21 as its source and nothing routes to the dummy address, leaving inbound traffic unchanged. The request is built and sent over NETLINK_ROUTE directly rather than by running `ip addr`, because some base images ship no iproute2 and envd (unlike the orchestrator) has no netlink library dependency — golang.org/x/sys/unix is already one. Adding is idempotent, which covers both lifecycle paths: a cold boot runs envd again and ignores EEXIST, while a memory-snapshot resume keeps the address because it lives in kernel state. A failure only warns, since egress does not depend on it. Fixes #3780 --- packages/envd/internal/host/net_addr_linux.go | 171 ++++++++++++++++++ .../envd/internal/host/net_addr_linux_test.go | 152 ++++++++++++++++ packages/envd/internal/host/net_addr_other.go | 8 + packages/envd/main.go | 11 ++ 4 files changed, 342 insertions(+) create mode 100644 packages/envd/internal/host/net_addr_linux.go create mode 100644 packages/envd/internal/host/net_addr_linux_test.go create mode 100644 packages/envd/internal/host/net_addr_other.go diff --git a/packages/envd/internal/host/net_addr_linux.go b/packages/envd/internal/host/net_addr_linux.go new file mode 100644 index 00000000000..b4c1577d048 --- /dev/null +++ b/packages/envd/internal/host/net_addr_linux.go @@ -0,0 +1,171 @@ +//go:build linux + +package host + +import ( + "encoding/binary" + "errors" + "fmt" + "net" + + "golang.org/x/sys/unix" +) + +// RoutableAddrCIDR is the RFC 7600 IPv4 dummy address EnsureRoutableAddr adds +// to the guest's eth0. RFC 7600 reserves 192.0.0.8 for hosts with no other +// usable IPv4 address and it is never routed, so unlike 100.64.0.0/10 +// (Tailscale's own CGNAT range) or the RFC 1918 ranges it cannot clash with a +// user network or VPN. +const RoutableAddrCIDR = "192.0.0.8/32" + +// routableAddrLink is the guest's network interface, configured by the +// orchestrator through the kernel's ip= argument (169.254.0.21/30). +const routableAddrLink = "eth0" + +// EnsureRoutableAddr adds the RFC 7600 dummy address (192.0.0.8/32) to eth0 +// with IFA_F_NOPREFIXROUTE. +// +// The guest's only IPv4 address is link-local, and software that requires a +// non-link-local address to consider the host online treats the sandbox as +// offline. The reported case is Tailscale: its network monitor does not count +// 169.254.0.0/16 as usable, so tailscaled pauses its control client before +// login and auth keys silently never reach the control plane — even though +// outbound connectivity is fine. +// +// The address is added noprefixroute, so no prefix route is created: egress +// keeps selecting the gateway-subnet address (169.254.0.21) as its source and +// nothing routes to 192.0.0.8, leaving inbound traffic unchanged. +// +// Idempotent across the sandbox lifecycle — a cold boot runs envd again and a +// repeated add is a no-op (EEXIST), while a memory-snapshot resume keeps the +// address because it lives in kernel state. It talks to NETLINK_ROUTE directly +// rather than shelling out to `ip addr`, because some base images ship no +// iproute2 and envd (unlike the orchestrator) does not depend on a netlink +// library; golang.org/x/sys/unix is already an envd dependency. +func EnsureRoutableAddr() error { + return ensureRoutableAddr(routableAddrLink, RoutableAddrCIDR) +} + +func ensureRoutableAddr(linkName, cidr string) error { + ip, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("parse dummy address %s: %w", cidr, err) + } + ip4 := ip.To4() + if ip4 == nil { + return fmt.Errorf("dummy address %s is not IPv4", cidr) + } + link, err := net.InterfaceByName(linkName) + if err != nil { + return fmt.Errorf("find link %s: %w", linkName, err) + } + prefixLen, _ := ipnet.Mask.Size() + + msg := buildAddrMsg(unix.RTM_NEWADDR, unix.NLM_F_CREATE|unix.NLM_F_EXCL, + link.Index, uint8(prefixLen), ip4) + + return sendAddrMsg(msg) +} + +// buildAddrMsg encodes an RTM_NEWADDR request for a single IPv4 address, +// mirroring what `ip addr add /32 dev noprefixroute` sends. The +// 8-bit IfAddrmsg flags field cannot hold IFA_F_NOPREFIXROUTE (0x200), so it +// travels in the IFA_FLAGS attribute; IFA_ADDRESS and IFA_LOCAL carry the same +// address (this is not point-to-point); and a /32 gets no IFA_BROADCAST (the +// kernel only wants one for prefixes shorter than /31). +func buildAddrMsg(msgType uint16, extraFlags uint16, ifIndex int, prefixLen uint8, ip4 net.IP) []byte { + const ( + hdrLen = unix.SizeofNlMsghdr // 16 + ifaLen = unix.SizeofIfAddrmsg // 8 + ipLen = net.IPv4len // payload of IFA_LOCAL / IFA_ADDRESS + flagLen = 4 // payload of IFA_FLAGS (uint32) + ) + attrLen := func(payload int) int { return unix.SizeofRtAttr + payload + nlaPad(payload) } + total := hdrLen + ifaLen + + attrLen(ipLen) + // IFA_LOCAL + attrLen(ipLen) + // IFA_ADDRESS + attrLen(flagLen) // IFA_FLAGS + + buf := make([]byte, total) + + // nlmsghdr + binary.NativeEndian.PutUint32(buf[0:4], uint32(total)) + binary.NativeEndian.PutUint16(buf[4:6], msgType) + binary.NativeEndian.PutUint16(buf[6:8], unix.NLM_F_REQUEST|unix.NLM_F_ACK|extraFlags) + binary.NativeEndian.PutUint32(buf[8:12], 1) // seq + binary.NativeEndian.PutUint32(buf[12:16], 0) + + // ifaddrmsg + off := hdrLen + buf[off] = unix.AF_INET + buf[off+1] = prefixLen + buf[off+2] = 0 // legacy 8-bit flags; NOPREFIXROUTE goes into IFA_FLAGS + buf[off+3] = unix.RT_SCOPE_UNIVERSE + binary.NativeEndian.PutUint32(buf[off+4:off+ifaLen], uint32(ifIndex)) + off += ifaLen + + putAttr := func(typ uint16, payload []byte) { + binary.NativeEndian.PutUint16(buf[off:off+2], uint16(unix.SizeofRtAttr+len(payload))) + binary.NativeEndian.PutUint16(buf[off+2:off+4], typ) + copy(buf[off+unix.SizeofRtAttr:], payload) + off += unix.SizeofRtAttr + len(payload) + nlaPad(len(payload)) + } + putAttr(unix.IFA_LOCAL, ip4) + putAttr(unix.IFA_ADDRESS, ip4) + flags := make([]byte, flagLen) + binary.NativeEndian.PutUint32(flags, unix.IFA_F_NOPREFIXROUTE) + putAttr(unix.IFA_FLAGS, flags) + + return buf +} + +// nlaPad is the padding aligning the next netlink attribute to NLA_ALIGNTO. +func nlaPad(payloadLen int) int { + return -payloadLen & (unix.NLA_ALIGNTO - 1) +} + +// sendAddrMsg sends one address request to NETLINK_ROUTE and waits for the +// kernel ACK. EEXIST means the address is already configured (a re-exec'd envd +// after a live upgrade), which satisfies the caller just like success. +func sendAddrMsg(msg []byte) error { + fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE) + if err != nil { + return fmt.Errorf("open netlink route socket: %w", err) + } + defer func() { _ = unix.Close(fd) }() + + if err := unix.Bind(fd, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil { + return fmt.Errorf("bind netlink route socket: %w", err) + } + if err := unix.Sendto(fd, msg, 0, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil { + return fmt.Errorf("send netlink address request: %w", err) + } + + buf := make([]byte, 4096) + for { + n, _, err := unix.Recvfrom(fd, buf, 0) + if err != nil { + return fmt.Errorf("receive netlink ack: %w", err) + } + if n < unix.SizeofNlMsghdr+4 { + return errors.New("short netlink ack") + } + switch msgType := binary.NativeEndian.Uint16(buf[4:6]); msgType { + case unix.NLMSG_NOOP: + continue + case unix.NLMSG_ERROR: + errno := int32(binary.NativeEndian.Uint32(buf[unix.SizeofNlMsghdr : unix.SizeofNlMsghdr+4])) + if errno == 0 { + return nil // ACK, no error + } + nlErr := unix.Errno(-errno) + if errors.Is(nlErr, unix.EEXIST) { + return nil + } + + return fmt.Errorf("netlink address request: %w", nlErr) + default: + return fmt.Errorf("unexpected netlink reply type %d", msgType) + } + } +} diff --git a/packages/envd/internal/host/net_addr_linux_test.go b/packages/envd/internal/host/net_addr_linux_test.go new file mode 100644 index 00000000000..dca4332bd30 --- /dev/null +++ b/packages/envd/internal/host/net_addr_linux_test.go @@ -0,0 +1,152 @@ +//go:build linux + +package host + +import ( + "encoding/binary" + "net" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestNlaPad(t *testing.T) { + assert.Equal(t, 0, nlaPad(0)) + assert.Equal(t, 3, nlaPad(1)) + assert.Equal(t, 2, nlaPad(2)) + assert.Equal(t, 1, nlaPad(3)) + assert.Equal(t, 0, nlaPad(4)) + assert.Equal(t, 3, nlaPad(5)) +} + +// TestBuildAddrMsgEncodesNoPrefixRoute checks the request against the wire +// format `ip addr add 192.0.0.8/32 dev noprefixroute` produces: the +// flag must travel in the IFA_FLAGS attribute (it does not fit the 8-bit +// legacy field), and a /32 must carry no IFA_BROADCAST. +func TestBuildAddrMsgEncodesNoPrefixRoute(t *testing.T) { + ip4 := net.IPv4(192, 0, 0, 8).To4() + msg := buildAddrMsg(unix.RTM_NEWADDR, unix.NLM_F_CREATE|unix.NLM_F_EXCL, 7, 32, ip4) + + // nlmsghdr + assert.Len(t, msg, int(binary.NativeEndian.Uint32(msg[0:4]))) + assert.EqualValues(t, unix.RTM_NEWADDR, binary.NativeEndian.Uint16(msg[4:6])) + flags := binary.NativeEndian.Uint16(msg[6:8]) + for _, want := range []uint16{unix.NLM_F_REQUEST, unix.NLM_F_ACK, unix.NLM_F_CREATE, unix.NLM_F_EXCL} { + assert.NotZero(t, flags&want) + } + + // ifaddrmsg + off := unix.SizeofNlMsghdr + assert.EqualValues(t, unix.AF_INET, msg[off]) + assert.EqualValues(t, 32, msg[off+1], "prefixlen") + assert.Zero(t, msg[off+2], "legacy flags field must stay zero; NOPREFIXROUTE goes into IFA_FLAGS") + assert.EqualValues(t, unix.RT_SCOPE_UNIVERSE, msg[off+3]) + assert.EqualValues(t, 7, binary.NativeEndian.Uint32(msg[off+4:off+8])) + off += unix.SizeofIfAddrmsg + + // rtattr walk + attrs := map[uint16][]byte{} + for off < len(msg) { + attrLen := int(binary.NativeEndian.Uint16(msg[off : off+2])) + attrType := binary.NativeEndian.Uint16(msg[off+2 : off+4]) + require.GreaterOrEqual(t, attrLen, unix.SizeofRtAttr) + attrs[attrType] = msg[off+unix.SizeofRtAttr : off+attrLen] + off += attrLen + nlaPad(attrLen-unix.SizeofRtAttr) + } + + assert.Equal(t, ip4, net.IP(attrs[unix.IFA_LOCAL])) + assert.Equal(t, ip4, net.IP(attrs[unix.IFA_ADDRESS])) + require.Contains(t, attrs, uint16(unix.IFA_FLAGS)) + assert.EqualValues(t, unix.IFA_F_NOPREFIXROUTE, binary.NativeEndian.Uint32(attrs[unix.IFA_FLAGS])) + assert.NotContains(t, attrs, uint16(unix.IFA_BROADCAST), "a /32 gets no broadcast address") +} + +func TestEnsureRoutableAddrRejectsBadInput(t *testing.T) { + // Neither case needs root: parsing and the link lookup fail first. + assert.Error(t, ensureRoutableAddr("eth0", "not-a-cidr")) + assert.Error(t, ensureRoutableAddr("e2b-nonexistent0", RoutableAddrCIDR)) + assert.Error(t, ensureRoutableAddr("lo", "2001:db8::1/128"), "the dummy address contract is IPv4-only") +} + +// TestEnsureRoutableAddrRoundTrip exercises the real netlink path against the +// loopback interface: add, re-add (the EEXIST branch a re-exec'd envd hits), +// then delete. Needs root and mutates only this interface's address list. +func TestEnsureRoutableAddrRoundTrip(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("skipping netlink address round trip: needs root") + } + + ip4 := net.IPv4(192, 0, 0, 8).To4() + lo, err := net.InterfaceByName("lo") + require.NoError(t, err) + + del := buildAddrMsg(unix.RTM_DELADDR, 0, lo.Index, 32, ip4) + // Start and end clean regardless of how the test exits. + _ = sendAddrMsg(del) // best-effort cleanup of a possibly absent address + t.Cleanup(func() { _ = sendAddrMsg(del) }) + + require.NoError(t, ensureRoutableAddr("lo", RoutableAddrCIDR)) + assert.True(t, hasAddr(t, lo.Index, "192.0.0.8"), "the dummy address must be on the link after the add") + + // A second add is the resume/re-exec case: EEXIST must read as success. + assert.NoError(t, ensureRoutableAddr("lo", RoutableAddrCIDR)) + + require.NoError(t, sendAddrMsg(del)) + assert.False(t, hasAddr(t, lo.Index, "192.0.0.8"), "the address must be gone after the delete") +} + +// hasAddr dumps the link's addresses over netlink and looks for ip. +func hasAddr(t *testing.T, ifIndex int, ip string) bool { + t.Helper() + + fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE) + require.NoError(t, err) + defer func() { _ = unix.Close(fd) }() + require.NoError(t, unix.Bind(fd, &unix.SockaddrNetlink{Family: unix.AF_NETLINK})) + + req := make([]byte, unix.SizeofNlMsghdr+unix.SizeofIfAddrmsg) + binary.NativeEndian.PutUint32(req[0:4], uint32(len(req))) + binary.NativeEndian.PutUint16(req[4:6], unix.RTM_GETADDR) + binary.NativeEndian.PutUint16(req[6:8], unix.NLM_F_REQUEST|unix.NLM_F_DUMP) + binary.NativeEndian.PutUint32(req[8:12], 1) + req[unix.SizeofNlMsghdr+0] = unix.AF_INET + require.NoError(t, unix.Sendto(fd, req, 0, &unix.SockaddrNetlink{Family: unix.AF_NETLINK})) + + want := net.ParseIP(ip).To4() + buf := make([]byte, 32*1024) + for { + n, _, err := unix.Recvfrom(fd, buf, 0) + require.NoError(t, err) + if n == 0 { + return false + } + for off := 0; off+unix.SizeofNlMsghdr <= n; { + msgLen := int(binary.NativeEndian.Uint32(buf[off : off+4])) + msgType := binary.NativeEndian.Uint16(buf[off+4 : off+6]) + if msgType == unix.NLMSG_DONE { + return false + } + require.EqualValues(t, unix.RTM_NEWADDR, msgType, "dump replies are RTM_NEWADDR messages") + body := buf[off+unix.SizeofNlMsghdr : off+msgLen] + if len(body) >= unix.SizeofIfAddrmsg && + int(binary.NativeEndian.Uint32(body[4:8])) == ifIndex { + for a := unix.SizeofIfAddrmsg; a+unix.SizeofRtAttr <= len(body); { + attrLen := int(binary.NativeEndian.Uint16(body[a : a+2])) + attrType := binary.NativeEndian.Uint16(body[a+2 : a+4]) + if attrLen < unix.SizeofRtAttr { + break + } + if attrType == unix.IFA_ADDRESS && + net.IP(body[a+unix.SizeofRtAttr:a+attrLen]).Equal(want) { + return true + } + a += attrLen + nlaPad(attrLen-unix.SizeofRtAttr) + } + } + off += msgLen + nlaPad(msgLen) + } + } +} diff --git a/packages/envd/internal/host/net_addr_other.go b/packages/envd/internal/host/net_addr_other.go new file mode 100644 index 00000000000..7e884277626 --- /dev/null +++ b/packages/envd/internal/host/net_addr_other.go @@ -0,0 +1,8 @@ +//go:build !linux + +package host + +// RoutableAddrCIDR mirrors the linux implementation for tests on other hosts. +const RoutableAddrCIDR = "192.0.0.8/32" + +func EnsureRoutableAddr() error { return nil } diff --git a/packages/envd/main.go b/packages/envd/main.go index d3bf0250d37..88d080843ea 100644 --- a/packages/envd/main.go +++ b/packages/envd/main.go @@ -213,6 +213,17 @@ func run(stop <-chan struct{}) error { } l := logs.NewLogger(verbose, logWriters...) + if !isNotFC { + // The guest's only IPv4 address is link-local, which software that + // requires a routable address to consider the host online (e.g. + // Tailscale's network monitor) reads as "no network". Add the RFC 7600 + // dummy address; a failure must not block startup — egress does not + // depend on it. + if err := host.EnsureRoutableAddr(); err != nil { + l.Warn().Err(err).Msg("failed to add the RFC 7600 dummy address to eth0") + } + } + m := chi.NewRouter() envLogger := l.With().Str("logger", "envd").Logger()