diff --git a/packages/envd/internal/host/net_addr_linux.go b/packages/envd/internal/host/net_addr_linux.go new file mode 100644 index 00000000000..b4c1577d048 --- /dev/null +++ b/packages/envd/internal/host/net_addr_linux.go @@ -0,0 +1,171 @@ +//go:build linux + +package host + +import ( + "encoding/binary" + "errors" + "fmt" + "net" + + "golang.org/x/sys/unix" +) + +// RoutableAddrCIDR is the RFC 7600 IPv4 dummy address EnsureRoutableAddr adds +// to the guest's eth0. RFC 7600 reserves 192.0.0.8 for hosts with no other +// usable IPv4 address and it is never routed, so unlike 100.64.0.0/10 +// (Tailscale's own CGNAT range) or the RFC 1918 ranges it cannot clash with a +// user network or VPN. +const RoutableAddrCIDR = "192.0.0.8/32" + +// routableAddrLink is the guest's network interface, configured by the +// orchestrator through the kernel's ip= argument (169.254.0.21/30). +const routableAddrLink = "eth0" + +// EnsureRoutableAddr adds the RFC 7600 dummy address (192.0.0.8/32) to eth0 +// with IFA_F_NOPREFIXROUTE. +// +// The guest's only IPv4 address is link-local, and software that requires a +// non-link-local address to consider the host online treats the sandbox as +// offline. The reported case is Tailscale: its network monitor does not count +// 169.254.0.0/16 as usable, so tailscaled pauses its control client before +// login and auth keys silently never reach the control plane — even though +// outbound connectivity is fine. +// +// The address is added noprefixroute, so no prefix route is created: egress +// keeps selecting the gateway-subnet address (169.254.0.21) as its source and +// nothing routes to 192.0.0.8, leaving inbound traffic unchanged. +// +// Idempotent across the sandbox lifecycle — a cold boot runs envd again and a +// repeated add is a no-op (EEXIST), while a memory-snapshot resume keeps the +// address because it lives in kernel state. It talks to NETLINK_ROUTE directly +// rather than shelling out to `ip addr`, because some base images ship no +// iproute2 and envd (unlike the orchestrator) does not depend on a netlink +// library; golang.org/x/sys/unix is already an envd dependency. +func EnsureRoutableAddr() error { + return ensureRoutableAddr(routableAddrLink, RoutableAddrCIDR) +} + +func ensureRoutableAddr(linkName, cidr string) error { + ip, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("parse dummy address %s: %w", cidr, err) + } + ip4 := ip.To4() + if ip4 == nil { + return fmt.Errorf("dummy address %s is not IPv4", cidr) + } + link, err := net.InterfaceByName(linkName) + if err != nil { + return fmt.Errorf("find link %s: %w", linkName, err) + } + prefixLen, _ := ipnet.Mask.Size() + + msg := buildAddrMsg(unix.RTM_NEWADDR, unix.NLM_F_CREATE|unix.NLM_F_EXCL, + link.Index, uint8(prefixLen), ip4) + + return sendAddrMsg(msg) +} + +// buildAddrMsg encodes an RTM_NEWADDR request for a single IPv4 address, +// mirroring what `ip addr add /32 dev noprefixroute` sends. The +// 8-bit IfAddrmsg flags field cannot hold IFA_F_NOPREFIXROUTE (0x200), so it +// travels in the IFA_FLAGS attribute; IFA_ADDRESS and IFA_LOCAL carry the same +// address (this is not point-to-point); and a /32 gets no IFA_BROADCAST (the +// kernel only wants one for prefixes shorter than /31). +func buildAddrMsg(msgType uint16, extraFlags uint16, ifIndex int, prefixLen uint8, ip4 net.IP) []byte { + const ( + hdrLen = unix.SizeofNlMsghdr // 16 + ifaLen = unix.SizeofIfAddrmsg // 8 + ipLen = net.IPv4len // payload of IFA_LOCAL / IFA_ADDRESS + flagLen = 4 // payload of IFA_FLAGS (uint32) + ) + attrLen := func(payload int) int { return unix.SizeofRtAttr + payload + nlaPad(payload) } + total := hdrLen + ifaLen + + attrLen(ipLen) + // IFA_LOCAL + attrLen(ipLen) + // IFA_ADDRESS + attrLen(flagLen) // IFA_FLAGS + + buf := make([]byte, total) + + // nlmsghdr + binary.NativeEndian.PutUint32(buf[0:4], uint32(total)) + binary.NativeEndian.PutUint16(buf[4:6], msgType) + binary.NativeEndian.PutUint16(buf[6:8], unix.NLM_F_REQUEST|unix.NLM_F_ACK|extraFlags) + binary.NativeEndian.PutUint32(buf[8:12], 1) // seq + binary.NativeEndian.PutUint32(buf[12:16], 0) + + // ifaddrmsg + off := hdrLen + buf[off] = unix.AF_INET + buf[off+1] = prefixLen + buf[off+2] = 0 // legacy 8-bit flags; NOPREFIXROUTE goes into IFA_FLAGS + buf[off+3] = unix.RT_SCOPE_UNIVERSE + binary.NativeEndian.PutUint32(buf[off+4:off+ifaLen], uint32(ifIndex)) + off += ifaLen + + putAttr := func(typ uint16, payload []byte) { + binary.NativeEndian.PutUint16(buf[off:off+2], uint16(unix.SizeofRtAttr+len(payload))) + binary.NativeEndian.PutUint16(buf[off+2:off+4], typ) + copy(buf[off+unix.SizeofRtAttr:], payload) + off += unix.SizeofRtAttr + len(payload) + nlaPad(len(payload)) + } + putAttr(unix.IFA_LOCAL, ip4) + putAttr(unix.IFA_ADDRESS, ip4) + flags := make([]byte, flagLen) + binary.NativeEndian.PutUint32(flags, unix.IFA_F_NOPREFIXROUTE) + putAttr(unix.IFA_FLAGS, flags) + + return buf +} + +// nlaPad is the padding aligning the next netlink attribute to NLA_ALIGNTO. +func nlaPad(payloadLen int) int { + return -payloadLen & (unix.NLA_ALIGNTO - 1) +} + +// sendAddrMsg sends one address request to NETLINK_ROUTE and waits for the +// kernel ACK. EEXIST means the address is already configured (a re-exec'd envd +// after a live upgrade), which satisfies the caller just like success. +func sendAddrMsg(msg []byte) error { + fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE) + if err != nil { + return fmt.Errorf("open netlink route socket: %w", err) + } + defer func() { _ = unix.Close(fd) }() + + if err := unix.Bind(fd, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil { + return fmt.Errorf("bind netlink route socket: %w", err) + } + if err := unix.Sendto(fd, msg, 0, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil { + return fmt.Errorf("send netlink address request: %w", err) + } + + buf := make([]byte, 4096) + for { + n, _, err := unix.Recvfrom(fd, buf, 0) + if err != nil { + return fmt.Errorf("receive netlink ack: %w", err) + } + if n < unix.SizeofNlMsghdr+4 { + return errors.New("short netlink ack") + } + switch msgType := binary.NativeEndian.Uint16(buf[4:6]); msgType { + case unix.NLMSG_NOOP: + continue + case unix.NLMSG_ERROR: + errno := int32(binary.NativeEndian.Uint32(buf[unix.SizeofNlMsghdr : unix.SizeofNlMsghdr+4])) + if errno == 0 { + return nil // ACK, no error + } + nlErr := unix.Errno(-errno) + if errors.Is(nlErr, unix.EEXIST) { + return nil + } + + return fmt.Errorf("netlink address request: %w", nlErr) + default: + return fmt.Errorf("unexpected netlink reply type %d", msgType) + } + } +} diff --git a/packages/envd/internal/host/net_addr_linux_test.go b/packages/envd/internal/host/net_addr_linux_test.go new file mode 100644 index 00000000000..dca4332bd30 --- /dev/null +++ b/packages/envd/internal/host/net_addr_linux_test.go @@ -0,0 +1,152 @@ +//go:build linux + +package host + +import ( + "encoding/binary" + "net" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestNlaPad(t *testing.T) { + assert.Equal(t, 0, nlaPad(0)) + assert.Equal(t, 3, nlaPad(1)) + assert.Equal(t, 2, nlaPad(2)) + assert.Equal(t, 1, nlaPad(3)) + assert.Equal(t, 0, nlaPad(4)) + assert.Equal(t, 3, nlaPad(5)) +} + +// TestBuildAddrMsgEncodesNoPrefixRoute checks the request against the wire +// format `ip addr add 192.0.0.8/32 dev noprefixroute` produces: the +// flag must travel in the IFA_FLAGS attribute (it does not fit the 8-bit +// legacy field), and a /32 must carry no IFA_BROADCAST. +func TestBuildAddrMsgEncodesNoPrefixRoute(t *testing.T) { + ip4 := net.IPv4(192, 0, 0, 8).To4() + msg := buildAddrMsg(unix.RTM_NEWADDR, unix.NLM_F_CREATE|unix.NLM_F_EXCL, 7, 32, ip4) + + // nlmsghdr + assert.Len(t, msg, int(binary.NativeEndian.Uint32(msg[0:4]))) + assert.EqualValues(t, unix.RTM_NEWADDR, binary.NativeEndian.Uint16(msg[4:6])) + flags := binary.NativeEndian.Uint16(msg[6:8]) + for _, want := range []uint16{unix.NLM_F_REQUEST, unix.NLM_F_ACK, unix.NLM_F_CREATE, unix.NLM_F_EXCL} { + assert.NotZero(t, flags&want) + } + + // ifaddrmsg + off := unix.SizeofNlMsghdr + assert.EqualValues(t, unix.AF_INET, msg[off]) + assert.EqualValues(t, 32, msg[off+1], "prefixlen") + assert.Zero(t, msg[off+2], "legacy flags field must stay zero; NOPREFIXROUTE goes into IFA_FLAGS") + assert.EqualValues(t, unix.RT_SCOPE_UNIVERSE, msg[off+3]) + assert.EqualValues(t, 7, binary.NativeEndian.Uint32(msg[off+4:off+8])) + off += unix.SizeofIfAddrmsg + + // rtattr walk + attrs := map[uint16][]byte{} + for off < len(msg) { + attrLen := int(binary.NativeEndian.Uint16(msg[off : off+2])) + attrType := binary.NativeEndian.Uint16(msg[off+2 : off+4]) + require.GreaterOrEqual(t, attrLen, unix.SizeofRtAttr) + attrs[attrType] = msg[off+unix.SizeofRtAttr : off+attrLen] + off += attrLen + nlaPad(attrLen-unix.SizeofRtAttr) + } + + assert.Equal(t, ip4, net.IP(attrs[unix.IFA_LOCAL])) + assert.Equal(t, ip4, net.IP(attrs[unix.IFA_ADDRESS])) + require.Contains(t, attrs, uint16(unix.IFA_FLAGS)) + assert.EqualValues(t, unix.IFA_F_NOPREFIXROUTE, binary.NativeEndian.Uint32(attrs[unix.IFA_FLAGS])) + assert.NotContains(t, attrs, uint16(unix.IFA_BROADCAST), "a /32 gets no broadcast address") +} + +func TestEnsureRoutableAddrRejectsBadInput(t *testing.T) { + // Neither case needs root: parsing and the link lookup fail first. + assert.Error(t, ensureRoutableAddr("eth0", "not-a-cidr")) + assert.Error(t, ensureRoutableAddr("e2b-nonexistent0", RoutableAddrCIDR)) + assert.Error(t, ensureRoutableAddr("lo", "2001:db8::1/128"), "the dummy address contract is IPv4-only") +} + +// TestEnsureRoutableAddrRoundTrip exercises the real netlink path against the +// loopback interface: add, re-add (the EEXIST branch a re-exec'd envd hits), +// then delete. Needs root and mutates only this interface's address list. +func TestEnsureRoutableAddrRoundTrip(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("skipping netlink address round trip: needs root") + } + + ip4 := net.IPv4(192, 0, 0, 8).To4() + lo, err := net.InterfaceByName("lo") + require.NoError(t, err) + + del := buildAddrMsg(unix.RTM_DELADDR, 0, lo.Index, 32, ip4) + // Start and end clean regardless of how the test exits. + _ = sendAddrMsg(del) // best-effort cleanup of a possibly absent address + t.Cleanup(func() { _ = sendAddrMsg(del) }) + + require.NoError(t, ensureRoutableAddr("lo", RoutableAddrCIDR)) + assert.True(t, hasAddr(t, lo.Index, "192.0.0.8"), "the dummy address must be on the link after the add") + + // A second add is the resume/re-exec case: EEXIST must read as success. + assert.NoError(t, ensureRoutableAddr("lo", RoutableAddrCIDR)) + + require.NoError(t, sendAddrMsg(del)) + assert.False(t, hasAddr(t, lo.Index, "192.0.0.8"), "the address must be gone after the delete") +} + +// hasAddr dumps the link's addresses over netlink and looks for ip. +func hasAddr(t *testing.T, ifIndex int, ip string) bool { + t.Helper() + + fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE) + require.NoError(t, err) + defer func() { _ = unix.Close(fd) }() + require.NoError(t, unix.Bind(fd, &unix.SockaddrNetlink{Family: unix.AF_NETLINK})) + + req := make([]byte, unix.SizeofNlMsghdr+unix.SizeofIfAddrmsg) + binary.NativeEndian.PutUint32(req[0:4], uint32(len(req))) + binary.NativeEndian.PutUint16(req[4:6], unix.RTM_GETADDR) + binary.NativeEndian.PutUint16(req[6:8], unix.NLM_F_REQUEST|unix.NLM_F_DUMP) + binary.NativeEndian.PutUint32(req[8:12], 1) + req[unix.SizeofNlMsghdr+0] = unix.AF_INET + require.NoError(t, unix.Sendto(fd, req, 0, &unix.SockaddrNetlink{Family: unix.AF_NETLINK})) + + want := net.ParseIP(ip).To4() + buf := make([]byte, 32*1024) + for { + n, _, err := unix.Recvfrom(fd, buf, 0) + require.NoError(t, err) + if n == 0 { + return false + } + for off := 0; off+unix.SizeofNlMsghdr <= n; { + msgLen := int(binary.NativeEndian.Uint32(buf[off : off+4])) + msgType := binary.NativeEndian.Uint16(buf[off+4 : off+6]) + if msgType == unix.NLMSG_DONE { + return false + } + require.EqualValues(t, unix.RTM_NEWADDR, msgType, "dump replies are RTM_NEWADDR messages") + body := buf[off+unix.SizeofNlMsghdr : off+msgLen] + if len(body) >= unix.SizeofIfAddrmsg && + int(binary.NativeEndian.Uint32(body[4:8])) == ifIndex { + for a := unix.SizeofIfAddrmsg; a+unix.SizeofRtAttr <= len(body); { + attrLen := int(binary.NativeEndian.Uint16(body[a : a+2])) + attrType := binary.NativeEndian.Uint16(body[a+2 : a+4]) + if attrLen < unix.SizeofRtAttr { + break + } + if attrType == unix.IFA_ADDRESS && + net.IP(body[a+unix.SizeofRtAttr:a+attrLen]).Equal(want) { + return true + } + a += attrLen + nlaPad(attrLen-unix.SizeofRtAttr) + } + } + off += msgLen + nlaPad(msgLen) + } + } +} diff --git a/packages/envd/internal/host/net_addr_other.go b/packages/envd/internal/host/net_addr_other.go new file mode 100644 index 00000000000..7e884277626 --- /dev/null +++ b/packages/envd/internal/host/net_addr_other.go @@ -0,0 +1,8 @@ +//go:build !linux + +package host + +// RoutableAddrCIDR mirrors the linux implementation for tests on other hosts. +const RoutableAddrCIDR = "192.0.0.8/32" + +func EnsureRoutableAddr() error { return nil } diff --git a/packages/envd/main.go b/packages/envd/main.go index d3bf0250d37..88d080843ea 100644 --- a/packages/envd/main.go +++ b/packages/envd/main.go @@ -213,6 +213,17 @@ func run(stop <-chan struct{}) error { } l := logs.NewLogger(verbose, logWriters...) + if !isNotFC { + // The guest's only IPv4 address is link-local, which software that + // requires a routable address to consider the host online (e.g. + // Tailscale's network monitor) reads as "no network". Add the RFC 7600 + // dummy address; a failure must not block startup — egress does not + // depend on it. + if err := host.EnsureRoutableAddr(); err != nil { + l.Warn().Err(err).Msg("failed to add the RFC 7600 dummy address to eth0") + } + } + m := chi.NewRouter() envLogger := l.With().Str("logger", "envd").Logger()