Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
171 changes: 171 additions & 0 deletions packages/envd/internal/host/net_addr_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,171 @@
//go:build linux

package host

import (
"encoding/binary"
"errors"
"fmt"
"net"

"golang.org/x/sys/unix"
)

// RoutableAddrCIDR is the RFC 7600 IPv4 dummy address EnsureRoutableAddr adds
// to the guest's eth0. RFC 7600 reserves 192.0.0.8 for hosts with no other
// usable IPv4 address and it is never routed, so unlike 100.64.0.0/10
// (Tailscale's own CGNAT range) or the RFC 1918 ranges it cannot clash with a
// user network or VPN.
const RoutableAddrCIDR = "192.0.0.8/32"

// routableAddrLink is the guest's network interface, configured by the
// orchestrator through the kernel's ip= argument (169.254.0.21/30).
const routableAddrLink = "eth0"

// EnsureRoutableAddr adds the RFC 7600 dummy address (192.0.0.8/32) to eth0
// with IFA_F_NOPREFIXROUTE.
//
// The guest's only IPv4 address is link-local, and software that requires a
// non-link-local address to consider the host online treats the sandbox as
// offline. The reported case is Tailscale: its network monitor does not count
// 169.254.0.0/16 as usable, so tailscaled pauses its control client before
// login and auth keys silently never reach the control plane — even though
// outbound connectivity is fine.
//
// The address is added noprefixroute, so no prefix route is created: egress
// keeps selecting the gateway-subnet address (169.254.0.21) as its source and
// nothing routes to 192.0.0.8, leaving inbound traffic unchanged.
//
// Idempotent across the sandbox lifecycle — a cold boot runs envd again and a
// repeated add is a no-op (EEXIST), while a memory-snapshot resume keeps the
// address because it lives in kernel state. It talks to NETLINK_ROUTE directly
// rather than shelling out to `ip addr`, because some base images ship no
// iproute2 and envd (unlike the orchestrator) does not depend on a netlink
// library; golang.org/x/sys/unix is already an envd dependency.
func EnsureRoutableAddr() error {
return ensureRoutableAddr(routableAddrLink, RoutableAddrCIDR)
}

func ensureRoutableAddr(linkName, cidr string) error {
ip, ipnet, err := net.ParseCIDR(cidr)
if err != nil {
return fmt.Errorf("parse dummy address %s: %w", cidr, err)
}
ip4 := ip.To4()
if ip4 == nil {
return fmt.Errorf("dummy address %s is not IPv4", cidr)
}
link, err := net.InterfaceByName(linkName)
if err != nil {
return fmt.Errorf("find link %s: %w", linkName, err)
}
prefixLen, _ := ipnet.Mask.Size()

msg := buildAddrMsg(unix.RTM_NEWADDR, unix.NLM_F_CREATE|unix.NLM_F_EXCL,
link.Index, uint8(prefixLen), ip4)

return sendAddrMsg(msg)
}

// buildAddrMsg encodes an RTM_NEWADDR request for a single IPv4 address,
// mirroring what `ip addr add <ip>/32 dev <link> noprefixroute` sends. The
// 8-bit IfAddrmsg flags field cannot hold IFA_F_NOPREFIXROUTE (0x200), so it
// travels in the IFA_FLAGS attribute; IFA_ADDRESS and IFA_LOCAL carry the same
// address (this is not point-to-point); and a /32 gets no IFA_BROADCAST (the
// kernel only wants one for prefixes shorter than /31).
func buildAddrMsg(msgType uint16, extraFlags uint16, ifIndex int, prefixLen uint8, ip4 net.IP) []byte {
const (
hdrLen = unix.SizeofNlMsghdr // 16
ifaLen = unix.SizeofIfAddrmsg // 8
ipLen = net.IPv4len // payload of IFA_LOCAL / IFA_ADDRESS
flagLen = 4 // payload of IFA_FLAGS (uint32)
)
attrLen := func(payload int) int { return unix.SizeofRtAttr + payload + nlaPad(payload) }
total := hdrLen + ifaLen +
attrLen(ipLen) + // IFA_LOCAL
attrLen(ipLen) + // IFA_ADDRESS
attrLen(flagLen) // IFA_FLAGS

buf := make([]byte, total)

// nlmsghdr
binary.NativeEndian.PutUint32(buf[0:4], uint32(total))
binary.NativeEndian.PutUint16(buf[4:6], msgType)
binary.NativeEndian.PutUint16(buf[6:8], unix.NLM_F_REQUEST|unix.NLM_F_ACK|extraFlags)
binary.NativeEndian.PutUint32(buf[8:12], 1) // seq
binary.NativeEndian.PutUint32(buf[12:16], 0)

// ifaddrmsg
off := hdrLen
buf[off] = unix.AF_INET
buf[off+1] = prefixLen
buf[off+2] = 0 // legacy 8-bit flags; NOPREFIXROUTE goes into IFA_FLAGS
buf[off+3] = unix.RT_SCOPE_UNIVERSE
binary.NativeEndian.PutUint32(buf[off+4:off+ifaLen], uint32(ifIndex))
off += ifaLen

putAttr := func(typ uint16, payload []byte) {
binary.NativeEndian.PutUint16(buf[off:off+2], uint16(unix.SizeofRtAttr+len(payload)))
binary.NativeEndian.PutUint16(buf[off+2:off+4], typ)
copy(buf[off+unix.SizeofRtAttr:], payload)
off += unix.SizeofRtAttr + len(payload) + nlaPad(len(payload))
}
putAttr(unix.IFA_LOCAL, ip4)
putAttr(unix.IFA_ADDRESS, ip4)
flags := make([]byte, flagLen)
binary.NativeEndian.PutUint32(flags, unix.IFA_F_NOPREFIXROUTE)
putAttr(unix.IFA_FLAGS, flags)

return buf
}

// nlaPad is the padding aligning the next netlink attribute to NLA_ALIGNTO.
func nlaPad(payloadLen int) int {
return -payloadLen & (unix.NLA_ALIGNTO - 1)
}

// sendAddrMsg sends one address request to NETLINK_ROUTE and waits for the
// kernel ACK. EEXIST means the address is already configured (a re-exec'd envd
// after a live upgrade), which satisfies the caller just like success.
func sendAddrMsg(msg []byte) error {
fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE)
if err != nil {
return fmt.Errorf("open netlink route socket: %w", err)
}
defer func() { _ = unix.Close(fd) }()

if err := unix.Bind(fd, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil {
return fmt.Errorf("bind netlink route socket: %w", err)
}
if err := unix.Sendto(fd, msg, 0, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil {
return fmt.Errorf("send netlink address request: %w", err)
}

buf := make([]byte, 4096)
for {
n, _, err := unix.Recvfrom(fd, buf, 0)
if err != nil {
return fmt.Errorf("receive netlink ack: %w", err)
}
if n < unix.SizeofNlMsghdr+4 {
return errors.New("short netlink ack")
}
switch msgType := binary.NativeEndian.Uint16(buf[4:6]); msgType {
case unix.NLMSG_NOOP:
continue
case unix.NLMSG_ERROR:
errno := int32(binary.NativeEndian.Uint32(buf[unix.SizeofNlMsghdr : unix.SizeofNlMsghdr+4]))
if errno == 0 {
return nil // ACK, no error
}
nlErr := unix.Errno(-errno)
if errors.Is(nlErr, unix.EEXIST) {
return nil
}

return fmt.Errorf("netlink address request: %w", nlErr)
default:
return fmt.Errorf("unexpected netlink reply type %d", msgType)
}
}
}
152 changes: 152 additions & 0 deletions packages/envd/internal/host/net_addr_linux_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
//go:build linux

package host

import (
"encoding/binary"
"net"
"os"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
)

func TestNlaPad(t *testing.T) {
assert.Equal(t, 0, nlaPad(0))
assert.Equal(t, 3, nlaPad(1))
assert.Equal(t, 2, nlaPad(2))
assert.Equal(t, 1, nlaPad(3))
assert.Equal(t, 0, nlaPad(4))
assert.Equal(t, 3, nlaPad(5))
}

// TestBuildAddrMsgEncodesNoPrefixRoute checks the request against the wire
// format `ip addr add 192.0.0.8/32 dev <link> noprefixroute` produces: the
// flag must travel in the IFA_FLAGS attribute (it does not fit the 8-bit
// legacy field), and a /32 must carry no IFA_BROADCAST.
func TestBuildAddrMsgEncodesNoPrefixRoute(t *testing.T) {
ip4 := net.IPv4(192, 0, 0, 8).To4()
msg := buildAddrMsg(unix.RTM_NEWADDR, unix.NLM_F_CREATE|unix.NLM_F_EXCL, 7, 32, ip4)

// nlmsghdr
assert.Len(t, msg, int(binary.NativeEndian.Uint32(msg[0:4])))
assert.EqualValues(t, unix.RTM_NEWADDR, binary.NativeEndian.Uint16(msg[4:6]))
flags := binary.NativeEndian.Uint16(msg[6:8])
for _, want := range []uint16{unix.NLM_F_REQUEST, unix.NLM_F_ACK, unix.NLM_F_CREATE, unix.NLM_F_EXCL} {
assert.NotZero(t, flags&want)
}

// ifaddrmsg
off := unix.SizeofNlMsghdr
assert.EqualValues(t, unix.AF_INET, msg[off])
assert.EqualValues(t, 32, msg[off+1], "prefixlen")
assert.Zero(t, msg[off+2], "legacy flags field must stay zero; NOPREFIXROUTE goes into IFA_FLAGS")
assert.EqualValues(t, unix.RT_SCOPE_UNIVERSE, msg[off+3])
assert.EqualValues(t, 7, binary.NativeEndian.Uint32(msg[off+4:off+8]))
off += unix.SizeofIfAddrmsg

// rtattr walk
attrs := map[uint16][]byte{}
for off < len(msg) {
attrLen := int(binary.NativeEndian.Uint16(msg[off : off+2]))
attrType := binary.NativeEndian.Uint16(msg[off+2 : off+4])
require.GreaterOrEqual(t, attrLen, unix.SizeofRtAttr)
attrs[attrType] = msg[off+unix.SizeofRtAttr : off+attrLen]
off += attrLen + nlaPad(attrLen-unix.SizeofRtAttr)
}

assert.Equal(t, ip4, net.IP(attrs[unix.IFA_LOCAL]))
assert.Equal(t, ip4, net.IP(attrs[unix.IFA_ADDRESS]))
require.Contains(t, attrs, uint16(unix.IFA_FLAGS))
assert.EqualValues(t, unix.IFA_F_NOPREFIXROUTE, binary.NativeEndian.Uint32(attrs[unix.IFA_FLAGS]))
assert.NotContains(t, attrs, uint16(unix.IFA_BROADCAST), "a /32 gets no broadcast address")
}

func TestEnsureRoutableAddrRejectsBadInput(t *testing.T) {
// Neither case needs root: parsing and the link lookup fail first.
assert.Error(t, ensureRoutableAddr("eth0", "not-a-cidr"))
assert.Error(t, ensureRoutableAddr("e2b-nonexistent0", RoutableAddrCIDR))
assert.Error(t, ensureRoutableAddr("lo", "2001:db8::1/128"), "the dummy address contract is IPv4-only")
}

// TestEnsureRoutableAddrRoundTrip exercises the real netlink path against the
// loopback interface: add, re-add (the EEXIST branch a re-exec'd envd hits),
// then delete. Needs root and mutates only this interface's address list.
func TestEnsureRoutableAddrRoundTrip(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("skipping netlink address round trip: needs root")
}

ip4 := net.IPv4(192, 0, 0, 8).To4()
lo, err := net.InterfaceByName("lo")
require.NoError(t, err)

del := buildAddrMsg(unix.RTM_DELADDR, 0, lo.Index, 32, ip4)
// Start and end clean regardless of how the test exits.
_ = sendAddrMsg(del) // best-effort cleanup of a possibly absent address
t.Cleanup(func() { _ = sendAddrMsg(del) })

require.NoError(t, ensureRoutableAddr("lo", RoutableAddrCIDR))
assert.True(t, hasAddr(t, lo.Index, "192.0.0.8"), "the dummy address must be on the link after the add")

// A second add is the resume/re-exec case: EEXIST must read as success.
assert.NoError(t, ensureRoutableAddr("lo", RoutableAddrCIDR))

require.NoError(t, sendAddrMsg(del))
assert.False(t, hasAddr(t, lo.Index, "192.0.0.8"), "the address must be gone after the delete")
}

// hasAddr dumps the link's addresses over netlink and looks for ip.
func hasAddr(t *testing.T, ifIndex int, ip string) bool {
t.Helper()

fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE)
require.NoError(t, err)
defer func() { _ = unix.Close(fd) }()
require.NoError(t, unix.Bind(fd, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}))

req := make([]byte, unix.SizeofNlMsghdr+unix.SizeofIfAddrmsg)
binary.NativeEndian.PutUint32(req[0:4], uint32(len(req)))
binary.NativeEndian.PutUint16(req[4:6], unix.RTM_GETADDR)
binary.NativeEndian.PutUint16(req[6:8], unix.NLM_F_REQUEST|unix.NLM_F_DUMP)
binary.NativeEndian.PutUint32(req[8:12], 1)
req[unix.SizeofNlMsghdr+0] = unix.AF_INET
require.NoError(t, unix.Sendto(fd, req, 0, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}))

want := net.ParseIP(ip).To4()
buf := make([]byte, 32*1024)
for {
n, _, err := unix.Recvfrom(fd, buf, 0)
require.NoError(t, err)
if n == 0 {
return false
}
for off := 0; off+unix.SizeofNlMsghdr <= n; {
msgLen := int(binary.NativeEndian.Uint32(buf[off : off+4]))
msgType := binary.NativeEndian.Uint16(buf[off+4 : off+6])
if msgType == unix.NLMSG_DONE {
return false
}
require.EqualValues(t, unix.RTM_NEWADDR, msgType, "dump replies are RTM_NEWADDR messages")
body := buf[off+unix.SizeofNlMsghdr : off+msgLen]
if len(body) >= unix.SizeofIfAddrmsg &&
int(binary.NativeEndian.Uint32(body[4:8])) == ifIndex {
for a := unix.SizeofIfAddrmsg; a+unix.SizeofRtAttr <= len(body); {
attrLen := int(binary.NativeEndian.Uint16(body[a : a+2]))
attrType := binary.NativeEndian.Uint16(body[a+2 : a+4])
if attrLen < unix.SizeofRtAttr {
break
}
if attrType == unix.IFA_ADDRESS &&
net.IP(body[a+unix.SizeofRtAttr:a+attrLen]).Equal(want) {
return true
}
a += attrLen + nlaPad(attrLen-unix.SizeofRtAttr)
}
}
off += msgLen + nlaPad(msgLen)
}
}
}
8 changes: 8 additions & 0 deletions packages/envd/internal/host/net_addr_other.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
//go:build !linux

package host

// RoutableAddrCIDR mirrors the linux implementation for tests on other hosts.
const RoutableAddrCIDR = "192.0.0.8/32"

func EnsureRoutableAddr() error { return nil }
11 changes: 11 additions & 0 deletions packages/envd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,17 @@ func run(stop <-chan struct{}) error {
}
l := logs.NewLogger(verbose, logWriters...)

if !isNotFC {
// The guest's only IPv4 address is link-local, which software that
// requires a routable address to consider the host online (e.g.
// Tailscale's network monitor) reads as "no network". Add the RFC 7600
// dummy address; a failure must not block startup — egress does not
// depend on it.
if err := host.EnsureRoutableAddr(); err != nil {
l.Warn().Err(err).Msg("failed to add the RFC 7600 dummy address to eth0")
}
}

m := chi.NewRouter()

envLogger := l.With().Str("logger", "envd").Logger()
Expand Down