From 90260e20888458149284a3f73003a333610658d6 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 01:25:26 +0800 Subject: [PATCH 01/10] sandboxd: tunnet, the advanced network mode's packet path over VXLAN The guest's IP packets leave over VXLAN on an ordinary UDP socket to an exit that is plain Linux configuration; sandboxd keeps identity, policy and the source rewrite. Frames, virtio-net headers and IPv4 headers are bounds checked and every refusal is counted by reason; a partial checksum must sit in the L4 checksum field, which closes a fuzz-found bypass where a guest aimed the checksum write at the IPv4 destination after the policy check. A GSO frame is segmented with full checksums straight into the send buffer and charged for every packet it puts on the wire; a packet or segment over the MTU is dropped. Per-claim limits and flows outlive a stream reconnect. VXLAN and ARP are accepted only from the configured exits. Inbound TCP is merged into GSO frames after checksum verification, and ICMP errors quote the guest address. Overlay addresses come from an in-memory local allocator with an append-only release log, or from one statement per claim against a cell's PostgreSQL table. --- sandboxd/egress/policy.go | 19 + sandboxd/egress/policy_test.go | 23 + sandboxd/tunnet/addrs.go | 171 ++++++ sandboxd/tunnet/addrs_test.go | 135 +++++ sandboxd/tunnet/bench_test.go | 69 +++ sandboxd/tunnet/flow.go | 284 +++++++++ sandboxd/tunnet/fuzz_test.go | 72 +++ sandboxd/tunnet/gro.go | 132 ++++ sandboxd/tunnet/packet.go | 271 +++++++++ sandboxd/tunnet/packet_test.go | 161 +++++ sandboxd/tunnet/pg/pg.go | 102 ++++ sandboxd/tunnet/pg/pg_test.go | 136 +++++ sandboxd/tunnet/pg/schema.sql | 10 + sandboxd/tunnet/router.go | 565 ++++++++++++++++++ sandboxd/tunnet/router_test.go | 390 ++++++++++++ sandboxd/tunnet/segment.go | 133 +++++ sandboxd/tunnet/segment_test.go | 109 ++++ .../fuzz/FuzzParseAndRewrite/a7fe022fecd7d8ec | 2 + .../fuzz/FuzzSegment/3560257eabc72bdd | 2 + sandboxd/tunnet/tunnet.go | 2 + sandboxd/tunnet/udp/exit_linux_test.go | 263 ++++++++ sandboxd/tunnet/udp/udp.go | 128 ++++ sandboxd/tunnet/udp/udp_linux.go | 53 ++ sandboxd/tunnet/udp/udp_linux_test.go | 55 ++ sandboxd/tunnet/udp/udp_other.go | 21 + sandboxd/tunnet/vxlan.go | 80 +++ 26 files changed, 3388 insertions(+) create mode 100644 sandboxd/tunnet/addrs.go create mode 100644 sandboxd/tunnet/addrs_test.go create mode 100644 sandboxd/tunnet/bench_test.go create mode 100644 sandboxd/tunnet/flow.go create mode 100644 sandboxd/tunnet/fuzz_test.go create mode 100644 sandboxd/tunnet/gro.go create mode 100644 sandboxd/tunnet/packet.go create mode 100644 sandboxd/tunnet/packet_test.go create mode 100644 sandboxd/tunnet/pg/pg.go create mode 100644 sandboxd/tunnet/pg/pg_test.go create mode 100644 sandboxd/tunnet/pg/schema.sql create mode 100644 sandboxd/tunnet/router.go create mode 100644 sandboxd/tunnet/router_test.go create mode 100644 sandboxd/tunnet/segment.go create mode 100644 sandboxd/tunnet/segment_test.go create mode 100644 sandboxd/tunnet/testdata/fuzz/FuzzParseAndRewrite/a7fe022fecd7d8ec create mode 100644 sandboxd/tunnet/testdata/fuzz/FuzzSegment/3560257eabc72bdd create mode 100644 sandboxd/tunnet/tunnet.go create mode 100644 sandboxd/tunnet/udp/exit_linux_test.go create mode 100644 sandboxd/tunnet/udp/udp.go create mode 100644 sandboxd/tunnet/udp/udp_linux.go create mode 100644 sandboxd/tunnet/udp/udp_linux_test.go create mode 100644 sandboxd/tunnet/udp/udp_other.go create mode 100644 sandboxd/tunnet/vxlan.go diff --git a/sandboxd/egress/policy.go b/sandboxd/egress/policy.go index d07b4dba8..db154fd9c 100644 --- a/sandboxd/egress/policy.go +++ b/sandboxd/egress/policy.go @@ -182,6 +182,16 @@ func (p Policy) EvalInner(host, method string, port uint16) (Rule, Decision) { return Rule{}, DecisionDeny } +// EvalFlow judges an L3 flow to port: only a bare "*" rule has nothing an address and a port cannot carry. +func (p Policy) EvalFlow(port uint16) Decision { + if slices.ContainsFunc(p.Allow, func(r Rule) bool { + return r.Host == "*" && len(r.Methods) == 0 && r.Secret == "" && r.Intercept == InterceptOff && r.matchPort(port) + }) { + return DecisionAllow + } + return DecisionDeny +} + // ServesSocks reports whether the policy opted into the SOCKS5 door; Validate holds the rule that makes the door usable. func (p Policy) ServesSocks() bool { return p.Socks5 @@ -208,8 +218,10 @@ type Evaluator interface { EvalInner(host, method string, port uint16) (Rule, Decision) } +// PolicyEvaluator is a claim's effective policy: the per-request checks plus the door and flow decisions. type PolicyEvaluator interface { Evaluator + EvalFlow(port uint16) Decision ServesSocks() bool ServesDNS() bool InterceptsHost(pattern string) bool @@ -254,6 +266,13 @@ func (c composite) EvalInner(host, method string, port uint16) (Rule, Decision) return Rule{}, DecisionDeny } +func (c composite) EvalFlow(port uint16) Decision { + if c.pool.EvalFlow(port) == DecisionAllow && c.tenant.EvalFlow(port) == DecisionAllow { + return DecisionAllow + } + return DecisionDeny +} + // ServesSocks is the pool's call: the tenant's rules already gate every tunnel through Eval. func (c composite) ServesSocks() bool { return c.pool.ServesSocks() diff --git a/sandboxd/egress/policy_test.go b/sandboxd/egress/policy_test.go index 6c76b79c8..f53d9274e 100644 --- a/sandboxd/egress/policy_test.go +++ b/sandboxd/egress/policy_test.go @@ -165,6 +165,29 @@ func TestWildcardApexIsNotMatched(t *testing.T) { } } +func TestEvalFlowMatchesOnlyBareWildcardRules(t *testing.T) { + p := Policy{Allow: []Rule{ + {Host: "api.example.com", Ports: []uint16{443}}, + {Host: "*", Methods: []string{"GET"}, Ports: []uint16{80}}, + {Host: "*", Secret: "s", Ports: []uint16{8080}}, + {Host: "*", Intercept: InterceptInject, Ports: []uint16{8443}}, + {Host: "*", Ports: []uint16{53, 3478}}, + }} + for port, want := range map[uint16]Decision{443: DecisionDeny, 80: DecisionDeny, 8080: DecisionDeny, 8443: DecisionDeny, 53: DecisionAllow, 3478: DecisionAllow, 22: DecisionDeny} { + if got := p.EvalFlow(port); got != want { + t.Errorf("EvalFlow(%d) = %v, want %v", port, got, want) + } + } + if (Policy{Allow: []Rule{{Host: "*"}}}).EvalFlow(22) != DecisionAllow { + t.Error("a bare * rule without ports must allow every port") + } + pool := Policy{Allow: []Rule{{Host: "*"}}} + tenant := Policy{Allow: []Rule{{Host: "*", Ports: []uint16{443}}}} + if c := Compose(pool, tenant); c.EvalFlow(443) != DecisionAllow || c.EvalFlow(22) != DecisionDeny { + t.Error("a composite flow verdict must intersect pool and tenant") + } +} + func TestPolicyValidate(t *testing.T) { tests := []struct { name string diff --git a/sandboxd/tunnet/addrs.go b/sandboxd/tunnet/addrs.go new file mode 100644 index 000000000..8fccc280f --- /dev/null +++ b/sandboxd/tunnet/addrs.go @@ -0,0 +1,171 @@ +package tunnet + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "io/fs" + "maps" + "net/netip" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/utils" +) + +const ( + freedFile = "tun-freed.log" + // compactAfter is how far the log may grow past twice its holds before it is rewritten with the live ones. + compactAfter = 1024 +) + +// ErrNoAddress means every overlay address is held or still inside its reuse window. +var ErrNoAddress = errors.New("no free overlay address") + +// Addresses hands out the claims' overlay addresses. +type Addresses interface { + // Assign returns claimID's address, allocating one on the first call. + Assign(ctx context.Context, claimID string) (netip.Addr, error) + // Release frees addr; it becomes reusable once the reuse window has passed. + Release(ctx context.Context, claimID string, addr netip.Addr) error +} + +var _ Addresses = (*Local)(nil) + +// Local allocates one node's overlay addresses in memory; held addresses come from Seed, and each release is appended to a log in the data dir so its reuse window outlives a restart. +type Local struct { + path string + subnet netip.Prefix + reuse time.Duration + + mu sync.Mutex + next uint64 + lines int + compactAt int + freed map[netip.Addr]time.Time + held map[netip.Addr]string + owned map[string]netip.Addr +} + +// NewLocal loads the releases still inside the reuse window from dir. +func NewLocal(dir string, subnet netip.Prefix, reuse time.Duration) (*Local, error) { + l := &Local{ + path: filepath.Join(dir, freedFile), subnet: subnet.Masked(), reuse: reuse, + freed: map[netip.Addr]time.Time{}, held: map[netip.Addr]string{}, owned: map[string]netip.Addr{}, + } + raw, err := os.ReadFile(l.path) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("read %s: %w", freedFile, err) + } + now := time.Now() + for line := range bytes.Lines(raw) { + a, sec, _ := strings.Cut(strings.TrimSpace(string(line)), " ") + addr, addrErr := netip.ParseAddr(a) + at, atErr := strconv.ParseInt(sec, 10, 64) + if addrErr == nil && atErr == nil && now.Sub(time.Unix(at, 0)) < reuse { + l.freed[addr] = time.Unix(at, 0) + } + } + if err = l.compact(); err != nil { + return nil, err + } + return l, nil +} + +// Seed marks the addresses the node's claims already hold. +func (l *Local) Seed(held map[string]netip.Addr) { + l.mu.Lock() + defer l.mu.Unlock() + for id, addr := range held { + l.held[addr], l.owned[id] = id, addr + } +} + +// Assign hands claimID the next free address after the cursor, or the one it already holds; it does no I/O. +func (l *Local) Assign(_ context.Context, claimID string) (netip.Addr, error) { + l.mu.Lock() + defer l.mu.Unlock() + if addr, ok := l.owned[claimID]; ok { + return addr, nil + } + now := time.Now() + // Among blocked+1 consecutive candidates one is free unless the subnet itself is full. + tries := min(uint64(len(l.held)+len(l.freed))+1, uint64(1)<<(32-l.subnet.Bits())-2) + for range tries { + addr := HostAt(l.subnet, l.next) + l.next++ + if _, ok := l.held[addr]; ok { + continue + } + if at, ok := l.freed[addr]; ok { + if now.Sub(at) < l.reuse { + continue + } + delete(l.freed, addr) + } + l.held[addr], l.owned[claimID] = claimID, addr + return addr, nil + } + return netip.Addr{}, ErrNoAddress +} + +// Release frees addr unless another claim holds it, and appends the release to the log. +func (l *Local) Release(_ context.Context, claimID string, addr netip.Addr) error { + l.mu.Lock() + defer l.mu.Unlock() + if owner, ok := l.held[addr]; ok && owner != claimID { + return nil + } + delete(l.held, addr) + if l.owned[claimID] == addr { + delete(l.owned, claimID) + } + now := time.Now() + l.freed[addr] = now + if l.lines++; l.lines > l.compactAt { + maps.DeleteFunc(l.freed, func(_ netip.Addr, at time.Time) bool { return now.Sub(at) >= l.reuse }) + return l.compact() + } + f, err := os.OpenFile(l.path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o600) + if err != nil { + return fmt.Errorf("open %s: %w", freedFile, err) + } + _, err = f.Write(appendFreed(nil, addr, now)) + if closeErr := f.Close(); err == nil { + err = closeErr + } + if err != nil { + return fmt.Errorf("append %s: %w", freedFile, err) + } + return nil +} + +func (l *Local) compact() error { + var b []byte + for addr, at := range l.freed { + b = appendFreed(b, addr, at) + } + l.lines, l.compactAt = len(l.freed), 2*len(l.freed)+compactAfter + return utils.WriteFileSync(l.path, b, 0o600) +} + +// HostAt maps a counter onto subnet's host addresses in order, skipping the network and broadcast addresses. +func HostAt(subnet netip.Prefix, n uint64) netip.Addr { + base := binary.BigEndian.Uint32(subnet.Masked().Addr().AsSlice()) + hosts := uint64(1)<<(32-subnet.Bits()) - 2 + var b [4]byte + binary.BigEndian.PutUint32(b[:], base+1+uint32(n%hosts)) //nolint:gosec // n%hosts is below 2^28 + return netip.AddrFrom4(b) +} + +func appendFreed(b []byte, addr netip.Addr, at time.Time) []byte { + b = addr.AppendTo(b) + b = strconv.AppendInt(append(b, ' '), at.Unix(), 10) + return append(b, '\n') +} diff --git a/sandboxd/tunnet/addrs_test.go b/sandboxd/tunnet/addrs_test.go new file mode 100644 index 000000000..b169ed749 --- /dev/null +++ b/sandboxd/tunnet/addrs_test.go @@ -0,0 +1,135 @@ +package tunnet + +import ( + "bytes" + "errors" + "fmt" + "net/netip" + "os" + "path/filepath" + "testing" + "time" +) + +func TestHostAtSkipsNetworkAndBroadcastAndWraps(t *testing.T) { + p := netip.MustParsePrefix("10.1.2.0/24") + for n, want := range map[uint64]string{0: "10.1.2.1", 1: "10.1.2.2", 253: "10.1.2.254", 254: "10.1.2.1", 1<<40 + 5: "10.1.2.38"} { + if got := HostAt(p, n); got != netip.MustParseAddr(want) { + t.Errorf("HostAt(%d) = %s, want %s", n, got, want) + } + } + if got := HostAt(netip.MustParsePrefix("240.0.0.0/4"), 0); got != netip.MustParseAddr("240.0.0.1") { + t.Errorf("HostAt on a /4 = %s, want 240.0.0.1", got) + } +} + +func TestLocalAssignsInOrderAndIdempotently(t *testing.T) { + dir := t.TempDir() + l := newLocal(t, dir, time.Hour) + a, b := assign(t, l, "a"), assign(t, l, "b") + if a != netip.MustParseAddr("10.1.2.1") || b != netip.MustParseAddr("10.1.2.2") { + t.Fatalf("assigned %s %s, want .1 .2", a, b) + } + if again := assign(t, l, "a"); again != a { + t.Errorf("a second assign for a = %s, want %s", again, a) + } + next := newLocal(t, dir, time.Hour) + next.Seed(map[string]netip.Addr{"a": a, "b": b}) + if c := assign(t, next, "c"); c != netip.MustParseAddr("10.1.2.3") { + t.Errorf("after a restart c = %s, want .3 past the seeded claims", c) + } + if again := assign(t, next, "b"); again != b { + t.Errorf("a seeded claim assigned %s, want %s", again, b) + } +} + +func TestLocalSkipsHeldAddressesAndHoldsFreedOnesForTheReuseWindow(t *testing.T) { + dir := t.TempDir() + l := newLocal(t, dir, time.Hour) + l.Seed(map[string]netip.Addr{"x": netip.MustParseAddr("10.1.2.1")}) + a := assign(t, l, "a") + if a != netip.MustParseAddr("10.1.2.2") { + t.Fatalf("assigned %s, want .2 past the held .1", a) + } + if err := l.Release(t.Context(), "y", netip.MustParseAddr("10.1.2.1")); err != nil { + t.Fatalf("release of x's address by y: %v", err) + } + if err := l.Release(t.Context(), "a", a); err != nil { + t.Fatalf("release: %v", err) + } + l.next = 0 + if b := assign(t, l, "b"); b != netip.MustParseAddr("10.1.2.3") { + t.Errorf("assigned %s, want .3: x still holds .1 and .2 waits out its window", b) + } + held := newLocal(t, dir, time.Hour) + held.Seed(map[string]netip.Addr{"x": netip.MustParseAddr("10.1.2.1"), "b": netip.MustParseAddr("10.1.2.3")}) + if c := assign(t, held, "c"); c != netip.MustParseAddr("10.1.2.4") { + t.Errorf("after a restart c = %s, want .4: the freed .2 survives in the log", c) + } + expired := newLocal(t, dir, 0) + expired.next = 1 + if d := assign(t, expired, "d"); d != a { + t.Errorf("past the window d = %s, want the freed %s", d, a) + } +} + +func TestLocalCompactsTheLogToTheLiveHolds(t *testing.T) { + dir := t.TempDir() + l, err := NewLocal(dir, netip.MustParsePrefix("10.1.0.0/16"), 20*time.Millisecond) + if err != nil { + t.Fatal(err) + } + churn := func(from, to int) { + for i := from; i < to; i++ { + id := fmt.Sprint(i) + if relErr := l.Release(t.Context(), id, assign(t, l, id)); relErr != nil { + t.Fatalf("release %d: %v", i, relErr) + } + } + } + churn(0, compactAfter-24) + time.Sleep(30 * time.Millisecond) + churn(compactAfter-24, compactAfter+100) + raw, err := os.ReadFile(filepath.Join(dir, freedFile)) + if err != nil { + t.Fatal(err) + } + if n := bytes.Count(raw, []byte("\n")); n > 200 { + t.Errorf("log holds %d lines after its expired holds were compacted away, want at most 200", n) + } +} + +func TestLocalRefusesWhenTheSubnetIsFull(t *testing.T) { + l := newLocal(t, t.TempDir(), time.Hour) + p := netip.MustParsePrefix("10.1.2.0/24") + held := map[string]netip.Addr{} + for n := range uint64(253) { + held[HostAt(p, n).String()] = HostAt(p, n) + } + l.Seed(held) + last := assign(t, l, "last") + if last != netip.MustParseAddr("10.1.2.254") { + t.Fatalf("assigned %s, want the one free .254", last) + } + if _, err := l.Assign(t.Context(), "over"); !errors.Is(err, ErrNoAddress) { + t.Errorf("assign on a full subnet: %v, want ErrNoAddress", err) + } +} + +func newLocal(t *testing.T, dir string, reuse time.Duration) *Local { + t.Helper() + l, err := NewLocal(dir, netip.MustParsePrefix("10.1.2.0/24"), reuse) + if err != nil { + t.Fatalf("NewLocal: %v", err) + } + return l +} + +func assign(t *testing.T, l *Local, id string) netip.Addr { + t.Helper() + addr, err := l.Assign(t.Context(), id) + if err != nil { + t.Fatalf("assign %s: %v", id, err) + } + return addr +} diff --git a/sandboxd/tunnet/bench_test.go b/sandboxd/tunnet/bench_test.go new file mode 100644 index 000000000..407fcd6df --- /dev/null +++ b/sandboxd/tunnet/bench_test.go @@ -0,0 +1,69 @@ +package tunnet + +import ( + "net/netip" + "testing" +) + +func BenchmarkForwardGSO(b *testing.B) { + l := benchLink() + f := gsoFrame(ProtoTCP, 63000, 1398) + buf := make([]byte, len(f)) + b.SetBytes(int64(len(f))) + b.ReportAllocs() + for b.Loop() { + copy(buf, f) + if d := l.forward(buf); d != DropNone { + b.Fatal(d) + } + } +} + +func BenchmarkForwardSmall(b *testing.B) { + l := benchLink() + f := frame(ProtoUDP, GuestAddr, remote, false) + buf := make([]byte, len(f)) + b.SetBytes(int64(len(f))) + b.ReportAllocs() + for b.Loop() { + copy(buf, f) + if d := l.forward(buf); d != DropNone { + b.Fatal(d) + } + } +} + +func BenchmarkDeliver(b *testing.B) { + l := benchLink() + l.forward(frame(ProtoTCP, GuestAddr, remote, false)) + d := reply(remote) + inner, _, _ := parseVXLAN(d, 7) + go func() { + for bp := range l.send { + l.r.pool.Put(bp) + } + }() + b.SetBytes(int64(len(inner))) + b.ReportAllocs() + for b.Loop() { + if drop := l.r.deliver(inner); drop != DropNone && drop != DropQueueFull { + b.Fatal(drop) + } + } +} + +func benchLink() *link { + r := New(nopUnder{}, &Overlay{Subnet: subnet, VNI: 7, Port: 4789, MTU: 1450, Exits: map[string]Exit{"default": {VTEP: vtep, MAC: exitMAC}}}) + c := Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }} + r.Arm(c) + a := r.claims[claimAddr] + l := &link{r: r, a: a, send: make(chan *[]byte, sendQueue), done: make(chan struct{})} + a.attach(l) + return l +} + +type nopUnder struct{} + +func (nopUnder) Send([]byte, int, netip.AddrPort, uint64) error { return nil } + +func (nopUnder) Recv(func([]byte, netip.AddrPort)) error { select {} } diff --git a/sandboxd/tunnet/flow.go b/sandboxd/tunnet/flow.go new file mode 100644 index 000000000..b6f4a6b75 --- /dev/null +++ b/sandboxd/tunnet/flow.go @@ -0,0 +1,284 @@ +package tunnet + +import ( + "net/netip" + "sync" + "sync/atomic" + "time" + + "golang.org/x/time/rate" +) + +const ( + icmpEchoReply = 0 + icmpUnreachable = 3 + icmpTimeExceed = 11 + + idleTCP = time.Hour + idleTCPEnd = 2 * time.Minute + idleUDP = 2 * time.Minute + idleOther = 30 * time.Second + + // packetBurst lets one GSO frame of small segments through a low packet rate. + packetBurst = 128 +) + +// AllowFunc judges a new flow's destination; port 0 asks for a rule that covers every port. +type AllowFunc func(dst netip.Addr, port uint16) bool + +// RecordFunc writes one flow's audit row. +type RecordFunc func(FlowRecord) + +// Limits caps one claim's traffic on the tun path; zero leaves a dimension open. +type Limits struct { + BytesPerSecond float64 `json:"bytes_per_second,omitzero"` + PacketsPerSecond float64 `json:"packets_per_second,omitzero"` + NewFlowsPerSecond float64 `json:"new_flows_per_second,omitzero"` + MaxFlows int `json:"max_flows,omitzero"` + QuotaBytes int64 `json:"quota_bytes,omitzero"` +} + +// FlowRecord is one flow's audit row, written when the flow ends. +type FlowRecord struct { + Proto uint8 + Dst netip.Addr + Port uint16 + Allowed bool + Up int64 + Down int64 + Start time.Time + End time.Time +} + +// flowKey is a flow in the outbound direction; the claim is the source. +type flowKey struct { + proto uint8 + dst netip.Addr + dport uint16 + sport uint16 +} + +type peerKey struct { + proto uint8 + addr netip.Addr +} + +type flow struct { + allowed bool + start time.Time + up atomic.Int64 + down atomic.Int64 + last atomic.Int64 + ended atomic.Bool +} + +type direction struct { + bytes, packets *rate.Limiter +} + +func newDirection(l Limits) *direction { + d := &direction{} + if l.BytesPerSecond > 0 { + d.bytes = rate.NewLimiter(rate.Limit(l.BytesPerSecond), max(int(l.BytesPerSecond), 2*MaxFrame)) + } + if l.PacketsPerSecond > 0 { + d.packets = rate.NewLimiter(rate.Limit(l.PacketsPerSecond), max(int(l.PacketsPerSecond), packetBurst)) + } + return d +} + +func (d *direction) admit(now time.Time, packets, bytes int) bool { + return (d.packets == nil || d.packets.AllowN(now, packets)) && (d.bytes == nil || d.bytes.AllowN(now, bytes)) +} + +// flows is one claim's flow table and limits: the claim's reader adds, the VXLAN receiver looks up. +type flows struct { + limits Limits + used *atomic.Int64 + allow AllowFunc + record RecordFunc + + newFlows *rate.Limiter + up, down *direction + + mu sync.RWMutex + table map[flowKey]*flow + peers map[peerKey]int +} + +func newFlows(c Claim) *flows { + f := &flows{ + limits: c.Limits, + used: c.Used, + allow: c.Allow, + record: c.Record, + up: newDirection(c.Limits), + down: newDirection(c.Limits), + table: map[flowKey]*flow{}, + peers: map[peerKey]int{}, + } + if r := c.Limits.NewFlowsPerSecond; r > 0 { + f.newFlows = rate.NewLimiter(rate.Limit(r), max(int(r), 1)) + } + if f.used == nil { + f.used = new(atomic.Int64) + } + return f +} + +// outbound decides a guest packet the caller already charged, opening its flow on first sight. +func (f *flows) outbound(p Packet, n int, now time.Time) Drop { + key := flowKey{proto: p.Proto, dst: p.Dst} + switch p.Proto { + case ProtoTCP, ProtoUDP: + key.dport, key.sport = p.DstPort, p.SrcPort + case ProtoICMP: + default: + return DropProto + } + f.mu.RLock() + fl := f.table[key] + f.mu.RUnlock() + if fl == nil { + if fl = f.open(key, now); fl == nil { + return DropLimit + } + } + fl.last.Store(now.UnixNano()) + if !fl.allowed { + return DropPolicy + } + fl.up.Add(int64(n)) + if p.End { + fl.ended.Store(true) + } + return DropNone +} + +// inbound passes a packet only for a flow the claim opened, and ICMP errors about one, then charges it. +func (f *flows) inbound(p Packet, frame []byte, n int, now time.Time) Drop { + fl, ok := f.match(p, frame) + if !ok { + return DropPolicy + } + if !f.charge(f.down, 1, n, now) { + return DropLimit + } + if fl != nil { + fl.down.Add(int64(n)) + fl.last.Store(now.UnixNano()) + if p.End { + fl.ended.Store(true) + } + } + return DropNone +} + +// match finds the allowed flow an inbound packet answers; an ICMP error about a peer passes without one. +func (f *flows) match(p Packet, frame []byte) (*flow, bool) { + f.mu.RLock() + defer f.mu.RUnlock() + key := flowKey{proto: p.Proto, dst: p.Src} + switch p.Proto { + case ProtoTCP, ProtoUDP: + key.dport, key.sport = p.SrcPort, p.DstPort + case ProtoICMP: + if p.ICMPType == icmpUnreachable || p.ICMPType == icmpTimeExceed { + inner, ok := innerDst(frame) + return nil, ok && (f.peers[peerKey{ProtoTCP, inner}] > 0 || f.peers[peerKey{ProtoUDP, inner}] > 0 || f.peers[peerKey{ProtoICMP, inner}] > 0) + } + if p.ICMPType != icmpEchoReply { + return nil, false + } + default: + return nil, false + } + fl := f.table[key] + return fl, fl != nil && fl.allowed +} + +// charge takes packets and bytes from the direction's buckets and the bytes from the quota. +func (f *flows) charge(d *direction, packets, n int, now time.Time) bool { + if !d.admit(now, packets, n) { + return false + } + if q := f.limits.QuotaBytes; q > 0 && f.used.Load()+int64(n) > q { + return false + } + f.used.Add(int64(n)) + return true +} + +func (f *flows) open(key flowKey, now time.Time) *flow { + f.mu.Lock() + defer f.mu.Unlock() + if fl := f.table[key]; fl != nil { + return fl + } + if f.limits.MaxFlows > 0 && len(f.table) >= f.limits.MaxFlows { + return nil + } + if f.newFlows != nil && !f.newFlows.AllowN(now, 1) { + return nil + } + fl := &flow{allowed: f.allow(key.dst, key.dport), start: now} + fl.last.Store(now.UnixNano()) + f.table[key] = fl + if fl.allowed { + f.peers[peerKey{key.proto, key.dst}]++ + } + return fl +} + +// sweep ends idle flows, or every flow when all is set, and writes their records. +func (f *flows) sweep(now time.Time, all bool) { + var ended []FlowRecord + f.mu.Lock() + for key, fl := range f.table { + idle := now.Sub(time.Unix(0, fl.last.Load())) + if !all && idle < idleFor(key.proto, fl) { + continue + } + delete(f.table, key) + if fl.allowed { + if pk := (peerKey{key.proto, key.dst}); f.peers[pk] <= 1 { + delete(f.peers, pk) + } else { + f.peers[pk]-- + } + } + ended = append(ended, FlowRecord{ + Proto: key.proto, Dst: key.dst, Port: key.dport, Allowed: fl.allowed, + Up: fl.up.Load(), Down: fl.down.Load(), Start: fl.start, End: time.Unix(0, fl.last.Load()), + }) + } + f.mu.Unlock() + if f.record != nil { + for _, r := range ended { + f.record(r) + } + } +} + +func idleFor(proto uint8, fl *flow) time.Duration { + switch { + case proto == ProtoTCP && fl.ended.Load(): + return idleTCPEnd + case proto == ProtoTCP && fl.allowed: + return idleTCP + case proto == ProtoUDP && fl.allowed: + return idleUDP + default: + return idleOther + } +} + +// innerDst is the destination of the packet an ICMP error quotes. +func innerDst(frame []byte) (netip.Addr, bool) { + ip := frame[VnetHdrLen:] + inner := int(ip[0]&0x0f)*4 + 8 + if len(ip) < inner+20 || ip[inner]>>4 != 4 { + return netip.Addr{}, false + } + return netip.AddrFrom4([4]byte(ip[inner+dstAt : inner+dstAt+4])), true +} diff --git a/sandboxd/tunnet/fuzz_test.go b/sandboxd/tunnet/fuzz_test.go new file mode 100644 index 000000000..275775ca7 --- /dev/null +++ b/sandboxd/tunnet/fuzz_test.go @@ -0,0 +1,72 @@ +package tunnet + +import ( + "encoding/binary" + "net/netip" + "testing" +) + +func FuzzFrameLen(f *testing.F) { + f.Add(uint32(10)) + f.Add(uint32(MaxFrame + 1)) + f.Fuzz(func(t *testing.T, n uint32) { + var head [4]byte + binary.LittleEndian.PutUint32(head[:], n) + got, err := FrameLen(head) + if err == nil && (got < VnetHdrLen || got > MaxFrame) { + t.Fatalf("FrameLen accepted %d", got) + } + }) +} + +func FuzzParseAndRewrite(f *testing.F) { + f.Add(frame(ProtoTCP, GuestAddr, remote, true)) + f.Add(frame(ProtoUDP, GuestAddr, remote, false)) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) > MaxFrame { + return + } + p, drop := Parse(b) + if drop != DropNone { + return + } + if p.Proto != ProtoTCP && p.Proto != ProtoUDP && p.Proto != ProtoICMP { + t.Fatalf("Parse passed protocol %d", p.Proto) + } + RewriteSrc(b, claimAddr) + RewriteDst(b, GuestAddr) + if q, _ := Parse(b); q.Src != claimAddr || q.Dst != GuestAddr { + t.Fatalf("rewrite left %v -> %v", q.Src, q.Dst) + } + }) +} + +func FuzzSegment(f *testing.F) { + f.Add(gsoFrame(ProtoTCP, 3000, 1400)) + f.Add(gsoFrame(ProtoUDP, 3000, 1000)) + f.Add(frame(ProtoTCP, GuestAddr, remote, true)) + f.Fuzz(func(t *testing.T, b []byte) { + if len(b) > MaxFrame { + return + } + out, stride, n := appendSegments(nil, nil, b) + if n > 0 && (stride > len(b) || len(out) > n*stride) { + t.Fatalf("%d segments of stride %d, %d bytes, from a %d-byte frame", n, stride, len(out), len(b)) + } + }) +} + +func FuzzVXLANAndARP(f *testing.F) { + f.Add(gratuitousARP(7, claimAddr)) + f.Add(arpReplyFor(7, claimAddr, netip.MustParseAddr("100.80.0.1"), broadcast)) + f.Fuzz(func(_ *testing.T, d []byte) { + inner, etype, ok := parseVXLAN(d, 7) + if !ok { + return + } + if etype == etherARP { + _, _, _, _ = arpRequest(inner) + } + _, _ = Parse(append(make([]byte, VnetHdrLen), inner...)) + }) +} diff --git a/sandboxd/tunnet/gro.go b/sandboxd/tunnet/gro.go new file mode 100644 index 000000000..4664baba4 --- /dev/null +++ b/sandboxd/tunnet/gro.go @@ -0,0 +1,132 @@ +package tunnet + +import ( + "bytes" + "encoding/binary" +) + +const ( + tcpAck = 0x10 + + ipTTL = 8 +) + +// coalescer merges in-order TCP segments of one flow into a GSO frame for the guest, as kernel GRO does; only the receive goroutine uses it. +type coalescer struct { + l *link + bp *[]byte + n int + segs int + mss int + next uint32 + push bool +} + +// add takes an admitted frame held in bp; false leaves both with the caller to enqueue unmerged. +func (c *coalescer) add(r *Router, l *link, bp *[]byte, frame []byte) bool { + seg, ok := segmentOf(frame) + if !ok { + c.flush(r) + return false + } + if c.joins(l, frame, seg) { + c.append(frame, seg) + r.pool.Put(bp) + if seg.push || seg.payload < c.mss { + c.flush(r) + } + return true + } + c.flush(r) + *bp = (*bp)[:cap(*bp)] + c.l, c.bp, c.n, c.segs = l, bp, len(frame), 1 + c.mss, c.next, c.push = seg.payload, seg.seq+uint32(seg.payload), seg.push //nolint:gosec // a payload fits a sequence step + if seg.push { + c.flush(r) + } + return true +} + +// flush hands the pending frame to its link, as one GSO frame when it holds more than one segment. +func (c *coalescer) flush(r *Router) { + if c.bp == nil { + return + } + b := (*c.bp)[:4+c.n] + frame := b[4:] + if c.segs > 1 { + c.finish(frame) + } + binary.LittleEndian.PutUint32(b, uint32(c.n)) //nolint:gosec // bounded by MaxFrame + *c.bp = b + if !c.l.enqueue(c.bp) { + r.drops[DropQueueFull].Add(1) + r.pool.Put(c.bp) + } + c.l, c.bp = nil, nil +} + +// joins reports whether frame continues the pending train: same flow, the next sequence, the same ack, TTL, TOS and options, and room left. +func (c *coalescer) joins(l *link, frame []byte, seg tcpSegment) bool { + if c.bp == nil || c.l != l || seg.seq != c.next || seg.payload > c.mss || c.n+seg.payload > MaxFrame { + return false + } + p := (*c.bp)[4 : 4+c.n] + ip, pip := frame[VnetHdrLen:], p[VnetHdrLen:] + return int(pip[32]>>4)*4 == seg.hdr && ip[1] == pip[1] && ip[ipTTL] == pip[ipTTL] && + bytes.Equal(ip[srcAt:dstAt+4], pip[srcAt:dstAt+4]) && bytes.Equal(ip[20:24], pip[20:24]) && + bytes.Equal(ip[28:32], pip[28:32]) && bytes.Equal(ip[40:20+seg.hdr], pip[40:20+seg.hdr]) +} + +func (c *coalescer) append(frame []byte, seg tcpSegment) { + payload := frame[VnetHdrLen+20+seg.hdr:] + copy((*c.bp)[4+c.n:], payload) + c.n += len(payload) + c.segs++ + c.next += uint32(len(payload)) //nolint:gosec // a payload fits a sequence step + c.push = c.push || seg.push +} + +// finish turns the merged packet into a GSO frame with a partial checksum, which the guest takes as already verified. +func (c *coalescer) finish(frame []byte) { + ip := frame[VnetHdrLen:] + hdr := int(ip[32]>>4) * 4 + binary.BigEndian.PutUint16(ip[2:], uint16(len(ip))) //nolint:gosec // bounded by MaxFrame + binary.BigEndian.PutUint16(ip[10:], 0) + binary.BigEndian.PutUint16(ip[10:], ^fold(onesSum(ip[:20], 0))) + if c.push { + ip[20+13] |= tcpPsh + } + binary.BigEndian.PutUint16(ip[20+16:], fold(pseudoSum(ip, len(ip)-20))) + frame[0], frame[1] = needsCsum, gsoTCPv4 + binary.LittleEndian.PutUint16(frame[2:], uint16(20+hdr)) + binary.LittleEndian.PutUint16(frame[4:], uint16(c.mss)) //nolint:gosec // an MSS fits + binary.LittleEndian.PutUint16(frame[6:], 20) + binary.LittleEndian.PutUint16(frame[8:], 16) +} + +// tcpSegment is what the coalescer needs from one TCP segment. +type tcpSegment struct { + seq uint32 + hdr int + payload int + push bool +} + +// segmentOf accepts a data segment with only ACK or ACK+PSH set and a valid checksum; GRO never vouches for a checksum it did not check. +func segmentOf(frame []byte) (tcpSegment, bool) { + ip := frame[VnetHdrLen:] + if ip[9] != ProtoTCP || frame[1] != gsoNone || frame[0] != 0 || len(ip) < 40 { + return tcpSegment{}, false + } + tcp := ip[20:] + hdr := int(tcp[12]>>4) * 4 + flags := tcp[13] + if hdr < 20 || len(tcp) <= hdr || flags&^tcpPsh != tcpAck { + return tcpSegment{}, false + } + if fold(onesSum(tcp, pseudoSum(ip, len(tcp)))) != 0xffff { + return tcpSegment{}, false + } + return tcpSegment{seq: binary.BigEndian.Uint32(tcp[4:]), hdr: hdr, payload: len(tcp) - hdr, push: flags&tcpPsh != 0}, true +} diff --git a/sandboxd/tunnet/packet.go b/sandboxd/tunnet/packet.go new file mode 100644 index 000000000..71e0a89cf --- /dev/null +++ b/sandboxd/tunnet/packet.go @@ -0,0 +1,271 @@ +package tunnet + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "net/netip" +) + +const ( + // HelloLen is the size of the hello each side sends first. + HelloLen = 12 + // VnetHdrLen is the virtio-net header that leads every packet in a frame. + VnetHdrLen = 10 + // MaxFrame is the largest frame body: a virtio-net header and a 64 KiB GSO packet. + MaxFrame = VnetHdrLen + 65535 + // Version is the packet protocol version this side speaks. + Version = 1 + // FlagOn in the host hello turns packet mode on for the guest. + FlagOn = 0x01 + + // Offload bits a hello carries, equal to the kernel's TUN_F_* values. + OffloadCSUM = 0x01 + OffloadTSO4 = 0x02 + OffloadTSO6 = 0x04 + OffloadTSOECN = 0x08 + OffloadUSO4 = 0x20 + OffloadUSO6 = 0x40 + + // IPv4 protocols the tun path carries. + ProtoICMP = 1 + ProtoTCP = 6 + ProtoUDP = 17 + + needsCsum = 0x01 + srcAt = 12 + dstAt = 16 + tcpFin = 0x01 + tcpRst = 0x04 +) + +var ( + // GuestAddr is the address every guest's tun0 carries. + GuestAddr = netip.AddrFrom4([4]byte{100, 64, 0, 2}) + + errMagic = errors.New("tun hello: bad magic") + + magic = [4]byte{'S', 'L', 'K', 'T'} +) + +// Hello is the 12 bytes each side sends first: version, the offloads its tun accepts, the MTU and flags (host only). +type Hello struct { + Version uint16 + Offloads uint16 + MTU uint16 + Flags uint16 +} + +// Encode lays the hello out on the wire. +func (h Hello) Encode() [HelloLen]byte { + var b [HelloLen]byte + copy(b[:4], magic[:]) + binary.LittleEndian.PutUint16(b[4:], h.Version) + binary.LittleEndian.PutUint16(b[6:], h.Offloads) + binary.LittleEndian.PutUint16(b[8:], h.MTU) + binary.LittleEndian.PutUint16(b[10:], h.Flags) + return b +} + +// ReadHello reads and decodes one hello. +func ReadHello(r io.Reader) (Hello, error) { + var b [HelloLen]byte + if _, err := io.ReadFull(r, b[:]); err != nil { + return Hello{}, err + } + if [4]byte(b[:4]) != magic { + return Hello{}, errMagic + } + return Hello{ + Version: binary.LittleEndian.Uint16(b[4:]), + Offloads: binary.LittleEndian.Uint16(b[6:]), + MTU: binary.LittleEndian.Uint16(b[8:]), + Flags: binary.LittleEndian.Uint16(b[10:]), + }, nil +} + +// FrameLen validates a frame's length prefix. +func FrameLen(head [4]byte) (int, error) { + n := int(binary.LittleEndian.Uint32(head[:])) + if n < VnetHdrLen || n > MaxFrame { + return 0, fmt.Errorf("tun frame length %d", n) + } + return n, nil +} + +// Packet is the IPv4 header view of one frame body: a virtio-net header, then the packet. +type Packet struct { + Proto uint8 + Src, Dst netip.Addr + SrcPort, DstPort uint16 + // End marks a TCP FIN or RST. + End bool + // ICMPType is the type of an ICMP packet. + ICMPType uint8 +} + +// Parse reads frame's IPv4 header and refuses, with the reason, what the first version does not carry: other versions, IP options, fragments, a length that disagrees with the frame. +func Parse(frame []byte) (Packet, Drop) { + if len(frame) < VnetHdrLen+20 { + return Packet{}, DropMalformed + } + ip := frame[VnetHdrLen:] + switch { + case ip[0]>>4 != 4: + return Packet{}, DropNotIPv4 + case ip[0]&0x0f != 5: + return Packet{}, DropOptions + case int(binary.BigEndian.Uint16(ip[2:])) != len(ip): + return Packet{}, DropMalformed + case binary.BigEndian.Uint16(ip[6:])&0x3fff != 0: + return Packet{}, DropFragment + } + p := Packet{ + Proto: ip[9], + Src: netip.AddrFrom4([4]byte(ip[srcAt : srcAt+4])), + Dst: netip.AddrFrom4([4]byte(ip[dstAt : dstAt+4])), + } + if !headerFits(frame, p.Proto) { + return Packet{}, DropGSO + } + l4 := ip[20:] + switch p.Proto { + case ProtoTCP: + if len(l4) < 20 || int(l4[12]>>4)*4 < 20 || int(l4[12]>>4)*4 > len(l4) { + return Packet{}, DropMalformed + } + p.SrcPort, p.DstPort = binary.BigEndian.Uint16(l4), binary.BigEndian.Uint16(l4[2:]) + p.End = l4[13]&(tcpFin|tcpRst) != 0 + case ProtoUDP: + if len(l4) < 8 { + return Packet{}, DropMalformed + } + p.SrcPort, p.DstPort = binary.BigEndian.Uint16(l4), binary.BigEndian.Uint16(l4[2:]) + case ProtoICMP: + if len(l4) < 8 { + return Packet{}, DropMalformed + } + p.ICMPType = l4[0] + default: + return Packet{}, DropProto + } + return p, DropNone +} + +// RewriteSrc overwrites the source address whatever it was. +func RewriteSrc(frame []byte, to netip.Addr) { + rewrite(frame, srcAt, to.As4()) +} + +// RewriteDst overwrites the destination address. +func RewriteDst(frame []byte, to netip.Addr) { + rewrite(frame, dstAt, to.As4()) +} + +// wireCost is what a parsed frame puts on the wire once segmented: its packet count and IP bytes; false when a packet would exceed mtu. +func wireCost(frame []byte, mtu int) (segs, bytes int, ok bool) { + ip := frame[VnetHdrLen:] + if frame[1]&^gsoECN == gsoNone { + return 1, len(ip), len(ip) <= mtu + } + hdrs := 20 + 8 + if ip[9] == ProtoTCP { + hdrs = 20 + int(ip[32]>>4)*4 + } + size := int(binary.LittleEndian.Uint16(frame[4:])) + if hdrs+size > mtu { + return 0, 0, false + } + segs = max((len(ip)-hdrs+size-1)/size, 1) + return segs, len(ip) + (segs-1)*hdrs, true +} + +// rewriteQuotedSrc moves the source of the packet an ICMP error quotes from one address to another and refreshes both checksums, so the guest's socket lookup finds it. +func rewriteQuotedSrc(frame []byte, from, to netip.Addr) { + ip := frame[VnetHdrLen:] + if len(ip) < 28+20 || [4]byte(ip[28+srcAt:28+srcAt+4]) != from.As4() { + return + } + q, f, t := ip[28:], from.As4(), to.As4() + copy(q[srcAt:], t[:]) + binary.BigEndian.PutUint16(q[10:], ^adjust(^binary.BigEndian.Uint16(q[10:]), f, t)) + binary.BigEndian.PutUint16(ip[22:], 0) + binary.BigEndian.PutUint16(ip[22:], ^fold(onesSum(ip[20:], 0))) +} + +// headerFits checks the virtio-net header against the packet: a GSO type matches the protocol, and a partial checksum sits in the L4 header's checksum field. +func headerFits(frame []byte, proto uint8) bool { + field := 16 + switch frame[1] &^ gsoECN { + case gsoNone: + if proto == ProtoUDP { + field = 6 + } + case gsoTCPv4: + if proto != ProtoTCP || binary.LittleEndian.Uint16(frame[4:]) == 0 { + return false + } + case gsoUDPL4: + if proto != ProtoUDP || binary.LittleEndian.Uint16(frame[4:]) == 0 { + return false + } + field = 6 + default: + return false + } + if frame[0]&needsCsum == 0 { + return true + } + return proto != ProtoICMP && binary.LittleEndian.Uint16(frame[6:]) == 20 && int(binary.LittleEndian.Uint16(frame[8:])) == field +} + +// rewrite patches the IPv4 header checksum and the L4 checksum: a partial one holds the uncomplemented pseudo-header sum, a UDP checksum of 0 stays 0. +func rewrite(frame []byte, at int, to [4]byte) { + ip := frame[VnetHdrLen:] + from := [4]byte(ip[at : at+4]) + if from == to { + return + } + copy(ip[at:at+4], to[:]) + check := binary.BigEndian.Uint16(ip[10:]) + binary.BigEndian.PutUint16(ip[10:], ^adjust(^check, from, to)) + var field int + partial := frame[0]&needsCsum != 0 + l4 := int(ip[0]&0x0f) * 4 + switch { + case partial: + field = int(binary.LittleEndian.Uint16(frame[6:])) + int(binary.LittleEndian.Uint16(frame[8:])) + case ip[9] == ProtoTCP: + field = l4 + 16 + case ip[9] == ProtoUDP: + field = l4 + 6 + default: + return + } + if field+2 > len(ip) { + return + } + sum := binary.BigEndian.Uint16(ip[field:]) + switch { + case partial: + sum = adjust(sum, from, to) + case ip[9] == ProtoUDP && sum == 0: + default: + sum = ^adjust(^sum, from, to) + } + binary.BigEndian.PutUint16(ip[field:], sum) +} + +// adjust moves a ones'-complement sum from one address to another (RFC 1624). +func adjust(sum uint16, from, to [4]byte) uint16 { + s := uint32(sum) + for i := 0; i < 4; i += 2 { + s += uint32(^binary.BigEndian.Uint16(from[i:])) + s += uint32(binary.BigEndian.Uint16(to[i:])) + } + for s>>16 != 0 { + s = s&0xffff + s>>16 + } + return uint16(s) +} diff --git a/sandboxd/tunnet/packet_test.go b/sandboxd/tunnet/packet_test.go new file mode 100644 index 000000000..01c5b5beb --- /dev/null +++ b/sandboxd/tunnet/packet_test.go @@ -0,0 +1,161 @@ +package tunnet + +import ( + "bytes" + "encoding/binary" + "net/netip" + "testing" +) + +var ( + claimAddr = netip.AddrFrom4([4]byte{100, 80, 0, 7}) + remote = netip.AddrFrom4([4]byte{203, 0, 113, 9}) +) + +func TestRewriteKeepsFullChecksumsValid(t *testing.T) { + for _, proto := range []uint8{ProtoTCP, ProtoUDP} { + f := frame(proto, GuestAddr, remote, false) + RewriteSrc(f, claimAddr) + p, drop := Parse(f) + if drop != DropNone || p.Src != claimAddr || p.Dst != remote { + t.Fatalf("proto %d: parse after rewrite = %+v %v", proto, p, drop) + } + if !validIP(f) || !validL4(f) { + t.Errorf("proto %d: checksum broken by the source rewrite", proto) + } + RewriteDst(f, GuestAddr) + if !validIP(f) { + t.Errorf("proto %d: ip checksum broken by the destination rewrite", proto) + } + } +} + +func TestRewriteMovesAPartialChecksumAsAnUncomplementedSum(t *testing.T) { + f := frame(ProtoTCP, GuestAddr, remote, true) + RewriteSrc(f, claimAddr) + ip := f[VnetHdrLen:] + got := binary.BigEndian.Uint16(ip[20+16:]) + if want := pseudo(ip); got != want { + t.Errorf("partial checksum = %#x, want the pseudo-header sum %#x", got, want) + } +} + +func TestRewriteLeavesAZeroUDPChecksumZero(t *testing.T) { + f := frame(ProtoUDP, GuestAddr, remote, false) + ip := f[VnetHdrLen:] + binary.BigEndian.PutUint16(ip[20+6:], 0) + RewriteSrc(f, claimAddr) + if binary.BigEndian.Uint16(ip[20+6:]) != 0 { + t.Error("a UDP checksum of 0 means none and must stay 0") + } + if !validIP(f) { + t.Error("ip checksum broken") + } +} + +func TestParseReadsPortsAndRefusesWhatV1DoesNotCarry(t *testing.T) { + f := frame(ProtoTCP, GuestAddr, remote, false) + f[VnetHdrLen+20+13] = tcpFin + p, drop := Parse(f) + if drop != DropNone || p.SrcPort != 40000 || p.DstPort != 443 || !p.End { + t.Fatalf("Parse = %+v %v", p, drop) + } + cases := map[Drop]func([]byte) []byte{ + DropFragment: func(b []byte) []byte { b[VnetHdrLen+6] = 0x20; return b }, + DropOptions: func(b []byte) []byte { b[VnetHdrLen] = 0x46; return b }, + DropNotIPv4: func(b []byte) []byte { b[VnetHdrLen] = 0x65; return b }, + DropMalformed: func(b []byte) []byte { return b[:len(b)-1] }, + DropProto: func(b []byte) []byte { b[VnetHdrLen+9] = 47; return b }, + } + for want, mutate := range cases { + if _, got := Parse(mutate(frame(ProtoTCP, GuestAddr, remote, false))); got != want { + t.Errorf("Parse = %v, want %v", got, want) + } + } + spoof := frame(ProtoTCP, GuestAddr, remote, true) + binary.LittleEndian.PutUint16(spoof[8:], 0) + binary.LittleEndian.PutUint16(spoof[6:], 16) + if _, got := Parse(spoof); got != DropGSO { + t.Errorf("a partial checksum aimed at the IP header: %v, want refused", got) + } + mismatch := gsoFrame(ProtoTCP, 3000, 1400) + mismatch[1] = gsoUDPL4 + if _, got := Parse(mismatch); got != DropGSO { + t.Errorf("a UDP GSO type on TCP: %v, want refused", got) + } + if _, got := Parse(f[:VnetHdrLen+19]); got != DropMalformed { + t.Errorf("truncated header: %v", got) + } +} + +func TestHelloRoundTripsAndFrameLenBounds(t *testing.T) { + h := Hello{Version: Version, Offloads: OffloadCSUM | OffloadTSO4, MTU: 1460, Flags: FlagOn} + b := h.Encode() + got, err := ReadHello(bytes.NewReader(b[:])) + if err != nil || got != h { + t.Fatalf("ReadHello = %+v %v", got, err) + } + b[0] = 'X' + if _, err = ReadHello(bytes.NewReader(b[:])); err == nil { + t.Error("bad magic accepted") + } + for n, ok := range map[uint32]bool{9: false, 10: true, MaxFrame: true, MaxFrame + 1: false} { + var head [4]byte + binary.LittleEndian.PutUint32(head[:], n) + if _, err := FrameLen(head); (err == nil) != ok { + t.Errorf("FrameLen(%d) err = %v", n, err) + } + } +} + +func frame(proto uint8, src, dst netip.Addr, partial bool) []byte { + l4 := 20 + if proto == ProtoUDP { + l4 = 8 + } + f := make([]byte, VnetHdrLen+20+l4+16) + ip := f[VnetHdrLen:] + ip[0] = 0x45 + binary.BigEndian.PutUint16(ip[2:], uint16(len(ip))) + binary.BigEndian.PutUint16(ip[4:], 7) + ip[8], ip[9] = 64, proto + s, d := src.As4(), dst.As4() + copy(ip[srcAt:], s[:]) + copy(ip[dstAt:], d[:]) + binary.BigEndian.PutUint16(ip[10:], ^fold(onesSum(ip[:20], 0))) + seg := ip[20:] + binary.BigEndian.PutUint16(seg, 40000) + binary.BigEndian.PutUint16(seg[2:], 443) + for i := l4; i < len(seg); i++ { + seg[i] = byte(i) + } + at := 16 + if proto == ProtoUDP { + binary.BigEndian.PutUint16(seg[4:], uint16(len(seg))) + at = 6 + } else { + seg[12] = 5 << 4 + } + if partial { + f[0] = needsCsum + binary.LittleEndian.PutUint16(f[6:], 20) + binary.LittleEndian.PutUint16(f[8:], uint16(at)) + binary.BigEndian.PutUint16(seg[at:], pseudo(ip)) + return f + } + binary.BigEndian.PutUint16(seg[at:], ^fold(onesSum(seg, pseudoSum(ip, len(ip)-20)))) + return f +} + +func validIP(f []byte) bool { + return fold(onesSum(f[VnetHdrLen:VnetHdrLen+20], 0)) == 0xffff +} + +func validL4(f []byte) bool { + ip := f[VnetHdrLen:] + return fold(onesSum(ip[20:], pseudoSum(ip, len(ip)-20))) == 0xffff +} + +func pseudo(ip []byte) uint16 { + return fold(pseudoSum(ip, len(ip)-20)) +} diff --git a/sandboxd/tunnet/pg/pg.go b/sandboxd/tunnet/pg/pg.go new file mode 100644 index 000000000..530f0ff56 --- /dev/null +++ b/sandboxd/tunnet/pg/pg.go @@ -0,0 +1,102 @@ +// Package pg keeps a cell's overlay addresses in one shared PostgreSQL table, so two nodes never hand out the same one. +package pg + +import ( + "context" + _ "embed" + "errors" + "fmt" + "net/netip" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/cocoonstack/sandbox/sandboxd/metastore" + "github.com/cocoonstack/sandbox/sandboxd/tunnet" +) + +const ( + table = "sandboxd_tun_addrs" + // assignTries bounds the candidates one Assign tries before it reports the subnet full. + assignTries = 64 + + assignSQL = `WITH held AS ( + SELECT address FROM sandboxd_tun_addrs WHERE cell = $1 AND claim = $2 AND released_at IS NULL + ), taken AS ( + INSERT INTO sandboxd_tun_addrs (cell, address, claim) + SELECT $1, host($3::inet + (1 + nextval('sandboxd_tun_addrs_cursor') % $4)), $2 WHERE NOT EXISTS (SELECT 1 FROM held) + ON CONFLICT (cell, address) DO UPDATE SET claim = EXCLUDED.claim, allocated_at = now(), released_at = NULL + WHERE sandboxd_tun_addrs.released_at < now() - make_interval(secs => $5) + RETURNING address + ) + SELECT address FROM held UNION ALL SELECT address FROM taken` +) + +//go:embed schema.sql +var schema string + +var _ tunnet.Addresses = (*Store)(nil) + +// Store is the cell's address table as one node sees it. +type Store struct { + pool *pgxpool.Pool + cell string + subnet netip.Prefix + reuse time.Duration +} + +// Open connects to cell's addresses and creates the table when it is missing. +func Open(ctx context.Context, dsn, cell string, subnet netip.Prefix, reuse time.Duration) (*Store, error) { + pool, _, err := metastore.Open(ctx, dsn, table, schema) + if err != nil { + return nil, err + } + return &Store{pool: pool, cell: cell, subnet: subnet.Masked(), reuse: reuse}, nil +} + +// Assign returns claimID's active address, or claims the next candidate from the shared cursor that is free and past its reuse window, one round trip per candidate. +func (s *Store) Assign(ctx context.Context, claimID string) (netip.Addr, error) { + hosts := int64(1)<<(32-s.subnet.Bits()) - 2 + for range assignTries { + var addr string + err := s.scan(ctx, &addr, assignSQL, s.cell, claimID, s.subnet.Addr().String(), hosts, s.reuse.Seconds()) + switch { + case err == nil: + return netip.ParseAddr(addr) + case !errors.Is(err, pgx.ErrNoRows): + return netip.Addr{}, down(err) + } + } + return netip.Addr{}, tunnet.ErrNoAddress +} + +// Release stamps addr released; the stamp starts its reuse window. +func (s *Store) Release(ctx context.Context, claimID string, addr netip.Addr) error { + return s.exec(ctx, `UPDATE sandboxd_tun_addrs SET released_at = now() + WHERE cell = $1 AND address = $2 AND claim = $3 AND released_at IS NULL`, s.cell, addr.String(), claimID) +} + +// Close releases the connection pool. +func (s *Store) Close() { + s.pool.Close() +} + +func (s *Store) scan(ctx context.Context, dest any, sql string, args ...any) error { + qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) + defer cancel() + return s.pool.QueryRow(qctx, sql, args...).Scan(dest) +} + +func (s *Store) exec(ctx context.Context, sql string, args ...any) error { + qctx, cancel := context.WithTimeout(ctx, metastore.QueryTimeout) + defer cancel() + if _, err := s.pool.Exec(qctx, sql, args...); err != nil { + return down(err) + } + return nil +} + +func down(err error) error { + return fmt.Errorf("tun address store: %w", err) +} diff --git a/sandboxd/tunnet/pg/pg_test.go b/sandboxd/tunnet/pg/pg_test.go new file mode 100644 index 000000000..381559dc7 --- /dev/null +++ b/sandboxd/tunnet/pg/pg_test.go @@ -0,0 +1,136 @@ +package pg + +import ( + "encoding/binary" + "errors" + "fmt" + "net/netip" + "sync" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/metastore/metastoretest" + "github.com/cocoonstack/sandbox/sandboxd/tunnet" +) + +func TestAssignIsUniqueAcrossNodesAndIdempotentPerClaim(t *testing.T) { + dsn := metastoretest.PGSchema(t) + nodes := []*Store{open(t, dsn, "c1", time.Hour), open(t, dsn, "c1", time.Hour)} + var mu sync.Mutex + got := map[netip.Addr]string{} + var wg sync.WaitGroup + for i := range 40 { + wg.Go(func() { + id := fmt.Sprintf("sb_%d", i) + addr, err := nodes[i%2].Assign(t.Context(), id) + if err != nil { + t.Errorf("assign %s: %v", id, err) + return + } + mu.Lock() + defer mu.Unlock() + if other, dup := got[addr]; dup { + t.Errorf("%s and %s share %s", id, other, addr) + } + got[addr] = id + }) + } + wg.Wait() + for addr, id := range got { + if again, err := nodes[1].Assign(t.Context(), id); err != nil || again != addr { + t.Errorf("a second assign of %s = %s %v, want %s", id, again, err, addr) + } + } +} + +func TestReleasedAddressWaitsOutTheReuseWindow(t *testing.T) { + s := open(t, metastoretest.PGSchema(t), "c1", time.Hour) + ctx := t.Context() + addr, err := s.Assign(ctx, "sb_a") + if err != nil { + t.Fatalf("assign: %v", err) + } + if err = s.Release(ctx, "sb_b", addr); err != nil { + t.Fatalf("release by another claim: %v", err) + } + rewind(t, s, addr) + if b, assignErr := s.Assign(ctx, "sb_b"); assignErr != nil || b == addr { + t.Fatalf("assign over a held address = %s %v, want another address", b, assignErr) + } + if err = s.Release(ctx, "sb_a", addr); err != nil { + t.Fatalf("release: %v", err) + } + rewind(t, s, addr) + if c, assignErr := s.Assign(ctx, "sb_c"); assignErr != nil || c == addr { + t.Fatalf("assign inside the reuse window = %s %v, want another address", c, assignErr) + } + if _, err = s.pool.Exec(ctx, `UPDATE sandboxd_tun_addrs SET released_at = now() - interval '2 hours' WHERE address = $1`, addr.String()); err != nil { + t.Fatalf("age the release: %v", err) + } + rewind(t, s, addr) + if d, assignErr := s.Assign(ctx, "sb_d"); assignErr != nil || d != addr { + t.Errorf("assign past the reuse window = %s %v, want %s", d, assignErr, addr) + } +} + +func TestAssignGivesUpOnARunOfHeldAddresses(t *testing.T) { + s := open(t, metastoretest.PGSchema(t), "c1", time.Hour) + ctx := t.Context() + first, err := s.Assign(ctx, "sb_0") + if err != nil { + t.Fatalf("assign: %v", err) + } + for i := 1; i < assignTries; i++ { + if _, err = s.Assign(ctx, fmt.Sprintf("sb_%d", i)); err != nil { + t.Fatalf("assign %d: %v", i, err) + } + } + rewind(t, s, first) + if addr, assignErr := s.Assign(ctx, "sb_over"); !errors.Is(assignErr, tunnet.ErrNoAddress) { + t.Errorf("assign over %d held addresses = %s %v, want ErrNoAddress", assignTries, addr, assignErr) + } +} + +func TestCellsKeepTheirOwnAddresses(t *testing.T) { + dsn := metastoretest.PGSchema(t) + c1, c2 := open(t, dsn, "c1", time.Hour), open(t, dsn, "c2", time.Hour) + ctx := t.Context() + addr, err := c1.Assign(ctx, "sb_1") + if err != nil { + t.Fatalf("assign in c1: %v", err) + } + rewind(t, c2, addr) + if other, assignErr := c2.Assign(ctx, "sb_2"); assignErr != nil || other != addr { + t.Errorf("c2 assign = %s %v, want %s: cells do not share addresses", other, assignErr, addr) + } +} + +func TestADownDatabaseFailsClosed(t *testing.T) { + s := open(t, metastoretest.PGSchema(t), "c1", time.Hour) + s.Close() + ctx := t.Context() + if addr, err := s.Assign(ctx, "sb_1"); err == nil || addr.IsValid() { + t.Errorf("assign on a closed pool = %s %v, want an error and no address", addr, err) + } + if err := s.Release(ctx, "sb_1", netip.MustParseAddr("240.0.0.2")); err == nil { + t.Error("release on a closed pool succeeded") + } +} + +func open(t *testing.T, dsn, cell string, reuse time.Duration) *Store { + t.Helper() + s, err := Open(t.Context(), dsn, cell, netip.MustParsePrefix("240.0.0.0/4"), reuse) + if err != nil { + t.Fatalf("open: %v", err) + } + t.Cleanup(s.Close) + return s +} + +func rewind(t *testing.T, s *Store, addr netip.Addr) { + t.Helper() + off := binary.BigEndian.Uint32(addr.AsSlice()) - binary.BigEndian.Uint32(s.subnet.Addr().AsSlice()) - 1 + if _, err := s.pool.Exec(t.Context(), `SELECT setval('sandboxd_tun_addrs_cursor', $1, false)`, int64(off)); err != nil { + t.Fatalf("rewind the cursor: %v", err) + } +} diff --git a/sandboxd/tunnet/pg/schema.sql b/sandboxd/tunnet/pg/schema.sql new file mode 100644 index 000000000..da79f9517 --- /dev/null +++ b/sandboxd/tunnet/pg/schema.sql @@ -0,0 +1,10 @@ +CREATE SEQUENCE IF NOT EXISTS sandboxd_tun_addrs_cursor; +CREATE TABLE IF NOT EXISTS sandboxd_tun_addrs ( + cell text NOT NULL, + address text NOT NULL, + claim text NOT NULL, + allocated_at timestamptz NOT NULL DEFAULT now(), + released_at timestamptz, + PRIMARY KEY (cell, address) +); +CREATE UNIQUE INDEX IF NOT EXISTS sandboxd_tun_addrs_claim ON sandboxd_tun_addrs (cell, claim) WHERE released_at IS NULL; diff --git a/sandboxd/tunnet/router.go b/sandboxd/tunnet/router.go new file mode 100644 index 000000000..5e9564c3c --- /dev/null +++ b/sandboxd/tunnet/router.go @@ -0,0 +1,565 @@ +package tunnet + +import ( + "bufio" + "cmp" + "context" + "encoding/binary" + "errors" + "fmt" + "hash/maphash" + "io" + "maps" + "net" + "net/netip" + "slices" + "sync" + "sync/atomic" + "time" +) + +const ( + // Drop reasons, one counter each. + DropNone Drop = iota + DropMalformed + DropNotIPv4 + DropOptions + DropFragment + DropProto + DropGSO + DropPolicy + DropNoExit + DropUnknownClaim + DropQueueFull + DropSize + DropLimit + DropSend + DropSender + dropKinds + + helloTimeout = 5 * time.Second + sweepEvery = 10 * time.Second + refreshEvery = 45 * time.Second + // sendQueue bounds the frames waiting for one guest, about 64 MiB at worst; a full queue drops, as a congested link would. + sendQueue = 1024 + readBuf = 256 << 10 + // segmentable is what the segmenter takes from a guest. + segmentable = OffloadCSUM | OffloadTSO4 | OffloadTSOECN | OffloadUSO4 +) + +var ( + // GuestRange holds the fixed guest address; no overlay subnet may overlap it. + GuestRange = netip.MustParsePrefix("100.64.0.0/30") + + seed = maphash.MakeSeed() + dropNames = [dropKinds]string{"", "malformed", "not_ipv4", "options", "fragment", "proto", "gso", "policy", "no_exit", "unknown_claim", "queue_full", "size", "limit", "send", "sender"} +) + +// Drop names why a packet was not carried. +type Drop uint8 + +// String is the reason's metric label; DropNone has none. +func (d Drop) String() string { + if d >= dropKinds { + return "" + } + return dropNames[d] +} + +// Underlay carries VXLAN datagrams: the transport sandboxd's exits listen on, kept apart from identity, policy and rewrite. +type Underlay interface { + // Send writes the datagrams packed in b, each seg bytes but the last, to dst; flow picks the outer source port. + Send(b []byte, seg int, dst netip.AddrPort, flow uint64) error + // Recv reads one batch and calls each per datagram with its sender. + Recv(each func(d []byte, from netip.AddrPort)) error +} + +// Exit is one exit's VTEP and the MAC of its VXLAN device. +type Exit struct { + VTEP netip.Addr + MAC [6]byte +} + +// Overlay is the reloadable part of the node's tun config. +type Overlay struct { + Subnet netip.Prefix + VNI uint32 + Port uint16 + MTU uint16 + // Exits maps a profile label to its exit; a claim without a label uses "default". + Exits map[string]Exit +} + +func (o *Overlay) exit(profile string) (Exit, bool) { + e, ok := o.Exits[cmp.Or(profile, "default")] + return e, ok +} + +// overlay is a loaded Overlay with the set of exit VTEPs the router accepts VXLAN from. +type overlay struct { + *Overlay + vteps map[netip.Addr]bool +} + +func load(o *Overlay) *overlay { + v := &overlay{Overlay: o, vteps: make(map[netip.Addr]bool, len(o.Exits))} + for _, e := range o.Exits { + v.vteps[e.VTEP] = true + } + return v +} + +// Claim is one tun-enabled claim as the router sees it. +type Claim struct { + ID string + Addr netip.Addr + // Enabled is asked once per packet stream; false answers the guest off. + Enabled func() bool + // Profile is the claim's exit label; empty means "default". + Profile string + Allow AllowFunc + Limits Limits + // Used counts the claim's bytes both ways; the caller persists it. + Used *atomic.Int64 + Record RecordFunc +} + +// Stats counts what crossed each hop and why packets were dropped. +type Stats struct { + FramesIn, FramesOut uint64 + DgramsOut, DgramsIn uint64 + ARPAnswered, ARPSent uint64 + Drops [dropKinds]uint64 +} + +// Router moves packets between claims' vsock streams and the VXLAN underlay. +type Router struct { + under Underlay + conf atomic.Pointer[overlay] + pool sync.Pool + + framesIn, framesOut atomic.Uint64 + dgramsOut, dgramsIn atomic.Uint64 + arpAnswered, arpSent atomic.Uint64 + drops [dropKinds]atomic.Uint64 + + gro coalescer + + mu sync.RWMutex + claims map[netip.Addr]*armed +} + +// New builds a router over under with the overlay o. +func New(under Underlay, o *Overlay) *Router { + r := &Router{under: under, claims: map[netip.Addr]*armed{}} + r.conf.Store(load(o)) + r.pool.New = func() any { b := make([]byte, 4+MaxFrame); return &b } + return r +} + +// Reload swaps the overlay config and announces every armed claim again; live streams pick the new exits up on their next packet. +func (r *Router) Reload(o *Overlay) { + r.conf.Store(load(o)) + r.announceAll() +} + +// Arm makes c's address answer ARP on this node and announces it to its exit. +func (r *Router) Arm(c Claim) { + a := &armed{c: c, flows: newFlows(c)} + r.mu.Lock() + old := r.claims[c.Addr] + r.claims[c.Addr] = a + r.mu.Unlock() + if old != nil { + old.close() + } + r.announce(a) +} + +// Disarm stops answering for addr, ends its packet stream and closes its flows. +func (r *Router) Disarm(addr netip.Addr) { + r.mu.Lock() + a := r.claims[addr] + delete(r.claims, addr) + r.mu.Unlock() + if a != nil { + a.close() + } +} + +// Serve runs one packet stream for the claim armed at addr until it ends; a newer stream replaces it. +func (r *Router) Serve(conn net.Conn, addr netip.Addr) error { + defer func() { _ = conn.Close() }() + _ = conn.SetDeadline(time.Now().Add(helloTimeout)) + guest, err := ReadHello(conn) + if err != nil { + return err + } + r.mu.RLock() + a := r.claims[addr] + r.mu.RUnlock() + on := a != nil && (a.c.Enabled == nil || a.c.Enabled()) + o := r.conf.Load() + hello := Hello{Version: Version, Offloads: segmentable, MTU: o.MTU} + if on { + hello.Flags = FlagOn + } + b := hello.Encode() + if guest.Version < Version { + return fmt.Errorf("tun hello: guest speaks version %d", guest.Version) + } + if _, err = conn.Write(b[:]); err != nil || !on { + return err + } + _ = conn.SetDeadline(time.Time{}) + l := &link{r: r, a: a, conn: conn, send: make(chan *[]byte, sendQueue), done: make(chan struct{})} + if !a.attach(l) { + return nil + } + go l.write() + go l.sweep() + err = l.read(bufio.NewReaderSize(conn, readBuf)) + if a.detach(l) { + a.flows.sweep(time.Now(), true) + } + l.close() + if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) { + return nil + } + return err +} + +// Run receives VXLAN until ctx ends or the underlay fails, and refreshes every armed claim's announcement. +func (r *Router) Run(ctx context.Context) error { + go func() { + tick := time.Tick(refreshEvery) + for { + select { + case <-ctx.Done(): + return + case <-tick: + r.announceAll() + } + } + }() + for ctx.Err() == nil { + err := r.under.Recv(r.receive) + r.gro.flush(r) + if err != nil { + if ctx.Err() != nil { + return nil + } + return fmt.Errorf("vxlan receive: %w", err) + } + } + return nil +} + +// Stats snapshots the counters. +func (r *Router) Stats() Stats { + s := Stats{ + FramesIn: r.framesIn.Load(), FramesOut: r.framesOut.Load(), + DgramsOut: r.dgramsOut.Load(), DgramsIn: r.dgramsIn.Load(), + ARPAnswered: r.arpAnswered.Load(), ARPSent: r.arpSent.Load(), + } + for i := range s.Drops { + s.Drops[i] = r.drops[i].Load() + } + return s +} + +func (r *Router) receive(d []byte, from netip.AddrPort) { + r.dgramsIn.Add(1) + o := r.conf.Load() + if !o.vteps[from.Addr()] { + r.drops[DropSender].Add(1) + return + } + inner, etype, ok := parseVXLAN(d, o.VNI) + if !ok { + r.drops[DropMalformed].Add(1) + return + } + switch etype { + case etherARP: + sender, mac, target, ok := arpRequest(inner) + if !ok { + return + } + r.mu.RLock() + a := r.claims[target] + r.mu.RUnlock() + if a == nil { + return + } + reply := arpReplyFor(o.VNI, target, sender, mac) + if r.under.Send(reply, len(reply), netip.AddrPortFrom(from.Addr(), o.Port), 0) == nil { + r.arpAnswered.Add(1) + } + case etherIPv4: + if drop := r.deliver(inner); drop != DropNone { + r.drops[drop].Add(1) + } + } +} + +// deliver hands an inbound packet to its claim's stream, rewritten to the guest address. +func (r *Router) deliver(inner []byte) Drop { + if len(inner) > MaxFrame-VnetHdrLen { + return DropMalformed + } + bp := r.pool.Get().(*[]byte) + b := (*bp)[:4+VnetHdrLen+len(inner)] + clear(b[4 : 4+VnetHdrLen]) + copy(b[4+VnetHdrLen:], inner) + frame := b[4:] + l, drop := r.admit(frame) + if drop == DropNone { + *bp = b + if r.gro.add(r, l, bp, frame) { + return DropNone + } + binary.LittleEndian.PutUint32(b, uint32(len(frame))) //nolint:gosec // bounded by MaxFrame + if l.enqueue(bp) { + return DropNone + } + drop = DropQueueFull + } + r.pool.Put(bp) + return drop +} + +// admit parses an inbound frame, checks it against its claim's flows and rewrites it to the guest. +func (r *Router) admit(frame []byte) (*link, Drop) { + p, drop := Parse(frame) + if drop != DropNone { + return nil, drop + } + r.mu.RLock() + a := r.claims[p.Dst] + r.mu.RUnlock() + l := a.live() + if l == nil { + return nil, DropUnknownClaim + } + if drop = a.flows.inbound(p, frame, len(frame), time.Now()); drop != DropNone { + return nil, drop + } + RewriteDst(frame, GuestAddr) + if p.Proto == ProtoICMP && p.ICMPType != icmpEchoReply { + rewriteQuotedSrc(frame, p.Dst, GuestAddr) + } + return l, DropNone +} + +func (r *Router) announceAll() { + r.mu.RLock() + all := slices.Collect(maps.Values(r.claims)) + r.mu.RUnlock() + for _, a := range all { + r.announce(a) + } +} + +func (r *Router) announce(a *armed) { + o := r.conf.Load() + exit, ok := o.exit(a.c.Profile) + if !ok { + return + } + g := gratuitousARP(o.VNI, a.c.Addr) + if r.under.Send(g, len(g), netip.AddrPortFrom(exit.VTEP, o.Port), 0) == nil { + r.arpSent.Add(1) + } +} + +// armed is a claim this node answers for, with its flows and limits, which outlive a stream, and its live stream if one runs. +type armed struct { + c Claim + flows *flows + mu sync.Mutex + l *link + gone bool +} + +func (a *armed) live() *link { + if a == nil { + return nil + } + a.mu.Lock() + defer a.mu.Unlock() + return a.l +} + +func (a *armed) attach(l *link) bool { + a.mu.Lock() + old, gone := a.l, a.gone + if !gone { + a.l = l + } + a.mu.Unlock() + if old != nil { + old.close() + } + return !gone +} + +// detach drops l as the live stream and reports whether the claim was disarmed meanwhile. +func (a *armed) detach(l *link) bool { + a.mu.Lock() + defer a.mu.Unlock() + if a.l == l { + a.l = nil + } + return a.gone +} + +func (a *armed) close() { + a.mu.Lock() + l := a.l + a.l, a.gone = nil, true + a.mu.Unlock() + if l != nil { + l.close() + } + a.flows.sweep(time.Now(), true) +} + +// link is one claim's live packet stream. +type link struct { + r *Router + a *armed + conn net.Conn + out []byte + send chan *[]byte + done chan struct{} + once sync.Once +} + +func (l *link) read(br *bufio.Reader) error { + var head [4]byte + frame := make([]byte, MaxFrame) + for { + if _, err := io.ReadFull(br, head[:]); err != nil { + return err + } + n, err := FrameLen(head) + if err != nil { + return err + } + pkt := frame[:n] + if _, err = io.ReadFull(br, pkt); err != nil { + return err + } + l.r.framesIn.Add(1) + if drop := l.forward(pkt); drop != DropNone { + l.r.drops[drop].Add(1) + } + } +} + +// forward carries one guest frame out: parse, size, charge and policy, then rewrite, segment and send. +func (l *link) forward(frame []byte) Drop { + p, drop := Parse(frame) + if drop != DropNone { + return drop + } + o := l.r.conf.Load() + segs, wire, ok := wireCost(frame, int(o.MTU)) + if !ok { + return DropSize + } + now, f := time.Now(), l.a.flows + if !f.charge(f.up, segs, wire, now) { + return DropLimit + } + if o.Subnet.Contains(p.Dst) || GuestRange.Contains(p.Dst) { + return DropPolicy + } + if drop = f.outbound(p, wire, now); drop != DropNone { + return drop + } + exit, ok := o.exit(l.a.c.Profile) + if !ok { + return DropNoExit + } + RewriteSrc(frame, l.a.c.Addr) + var hdr [Overhead]byte + appendHeader(hdr[:0], o.VNI, exit.MAC, MAC(l.a.c.Addr), etherIPv4) + out, stride, n := appendSegments(l.out[:0], hdr[:], frame) + l.out = out + if n == 0 { + return DropGSO + } + flow := maphash.Comparable(seed, [3]uint64{uint64(p.Proto), uint64(p.SrcPort)<<16 | uint64(p.DstPort), dst4(p.Dst)}) + if l.r.under.Send(out, stride, netip.AddrPortFrom(exit.VTEP, o.Port), flow) != nil { + return DropSend + } + l.r.dgramsOut.Add(uint64(n)) //nolint:gosec // a count + return DropNone +} + +func (l *link) enqueue(bp *[]byte) bool { + select { + case l.send <- bp: + return true + default: + return false + } +} + +func (l *link) write() { + batch := make(net.Buffers, 0, 64) + held := make([]*[]byte, 0, 64) + for { + select { + case <-l.done: + return + case bp := <-l.send: + batch, held = append(batch[:0], *bp), append(held[:0], bp) + drain: + for len(held) < cap(held) { + select { + case bp = <-l.send: + batch, held = append(batch, *bp), append(held, bp) + default: + break drain + } + } + bufs := batch + _, err := bufs.WriteTo(l.conn) + l.r.framesOut.Add(uint64(len(held))) + for _, bp := range held { + l.r.pool.Put(bp) + } + if err != nil { + l.close() + return + } + } + } +} + +func (l *link) sweep() { + tick := time.Tick(sweepEvery) + for { + select { + case <-l.done: + return + case now := <-tick: + l.a.flows.sweep(now, false) + } + } +} + +func (l *link) close() { + l.once.Do(func() { + close(l.done) + _ = l.conn.Close() + }) +} + +func dst4(a netip.Addr) uint64 { + b := a.As4() + return uint64(binary.BigEndian.Uint32(b[:])) +} diff --git a/sandboxd/tunnet/router_test.go b/sandboxd/tunnet/router_test.go new file mode 100644 index 000000000..e322d9ac4 --- /dev/null +++ b/sandboxd/tunnet/router_test.go @@ -0,0 +1,390 @@ +package tunnet + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "io" + "net" + "net/netip" + "sync" + "sync/atomic" + "testing" + "time" +) + +var ( + vtep = netip.MustParseAddr("10.0.0.9") + exitMAC = [6]byte{0x02, 0xee, 0, 0, 0, 1} + subnet = netip.MustParsePrefix("100.80.0.0/12") +) + +func TestRouterAnnouncesRewritesAndPassesOnlyReplies(t *testing.T) { + u, r := newRouter() + var records []FlowRecord + var mu sync.Mutex + r.Arm(Claim{ + ID: "c1", Addr: claimAddr, + Allow: func(_ netip.Addr, port uint16) bool { return port == 443 }, + Record: func(fr FlowRecord) { mu.Lock(); records = append(records, fr); mu.Unlock() }, + }) + if inner, etype, to := u.sent(t); etype != etherARP || to.Addr() != vtep { + t.Fatalf("arm sent %v to %v, want a gratuitous ARP to the exit", etype, to) + } else if _, _, target, ok := arpRequest(inner); !ok || target != claimAddr { + t.Errorf("gratuitous ARP announces %v", target) + } + guest := serve(t, r, claimAddr) + if h := guestHello(t, guest); h.Flags&FlagOn == 0 || h.MTU != 1450 || h.Offloads&OffloadTSO4 == 0 { + t.Fatalf("host hello = %+v", h) + } + sendFrame(t, guest, frame(ProtoTCP, netip.MustParseAddr("100.80.0.99"), remote, true)) + inner, etype, _ := u.sent(t) + pkt := append(make([]byte, VnetHdrLen), inner...) + if p, _ := Parse(pkt); etype != etherIPv4 || p.Src != claimAddr || !validIP(pkt) || !validL4(pkt) { + t.Fatalf("sent src %v, want the claim address with full checksums", p.Src) + } + denied := frame(ProtoTCP, GuestAddr, remote, false) + binary.BigEndian.PutUint16(denied[VnetHdrLen+20+2:], 22) + sendFrame(t, guest, denied) + sendFrame(t, guest, frame(ProtoTCP, GuestAddr, netip.MustParseAddr("100.80.0.5"), false)) + u.inject(t, reply(netip.MustParseAddr("198.51.100.1"))) + u.inject(t, reply(remote)) + got := readFrame(t, guest) + if p, _ := Parse(got); p.Dst != GuestAddr || p.Src != remote || !validIP(got) { + t.Fatalf("delivered %+v, want the reply rewritten to the guest", p) + } + waitFor(t, func() bool { return r.Stats().Drops[DropPolicy] == 3 }) + if n := len(u.out); n != 0 { + t.Errorf("%d more datagrams sent, want none for denied traffic", n) + } + _ = guest.Close() + r.Disarm(claimAddr) + waitFor(t, func() bool { mu.Lock(); defer mu.Unlock(); return len(records) == 2 }) +} + +func TestRouterAnswersARPOnlyForArmedClaims(t *testing.T) { + u, r := newRouter() + r.Arm(Claim{ID: "c1", Addr: claimAddr}) + u.sent(t) + asker := netip.MustParseAddr("100.80.0.1") + for _, target := range []netip.Addr{netip.MustParseAddr("100.80.0.8"), claimAddr} { + b := appendHeader(nil, 7, broadcast, exitMAC, etherARP) + u.inject(t, appendARP(b, arpReq, exitMAC, asker, [6]byte{}, target)) + } + inner, _, to := u.sent(t) + if to.Addr() != vtep || binary.BigEndian.Uint16(inner[6:]) != arpReply || netip.AddrFrom4([4]byte(inner[14:18])) != claimAddr { + t.Fatalf("arp reply to %v: %x", to, inner) + } + b := appendHeader(nil, 7, broadcast, exitMAC, etherARP) + u.injectFrom(t, appendARP(b, arpReq, exitMAC, asker, [6]byte{}, claimAddr), netip.MustParseAddrPort("10.0.0.77:4789")) + waitFor(t, func() bool { return r.Stats().Drops[DropSender] == 1 }) + r.Disarm(claimAddr) + b = appendHeader(nil, 7, broadcast, exitMAC, etherARP) + u.inject(t, appendARP(b, arpReq, exitMAC, asker, [6]byte{}, claimAddr)) + time.Sleep(50 * time.Millisecond) + if len(u.out) != 0 { + t.Error("a disarmed claim, or a sender that is no exit, got an ARP answer") + } +} + +func TestRouterSegmentsGSOIntoOneBatch(t *testing.T) { + u, r := newRouter() + r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) + u.sent(t) + guest := serve(t, r, claimAddr) + guestHello(t, guest) + sendFrame(t, guest, gsoFrame(ProtoTCP, 4000, 1400)) + for i := range 3 { + inner, _, _ := u.sent(t) + if pkt := append(make([]byte, VnetHdrLen), inner...); !validIP(pkt) || !validL4(pkt) { + t.Errorf("segment %d invalid", i) + } + } + if b := u.batches.Load(); b != 2 { + t.Errorf("sends = %d, want the ARP and one batch for the GSO frame", b) + } +} + +func TestRouterRefusesUnarmedAndDisabledAndDisarmEndsStream(t *testing.T) { + _, r := newRouter() + if h := guestHello(t, serve(t, r, claimAddr)); h.Flags&FlagOn != 0 { + t.Error("an unarmed address got on") + } + r.Arm(Claim{ID: "c1", Addr: claimAddr, Enabled: func() bool { return false }}) + off := serve(t, r, claimAddr) + if h := guestHello(t, off); h.Flags&FlagOn != 0 { + t.Error("a disabled claim got on") + } + if _, err := off.Read(make([]byte, 1)); !errors.Is(err, io.EOF) { + t.Errorf("stream after off: %v", err) + } + r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) + on := serve(t, r, claimAddr) + guestHello(t, on) + waitFor(t, func() bool { return r.claims[claimAddr].live() != nil }) + r.Disarm(claimAddr) + if _, err := on.Read(make([]byte, 1)); err == nil { + t.Error("stream still open after disarm") + } +} + +func TestRouterQuotaStopsTraffic(t *testing.T) { + u, r := newRouter() + used := new(atomic.Int64) + r.Arm(Claim{ID: "c1", Addr: claimAddr, Used: used, Limits: Limits{QuotaBytes: 100}, Allow: func(netip.Addr, uint16) bool { return true }}) + u.sent(t) + guest := serve(t, r, claimAddr) + guestHello(t, guest) + f := frame(ProtoTCP, GuestAddr, remote, false) + for range 3 { + sendFrame(t, guest, f) + } + u.sent(t) + waitFor(t, func() bool { return r.Stats().Drops[DropLimit] == 2 }) + if used.Load() > 100 { + t.Errorf("used %d past the quota", used.Load()) + } +} + +func TestDeliverCoalescesInOrderSegmentsIntoOneGSOFrame(t *testing.T) { + u, r := newRouter() + r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) + u.sent(t) + guest := serve(t, r, claimAddr) + guestHello(t, guest) + sendFrame(t, guest, frame(ProtoTCP, GuestAddr, remote, false)) + u.sent(t) + bad := dataSegment(5000, 1000) + bad[len(bad)-1] ^= 0xff + batch := [][]byte{dataSegment(1000, 1000), dataSegment(2000, 1000), dataSegment(3000, 1000), dataSegment(4000, 400), bad} + for _, d := range batch { + u.inject(t, d) + } + merged := readFrame(t, guest) + ip := merged[VnetHdrLen:] + if merged[1] != gsoTCPv4 || binary.LittleEndian.Uint16(merged[4:]) != 1000 || len(ip) != 40+3400 { + t.Fatalf("frame gso %d size %d len %d, want one TCPv4 GSO frame of 3400 payload bytes at 1000", merged[1], binary.LittleEndian.Uint16(merged[4:]), len(ip)) + } + if got, want := binary.BigEndian.Uint16(ip[36:]), fold(pseudoSum(ip, len(ip)-20)); got != want || !validIP(merged) { + t.Errorf("merged checksum %#x want the pseudo-header sum %#x, ip valid %v", got, want, validIP(merged)) + } + if p, _ := Parse(merged); p.Dst != GuestAddr { + t.Errorf("merged frame goes to %v", p.Dst) + } + if lone := readFrame(t, guest); lone[1] != gsoNone || validL4(lone) { + t.Error("a segment with a bad checksum was merged or repaired instead of passed through") + } +} + +func TestRouterChargesSegmentsAndRefusesOversizedPackets(t *testing.T) { + u, r := newRouter() + r.Arm(Claim{ID: "c1", Addr: claimAddr, Limits: Limits{PacketsPerSecond: 10}, Allow: func(netip.Addr, uint16) bool { return true }}) + u.sent(t) + guest := serve(t, r, claimAddr) + guestHello(t, guest) + sendFrame(t, guest, gsoFrame(ProtoTCP, 4000, 1420)) + big := gsoFrame(ProtoUDP, 1500, 1500) + big[1] = gsoNone + sendFrame(t, guest, big) + sendFrame(t, guest, gsoFrame(ProtoTCP, 4000, 8)) + waitFor(t, func() bool { s := r.Stats(); return s.Drops[DropSize] == 2 && s.Drops[DropLimit] == 1 }) + if s := r.Stats(); s.DgramsOut != 0 { + t.Errorf("sent %d datagrams, want none", s.DgramsOut) + } +} + +func TestRouterLimitsOutliveAReconnect(t *testing.T) { + u, r := newRouter() + r.Arm(Claim{ID: "c1", Addr: claimAddr, Limits: Limits{NewFlowsPerSecond: 0.01}, Allow: func(netip.Addr, uint16) bool { return true }}) + u.sent(t) + first := serve(t, r, claimAddr) + guestHello(t, first) + sendFrame(t, first, frame(ProtoTCP, GuestAddr, remote, false)) + u.sent(t) + _ = first.Close() + second := serve(t, r, claimAddr) + guestHello(t, second) + other := frame(ProtoTCP, GuestAddr, remote, false) + binary.BigEndian.PutUint16(other[VnetHdrLen+20:], 40001) + sendFrame(t, second, other) + waitFor(t, func() bool { return r.Stats().Drops[DropLimit] == 1 }) +} + +func TestRouterHandsAnICMPErrorBackWithTheQuotedGuestAddress(t *testing.T) { + u, r := newRouter() + r.Arm(Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) + u.sent(t) + guest := serve(t, r, claimAddr) + guestHello(t, guest) + sendFrame(t, guest, frame(ProtoTCP, GuestAddr, remote, false)) + inner, _, _ := u.sent(t) + icmp := make([]byte, 20+8+28) + icmp[0], icmp[8], icmp[9] = 0x45, 64, ProtoICMP + binary.BigEndian.PutUint16(icmp[2:], uint16(len(icmp))) + s, d := remote.As4(), claimAddr.As4() + copy(icmp[srcAt:], s[:]) + copy(icmp[dstAt:], d[:]) + binary.BigEndian.PutUint16(icmp[10:], ^fold(onesSum(icmp[:20], 0))) + icmp[20], icmp[21] = icmpUnreachable, 4 + copy(icmp[28:], inner[:28]) + binary.BigEndian.PutUint16(icmp[22:], ^fold(onesSum(icmp[20:], 0))) + u.inject(t, append(appendHeader(nil, 7, MAC(claimAddr), exitMAC, etherIPv4), icmp...)) + got := readFrame(t, guest) + ip := got[VnetHdrLen:] + if netip.AddrFrom4([4]byte(ip[28+srcAt:])) != GuestAddr || fold(onesSum(ip[20:], 0)) != 0xffff || fold(onesSum(ip[28:48], 0)) != 0xffff || !validIP(got) { + t.Fatalf("quoted source %v, checksums icmp %#x quoted %#x", netip.AddrFrom4([4]byte(ip[28+srcAt:])), fold(onesSum(ip[20:], 0)), fold(onesSum(ip[28:48], 0))) + } +} + +func newRouter() (*fakeUnder, *Router) { + u := &fakeUnder{out: make(chan sent, 64), in: make(chan sent, 16)} + r := New(u, &Overlay{Subnet: subnet, VNI: 7, Port: 4789, MTU: 1450, Exits: map[string]Exit{"default": {VTEP: vtep, MAC: exitMAC}}}) + go func() { _ = r.Run(context.Background()) }() + return u, r +} + +func serve(t *testing.T, r *Router, addr netip.Addr) net.Conn { + t.Helper() + host, guest := net.Pipe() + go func() { _ = r.Serve(host, addr) }() + h := Hello{Version: Version, Offloads: OffloadCSUM | OffloadTSO4 | OffloadTSO6 | OffloadTSOECN}.Encode() + if _, err := guest.Write(h[:]); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = guest.Close() }) + return guest +} + +func guestHello(t *testing.T, guest net.Conn) Hello { + t.Helper() + h, err := ReadHello(guest) + if err != nil { + t.Fatal(err) + } + return h +} + +func sendFrame(t *testing.T, guest net.Conn, f []byte) { + t.Helper() + b := binary.LittleEndian.AppendUint32(nil, uint32(len(f))) + if _, err := guest.Write(append(b, f...)); err != nil { + t.Fatal(err) + } +} + +func readFrame(t *testing.T, guest net.Conn) []byte { + t.Helper() + _ = guest.SetReadDeadline(time.Now().Add(2 * time.Second)) + var head [4]byte + if _, err := io.ReadFull(guest, head[:]); err != nil { + t.Fatal(err) + } + b := make([]byte, binary.LittleEndian.Uint32(head[:])) + if _, err := io.ReadFull(guest, b); err != nil { + t.Fatal(err) + } + return b +} + +func reply(from netip.Addr) []byte { + f := frame(ProtoTCP, from, claimAddr, false) + seg := f[VnetHdrLen+20:] + binary.BigEndian.PutUint16(seg, 443) + binary.BigEndian.PutUint16(seg[2:], 40000) + ip := f[VnetHdrLen:] + binary.BigEndian.PutUint16(seg[16:], 0) + binary.BigEndian.PutUint16(seg[16:], ^fold(onesSum(seg, pseudoSum(ip, len(ip)-20)))) + b := appendHeader(nil, 7, MAC(claimAddr), exitMAC, etherIPv4) + return append(b, ip...) +} + +func dataSegment(seq uint32, payload int) []byte { + ip := make([]byte, 40+payload) + ip[0], ip[8], ip[9] = 0x45, 64, ProtoTCP + binary.BigEndian.PutUint16(ip[2:], uint16(len(ip))) + s, d := remote.As4(), claimAddr.As4() + copy(ip[srcAt:], s[:]) + copy(ip[dstAt:], d[:]) + binary.BigEndian.PutUint16(ip[10:], ^fold(onesSum(ip[:20], 0))) + tcp := ip[20:] + binary.BigEndian.PutUint16(tcp, 443) + binary.BigEndian.PutUint16(tcp[2:], 40000) + binary.BigEndian.PutUint32(tcp[4:], seq) + binary.BigEndian.PutUint32(tcp[8:], 77) + tcp[12], tcp[13] = 5<<4, tcpAck + for i := 20; i < len(tcp); i++ { + tcp[i] = byte(seq) + byte(i) + } + binary.BigEndian.PutUint16(tcp[16:], ^fold(onesSum(tcp, pseudoSum(ip, len(tcp))))) + return append(appendHeader(nil, 7, MAC(claimAddr), exitMAC, etherIPv4), ip...) +} + +func waitFor(t *testing.T, ok func() bool) { + t.Helper() + for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(5 * time.Millisecond) { + if ok() { + return + } + } + t.Fatal("condition not reached") +} + +type sent struct { + d []byte + to netip.AddrPort +} + +type fakeUnder struct { + out chan sent + in chan sent + batches atomic.Int64 +} + +func (u *fakeUnder) Send(b []byte, seg int, dst netip.AddrPort, _ uint64) error { + u.batches.Add(1) + for len(b) > 0 { + n := min(seg, len(b)) + u.out <- sent{d: bytes.Clone(b[:n]), to: dst} + b = b[n:] + } + return nil +} + +func (u *fakeUnder) Recv(each func(d []byte, from netip.AddrPort)) error { + s := <-u.in + each(s.d, s.to) + for { + select { + case s = <-u.in: + each(s.d, s.to) + default: + return nil + } + } +} + +func (u *fakeUnder) inject(t *testing.T, d []byte) { + t.Helper() + u.injectFrom(t, d, netip.AddrPortFrom(vtep, 50000)) +} + +func (u *fakeUnder) injectFrom(t *testing.T, d []byte, from netip.AddrPort) { + t.Helper() + u.in <- sent{d: d, to: from} +} + +func (u *fakeUnder) sent(t *testing.T) ([]byte, uint16, netip.AddrPort) { + t.Helper() + select { + case s := <-u.out: + inner, etype, ok := parseVXLAN(s.d, 7) + if !ok { + t.Fatalf("sent a datagram that is not VXLAN for vni 7: %x", s.d) + } + return inner, etype, s.to + case <-time.After(2 * time.Second): + t.Fatal("nothing sent") + return nil, 0, netip.AddrPort{} + } +} diff --git a/sandboxd/tunnet/segment.go b/sandboxd/tunnet/segment.go new file mode 100644 index 000000000..9e8bbe524 --- /dev/null +++ b/sandboxd/tunnet/segment.go @@ -0,0 +1,133 @@ +package tunnet + +import ( + "encoding/binary" +) + +const ( + gsoNone = 0 + gsoTCPv4 = 1 + gsoUDPL4 = 5 + gsoECN = 0x80 + + tcpPsh = 0x08 + tcpCwr = 0x80 +) + +// appendSegments appends frame's packets to out behind hdr, splitting a GSO frame and finishing a partial checksum; it returns out, the length of every datagram but the last, and the count, zero for a dropped frame. +func appendSegments(out, hdr, frame []byte) ([]byte, int, int) { + if len(frame) < VnetHdrLen+20 { + return out, 0, 0 + } + ip := frame[VnetHdrLen:] + ihl := int(ip[0]&0x0f) * 4 + if ihl < 20 || len(ip) < ihl { + return out, 0, 0 + } + gsoSize := int(binary.LittleEndian.Uint16(frame[4:])) + switch frame[1] &^ gsoECN { + case gsoNone: + if frame[0]&needsCsum != 0 { + start := int(binary.LittleEndian.Uint16(frame[6:])) + field := start + int(binary.LittleEndian.Uint16(frame[8:])) + if field+2 > len(ip) || start > field { + return out, 0, 0 + } + sum := ^fold(onesSum(ip[start:], 0)) + if sum == 0 && ip[9] == ProtoUDP { + sum = 0xffff + } + binary.BigEndian.PutUint16(ip[field:], sum) + } + return append(append(out, hdr...), ip...), len(hdr) + len(ip), 1 + case gsoTCPv4: + if len(ip) < ihl+20 || gsoSize == 0 || ip[9] != ProtoTCP { + return out, 0, 0 + } + return split(out, hdr, ip, ihl, int(ip[ihl+12]>>4)*4, gsoSize) + case gsoUDPL4: + if len(ip) < ihl+8 || gsoSize == 0 || ip[9] != ProtoUDP { + return out, 0, 0 + } + return split(out, hdr, ip, ihl, 8, gsoSize) + default: + return out, 0, 0 + } +} + +// split cuts the payload after ihl+l4len header bytes into size-byte pieces, each with its own IP and L4 header, straight into out. +func split(out, hdr, ip []byte, ihl, l4len, size int) ([]byte, int, int) { + hdrs := ihl + l4len + if l4len < 8 || len(ip) < hdrs || ip[9] == ProtoTCP && l4len < 20 { + return out, 0, 0 + } + payload := ip[hdrs:] + tcp := ip[9] == ProtoTCP + id := binary.BigEndian.Uint16(ip[4:]) + var seq uint32 + if tcp { + seq = binary.BigEndian.Uint32(ip[ihl+4:]) + } + segs := 0 + for off := 0; off < len(payload) || segs == 0; off += size { + n := min(size, len(payload)-off) + out = append(out, hdr...) + at := len(out) + out = append(append(out, ip[:hdrs]...), payload[off:off+n]...) + seg := out[at:] + binary.BigEndian.PutUint16(seg[2:], uint16(hdrs+n)) //nolint:gosec // a segment fits a packet + binary.BigEndian.PutUint16(seg[4:], id+uint16(segs)) + binary.BigEndian.PutUint16(seg[10:], 0) + binary.BigEndian.PutUint16(seg[10:], ^fold(onesSum(seg[:ihl], 0))) + l4 := seg[ihl:] + field := 6 + if tcp { + field = 16 + binary.BigEndian.PutUint32(l4[4:], seq+uint32(off)) + if off+n < len(payload) { + l4[13] &^= tcpFin | tcpPsh + } + if segs > 0 { + l4[13] &^= tcpCwr + } + } else { + binary.BigEndian.PutUint16(l4[4:], uint16(l4len+n)) //nolint:gosec // a segment fits a datagram + } + binary.BigEndian.PutUint16(l4[field:], 0) + sum := ^fold(onesSum(l4, pseudoSum(seg, l4len+n))) + if sum == 0 && !tcp { + sum = 0xffff + } + binary.BigEndian.PutUint16(l4[field:], sum) + segs++ + } + return out, len(hdr) + min(hdrs+size, len(ip)), segs +} + +// pseudoSum is the IPv4 pseudo-header sum for an L4 segment of l4len bytes. +func pseudoSum(ip []byte, l4len int) uint64 { + return onesSum(ip[srcAt:dstAt+4], uint64(ip[9])+uint64(l4len)) //nolint:gosec // l4len is a segment length +} + +// onesSum adds b to s as big-endian 16-bit words, four at a time. +func onesSum(b []byte, s uint64) uint64 { + for len(b) >= 8 { + s += uint64(binary.BigEndian.Uint32(b)) + uint64(binary.BigEndian.Uint32(b[4:])) + b = b[8:] + } + for len(b) >= 2 { + s += uint64(binary.BigEndian.Uint16(b)) + b = b[2:] + } + if len(b) == 1 { + s += uint64(b[0]) << 8 + } + return s +} + +func fold(s uint64) uint16 { + for s>>16 != 0 { + s = s&0xffff + s>>16 + } + return uint16(s) +} diff --git a/sandboxd/tunnet/segment_test.go b/sandboxd/tunnet/segment_test.go new file mode 100644 index 000000000..0f231909d --- /dev/null +++ b/sandboxd/tunnet/segment_test.go @@ -0,0 +1,109 @@ +package tunnet + +import ( + "bytes" + "encoding/binary" + "testing" +) + +func TestSegmentSplitsTCPAtGSOSizeWithValidHeaders(t *testing.T) { + f := gsoFrame(ProtoTCP, 5000, 1400) + ip := f[VnetHdrLen:] + ip[20+13] = tcpFin | tcpPsh | tcpCwr + payload := append([]byte(nil), ip[40:]...) + got := segments(f) + if len(got) != 4 { + t.Fatalf("segments = %d, want 4", len(got)) + } + var joined []byte + for i, seg := range got { + pkt := append(make([]byte, VnetHdrLen), seg...) + if !validIP(pkt) || !validL4(pkt) { + t.Errorf("segment %d checksum invalid", i) + } + if seq := binary.BigEndian.Uint32(seg[24:]); seq != 1000+uint32(i*1400) { + t.Errorf("segment %d seq %d", i, seq) + } + last := i == len(got)-1 + if fl := seg[33]; (fl&tcpFin != 0) != last || (fl&tcpPsh != 0) != last || (fl&tcpCwr != 0) != (i == 0) { + t.Errorf("segment %d flags %#x", i, fl) + } + joined = append(joined, seg[40:]...) + } + if !bytes.Equal(joined, payload) { + t.Error("payload not preserved across segments") + } +} + +func TestSegmentSplitsUDPAndFinishesPartialChecksums(t *testing.T) { + got := segments(gsoFrame(ProtoUDP, 3000, 1000)) + for i, pkt := range got { + full := append(make([]byte, VnetHdrLen), pkt...) + if !validIP(full) || !validL4(full) || binary.BigEndian.Uint16(pkt[24:]) != 1008 { + t.Errorf("udp segment %d invalid", i) + } + } + if len(got) != 3 { + t.Errorf("udp segments = %d, want 3", len(got)) + } + for _, pkt := range segments(frame(ProtoTCP, GuestAddr, remote, true)) { + if !validL4(append(make([]byte, VnetHdrLen), pkt...)) { + t.Error("partial checksum not finished") + } + } +} + +func TestSegmentWritesEachPacketBehindTheHeader(t *testing.T) { + hdr := []byte("vxlan-and-eth-header..") + out, stride, n := appendSegments([]byte("kept"), hdr, gsoFrame(ProtoTCP, 3000, 1400)) + if n != 3 || stride != len(hdr)+1440 || len(out) != 4+2*stride+len(hdr)+40+200 { + t.Fatalf("n %d stride %d len %d", n, stride, len(out)) + } + for at := 4; at < len(out); at += stride { + if !bytes.HasPrefix(out[at:], hdr) { + t.Errorf("datagram at %d does not start with the header", at) + } + } +} + +func segments(f []byte) [][]byte { + out, stride, n := appendSegments(nil, nil, f) + got := make([][]byte, 0, n) + for range n { + k := min(stride, len(out)) + got, out = append(got, out[:k]), out[k:] + } + return got +} + +func gsoFrame(proto uint8, payload, size int) []byte { + l4 := 20 + if proto == ProtoUDP { + l4 = 8 + } + f := make([]byte, VnetHdrLen+20+l4+payload) + ip := f[VnetHdrLen:] + ip[0], ip[8], ip[9] = 0x45, 64, proto + binary.BigEndian.PutUint16(ip[2:], uint16(len(ip))) + s, d := GuestAddr.As4(), remote.As4() + copy(ip[srcAt:], s[:]) + copy(ip[dstAt:], d[:]) + seg := ip[20:] + binary.BigEndian.PutUint16(seg, 40000) + binary.BigEndian.PutUint16(seg[2:], 443) + for i := l4; i < len(seg); i++ { + seg[i] = byte(i * 7) + } + f[1] = gsoUDPL4 + binary.LittleEndian.PutUint16(f[8:], 6) + if proto == ProtoTCP { + f[1] = gsoTCPv4 + binary.BigEndian.PutUint32(seg[4:], 1000) + seg[12] = 5 << 4 + binary.LittleEndian.PutUint16(f[8:], 16) + } + f[0] = needsCsum + binary.LittleEndian.PutUint16(f[6:], 20) + binary.LittleEndian.PutUint16(f[4:], uint16(size)) + return f +} diff --git a/sandboxd/tunnet/testdata/fuzz/FuzzParseAndRewrite/a7fe022fecd7d8ec b/sandboxd/tunnet/testdata/fuzz/FuzzParseAndRewrite/a7fe022fecd7d8ec new file mode 100644 index 000000000..26e8282a5 --- /dev/null +++ b/sandboxd/tunnet/testdata/fuzz/FuzzParseAndRewrite/a7fe022fecd7d8ec @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("\x01\x00@\x00\x02\xcb\x00\x00\x10\x00E\x00\x008\x00\a\x00\x00@\x06\xd6\x7fdP\x00\a`\"\x00\x02000000000000X00000000000000000000000") diff --git a/sandboxd/tunnet/testdata/fuzz/FuzzSegment/3560257eabc72bdd b/sandboxd/tunnet/testdata/fuzz/FuzzSegment/3560257eabc72bdd new file mode 100644 index 000000000..b2228e737 --- /dev/null +++ b/sandboxd/tunnet/testdata/fuzz/FuzzSegment/3560257eabc72bdd @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("0\x0500000000%00000000\x06000000000000000000") diff --git a/sandboxd/tunnet/tunnet.go b/sandboxd/tunnet/tunnet.go new file mode 100644 index 000000000..67ce2536b --- /dev/null +++ b/sandboxd/tunnet/tunnet.go @@ -0,0 +1,2 @@ +// Package tunnet carries the advanced network mode on the host: a shared TUN, the per-claim packet streams, and the 1:1 address rewrite between them. +package tunnet diff --git a/sandboxd/tunnet/udp/exit_linux_test.go b/sandboxd/tunnet/udp/exit_linux_test.go new file mode 100644 index 000000000..89e35b720 --- /dev/null +++ b/sandboxd/tunnet/udp/exit_linux_test.go @@ -0,0 +1,263 @@ +//go:build linux + +package udp_test + +import ( + "bytes" + "encoding/binary" + "io" + "net" + "net/netip" + "os" + "os/exec" + "runtime" + "strings" + "testing" + "time" + + "golang.org/x/sys/unix" + + "github.com/cocoonstack/sandbox/sandboxd/tunnet" + "github.com/cocoonstack/sandbox/sandboxd/tunnet/udp" +) + +const bulk = 64 << 20 + +var ( + claimAddr = netip.MustParseAddr("100.80.0.7") + otherAddr = netip.MustParseAddr("100.80.0.8") + exitMAC = [6]byte{0x02, 0xee, 0, 0, 0, 1} +) + +func TestTrafficCrossesAKernelExit(t *testing.T) { + if os.Getenv("SANDBOXD_TUN_EXIT_TEST") == "" { + t.Skip("needs root, iproute2, nftables and conntrack; set SANDBOXD_TUN_EXIT_TEST=1") + } + exitNetns(t) + under, err := udp.Listen(netip.MustParseAddrPort("10.99.0.1:4789"), 4) + if err != nil { + t.Fatal(err) + } + defer under.Close() + r := tunnet.New(under, &tunnet.Overlay{ + Subnet: netip.MustParsePrefix("100.80.0.0/12"), VNI: 7, Port: 4789, MTU: 1450, + Exits: map[string]tunnet.Exit{"default": {VTEP: netip.MustParseAddr("10.99.0.2"), MAC: exitMAC}}, + }) + go func() { _ = r.Run(t.Context()) }() + r.Arm(tunnet.Claim{ID: "c1", Addr: claimAddr, Allow: func(netip.Addr, uint16) bool { return true }}) + tun := guestTun(t) + host, guest := net.Pipe() + go func() { _ = r.Serve(host, claimAddr) }() + go pumpGuest(tun, guest) + + inNetns(t, "inet", func() { + l, err := net.Listen("tcp", "10.98.0.2:7000") + if err != nil { + t.Fatal(err) + } + c, err := net.ListenPacket("udp", "10.98.0.2:7001") + if err != nil { + t.Fatal(err) + } + go echoTCP(l) + go echoUDP(c) + }) + time.Sleep(300 * time.Millisecond) + waitNeigh(t) + + var got []byte + inNetns(t, "guest", func() { + c, err := net.DialTimeout("tcp", "10.98.0.2:7000", 5*time.Second) + if err != nil { + t.Log(run(t, "sh", "-c", "ip netns exec guest cat /proc/net/snmp | grep -E '^(Ip|Tcp|Udp):'; ip netns exec exit cat /proc/net/snmp | grep -E '^(Ip|Tcp):'; ip netns exec exit conntrack -L 2>&1 | head; ip -n exit -s link show vxlan0 | head -8")) + t.Fatalf("tcp through the exit: %v", err) + } + defer func() { _ = c.Close() }() + start := time.Now() + go func() { _, _ = c.Write(bytes.Repeat([]byte{0x5a}, bulk)); _ = c.(*net.TCPConn).CloseWrite() }() + got, err = io.ReadAll(c) + if err != nil || len(got) != bulk { + t.Fatalf("tcp echo %d bytes, err %v", len(got), err) + } + t.Logf("tcp echo %d MiB both ways in %v", bulk>>20, time.Since(start)) + u, err := net.Dial("udp", "10.98.0.2:7001") + if err != nil { + t.Fatal(err) + } + defer func() { _ = u.Close() }() + _ = u.SetReadDeadline(time.Now().Add(3 * time.Second)) + if _, err = u.Write([]byte("ping")); err != nil { + t.Fatal(err) + } + b := make([]byte, 64) + n, err := u.Read(b) + if err != nil || !strings.HasPrefix(string(b[:n]), "10.98.0.1") { + t.Fatalf("udp echo %q %v, want the exit's masquerade address", b[:n], err) + } + other, _ := net.DialTimeout("tcp", otherAddr.String()+":7000", 500*time.Millisecond) + if other != nil { + t.Error("a claim reached another claim's overlay address") + } + }) + ct := run(t, "ip", "netns", "exec", "exit", "conntrack", "-L", "-s", claimAddr.String()) + if !strings.Contains(ct, "src="+claimAddr.String()) { + t.Errorf("exit conntrack has no flow from the claim's overlay address:\n%s", ct) + } + st := r.Stats() + t.Logf("stats %+v", st) + if st.Drops[tunnet.DropPolicy] == 0 { + t.Error("the packet to another claim was not dropped by policy") + } +} + +func exitNetns(t *testing.T) { + t.Helper() + script := ` +set -e +for n in guest exit inet; do ip netns del $n 2>/dev/null || true; ip netns add $n; ip -n $n link set lo up; done +ip link del vh 2>/dev/null || true +ip link add vh type veth peer name ve netns exit +ip addr add 10.99.0.1/24 dev vh; ip link set vh up +ip -n exit addr add 10.99.0.2/24 dev ve; ip -n exit link set ve up +ip netns exec exit ethtool -K ve tx off >/dev/null +ip -n exit link add xu type veth peer name iu netns inet +ip -n exit addr add 10.98.0.1/24 dev xu; ip -n exit link set xu up +ip -n inet addr add 10.98.0.2/24 dev iu; ip -n inet link set iu up +ip -n exit link add vxlan0 address 02:ee:00:00:00:01 type vxlan id 7 dstport 4789 local 10.99.0.2 dev ve learning +ip -n exit addr add 192.0.2.1/32 dev vxlan0; ip -n exit link set vxlan0 up +ip -n exit route add 100.80.0.0/12 dev vxlan0 +ip netns exec exit sysctl -qw net.ipv4.ip_forward=1 net.ipv4.conf.vxlan0.arp_accept=1 net.ipv4.neigh.vxlan0.base_reachable_time_ms=300000 +ip netns exec exit nft -f - <<'NFT' +table ip exit { + chain overlay_nat { type nat hook postrouting priority srcnat; ip saddr 100.80.0.0/12 oifname != "vxlan0" masquerade; } + chain overlay_forward { type filter hook forward priority filter; iifname "vxlan0" oifname "vxlan0" drop; } + chain overlay_input { type filter hook input priority filter; iifname "vxlan0" drop; } + chain underlay_input { type filter hook input priority filter; udp dport 4789 ip saddr != 10.99.0.1 drop; } +} +NFT +` + run(t, "sh", "-c", script) + t.Cleanup(func() { + _ = exec.Command("sh", "-c", "for n in guest exit inet; do ip netns del $n; done; ip link del vh").Run() + }) +} + +func guestTun(t *testing.T) *os.File { + t.Helper() + var f *os.File + inNetns(t, "guest", func() { + fd, err := unix.Open("/dev/net/tun", unix.O_RDWR|unix.O_CLOEXEC, 0) + if err != nil { + t.Fatal(err) + } + ifr, _ := unix.NewIfreq("tun0") + ifr.SetUint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_VNET_HDR) + if err = unix.IoctlIfreq(fd, unix.TUNSETIFF, ifr); err != nil { + t.Fatal(err) + } + if err = unix.IoctlSetInt(fd, unix.TUNSETOFFLOAD, tunnet.OffloadCSUM|tunnet.OffloadTSO4|tunnet.OffloadTSO6|tunnet.OffloadTSOECN); err != nil { + t.Fatal(err) + } + f = os.NewFile(uintptr(fd), "tun0") + }) + run(t, "sh", "-c", "ip -n guest addr add 100.64.0.2/30 dev tun0 && ip -n guest link set tun0 mtu 1450 up && ip -n guest route add default via 100.64.0.1") + return f +} + +func pumpGuest(tun *os.File, s net.Conn) { + h := tunnet.Hello{Version: tunnet.Version, Offloads: tunnet.OffloadCSUM | tunnet.OffloadTSO4 | tunnet.OffloadTSO6 | tunnet.OffloadTSOECN}.Encode() + if _, err := s.Write(h[:]); err != nil { + return + } + if _, err := tunnet.ReadHello(s); err != nil { + return + } + go func() { + buf := make([]byte, 4+tunnet.MaxFrame) + for { + n, err := tun.Read(buf[4:]) + if err != nil { + return + } + binary.LittleEndian.PutUint32(buf, uint32(n)) + if _, err = s.Write(buf[:4+n]); err != nil { + return + } + } + }() + var head [4]byte + buf := make([]byte, tunnet.MaxFrame) + for { + if _, err := io.ReadFull(s, head[:]); err != nil { + return + } + n := int(binary.LittleEndian.Uint32(head[:])) + if _, err := io.ReadFull(s, buf[:n]); err != nil { + return + } + _, _ = tun.Write(buf[:n]) + } +} + +func echoTCP(l net.Listener) { + for { + c, err := l.Accept() + if err != nil { + return + } + go func() { _, _ = io.Copy(c, c); _ = c.Close() }() + } +} + +func echoUDP(c net.PacketConn) { + b := make([]byte, 2048) + for { + _, from, err := c.ReadFrom(b) + if err != nil { + return + } + _, _ = c.WriteTo([]byte(from.String()), from) + } +} + +func waitNeigh(t *testing.T) { + t.Helper() + for range 50 { + if strings.Contains(run(t, "ip", "-n", "exit", "neigh", "show", claimAddr.String()), "02:00:64:50:00:07") { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatal("the exit never learned the claim from its gratuitous ARP") +} + +// inNetns runs fn on a thread inside netns name; sockets fn opens stay in that netns. +func inNetns(t *testing.T, name string, fn func()) { + t.Helper() + done := make(chan struct{}) + go func() { + defer close(done) + runtime.LockOSThread() + f, err := os.Open("/var/run/netns/" + name) + if err != nil { + t.Error(err) + return + } + defer func() { _ = f.Close() }() + if err = unix.Setns(int(f.Fd()), unix.CLONE_NEWNET); err != nil { + t.Error(err) + return + } + fn() + }() + <-done +} + +func run(t *testing.T, name string, args ...string) string { + t.Helper() + out, err := exec.Command(name, args...).CombinedOutput() + if err != nil { + t.Fatalf("%s %v: %v\n%s", name, args, err, out) + } + return string(out) +} diff --git a/sandboxd/tunnet/udp/udp.go b/sandboxd/tunnet/udp/udp.go new file mode 100644 index 000000000..8feb8bd8c --- /dev/null +++ b/sandboxd/tunnet/udp/udp.go @@ -0,0 +1,128 @@ +// Package udp is the VXLAN underlay on plain UDP sockets: one send call per segmented batch, one receive per coalesced batch, and several send sockets so flows leave from different source ports. +package udp + +import ( + "errors" + "fmt" + "net" + "net/netip" + "sync" + "sync/atomic" + + "golang.org/x/net/ipv4" + + "github.com/cocoonstack/sandbox/sandboxd/tunnet" +) + +const ( + // maxBatch keeps one GSO send under the kernel's 64 KiB and 64-segment caps. + maxBatch = 65000 + maxSegments = 64 + sockBuf = 8 << 20 + recvBuf = 1 << 16 + // recvBatch is the datagrams one receive call can return, each up to a GRO-coalesced 64 KiB. + recvBatch = 32 + oobLen = 64 +) + +var cmsgPool = sync.Pool{New: func() any { return new([cmsgCap]byte) }} + +var _ tunnet.Underlay = (*Underlay)(nil) + +// Underlay is a receive socket on the VXLAN port and a set of send sockets on ephemeral ports. +type Underlay struct { + recv *net.UDPConn + batch *ipv4.PacketConn + msgs []ipv4.Message + sends []*net.UDPConn + gso atomic.Bool +} + +// Listen binds the VXLAN port at addr and senders send sockets on the same address. +func Listen(addr netip.AddrPort, senders int) (*Underlay, error) { + recv, err := net.ListenUDP("udp4", net.UDPAddrFromAddrPort(addr)) + if err != nil { + return nil, fmt.Errorf("listen vxlan %s: %w", addr, err) + } + u := &Underlay{recv: recv, batch: ipv4.NewPacketConn(recv), msgs: make([]ipv4.Message, recvBatch)} + for i := range u.msgs { + u.msgs[i].Buffers = [][]byte{make([]byte, recvBuf)} + u.msgs[i].OOB = make([]byte, oobLen) + } + _ = recv.SetReadBuffer(sockBuf) + enableGRO(recv) + for range max(senders, 1) { + c, err := net.ListenUDP("udp4", net.UDPAddrFromAddrPort(netip.AddrPortFrom(addr.Addr(), 0))) + if err != nil { + u.Close() + return nil, fmt.Errorf("listen vxlan sender: %w", err) + } + _ = c.SetWriteBuffer(sockBuf) + u.sends = append(u.sends, c) + } + u.gso.Store(gsoSupported) + return u, nil +} + +func (u *Underlay) Send(b []byte, seg int, dst netip.AddrPort, flow uint64) error { + c := u.sends[flow%uint64(len(u.sends))] + per := min(maxBatch/seg, maxSegments) * seg + for len(b) > 0 { + n := min(len(b), per) + var err error + switch { + case n <= seg: + _, err = c.WriteToUDPAddrPort(b[:n], dst) + case u.gso.Load(): + oob := cmsgPool.Get().(*[cmsgCap]byte) + _, _, err = c.WriteMsgUDPAddrPort(b[:n], segmentCmsg(oob, seg), dst) + cmsgPool.Put(oob) + if errors.Is(err, errGSORefused) { + u.gso.Store(false) + continue + } + default: + for off := 0; off < n && err == nil; off += seg { + _, err = c.WriteToUDPAddrPort(b[off:min(off+seg, n)], dst) + } + } + if err != nil { + return err + } + b = b[n:] + } + return nil +} + +func (u *Underlay) Recv(each func(d []byte, from netip.AddrPort)) error { + got, err := u.batch.ReadBatch(u.msgs, 0) + if err != nil { + return err + } + for i := range got { + m := &u.msgs[i] + ua, ok := m.Addr.(*net.UDPAddr) + if !ok { + continue + } + from, buf := ua.AddrPort(), m.Buffers[0][:m.N] + seg := groSize(m.OOB[:m.NN]) + if seg <= 0 { + seg = len(buf) + } + for off := 0; off < len(buf); off += seg { + each(buf[off:min(off+seg, len(buf))], from) + } + } + return nil +} + +// Close closes every socket. +func (u *Underlay) Close() { + if u.recv != nil { + _ = u.recv.Close() + } + for _, c := range u.sends { + _ = c.Close() + } +} diff --git a/sandboxd/tunnet/udp/udp_linux.go b/sandboxd/tunnet/udp/udp_linux.go new file mode 100644 index 000000000..61b8f6fda --- /dev/null +++ b/sandboxd/tunnet/udp/udp_linux.go @@ -0,0 +1,53 @@ +//go:build linux + +package udp + +import ( + "encoding/binary" + "net" + "unsafe" + + "golang.org/x/sys/unix" +) + +const ( + gsoSupported = true + cmsgCap = 32 +) + +var errGSORefused = unix.EIO + +// segmentCmsg fills b with a control message asking the kernel to cut one send into seg-byte datagrams. +func segmentCmsg(b *[cmsgCap]byte, seg int) []byte { + h := (*unix.Cmsghdr)(unsafe.Pointer(&b[0])) //nolint:gosec // b holds a whole Cmsghdr + h.Level, h.Type = unix.SOL_UDP, unix.UDP_SEGMENT + h.SetLen(unix.CmsgLen(2)) + binary.NativeEndian.PutUint16(b[unix.CmsgLen(0):], uint16(seg)) //nolint:gosec // a datagram size fits + return b[:unix.CmsgSpace(2)] +} + +// groSize is the segment size the kernel coalesced a receive at, 0 when it did not. +func groSize(oob []byte) int { + for len(oob) >= unix.SizeofCmsghdr { + h := (*unix.Cmsghdr)(unsafe.Pointer(&oob[0])) //nolint:gosec // oob holds a whole Cmsghdr + l := int(h.Len) //nolint:gosec // a control message length fits + if l < unix.CmsgLen(0) || l > len(oob) { + return 0 + } + if h.Level == unix.SOL_UDP && h.Type == unix.UDP_GRO && l >= unix.CmsgLen(4) { + return int(binary.NativeEndian.Uint32(oob[unix.CmsgLen(0):])) + } + oob = oob[min(unix.CmsgSpace(l-unix.CmsgLen(0)), len(oob)):] + } + return 0 +} + +func enableGRO(c *net.UDPConn) { + rc, err := c.SyscallConn() + if err != nil { + return + } + _ = rc.Control(func(fd uintptr) { + _ = unix.SetsockoptInt(int(fd), unix.SOL_UDP, unix.UDP_GRO, 1) + }) +} diff --git a/sandboxd/tunnet/udp/udp_linux_test.go b/sandboxd/tunnet/udp/udp_linux_test.go new file mode 100644 index 000000000..c1938a32d --- /dev/null +++ b/sandboxd/tunnet/udp/udp_linux_test.go @@ -0,0 +1,55 @@ +//go:build linux + +package udp + +import ( + "net" + "net/netip" + "testing" +) + +func TestSendBatchArrivesAsSegments(t *testing.T) { + u, err := Listen(netip.MustParseAddrPort("127.0.0.1:0"), 2) + if err != nil { + t.Fatal(err) + } + defer u.Close() + dst := u.recv.LocalAddr().(*net.UDPAddr).AddrPort() + batch := make([]byte, 3*1000+400) + if err = u.Send(batch, 1000, dst, 7); err != nil { + t.Fatal(err) + } + var sizes []int + for len(sizes) < 4 { + if err = u.Recv(func(d []byte, _ netip.AddrPort) { sizes = append(sizes, len(d)) }); err != nil { + t.Fatal(err) + } + } + if sizes[0] != 1000 || sizes[3] != 400 { + t.Errorf("datagram sizes %v, want 1000 x3 then 400", sizes) + } +} + +func BenchmarkSendGSO(b *testing.B) { + u, err := Listen(netip.MustParseAddrPort("127.0.0.1:0"), 1) + if err != nil { + b.Fatal(err) + } + defer u.Close() + dst := u.recv.LocalAddr().(*net.UDPAddr).AddrPort() + go func() { + for { + if u.Recv(func([]byte, netip.AddrPort) {}) != nil { + return + } + } + }() + batch := make([]byte, 44*1472) + b.SetBytes(int64(len(batch))) + b.ReportAllocs() + for b.Loop() { + if err := u.Send(batch, 1472, dst, 1); err != nil { + b.Fatal(err) + } + } +} diff --git a/sandboxd/tunnet/udp/udp_other.go b/sandboxd/tunnet/udp/udp_other.go new file mode 100644 index 000000000..623535e81 --- /dev/null +++ b/sandboxd/tunnet/udp/udp_other.go @@ -0,0 +1,21 @@ +//go:build !linux + +package udp + +import ( + "errors" + "net" +) + +const ( + gsoSupported = false + cmsgCap = 32 +) + +var errGSORefused = errors.ErrUnsupported + +func segmentCmsg(*[cmsgCap]byte, int) []byte { return nil } + +func groSize([]byte) int { return 0 } + +func enableGRO(*net.UDPConn) {} diff --git a/sandboxd/tunnet/vxlan.go b/sandboxd/tunnet/vxlan.go new file mode 100644 index 000000000..c8e5c4a96 --- /dev/null +++ b/sandboxd/tunnet/vxlan.go @@ -0,0 +1,80 @@ +package tunnet + +import ( + "encoding/binary" + "net/netip" +) + +const ( + // VXLANHdrLen is the VXLAN header; EthHdrLen the inner Ethernet header sandboxd builds. + VXLANHdrLen = 8 + EthHdrLen = 14 + // Overhead is what a VXLAN datagram adds before the inner IP packet. + Overhead = VXLANHdrLen + EthHdrLen + + etherIPv4 = 0x0800 + etherARP = 0x0806 + vxlanI = 0x08 + arpLen = 28 + arpReq = 1 + arpReply = 2 +) + +var broadcast = [6]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff} + +// MAC derives a claim's MAC from its overlay address: a locally administered prefix and the IPv4 bytes. +func MAC(addr netip.Addr) [6]byte { + a := addr.As4() + return [6]byte{0x02, 0x00, a[0], a[1], a[2], a[3]} +} + +func appendHeader(b []byte, vni uint32, dst, src [6]byte, etype uint16) []byte { + b = binary.BigEndian.AppendUint32(append(b, vxlanI, 0, 0, 0), vni<<8) + b = append(b, dst[:]...) + b = append(b, src[:]...) + return binary.BigEndian.AppendUint16(b, etype) +} + +// parseVXLAN returns the Ethernet payload and type of a datagram for vni; false for anything else. +func parseVXLAN(d []byte, vni uint32) ([]byte, uint16, bool) { + if len(d) < Overhead || d[0]&vxlanI == 0 { + return nil, 0, false + } + if uint32(d[4])<<16|uint32(d[5])<<8|uint32(d[6]) != vni { + return nil, 0, false + } + return d[Overhead:], binary.BigEndian.Uint16(d[Overhead-2:]), true +} + +// arpRequest reads an IPv4-over-Ethernet ARP request: who asks, from which MAC, for which address. +func arpRequest(p []byte) (sender netip.Addr, mac [6]byte, target netip.Addr, ok bool) { + if len(p) < arpLen || binary.BigEndian.Uint16(p) != 1 || binary.BigEndian.Uint16(p[2:]) != etherIPv4 || + p[4] != 6 || p[5] != 4 || binary.BigEndian.Uint16(p[6:]) != arpReq { + return netip.Addr{}, mac, netip.Addr{}, false + } + copy(mac[:], p[8:14]) + return netip.AddrFrom4([4]byte(p[14:18])), mac, netip.AddrFrom4([4]byte(p[24:28])), true +} + +func appendARP(b []byte, op uint16, sha [6]byte, spa netip.Addr, tha [6]byte, tpa netip.Addr) []byte { + b = append(b, 0, 1, 0x08, 0x00, 6, 4) + b = binary.BigEndian.AppendUint16(b, op) + s, t := spa.As4(), tpa.As4() + b = append(b, sha[:]...) + b = append(b, s[:]...) + b = append(b, tha[:]...) + return append(b, t[:]...) +} + +// gratuitousARP is a broadcast ARP request announcing addr at its MAC, which a learning VXLAN device turns into an FDB and a neighbor entry. +func gratuitousARP(vni uint32, addr netip.Addr) []byte { + mac := MAC(addr) + b := appendHeader(make([]byte, 0, Overhead+arpLen), vni, broadcast, mac, etherARP) + return appendARP(b, arpReq, mac, addr, [6]byte{}, addr) +} + +func arpReplyFor(vni uint32, addr, sender netip.Addr, senderMAC [6]byte) []byte { + mac := MAC(addr) + b := appendHeader(make([]byte, 0, Overhead+arpLen), vni, senderMAC, mac, etherARP) + return appendARP(b, arpReply, mac, addr, senderMAC, sender) +} From 91707039e831030126dc075b475589256a9eb75b Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 01:25:33 +0800 Subject: [PATCH 02/10] sandboxd: wire the advanced network mode into claims, the API and metrics A pool with "tun": true gives each claim an overlay address before its claims commit and a packet door on vsock port 2051 next to its proxy doors; tun_up is sent in the background, so the claim path adds no RPC. The address, the switch and the exit label follow the claim through hibernate, archive, checkpoint, fork and move, and are released with it. PUT /v1/sandboxes/{id}/tun (root only) switches a claim; a disable runs beside any transition and the hello re-checks the switch under the manager lock, while an enable or a label change waits for the transition. A resend rewrites an archive row a failed write missed. The quota counter is persisted at disarm, at the move commit, on any claim write and at a clean stop. Ended flows write tun_flow audit records and, with egress_usage_bytes, egress_bytes usage events. CI runs the kernel exit test as root and each fuzz target for 30 seconds. --- .github/workflows/sandboxd.yml | 16 ++ e2e/fakeengine_test.go | 4 + sandboxd/backends.go | 54 +++- sandboxd/config/config.go | 153 ++++++++++- sandboxd/config/config_test.go | 58 +++++ sandboxd/config/reload.go | 39 +++ sandboxd/config/reload_test.go | 38 +++ sandboxd/engine/engine.go | 6 + sandboxd/engine/fakesilkd_test.go | 17 ++ sandboxd/engine/tun.go | 17 ++ sandboxd/engine/tun_test.go | 29 +++ sandboxd/main.go | 3 +- sandboxd/outbound/claim.go | 6 +- sandboxd/outbound/doors.go | 11 +- sandboxd/outbound/outbound.go | 28 +- sandboxd/outbound/outbound_test.go | 8 + sandboxd/outbound/tun.go | 176 +++++++++++++ sandboxd/outbound/view.go | 16 ++ sandboxd/pool/checkpoint.go | 3 + sandboxd/pool/claim.go | 8 +- sandboxd/pool/claims.go | 6 + sandboxd/pool/claims_test.go | 4 +- sandboxd/pool/egress.go | 4 + sandboxd/pool/failover.go | 1 + sandboxd/pool/fork.go | 3 +- sandboxd/pool/move_out.go | 3 + sandboxd/pool/operator.go | 2 +- sandboxd/pool/pool.go | 21 +- sandboxd/pool/pool_test.go | 15 ++ sandboxd/pool/reap.go | 8 +- sandboxd/pool/reconcile.go | 3 + sandboxd/pool/reload.go | 3 + sandboxd/pool/telemetry.go | 56 +++- sandboxd/pool/template.go | 3 + sandboxd/pool/tun.go | 233 +++++++++++++++++ sandboxd/pool/tun_test.go | 401 +++++++++++++++++++++++++++++ sandboxd/server/metrics.go | 22 ++ sandboxd/server/server.go | 25 ++ sandboxd/server/server_test.go | 53 ++++ sandboxd/types/types.go | 12 + 40 files changed, 1541 insertions(+), 27 deletions(-) create mode 100644 sandboxd/engine/tun.go create mode 100644 sandboxd/engine/tun_test.go create mode 100644 sandboxd/outbound/tun.go create mode 100644 sandboxd/pool/tun.go create mode 100644 sandboxd/pool/tun_test.go diff --git a/.github/workflows/sandboxd.yml b/.github/workflows/sandboxd.yml index 3cc13e983..47463cbf9 100644 --- a/.github/workflows/sandboxd.yml +++ b/.github/workflows/sandboxd.yml @@ -81,6 +81,22 @@ jobs: GOWORK=off go test -race -count=1 -v -run 'TestTwoNodesOnOneMetaStore|TestNodesOfACell|TestAMetaStoreNodeWithoutACell' ./pool/ | tee -a "$RUNNER_TEMP/pg-test.log" ! grep -- '--- SKIP' "$RUNNER_TEMP/pg-test.log" + - name: Test the tun path through a kernel exit + working-directory: sandboxd + shell: bash + run: | + sudo apt-get update -qq && sudo apt-get install -y -qq --no-install-recommends nftables conntrack ethtool + GOWORK=off go test -c -o "$RUNNER_TEMP/udp.test" ./tunnet/udp/ + sudo SANDBOXD_TUN_EXIT_TEST=1 "$RUNNER_TEMP/udp.test" -test.run TestTrafficCrossesAKernelExit -test.v | tee "$RUNNER_TEMP/exit-test.log" + ! grep -- '--- SKIP' "$RUNNER_TEMP/exit-test.log" + + - name: Fuzz the tun parsers + working-directory: sandboxd + run: | + for t in FuzzFrameLen FuzzParseAndRewrite FuzzSegment FuzzVXLANAndARP; do + GOWORK=off go test -run '^$' -fuzz "^$t\$" -fuzztime 30s ./tunnet/ + done + - name: Set up Python uses: actions/setup-python@v7.1.0 with: diff --git a/e2e/fakeengine_test.go b/e2e/fakeengine_test.go index afb0e23e7..2e7d57908 100644 --- a/e2e/fakeengine_test.go +++ b/e2e/fakeengine_test.go @@ -127,6 +127,10 @@ func (f *fakeEngine) ImageIDs(context.Context) (map[string]string, error) { retu func (f *fakeEngine) WriteInstanceMetadata(context.Context, string, []byte) error { return nil } +func (f *fakeEngine) TunUp(context.Context, string) error { return nil } + +func (f *fakeEngine) TunDown(context.Context, string) error { return nil } + func (f *fakeEngine) WriteGuestEnv(_ context.Context, vsockSocket string, doc []byte) error { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/backends.go b/sandboxd/backends.go index 7fbd3864d..10f55a615 100644 --- a/sandboxd/backends.go +++ b/sandboxd/backends.go @@ -7,6 +7,8 @@ import ( "path/filepath" "regexp" + "github.com/projecteru2/core/log" + archivespg "github.com/cocoonstack/sandbox/sandboxd/archives/pg" "github.com/cocoonstack/sandbox/sandboxd/config" movespg "github.com/cocoonstack/sandbox/sandboxd/moves/pg" @@ -18,39 +20,75 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/tenants" "github.com/cocoonstack/sandbox/sandboxd/tenants/file" tenantspg "github.com/cocoonstack/sandbox/sandboxd/tenants/pg" + "github.com/cocoonstack/sandbox/sandboxd/tunnet" + tunpg "github.com/cocoonstack/sandbox/sandboxd/tunnet/pg" + "github.com/cocoonstack/sandbox/sandboxd/tunnet/udp" usagepg "github.com/cocoonstack/sandbox/sandboxd/usage/pg" ) -func openBackends(ctx context.Context, cfg *config.Config) (pool.Backends, error) { +func openBackends(ctx context.Context, cfg *config.Config) (pool.Backends, func(context.Context), error) { var b pool.Backends var err error if b.Checkpoints, err = newStoreView(ctx, cfg, "checkpoint-staging", store.CheckpointIDRe, false); err != nil { - return pool.Backends{}, err + return pool.Backends{}, nil, err } b.SharedCheckpoints = cfg.S3Checkpoints() if b.Templates, err = newStoreView(ctx, cfg, "template-staging", store.TemplateIDRe, true); err != nil { - return pool.Backends{}, err + return pool.Backends{}, nil, err } if b.Tenants, err = openTenants(ctx, cfg); err != nil { - return pool.Backends{}, err + return pool.Backends{}, nil, err } ms := cfg.MetaStore if ms != nil && ms.Cell != "" { if b.CellPools, err = poolpg.Open(ctx, os.Getenv(ms.DSNEnv), ms.Cell); err != nil { - return pool.Backends{}, err + return pool.Backends{}, nil, err } } if cfg.ArchiveFailover { if b.Archives, err = archivespg.Open(ctx, os.Getenv(ms.DSNEnv), ms.Cell); err != nil { - return pool.Backends{}, err + return pool.Backends{}, nil, err } } if cfg.Movable() { if b.Moves, err = movespg.Open(ctx, os.Getenv(ms.DSNEnv), ms.Cell); err != nil { - return pool.Backends{}, err + return pool.Backends{}, nil, err + } + } + runTun := func(context.Context) {} + if cfg.Tun != nil { + if runTun, err = openTun(ctx, cfg, &b); err != nil { + return pool.Backends{}, nil, err } } - return b, nil + return b, runTun, nil +} + +// openTun returns run, which receives VXLAN until its context ends. +func openTun(ctx context.Context, cfg *config.Config, b *pool.Backends) (func(context.Context), error) { + t := cfg.Tun + under, err := udp.Listen(t.ListenAddr(), t.Senders) + if err != nil { + return nil, err + } + var addrs tunnet.Addresses + if cfg.Mesh != nil { + ms := cfg.MetaStore + addrs, err = tunpg.Open(ctx, os.Getenv(ms.DSNEnv), ms.Cell, t.Prefix(), t.ReuseAfter()) + } else { + addrs, err = tunnet.NewLocal(cfg.DataDir, t.Prefix(), t.ReuseAfter()) + } + if err != nil { + under.Close() + return nil, err + } + router := tunnet.New(under, t.Overlay()) + b.TunRouter, b.TunAddrs = router, addrs + return func(ctx context.Context) { + // closing the underlay is what ends the blocked receive + context.AfterFunc(ctx, under.Close) + log.WithFunc("main.openTun").Error(ctx, router.Run(ctx), "tun router stopped; the advanced network mode is down") + }, nil } func openUsage(ctx context.Context, cfg *config.Config) (*usagepg.Store, error) { diff --git a/sandboxd/config/config.go b/sandboxd/config/config.go index 0880d07c1..3fcf162d3 100644 --- a/sandboxd/config/config.go +++ b/sandboxd/config/config.go @@ -16,11 +16,13 @@ import ( "slices" "strconv" "strings" + "time" "github.com/docker/go-units" "github.com/cocoonstack/sandbox/sandboxd/egress" "github.com/cocoonstack/sandbox/sandboxd/store" + "github.com/cocoonstack/sandbox/sandboxd/tunnet" "github.com/cocoonstack/sandbox/sandboxd/types" "github.com/cocoonstack/sandbox/sandboxd/utils" ) @@ -35,6 +37,14 @@ const ( refillFloor = 4 refillCeiling = 256 minStorageBytes = 10 << 30 + + defaultTunVNI = 1 + defaultTunPort = 4789 + defaultTunMTU = 1450 + defaultTunSenders = 4 + defaultTunReuse = 5 * 24 * 3600 + defaultTunFlows = 16384 + defaultTunNewRate = 1000 ) // PoolSpec declares one warm pool and its target of claim-ready VMs. @@ -55,6 +65,8 @@ type PoolSpec struct { ReplicateSeconds int `json:"replicate_seconds,omitzero"` // Reader serves read-only file calls on this pool's hibernated or archived claims from a warm VM, without waking them. Reader bool `json:"reader,omitzero"` + // Tun gives this pool's claims the advanced network mode; the node's tun block must be set. + Tun bool `json:"tun,omitzero"` IdleHibernateSeconds int `json:"idle_hibernate_seconds,omitzero"` ArchiveAfterSeconds int `json:"archive_after_seconds,omitzero"` @@ -147,6 +159,121 @@ type EgressConnRate struct { Burst int `json:"burst"` } +// TunExit is one exit: its VTEP address and the MAC of its VXLAN device. +type TunExit struct { + VTEP string `json:"vtep"` + MAC string `json:"mac"` +} + +// TunConfig turns on the advanced network mode: the overlay subnet, the VXLAN socket and the exits claims leave through. +type TunConfig struct { + // Subnet holds the claims' overlay addresses; under a meta store every node of the cell sets the same subnet. + Subnet string `json:"subnet"` + // Listen is the ip:port the VXLAN socket binds; exits reach the node at this address. + Listen string `json:"listen"` + VNI uint32 `json:"vni,omitzero"` + // Port is the exits' VXLAN port. + Port uint16 `json:"port,omitzero"` + // MTU is the guest tun0 MTU. + MTU int `json:"mtu,omitzero"` + Senders int `json:"senders,omitzero"` + // Exits maps a claim's profile label to its exit; "default" serves a claim without one. + Exits map[string]TunExit `json:"exits"` + Limits tunnet.Limits `json:"limits,omitzero"` + // ReuseAfterSeconds holds a freed address back until exit conntrack entries for it can have expired. + ReuseAfterSeconds int `json:"reuse_after_seconds,omitzero"` +} + +// Prefix is the parsed overlay subnet; validation already rejected a bad one. +func (t *TunConfig) Prefix() netip.Prefix { + p, _ := netip.ParsePrefix(t.Subnet) + return p.Masked() +} + +// ListenAddr is the parsed VXLAN socket address. +func (t *TunConfig) ListenAddr() netip.AddrPort { + ap, _ := netip.ParseAddrPort(t.Listen) + return ap +} + +// ReuseAfter is the freed address reuse wait. +func (t *TunConfig) ReuseAfter() time.Duration { + return time.Duration(t.ReuseAfterSeconds) * time.Second +} + +// Overlay is the router's reloadable view of the block. +func (t *TunConfig) Overlay() *tunnet.Overlay { + o, _ := t.overlay() + return o +} + +func (t *TunConfig) applyDefaults() { + t.VNI = cmp.Or(t.VNI, defaultTunVNI) + t.Port = cmp.Or(t.Port, defaultTunPort) + t.MTU = cmp.Or(t.MTU, defaultTunMTU) + t.Senders = cmp.Or(t.Senders, defaultTunSenders) + t.ReuseAfterSeconds = cmp.Or(t.ReuseAfterSeconds, defaultTunReuse) + t.Limits.MaxFlows = cmp.Or(t.Limits.MaxFlows, defaultTunFlows) + t.Limits.NewFlowsPerSecond = cmp.Or(t.Limits.NewFlowsPerSecond, defaultTunNewRate) +} + +func (t *TunConfig) validate() error { + listen, err := netip.ParseAddrPort(t.Listen) + l := t.Limits + switch { + case err != nil || !listen.Addr().Is4() || listen.Addr().IsUnspecified() || listen.Port() == 0: + return fmt.Errorf("tun.listen %q must be a specific IPv4 address and port", t.Listen) + case t.VNI >= 1<<24: + return fmt.Errorf("tun.vni %d does not fit 24 bits", t.VNI) + case t.MTU < 576 || t.MTU > 9000: + return fmt.Errorf("tun.mtu %d must be between 576 and 9000", t.MTU) + case t.Senders < 1: + return fmt.Errorf("tun.senders must be at least 1, got %d", t.Senders) + case t.ReuseAfterSeconds < 0: + return fmt.Errorf("tun.reuse_after_seconds must not be negative") + case l.BytesPerSecond < 0 || l.PacketsPerSecond < 0 || l.NewFlowsPerSecond < 0 || l.MaxFlows < 0 || l.QuotaBytes < 0: + return fmt.Errorf("tun.limits must not be negative") + } + _, err = t.overlay() + return err +} + +func (t *TunConfig) overlay() (*tunnet.Overlay, error) { + subnet, err := netip.ParsePrefix(t.Subnet) + switch { + case err != nil || !subnet.Addr().Is4(): + return nil, fmt.Errorf("tun.subnet %q must be an IPv4 prefix", t.Subnet) + case subnet.Bits() < 4 || subnet.Bits() > 24: + return nil, fmt.Errorf("tun.subnet %q must be between /4 and /24", t.Subnet) + case subnet.Overlaps(tunnet.GuestRange): + return nil, fmt.Errorf("tun.subnet %q overlaps the guest range %s", t.Subnet, tunnet.GuestRange) + } + if _, ok := t.Exits["default"]; !ok { + return nil, fmt.Errorf(`tun.exits needs a "default" exit`) + } + exits := make(map[string]tunnet.Exit, len(t.Exits)) + for label, e := range t.Exits { + vtep, vtepErr := netip.ParseAddr(e.VTEP) + mac, macErr := net.ParseMAC(e.MAC) + switch { + case !types.NameRe.MatchString(label): + return nil, fmt.Errorf("tun.exits label %q must match %s", label, types.NameRe) + case vtepErr != nil || !vtep.Is4() || vtep.IsUnspecified(): + return nil, fmt.Errorf("tun.exits %q vtep %q must be an IPv4 address", label, e.VTEP) + case macErr != nil || len(mac) != 6 || mac[0]&1 != 0 || [6]byte(mac) == [6]byte{}: + return nil, fmt.Errorf("tun.exits %q mac %q must be a unicast 6-byte MAC", label, e.MAC) + } + exits[label] = tunnet.Exit{VTEP: vtep, MAC: [6]byte(mac)} + } + return &tunnet.Overlay{ + Subnet: subnet.Masked(), + VNI: t.VNI, + Port: t.Port, + MTU: uint16(t.MTU), //nolint:gosec // validate bounds it to 576..9000 + Exits: exits, + }, nil +} + // EgressCAConfig provisions HTTPS interception; the root private key never appears here. type EgressCAConfig struct { RootCert string `json:"root_cert"` @@ -303,6 +430,8 @@ type Config struct { DiskReserve *DiskReserve `json:"disk_reserve,omitempty"` MemoryLimit *MemoryLimit `json:"memory_limit,omitempty"` + Tun *TunConfig `json:"tun,omitempty"` + Volumes []VolumeSpec `json:"volumes,omitempty"` // VolumeRoots bounds runtime registration paths only; static volumes stay unchecked. VolumeRoots []string `json:"volume_roots,omitempty"` @@ -347,6 +476,9 @@ func (c *Config) applyDefaults() { for i := range c.Volumes { c.Volumes[i].DirectIO = cmp.Or(c.Volumes[i].DirectIO, types.DirectIOOff) } + if c.Tun != nil { + c.Tun.applyDefaults() + } } func (c *Config) validate() error { @@ -410,7 +542,7 @@ func (c *Config) validate() error { if c.MoveConcurrency > 0 && !c.Movable() { return fmt.Errorf("move_concurrency requires meta_store with a cell and an encrypted mesh (set mesh.cluster_key)") } - for _, check := range []func() error{c.validateNodeEgress, c.validateDiskReserve, c.validateMemoryLimit, c.validateMetaStore, c.validateVolumes, c.validateMesh, c.validateReplication} { + for _, check := range []func() error{c.validateNodeEgress, c.validateDiskReserve, c.validateMemoryLimit, c.validateMetaStore, c.validateVolumes, c.validateMesh, c.validateReplication, c.validateTun} { if err := check(); err != nil { return err } @@ -436,6 +568,25 @@ func (c *Config) validateReplication() error { return nil } +func (c *Config) validateTun() error { + for _, p := range c.Pools { + switch { + case !p.Tun: + case c.Tun == nil: + return fmt.Errorf("pool %q: tun needs the node tun block", p.Template) + case p.Net != types.NetNone: + return fmt.Errorf("pool %q: tun needs net %q", p.Template, types.NetNone) + } + } + if c.Tun == nil { + return nil + } + if c.Mesh != nil && (c.MetaStore == nil || c.MetaStore.Cell == "") { + return fmt.Errorf("tun with a mesh requires meta_store with a cell: overlay addresses must be unique across the cell") + } + return c.Tun.validate() +} + func (c *Config) validateVolumes() error { for i, root := range c.VolumeRoots { if !filepath.IsAbs(root) { diff --git a/sandboxd/config/config_test.go b/sandboxd/config/config_test.go index 9b99d3cba..aed30dc6e 100644 --- a/sandboxd/config/config_test.go +++ b/sandboxd/config/config_test.go @@ -1,14 +1,17 @@ package config import ( + "net/netip" "os" "path/filepath" "slices" "strconv" "strings" "testing" + "time" "github.com/cocoonstack/sandbox/sandboxd/egress" + "github.com/cocoonstack/sandbox/sandboxd/tunnet" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -290,6 +293,61 @@ func TestLoadMoveConcurrency(t *testing.T) { } } +func TestLoadTun(t *testing.T) { + t.Setenv("SANDBOX_TEST_DSN", "postgres://sandboxd@db/sandboxd") + block := func(over string) string { + return `"tun":{"subnet":"240.0.0.0/4","listen":"10.0.0.5:4789","exits":{"default":{"vtep":"10.0.0.9","mac":"02:aa:00:00:00:01"},"res":{"vtep":"10.0.0.10","mac":"02:aa:00:00:00:02"}}` + over + `}` + } + cfg := mustLoad(t, writeConfig(t, `{`+block("")+`,"pools":[{"template":"rt:24.04","tun":true}]}`)) + tun := cfg.Tun + if tun.VNI != 1 || tun.Port != 4789 || tun.MTU != 1450 || tun.Senders != 4 || tun.ReuseAfter() != 120*time.Hour { + t.Errorf("defaults %+v, want vni 1, port 4789, mtu 1450, senders 4, reuse 120h", tun) + } + if tun.Prefix() != netip.MustParsePrefix("240.0.0.0/4") || tun.ListenAddr() != netip.MustParseAddrPort("10.0.0.5:4789") { + t.Errorf("prefix %s listen %s", tun.Prefix(), tun.ListenAddr()) + } + o := tun.Overlay() + want := tunnet.Exit{VTEP: netip.MustParseAddr("10.0.0.10"), MAC: [6]byte{0x02, 0xaa, 0, 0, 0, 2}} + if o.Subnet != tun.Prefix() || o.VNI != 1 || o.Port != 4789 || o.MTU != 1450 || len(o.Exits) != 2 || o.Exits["res"] != want { + t.Errorf("overlay %+v", o) + } + if !cfg.Pools[0].Tun { + t.Error("pool tun not read") + } + meshed := `"api_token":"r","advertise_addr":"node1:7777","mesh":{"bind":"node1:7946"},` + cell := `"meta_store":{"kind":"pg","dsn_env":"SANDBOX_TEST_DSN","cell":"c1"},` + if _, err := Load(writeConfig(t, `{`+meshed+cell+block("")+`,"pools":[]}`)); err != nil { + t.Errorf("tun under a mesh with a cell: %v", err) + } + for name, tt := range map[string]struct{ body, want string }{ + "pool without the block": {`{"pools":[{"template":"rt:24.04","tun":true}]}`, "tun needs the node tun block"}, + "egress lane pool": {`{"bridges":["br0"],` + block("") + `,"pools":[{"template":"rt:24.04","net":"egress","tun":true}]}`, "tun needs net"}, + "mesh without meta store": {`{` + meshed + block("") + `,"pools":[]}`, "requires meta_store with a cell"}, + "mesh without a cell": {`{` + meshed + `"meta_store":{"kind":"pg","dsn_env":"SANDBOX_TEST_DSN"},` + block("") + `,"pools":[]}`, "requires meta_store with a cell"}, + "ipv6 subnet": {`{` + strings.Replace(block(""), "240.0.0.0/4", "fd00::/8", 1) + `,"pools":[]}`, "IPv4 prefix"}, + "subnet too wide": {`{` + strings.Replace(block(""), "240.0.0.0/4", "0.0.0.0/3", 1) + `,"pools":[]}`, "between /4 and /24"}, + "subnet too narrow": {`{` + strings.Replace(block(""), "240.0.0.0/4", "10.1.2.0/25", 1) + `,"pools":[]}`, "between /4 and /24"}, + "subnet over guests": {`{` + strings.Replace(block(""), "240.0.0.0/4", "100.64.0.0/10", 1) + `,"pools":[]}`, "overlaps the guest range"}, + "wildcard listen": {`{` + strings.Replace(block(""), "10.0.0.5:4789", "0.0.0.0:4789", 1) + `,"pools":[]}`, "tun.listen"}, + "listen without port": {`{` + strings.Replace(block(""), "10.0.0.5:4789", "10.0.0.5", 1) + `,"pools":[]}`, "tun.listen"}, + "vni past 24 bits": {`{` + block(`,"vni":16777216`) + `,"pools":[]}`, "tun.vni"}, + "mtu too small": {`{` + block(`,"mtu":500`) + `,"pools":[]}`, "tun.mtu"}, + "negative senders": {`{` + block(`,"senders":-1`) + `,"pools":[]}`, "tun.senders"}, + "negative reuse": {`{` + block(`,"reuse_after_seconds":-1`) + `,"pools":[]}`, "reuse_after_seconds"}, + "negative limit": {`{` + block(`,"limits":{"max_flows":-1}`) + `,"pools":[]}`, "tun.limits"}, + "no default exit": {`{` + strings.Replace(block(""), `"default"`, `"main"`, 1) + `,"pools":[]}`, `"default" exit`}, + "bad exit label": {`{` + strings.Replace(block(""), `"res"`, `"two words"`, 1) + `,"pools":[]}`, "label"}, + "ipv6 vtep": {`{` + strings.Replace(block(""), "10.0.0.9", "fd00::9", 1) + `,"pools":[]}`, "vtep"}, + "multicast mac": {`{` + strings.Replace(block(""), "02:aa:00:00:00:01", "01:00:5e:00:00:01", 1) + `,"pools":[]}`, "unicast"}, + "zero mac": {`{` + strings.Replace(block(""), "02:aa:00:00:00:01", "00:00:00:00:00:00", 1) + `,"pools":[]}`, "unicast"}, + "long mac": {`{` + strings.Replace(block(""), "02:aa:00:00:00:01", "02:00:00:00:00:00:00:01", 1) + `,"pools":[]}`, "unicast"}, + } { + if _, err := Load(writeConfig(t, tt.body)); err == nil || !strings.Contains(err.Error(), tt.want) { + t.Errorf("%s: %v, want an error containing %q", name, err, tt.want) + } + } +} + func TestLoadEgressUpstream(t *testing.T) { t.Setenv("SANDBOX_TEST_RES", "http://user:pw@res.example:3128") t.Setenv("SANDBOX_TEST_DC", "socks5://10.1.0.5:1080") diff --git a/sandboxd/config/reload.go b/sandboxd/config/reload.go index 6d6421133..190bcc525 100644 --- a/sandboxd/config/reload.go +++ b/sandboxd/config/reload.go @@ -10,6 +10,8 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/types" ) +const tunField = "tun" + var reloadableFields = map[string]bool{ "egress_classes": true, "egress_internal_allow": true, @@ -43,6 +45,9 @@ func (c *Config) ReloadDiff(next *Config) (changed, ignored []string, err error) case name == "pools": ch, ig, rs := diffPools(c.Pools, next.Pools) changed, ignored, restart = append(changed, ch...), append(ignored, ig...), append(restart, rs...) + case name == tunField: + ch, rs := diffTun(c.Tun, next.Tun) + changed, restart = append(changed, ch...), append(restart, rs...) case reflect.DeepEqual(cur.Field(i).Interface(), nxt.Field(i).Interface()): case reloadableFields[name]: changed = append(changed, name) @@ -73,6 +78,7 @@ func diffPools(cur, next []PoolSpec) (changed, ignored, restart []string) { {"storage", x.Storage == y.Storage}, {"reader", x.Reader == y.Reader}, {"egress_upstream_env", x.EgressUpstreamEnv == y.EgressUpstreamEnv}, + {"tun", x.Tun == y.Tun}, } { if !f.same { changed = append(changed, label+"."+f.name) @@ -88,6 +94,39 @@ func diffPools(cur, next []PoolSpec) (changed, ignored, restart []string) { return changed, ignored, restart } +// diffTun reloads the exits, MTU, exit port and limits; the socket, the subnet, the VNI and the allocator need a restart, as does adding or dropping the block. +func diffTun(cur, next *TunConfig) (changed, restart []string) { + switch { + case reflect.DeepEqual(cur, next): + return nil, nil + case cur == nil || next == nil: + return nil, []string{tunField} + } + for _, f := range []struct { + name string + restart, same bool + }{ + {"tun.subnet", true, cur.Prefix() == next.Prefix()}, + {"tun.listen", true, cur.ListenAddr() == next.ListenAddr()}, + {"tun.vni", true, cur.VNI == next.VNI}, + {"tun.senders", true, cur.Senders == next.Senders}, + {"tun.reuse_after_seconds", true, cur.ReuseAfterSeconds == next.ReuseAfterSeconds}, + {"tun.port", false, cur.Port == next.Port}, + {"tun.mtu", false, cur.MTU == next.MTU}, + {"tun.exits", false, maps.Equal(cur.Exits, next.Exits)}, + {"tun.limits", false, cur.Limits == next.Limits}, + } { + switch { + case f.same: + case f.restart: + restart = append(restart, f.name) + default: + changed = append(changed, f.name) + } + } + return changed, restart +} + func targetsOf(s PoolSpec) [5]int { return [5]int{s.Warm, s.WarmMax, s.IdleHibernateSeconds, s.ArchiveAfterSeconds, s.ArchiveDeleteAfterSeconds} } diff --git a/sandboxd/config/reload_test.go b/sandboxd/config/reload_test.go index 95aa8996d..54bf41520 100644 --- a/sandboxd/config/reload_test.go +++ b/sandboxd/config/reload_test.go @@ -71,3 +71,41 @@ func TestReloadDiffSortsReloadableOwnedAndRestartFields(t *testing.T) { t.Errorf("an identical config: %v %v %v, want nothing", changed, ignored, err) } } + +func TestReloadDiffReloadsTheExitsAndRefusesASubnetChange(t *testing.T) { + key := types.PoolKey{Template: "desktop:v12", Net: types.NetNone, Size: types.SizeSmall} + tun := TunConfig{Subnet: "240.0.0.0/4", Listen: "10.0.0.5:4789", VNI: 1, Port: 4789, MTU: 1450, Senders: 4, Exits: map[string]TunExit{"default": {VTEP: "10.0.0.9", MAC: "02:aa:00:00:00:01"}}} + cur := &Config{Tun: &tun, Pools: []PoolSpec{{PoolKey: key}}} + + exits := tun + exits.MTU = 1410 + exits.Exits = map[string]TunExit{"default": {VTEP: "10.0.0.10", MAC: "02:aa:00:00:00:01"}} + next := *cur + next.Tun, next.Pools = &exits, []PoolSpec{{PoolKey: key, Tun: true}} + changed, _, err := cur.ReloadDiff(&next) + if want := []string{"tun.mtu", "tun.exits", "pools[desktop:v12 none small].tun"}; err != nil || !slices.Equal(changed, want) { + t.Errorf("ReloadDiff over exits and the pool switch: %v %v, want %v", changed, err, want) + } + + for _, tt := range []struct { + edit func(*TunConfig) + want string + }{ + {func(c *TunConfig) { c.Subnet = "10.0.0.0/8" }, "tun.subnet change only at a restart"}, + {func(c *TunConfig) { c.Listen = "10.0.0.6:4789" }, "tun.listen change only at a restart"}, + {func(c *TunConfig) { c.VNI = 2 }, "tun.vni change only at a restart"}, + } { + moved := tun + tt.edit(&moved) + edited := *cur + edited.Tun = &moved + if _, _, err := cur.ReloadDiff(&edited); err == nil || err.Error() != tt.want { + t.Errorf("ReloadDiff: %v, want %q", err, tt.want) + } + } + dropped := *cur + dropped.Tun = nil + if _, _, err := cur.ReloadDiff(&dropped); err == nil || err.Error() != "tun change only at a restart" { + t.Errorf("ReloadDiff dropping the block: %v, want a restart refusal", err) + } +} diff --git a/sandboxd/engine/engine.go b/sandboxd/engine/engine.go index 3bd5fd0ef..9a65f3d14 100644 --- a/sandboxd/engine/engine.go +++ b/sandboxd/engine/engine.go @@ -52,6 +52,7 @@ const ( socksPort = 2050 dnsPort = 2052 dnsGramPort = 2053 + tunPort = 2051 cmdTimeout = 2 * time.Minute dumpTimeout = 15 * time.Minute cmdTermGrace = 30 * time.Second @@ -554,6 +555,11 @@ func SocksSocketPath(vsockSocket string) string { return fmt.Sprintf("%s_%d", vsockSocket, socksPort) } +// TunSocketPath is the host UDS the VMM connects when the guest dials CID2:tunPort. +func TunSocketPath(vsockSocket string) string { + return fmt.Sprintf("%s_%d", vsockSocket, tunPort) +} + // CapacitySignature returns the node-capacity signature err carries, or "". func CapacitySignature(err error) string { if _, guest := errors.AsType[*guestExitError](err); err == nil || guest { diff --git a/sandboxd/engine/fakesilkd_test.go b/sandboxd/engine/fakesilkd_test.go index ba79ce316..ad59fdccd 100644 --- a/sandboxd/engine/fakesilkd_test.go +++ b/sandboxd/engine/fakesilkd_test.go @@ -56,6 +56,9 @@ type fakeSilkd struct { statMisses map[string]int statErr map[string]string statCalls []string + + tunOps []string + tunErr string } func serveFakeSilkd(t *testing.T, path string) *fakeSilkd { @@ -138,6 +141,8 @@ func (f *fakeSilkd) serve(conn net.Conn) { f.handleStat(conn, req.Path) case "exec": f.handleExec(conn, req.Argv, req.Env) + case "tun_up", "tun_down": + f.handleTun(conn, req.Op) } } @@ -278,6 +283,18 @@ func (f *fakeSilkd) handleStat(conn net.Conn, path string) { writeFakeSilkdResponse(conn, &wire.Stat{Info: wire.FileInfo{Kind: wire.FileKindFile}}) } +func (f *fakeSilkd) handleTun(conn net.Conn, op string) { + f.mu.Lock() + f.tunOps = append(f.tunOps, op) + reply := f.tunErr + f.mu.Unlock() + if reply != "" { + writeFakeSilkdResponse(conn, &wire.ErrorResp{Kind: wire.KindInternal, Message: reply}) + return + } + writeFakeSilkdResponse(conn, &wire.Done{}) +} + func writeFakeSilkdResponse(conn net.Conn, resp wire.Response) { buf, err := wire.EncodeResponse(resp) if err != nil { diff --git a/sandboxd/engine/tun.go b/sandboxd/engine/tun.go new file mode 100644 index 000000000..8fc2489cc --- /dev/null +++ b/sandboxd/engine/tun.go @@ -0,0 +1,17 @@ +package engine + +import ( + "context" + + "github.com/cocoonstack/sandbox/protocol/wire" +) + +// TunUp brings the guest's tun0 up and starts its packet stream; the host hello still decides whether packets flow. +func (e *Engine) TunUp(ctx context.Context, vsockSocket string) error { + return e.silkdOnce[*wire.Done](ctx, vsockSocket, wire.TunUp{}) +} + +// TunDown removes the guest's tun0 and its routes so clients fail fast; it enforces nothing. +func (e *Engine) TunDown(ctx context.Context, vsockSocket string) error { + return e.silkdOnce[*wire.Done](ctx, vsockSocket, wire.TunDown{}) +} diff --git a/sandboxd/engine/tun_test.go b/sandboxd/engine/tun_test.go new file mode 100644 index 000000000..bd6258151 --- /dev/null +++ b/sandboxd/engine/tun_test.go @@ -0,0 +1,29 @@ +package engine + +import ( + "slices" + "strings" + "testing" +) + +func TestTunUpAndDownSendTheirVerbs(t *testing.T) { + path := sockPath(t) + fake := serveFakeSilkd(t, path) + e := newTestEngine() + if err := e.TunUp(t.Context(), path); err != nil { + t.Fatalf("TunUp: %v", err) + } + if err := e.TunDown(t.Context(), path); err != nil { + t.Fatalf("TunDown: %v", err) + } + fake.mu.Lock() + ops := slices.Clone(fake.tunOps) + fake.tunErr = "no /dev/net/tun" + fake.mu.Unlock() + if !slices.Equal(ops, []string{"tun_up", "tun_down"}) { + t.Errorf("silkd saw %q, want tun_up then tun_down", ops) + } + if err := e.TunUp(t.Context(), path); err == nil || !strings.Contains(err.Error(), "no /dev/net/tun") { + t.Errorf("TunUp on an error frame: %v, want the guest's error", err) + } +} diff --git a/sandboxd/main.go b/sandboxd/main.go index 948cb0857..1b68e7c1e 100644 --- a/sandboxd/main.go +++ b/sandboxd/main.go @@ -84,7 +84,7 @@ func main() { if err != nil { logger.Fatalf(ctx, err, "init egress secrets") } - backends, err := openBackends(ctx, cfg) + backends, runTun, err := openBackends(ctx, cfg) if err != nil { logger.Fatalf(ctx, err, "open backends") } @@ -103,6 +103,7 @@ func main() { if cfg.Mesh != nil { mgr.WithMovePeer(&server.MoveClient{Token: cfg.APIToken}) } + go runTun(ctx) if err := mgr.Reconcile(ctx); err != nil { logger.Fatalf(ctx, err, "reconcile") } diff --git a/sandboxd/outbound/claim.go b/sandboxd/outbound/claim.go index 5f9d78ee8..c4e23474a 100644 --- a/sandboxd/outbound/claim.go +++ b/sandboxd/outbound/claim.go @@ -20,7 +20,7 @@ const upstreamDirect = "direct" type resolvedPolicy struct { view *View - eval egress.Evaluator + eval egress.PolicyEvaluator } type connBucket struct { @@ -42,7 +42,7 @@ type claim struct { bucket atomic.Pointer[connBucket] } -func newClaim(h *Host, sb *types.Sandbox, v *View, eval egress.Evaluator) *claim { +func newClaim(h *Host, sb *types.Sandbox, v *View, eval egress.PolicyEvaluator) *claim { c := &claim{h: h, sb: sb} c.cur.Store(&resolvedPolicy{view: v, eval: eval}) return c @@ -141,7 +141,7 @@ func (c *claim) limiter(r config.EgressConnRate) *rate.Limiter { } // current denies everything once a reload leaves the claim with no policy. -func (c *claim) current() egress.Evaluator { +func (c *claim) current() egress.PolicyEvaluator { v := c.h.o.View() if r := c.cur.Load(); r.view == v { return r.eval diff --git a/sandboxd/outbound/doors.go b/sandboxd/outbound/doors.go index 742da4e2e..3d28b0478 100644 --- a/sandboxd/outbound/doors.go +++ b/sandboxd/outbound/doors.go @@ -33,7 +33,7 @@ func (s doorSpec) bind() error { return nil } -// doors are a sandbox's proxy door and optional SOCKS5 and DNS doors, served once armed. +// doors are a sandbox's proxy door and optional SOCKS5, DNS and packet doors, served once armed. type doors struct { srv *http.Server proxy *egress.Proxy @@ -42,6 +42,8 @@ type doors struct { socks net.Listener dns net.Listener dnsGram net.Listener + // tun is guarded by the host mutex. + tun *tunDoor } func (d *doors) close() { @@ -86,6 +88,13 @@ func (d *doors) specs(vsock string, p egress.PolicyEvaluator) [4]doorSpec { } } +func (d *doors) takeTun() (t *tunDoor) { + if d != nil { + t, d.tun = d.tun, nil + } + return t +} + // doorListener caps a door's live connections; doorConn's NetConn hands splice the *net.UnixConn the kernel copy needs. type doorListener struct { *net.UnixListener diff --git a/sandboxd/outbound/outbound.go b/sandboxd/outbound/outbound.go index 5ee4bce4c..d8100cca1 100644 --- a/sandboxd/outbound/outbound.go +++ b/sandboxd/outbound/outbound.go @@ -18,14 +18,17 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/egress" "github.com/cocoonstack/sandbox/sandboxd/netfilter" + "github.com/cocoonstack/sandbox/sandboxd/tunnet" "github.com/cocoonstack/sandbox/sandboxd/types" ) -// Engine is the VM engine surface the host needs: a VM's tap, and a guest's lane verdict and egress root. +// Engine is the VM engine surface the host needs: a VM's tap, a guest's lane verdict and egress root, and its tun0. type Engine interface { Inspect(ctx context.Context, name string) (types.VMRecord, bool, error) MarkLane(ctx context.Context, vsockSocket string, lane types.NetRoute) error InstallCACert(ctx context.Context, vsockSocket string, certPEM []byte) error + TunUp(ctx context.Context, vsockSocket string) error + TunDown(ctx context.Context, vsockSocket string) error } // TapFunc applies or drops the nft lock on one tap. @@ -47,6 +50,12 @@ type Options struct { LockNIC bool RecordEnds bool NoRxSplice bool + // Router carries the advanced network mode; nil when the node has no tun block. + Router *tunnet.Router + // Flow records one ended tun flow; TunBytes takes a disarmed claim's tun byte count back; TunOn reads the claim's tun switch under the owner's lock. + Flow func(ctx context.Context, id, tenant, profile string, r tunnet.FlowRecord) + TunBytes func(sb *types.Sandbox, used int64) + TunOn func(sb *types.Sandbox) bool // Dial, Verdict, Lock, Unlock, Sweep and Nameservers default to the internal-address guard, netfilter and /etc/resolv.conf; tests swap them. Dial egress.DialFunc Verdict VerdictFunc @@ -107,9 +116,11 @@ func (h *Host) ArmProxy(ctx context.Context, sb *types.Sandbox) error { d, displaced := h.prebound[sb.VMName], h.doors[sb.ID] delete(h.prebound, sb.VMName) delete(h.doors, sb.ID) + stale := displaced.takeTun() h.mu.Unlock() if displaced != nil { displaced.close() + h.dropTun(stale) } v := h.o.View() policy, ok := v.Resolve(sb) @@ -162,6 +173,7 @@ func (h *Host) ArmProxy(ctx context.Context, sb *types.Sandbox) error { go d.resolver.serve(d.dns, false) go d.resolver.serve(d.dnsGram, true) } + h.armTun(ctx, sb, d, v, c) return nil } @@ -203,9 +215,11 @@ func (h *Host) Disarm(id string, removed bool) { tap = h.taps[id] delete(h.taps, id) } + t := d.takeTun() h.mu.Unlock() if d != nil { d.close() + h.dropTun(t) } if tap != "" { _ = h.o.Unlock(tap) @@ -228,14 +242,24 @@ func (h *Host) ResolvedName(id string, ip netip.Addr) (string, bool) { return d.resolver.name(ip) } -// Reloaded follows a view swap: live proxies drop their pooled connections. +// Reloaded follows a view swap: live proxies drop their pooled connections, and claims of a pool whose tun switch went off lose their packet path. func (h *Host) Reloaded() { + v := h.o.View() + var off []*tunDoor h.mu.Lock() live := slices.Collect(maps.Values(h.doors)) + for _, d := range live { + if d.tun != nil && !v.Tun(d.tun.sb.PolicyKey()) { + off = append(off, d.takeTun()) + } + } h.mu.Unlock() for _, d := range live { d.proxy.ResetPools() } + for _, t := range off { + h.dropTun(t) + } } // KeepLock records sandbox id's locked tap, so its removal unlocks it. diff --git a/sandboxd/outbound/outbound_test.go b/sandboxd/outbound/outbound_test.go index ff5219f6b..b87b095b8 100644 --- a/sandboxd/outbound/outbound_test.go +++ b/sandboxd/outbound/outbound_test.go @@ -315,6 +315,14 @@ func (f *fakeEngine) MarkLane(context.Context, string, types.NetRoute) error { return nil } +func (f *fakeEngine) TunUp(context.Context, string) error { + return nil +} + +func (f *fakeEngine) TunDown(context.Context, string) error { + return nil +} + func (f *fakeEngine) lookups() int { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/outbound/tun.go b/sandboxd/outbound/tun.go new file mode 100644 index 000000000..478d97879 --- /dev/null +++ b/sandboxd/outbound/tun.go @@ -0,0 +1,176 @@ +package outbound + +import ( + "cmp" + "context" + "net" + "net/netip" + "sync/atomic" + "time" + + "github.com/projecteru2/core/log" + + "github.com/cocoonstack/sandbox/sandboxd/egress" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/tunnet" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +// tunVerbTimeout bounds a tun_up or tun_down; an old silkd that never answers costs the caller no more. +const tunVerbTimeout = 3 * time.Second + +// tunDoor is a claim's packet door and the byte counter its router claim charges. +type tunDoor struct { + ln net.Listener + addr netip.Addr + used *atomic.Int64 + sb *types.Sandbox +} + +// RearmTun re-arms an armed claim's packet door after its tun switch or profile changed; its proxy doors stay as they are. +func (h *Host) RearmTun(ctx context.Context, sb *types.Sandbox) { + if h.o.Router == nil { + return + } + h.mu.Lock() + d := h.doors[sb.ID] + old := d.takeTun() + h.mu.Unlock() + h.dropTun(old) + v := h.o.View() + policy, ok := v.Resolve(sb) + if d == nil || !ok { + return + } + h.armTun(ctx, sb, d, v, newClaim(h, sb, v, policy)) +} + +// DisarmTun revokes a claim's packet path at once, then asks the guest at sock, when it runs, to drop tun0. +func (h *Host) DisarmTun(ctx context.Context, id, sock string) { + h.mu.Lock() + t := h.doors[id].takeTun() + h.mu.Unlock() + if t == nil { + return + } + h.dropTun(t) + if sock == "" { + return + } + vctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), tunVerbTimeout) + defer cancel() + if err := h.o.Engine.TunDown(vctx, sock); err != nil { + log.WithFunc("outbound.DisarmTun").Warnf(ctx, "tun_down %s: %v", id, err) + } +} + +// TunUsed reads the live tun byte count of claim id; false when no packet door is armed. +func (h *Host) TunUsed(id string) (int64, bool) { + h.mu.Lock() + defer h.mu.Unlock() + if d := h.doors[id]; d != nil && d.tun != nil { + return d.tun.used.Load(), true + } + return 0, false +} + +// armTun binds the packet door of sb, whose doors d hold none, when its pool has tun and the claim is on and holds an address; then it arms the router and turns the guest's tun0 on. +func (h *Host) armTun(ctx context.Context, sb *types.Sandbox, d *doors, v *View, c *claim) { + key := sb.PolicyKey() + if h.o.Router == nil || !v.Tun(key) || !sb.TunAddr.IsValid() || sb.VsockSocket == "" || !h.tunOn(sb) { + return + } + logger := log.WithFunc("outbound.armTun") + ln, err := listenUnix(engine.TunSocketPath(sb.VsockSocket)) + if err != nil { + logger.Warnf(ctx, "bind the packet door of %s, it runs without tun: %v", sb.ID, err) + return + } + t := &tunDoor{ln: limitDoor(ln), addr: sb.TunAddr, used: new(atomic.Int64), sb: sb} + t.used.Store(sb.TunBytes) + id, tenant, profile, sock, usage := sb.ID, sb.Tenant, sb.TunProfile, sb.VsockSocket, v.usageBytes + evCtx := context.WithoutCancel(ctx) + tc := tunnet.Claim{ + ID: id, + Addr: t.addr, + Enabled: func() bool { return h.o.View().Tun(key) && h.tunLive(id, t) && h.tunOn(sb) }, + Profile: profile, + Allow: func(dst netip.Addr, port uint16) bool { + if c.current().EvalFlow(port) != egress.DecisionAllow { + return false + } + _, refused := h.o.Verdict(dst, port) + return refused == nil + }, + Limits: v.tunLimits, + Used: t.used, + Record: func(r tunnet.FlowRecord) { + if h.o.Flow != nil { + h.o.Flow(evCtx, id, tenant, profile, r) + } + if usage && r.Allowed && r.Up+r.Down > 0 { + h.o.Transfer(evCtx, id, tenant, egress.Event{Host: r.Dst.String(), Port: r.Port, Upstream: "tun/" + cmp.Or(profile, "default")}, r.Up, r.Down) + } + }, + } + h.o.Router.Arm(tc) + h.mu.Lock() + live := h.doors[id] == d + if live { + d.tun = t + } + h.mu.Unlock() + if !live { + h.dropTun(t) + return + } + go h.serveTun(evCtx, t) + go func() { + vctx, cancel := context.WithTimeout(evCtx, tunVerbTimeout) + defer cancel() + if err := h.o.Engine.TunUp(vctx, sock); err != nil { + logger.Warnf(ctx, "tun_up %s: %v", id, err) + } + if !h.tunLive(id, t) { + _ = h.o.Engine.TunDown(vctx, sock) + } + }() +} + +func (h *Host) serveTun(ctx context.Context, t *tunDoor) { + for { + conn, err := t.ln.Accept() + if err != nil { + return + } + go func() { + if err := h.o.Router.Serve(conn, t.addr); err != nil { + log.WithFunc("outbound.serveTun").Debugf(ctx, "packet stream of %s: %v", t.sb.ID, err) + } + }() + } +} + +func (h *Host) tunOn(sb *types.Sandbox) bool { + return h.o.TunOn == nil || h.o.TunOn(sb) +} + +// tunLive reports whether t is still the packet door of claim id. +func (h *Host) tunLive(id string, t *tunDoor) bool { + h.mu.Lock() + defer h.mu.Unlock() + d := h.doors[id] + return d != nil && d.tun == t +} + +// dropTun closes t's door, disarms its address and hands the claim's byte count back. +func (h *Host) dropTun(t *tunDoor) { + if t == nil { + return + } + _ = t.ln.Close() + h.o.Router.Disarm(t.addr) + if h.o.TunBytes != nil { + h.o.TunBytes(t.sb, t.used.Load()) + } +} diff --git a/sandboxd/outbound/view.go b/sandboxd/outbound/view.go index f1eb3e01c..45df86f72 100644 --- a/sandboxd/outbound/view.go +++ b/sandboxd/outbound/view.go @@ -9,6 +9,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/config" "github.com/cocoonstack/sandbox/sandboxd/egress" + "github.com/cocoonstack/sandbox/sandboxd/tunnet" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -26,6 +27,9 @@ type View struct { secrets *egress.SecretStore guarded bool usageBytes bool + + tunPools map[types.PoolKey]bool + tunLimits tunnet.Limits } // NewView reads cfg's egress settings; config validation already rejected bad entries. @@ -39,6 +43,10 @@ func NewView(cfg *config.Config, secrets *egress.SecretStore) *View { connRate: cfg.EgressConnRate, secrets: secrets, usageBytes: cfg.EgressUsageBytes, + tunPools: map[types.PoolKey]bool{}, + } + if cfg.Tun != nil { + v.tunLimits = cfg.Tun.Limits } if u := cfg.EgressUpstream; u != nil { v.upstreamEnv = u.ClaimEnv @@ -59,10 +67,18 @@ func NewView(cfg *config.Config, secrets *egress.SecretStore) *View { if spec.EgressUpstreamEnv != "" { v.poolUpstream[spec.PoolKey] = os.Getenv(spec.EgressUpstreamEnv) } + if spec.Tun { + v.tunPools[spec.PoolKey] = true + } } return v } +// Tun reports whether key's pool has the advanced network mode on. +func (v *View) Tun(key types.PoolKey) bool { + return v.tunPools[key] +} + // Intercepts reports whether key's pool policy may terminate HTTPS. func (v *View) Intercepts(key types.PoolKey) bool { return v.poolEgress[key].Intercepts() diff --git a/sandboxd/pool/checkpoint.go b/sandboxd/pool/checkpoint.go index ce7438057..eec16df88 100644 --- a/sandboxd/pool/checkpoint.go +++ b/sandboxd/pool/checkpoint.go @@ -180,6 +180,8 @@ func (m *Manager) publishCheckpoint(ctx context.Context, sb *types.Sandbox, ckID CreatedAt: time.Now(), PolicySource: sb.PolicySource, NoEgress: sb.NoEgress, + TunOff: sb.TunOff, + TunProfile: sb.TunProfile, Archive: archive, } if archive { @@ -231,6 +233,7 @@ func (m *Manager) claimLoaded(ctx context.Context, ckpt types.Checkpoint, o Clai o.apply(sb) sb.FromCheckpoint = ckpt.ID sb.PolicySource, sb.NoEgress = ckpt.PolicySource, ckpt.NoEgress + sb.TunOff, sb.TunProfile = ckpt.TunOff, ckpt.TunProfile out, err := m.finalize(ctx, sb, o.TTL, ckpt.GuestEnv, hold) if err == nil { m.counters.claimsClone.Add(1) diff --git a/sandboxd/pool/claim.go b/sandboxd/pool/claim.go index 1f2499525..62dc97010 100644 --- a/sandboxd/pool/claim.go +++ b/sandboxd/pool/claim.go @@ -309,7 +309,7 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand m.mu.Lock() } volumeHolds := m.cancelVolumeRun(sb) - snap, ck, vmName := sb.HibernateSnap, sb.ArchiveCk, sb.VMName + snap, ck, vmName, tunAddr := sb.HibernateSnap, sb.ArchiveCk, sb.VMName, sb.TunAddr if ck != "" { m.pendingCks[ck] = struct{}{} } @@ -357,6 +357,7 @@ func (m *Manager) releaseResolved(ctx context.Context, id string, sb *types.Sand } } m.out.Disarm(id, removed) + m.releaseTun(ctx, id, tunAddr) m.dropSnap(ctx, snap) return err } @@ -577,6 +578,9 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t if err == nil { err = m.deliverEnv(ctx, sbs, inherited) } + if err == nil { + m.assignTun(ctx, sbs) + } m.mu.Lock() m.dropHoldLocked(hold) if err == nil { @@ -585,6 +589,7 @@ func (m *Manager) finalizeBatch(ctx context.Context, sbs []*types.Sandbox, ttl t if err != nil { m.mu.Unlock() for _, sb := range sbs { + m.releaseTun(ctx, sb.ID, sb.TunAddr) // The rw mount dirtied the journal, so the clean umount is still owed td := m.quiesceVolumes(ctx, sb) m.removeOrRetry(ctx, sb.VMName, "", td) @@ -640,6 +645,7 @@ func (m *Manager) rollbackClaim(ctx context.Context, sbs []*types.Sandbox) { td := m.quiesceVolumes(ctx, sb) td.holds = holds[i] m.out.Disarm(sb.ID, m.removeOrRetry(ctx, sb.VMName, sb.ID, td)) + m.releaseTun(ctx, sb.ID, sb.TunAddr) } } diff --git a/sandboxd/pool/claims.go b/sandboxd/pool/claims.go index 50f448160..4792ea04d 100644 --- a/sandboxd/pool/claims.go +++ b/sandboxd/pool/claims.go @@ -7,6 +7,7 @@ import ( "fmt" "io/fs" "maps" + "net/netip" "os" "path/filepath" "slices" @@ -52,6 +53,10 @@ type claimDTO struct { Env types.Env `json:"env,omitempty"` VsockSocket string `json:"vsock_socket,omitempty"` TAP string `json:"tap,omitempty"` + TunAddr netip.Addr `json:"tun_addr,omitzero"` + TunOff bool `json:"tun_off,omitzero"` + TunProfile string `json:"tun_profile,omitempty"` + TunBytes int64 `json:"tun_bytes,omitzero"` HibernateSnap string `json:"hibernate_snap,omitempty"` PendingSnap string `json:"pending_snap,omitempty"` SaveSnap string `json:"save_snap,omitempty"` @@ -396,6 +401,7 @@ func dtoOf(sb *types.Sandbox) claimDTO { Tenant: sb.Tenant, EgressClass: sb.EgressClass, ClaimRef: sb.ClaimRef, Metadata: sb.Metadata, OnExpire: sb.OnExpire, Volumes: slices.Clone(sb.Volumes), VolumesWant: volumesWantDTO(sb), Env: sb.Env, VsockSocket: sb.VsockSocket, TAP: sb.TAP, HibernateSnap: sb.HibernateSnap, PendingSnap: sb.PendingSnap, SaveSnap: sb.SaveSnap, ReaderSafe: sb.ReaderSafe, + TunAddr: sb.TunAddr, TunOff: sb.TunOff, TunProfile: sb.TunProfile, TunBytes: sb.TunBytes, ArchiveCk: sb.ArchiveCk, ArchiveEpoch: sb.ArchiveEpoch, ArchivePending: sb.ArchivePending, ReplicaCk: sb.ReplicaCk, ReplicaPending: sb.ReplicaPending, FromCheckpoint: sb.FromCheckpoint, Restarts: sb.Restarts, RestartedAt: sb.RestartedAt, Failed: sb.Failed, MoveOut: sb.MoveOut, MoveIn: sb.MoveIn, } diff --git a/sandboxd/pool/claims_test.go b/sandboxd/pool/claims_test.go index 2d793a70b..b8fdd9141 100644 --- a/sandboxd/pool/claims_test.go +++ b/sandboxd/pool/claims_test.go @@ -4,6 +4,7 @@ import ( "errors" "io" "io/fs" + "net/netip" "os" "path/filepath" "reflect" @@ -82,6 +83,7 @@ func TestClaimStoreRoundTripsWithoutLeavingATempFile(t *testing.T) { claims := map[string]*types.Sandbox{ "sb_a": {ID: "sb_a", VMName: "sbx-1", Key: testKey, Token: "t1", Deadline: time.Now().Add(time.Minute).UTC(), ClaimedAt: time.Now().Add(-time.Minute).UTC(), ClaimRef: "ns/workload", Metadata: types.Metadata{"team": "a"}, OnExpire: types.ExpireArchive, Volumes: dataset, VolumesWant: dataset, VsockSocket: "/v/1"}, "sb_b": {ID: "sb_b", VMName: "sbx-2", Key: testKey, Token: "t2"}, + "sb_c": {ID: "sb_c", VMName: "sbx-3", Key: testKey, TunAddr: netip.MustParseAddr("240.0.0.7"), TunOff: true, TunProfile: "res", TunBytes: 1 << 33}, } for _, fsync := range []bool{false, true} { s := newClaimStore(t.TempDir(), fsync) @@ -131,7 +133,7 @@ func TestClaimRowOmitsUnsetFlags(t *testing.T) { if err != nil { t.Fatalf("encode: %v", err) } - for _, flag := range []string{"reader_safe", "no_egress"} { + for _, flag := range []string{"reader_safe", "no_egress", "tun_addr", "tun_off", "tun_profile", "tun_bytes"} { if strings.Contains(string(raw), flag) { t.Errorf("row %s carries the unset flag %s", raw, flag) } diff --git a/sandboxd/pool/egress.go b/sandboxd/pool/egress.go index ee1e2a729..4de053fac 100644 --- a/sandboxd/pool/egress.go +++ b/sandboxd/pool/egress.go @@ -70,5 +70,9 @@ func (m *Manager) outboundOptions(ca *egress.CA) outbound.Options { LockNIC: len(m.cfg.Bridges) > 0, RecordEnds: m.audit != nil, NoRxSplice: m.cfg.NoRxSplice, + Router: m.tunRouter, + Flow: m.recordTunFlow, + TunBytes: m.setTunBytes, + TunOn: m.tunOn, } } diff --git a/sandboxd/pool/failover.go b/sandboxd/pool/failover.go index e4e2e0f91..18a56cffb 100644 --- a/sandboxd/pool/failover.go +++ b/sandboxd/pool/failover.go @@ -541,6 +541,7 @@ func (m *Manager) purgeArchiveRow(ctx context.Context, rec archives.Record) { if !rec.Consumed { m.counters.archiveDeletes.Add(1) m.recordUsage(ctx, usage.Event{Event: "archive_delete", ID: rec.ID, Reference: rec.Checkpoint, Tenant: rec.Tenant}) + m.releaseRowTun(ctx, rec) } logger.Infof(ctx, "purged archive %s of %s", rec.ID, rec.Owner) } diff --git a/sandboxd/pool/fork.go b/sandboxd/pool/fork.go index e64818279..9684c97fe 100644 --- a/sandboxd/pool/fork.go +++ b/sandboxd/pool/fork.go @@ -41,12 +41,13 @@ func (m *Manager) Fork(ctx context.Context, id string, cred Cred, count int, ttl return nil, fmt.Errorf("fork %s: %w", sb.ID, err) } m.mu.Lock() - parentExpire, parentVM := sb.OnExpire, sb.VMName + parentExpire, parentVM, tunOff, tunProfile := sb.OnExpire, sb.VMName, sb.TunOff, sb.TunProfile m.mu.Unlock() for _, c := range children { c.Tenant, c.EgressClass = sb.Tenant, sb.EgressClass c.PolicySource = sb.PolicySource c.NoEgress = sb.NoEgress + c.TunOff, c.TunProfile = tunOff, tunProfile c.Metadata = sb.Metadata c.OnExpire = onExpire.Or(parentExpire) } diff --git a/sandboxd/pool/move_out.go b/sandboxd/pool/move_out.go index 42f9d5b28..ac6823574 100644 --- a/sandboxd/pool/move_out.go +++ b/sandboxd/pool/move_out.go @@ -118,6 +118,9 @@ func (m *Manager) moveOut(ctx context.Context, sb *types.Sandbox, mv *outMove, b if _, err = m.sendRound(ctx, sb.ID, mv, round); err != nil { return err } + if used, ok := m.out.TunUsed(sb.ID); ok { + m.setTunBytes(sb, used) + } m.mu.Lock() dto := dtoOf(sb) m.mu.Unlock() diff --git a/sandboxd/pool/operator.go b/sandboxd/pool/operator.go index 407b8681f..e04c8715b 100644 --- a/sandboxd/pool/operator.go +++ b/sandboxd/pool/operator.go @@ -22,7 +22,7 @@ func (m *Manager) Sandbox(id string) (SandboxSummary, bool) { return SandboxSummary{}, false } out := summarize(sb) - out.Token = sb.Token + out.Token, out.TunAddr, out.TunOff, out.TunProfile = sb.Token, sb.TunAddr, sb.TunOff, sb.TunProfile return out, true } diff --git a/sandboxd/pool/pool.go b/sandboxd/pool/pool.go index 332feab1a..24b1f5da1 100644 --- a/sandboxd/pool/pool.go +++ b/sandboxd/pool/pool.go @@ -10,6 +10,7 @@ import ( "fmt" "maps" "net" + "net/netip" "os" "path/filepath" "slices" @@ -31,6 +32,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/store" "github.com/cocoonstack/sandbox/sandboxd/store/peer" "github.com/cocoonstack/sandbox/sandboxd/tenants" + "github.com/cocoonstack/sandbox/sandboxd/tunnet" "github.com/cocoonstack/sandbox/sandboxd/types" "github.com/cocoonstack/sandbox/sandboxd/usage" ) @@ -104,6 +106,7 @@ var ( ErrNoEgress = errors.New("node has no egress attachment (bridge or network)") ErrNoEgressHibernate = errors.New("egress-lane sandboxes do not hibernate") ErrNoEgressFork = errors.New("egress-lane sandboxes cannot fork, checkpoint, or promote: a resumed guest egresses before its fresh tap can be locked") + ErrTunOffPromote = errors.New("a sandbox whose tun is turned off cannot be promoted: its template's claims would get tun back") ErrVolumeCapture = errors.New("sandboxes with volumes cannot hibernate, fork, checkpoint, or promote") ErrVolumeBusy = errors.New("volume is held by another claim") @@ -160,6 +163,8 @@ type Engine interface { DialGuestPort(ctx context.Context, vsockSocket string, port uint16) (net.Conn, error) InstallCACert(ctx context.Context, vsockSocket string, certPEM []byte) error MarkLane(ctx context.Context, vsockSocket string, lane types.NetRoute) error + TunUp(ctx context.Context, vsockSocket string) error + TunDown(ctx context.Context, vsockSocket string) error Warmup(ctx context.Context, vsockSocket string, argv []string) error SettleGuest(ctx context.Context, vsockSocket string) error TrimCow(ctx context.Context, vsockSocket string) error @@ -191,8 +196,11 @@ type SandboxSummary struct { ID string `json:"id"` Key types.PoolKey `json:"key"` Tenant string `json:"tenant,omitempty"` - // Token is the sandbox's own bearer token; only the root by-id read carries it. - Token string `json:"token,omitempty"` + // Token is the sandbox's own bearer token; only the root by-id read carries it, like the tun fields. + Token string `json:"token,omitempty"` + TunAddr netip.Addr `json:"tun_addr,omitzero"` + TunOff bool `json:"tun_off,omitzero"` + TunProfile string `json:"tun_profile,omitempty"` // ClaimRef echoes the caller reference; empty for checkpoint branches and unprefixed forks. ClaimRef string `json:"claim_ref,omitempty"` Metadata types.Metadata `json:"metadata,omitempty"` @@ -247,6 +255,9 @@ type Backends struct { CellPools poolset.Shared Archives archives.Registry Moves moves.Registry + // TunRouter and TunAddrs carry the advanced network mode; both nil when the node has no tun block. + TunRouter *tunnet.Router + TunAddrs tunnet.Addresses } // PeerDeleteFunc broadcasts a checkpoint delete to every peer. @@ -343,6 +354,9 @@ type Manager struct { notifyTemplates func() out *outbound.Host + // tunRouter and tunAddrs are nil when the node has no tun block. + tunRouter *tunnet.Router + tunAddrs tunnet.Addresses readers *readerSet @@ -394,6 +408,8 @@ func NewManager(ctx context.Context, cfg *config.Config, eng Engine, secrets *eg poolShared: b.CellPools, archives: b.Archives, moves: b.Moves, + tunRouter: b.TunRouter, + tunAddrs: b.TunAddrs, tenants: b.Tenants, ckpts: b.Checkpoints, ckptsShared: b.SharedCheckpoints, @@ -545,6 +561,7 @@ func (m *Manager) Run(ctx context.Context) { // FlushClaims synchronously persists the current claim set at shutdown. func (m *Manager) FlushClaims() error { + m.foldTunBytes() return m.store.commit(m.store.mark()) } diff --git a/sandboxd/pool/pool_test.go b/sandboxd/pool/pool_test.go index 3cd5ddecf..b56987652 100644 --- a/sandboxd/pool/pool_test.go +++ b/sandboxd/pool/pool_test.go @@ -1428,6 +1428,7 @@ type fakeEngine struct { metadataErr error guestEnvs map[string]string guestEnvErr error + tunVerbs []string migrateDir string migrateSeries map[string]string @@ -1801,6 +1802,20 @@ func (f *fakeEngine) MarkLane(_ context.Context, _ string, lane types.NetRoute) return nil } +func (f *fakeEngine) TunUp(_ context.Context, sock string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.tunVerbs = append(f.tunVerbs, "up "+sock) + return nil +} + +func (f *fakeEngine) TunDown(_ context.Context, sock string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.tunVerbs = append(f.tunVerbs, "down "+sock) + return nil +} + func (f *fakeEngine) SettleGuest(context.Context, string) error { f.mu.Lock() defer f.mu.Unlock() diff --git a/sandboxd/pool/reap.go b/sandboxd/pool/reap.go index 4922ce469..df30eab8d 100644 --- a/sandboxd/pool/reap.go +++ b/sandboxd/pool/reap.go @@ -3,6 +3,7 @@ package pool import ( "cmp" "context" + "net/netip" "slices" "time" @@ -29,6 +30,7 @@ func (m *Manager) reapOnce(ctx context.Context) { deadline time.Time sb *types.Sandbox holds []types.Volume + tun netip.Addr } m.mu.Lock() var expired []victim @@ -46,7 +48,7 @@ func (m *Manager) reapOnce(ctx context.Context) { unrows = append(unrows, sb) } case sb.ArchiveCk != "": - expired = append(expired, victim{action: reapPurge, id: id, ck: sb.ArchiveCk, tenant: sb.Tenant, deadline: sb.Deadline, sb: sb}) + expired = append(expired, victim{action: reapPurge, id: id, ck: sb.ArchiveCk, tenant: sb.Tenant, deadline: sb.Deadline, sb: sb, tun: sb.TunAddr}) purges = append(purges, sb.ArchiveCk) case m.archivesAtDeadline(sb): // Archive instead of destroy, kept in m.claimed for archive() to move @@ -60,7 +62,7 @@ func (m *Manager) reapOnce(ctx context.Context) { } expired = append(expired, victim{action: action, id: id, sb: sb}) default: - expired = append(expired, victim{action: reapDestroy, id: id, vmName: sb.VMName, snap: sb.HibernateSnap, tenant: sb.Tenant, deadline: sb.Deadline, sb: sb, holds: m.cancelVolumeRun(sb)}) + expired = append(expired, victim{action: reapDestroy, id: id, vmName: sb.VMName, snap: sb.HibernateSnap, tenant: sb.Tenant, deadline: sb.Deadline, sb: sb, holds: m.cancelVolumeRun(sb), tun: sb.TunAddr}) m.dropClaimLocked(sb) dropped = append(dropped, id) } @@ -137,6 +139,7 @@ func (m *Manager) reapOnce(ctx context.Context) { switch v.action { case reapPurge: m.out.Disarm(v.id, true) + m.releaseTun(ctx, v.id, v.tun) m.purgeArchiveCk(ctx, v.id, v.ck, v.tenant) m.untrack(m.pendingCks, v.ck) m.counters.reaps.Add(1) @@ -150,6 +153,7 @@ func (m *Manager) reapOnce(ctx context.Context) { td := m.quiesceVolumes(ctx, v.sb) td.holds = v.holds m.out.Disarm(v.id, m.removeOrRetry(ctx, v.vmName, v.id, td)) + m.releaseTun(ctx, v.id, v.tun) m.dropSnap(ctx, v.snap) m.counters.reaps.Add(1) m.recordUsage(ctx, usage.Event{Event: reapEvent, ID: v.id, VMName: v.vmName, Tenant: v.tenant, Deadline: v.deadline}) diff --git a/sandboxd/pool/reconcile.go b/sandboxd/pool/reconcile.go index f4b074270..be458b554 100644 --- a/sandboxd/pool/reconcile.go +++ b/sandboxd/pool/reconcile.go @@ -92,6 +92,7 @@ func (m *Manager) Reconcile(ctx context.Context) error { pooled[p.hash] = true } m.mu.Unlock() + m.seedTun() for _, sb := range adopted { m.recordHibernate(ctx, sb) } @@ -330,6 +331,7 @@ func (m *Manager) quarantineClaim(ctx context.Context, sb *types.Sandbox) bool { gone := m.removeOrRetry(ctx, sb.VMName, sb.ID, td) m.forgetClaim(ctx, sb) m.recordEnd(ctx, sb, "quarantined") + m.releaseTun(ctx, sb.ID, sb.TunAddr) return gone } @@ -340,6 +342,7 @@ func (m *Manager) recordReconciled(ctx context.Context, resumed, dropped []*type for _, sb := range dropped { if answered || !sb.HasReplica() { m.recordEnd(ctx, sb, "vm_gone") + m.releaseTun(ctx, sb.ID, sb.TunAddr) } } } diff --git a/sandboxd/pool/reload.go b/sandboxd/pool/reload.go index 1fbcc93b9..17223688a 100644 --- a/sandboxd/pool/reload.go +++ b/sandboxd/pool/reload.go @@ -62,6 +62,9 @@ func (m *Manager) ReloadConfig(ctx context.Context, next *config.Config) (Reload } m.mu.Unlock() m.store.setSync(next.SyncClaims) + if m.tunRouter != nil { + m.tunRouter.Reload(next.Tun.Overlay()) + } m.out.Reloaded() m.destroyAll(context.WithoutCancel(ctx), trim).Wait() m.cfg = next diff --git a/sandboxd/pool/telemetry.go b/sandboxd/pool/telemetry.go index 8bb98ac08..f6ca18f81 100644 --- a/sandboxd/pool/telemetry.go +++ b/sandboxd/pool/telemetry.go @@ -8,6 +8,7 @@ import ( "maps" "net/netip" "slices" + "strconv" "sync/atomic" "time" @@ -15,6 +16,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/egress" "github.com/cocoonstack/sandbox/sandboxd/outbound" + "github.com/cocoonstack/sandbox/sandboxd/tunnet" "github.com/cocoonstack/sandbox/sandboxd/usage" ) @@ -22,9 +24,11 @@ const ( // AuditLineCap bounds how much of a relayed request frame the audit tap captures and records. AuditLineCap = 4096 - egressEvent = "egress" - dnsEvent = "dns" - reapEvent = "reap" + egressEvent = "egress" + dnsEvent = "dns" + reapEvent = "reap" + tunFlowEvent = "tun_flow" + tunFlowPath = "tun" ) // ClaimBuckets are the upper bounds of the claim latency histogram. @@ -136,6 +140,20 @@ type dnsFrame struct { Tenant string `json:"tenant,omitempty"` } +type tunFlowFrame struct { + Op string `json:"op"` + Path string `json:"path"` + Tenant string `json:"tenant,omitempty"` + Proto string `json:"proto"` + Dst netip.AddrPort `json:"dst_addr"` + Decision string `json:"decision"` + Sent int64 `json:"sent"` + Received int64 `json:"received"` + Start time.Time `json:"start"` + End time.Time `json:"end"` + Profile string `json:"profile"` +} + func (m *Manager) Counters() Counters { c := &m.counters warm, clone, cold := c.claimWarmLatency.snapshot(), c.claimCloneLatency.snapshot(), c.claimColdLatency.snapshot() @@ -242,6 +260,25 @@ func (m *Manager) recordDNS(ctx context.Context, id, tenant string, ev outbound. }{time.Now(), id, dnsFrame{Op: dnsEvent, Name: ev.Name, Type: ev.Type, Answers: ev.Answers, Decision: ev.Decision, Tenant: tenant}}) } +func (m *Manager) recordTunFlow(ctx context.Context, id, tenant, profile string, r tunnet.FlowRecord) { + if m.audit == nil { + return + } + decision := "deny" + if r.Allowed { + decision = "allow" + } + m.appendAudit(ctx, struct { + Time time.Time `json:"t"` + ID string `json:"id"` + tunFlowFrame + }{ + Time: r.End, ID: id, + Op: tunFlowEvent, Path: tunFlowPath, Tenant: tenant, Proto: protoName(r.Proto), Dst: netip.AddrPortFrom(r.Dst, r.Port), + Decision: decision, Sent: r.Up, Received: r.Down, Start: r.Start, End: r.End, Profile: cmp.Or(profile, "default"), + }) +} + func (m *Manager) recordUsage(ctx context.Context, ev usage.Event) { ev.Time = cmp.Or(ev.Time, time.Now()) if err := m.usage.append(ev); err != nil { @@ -261,3 +298,16 @@ func memberwiseFrame(line []byte) requestFrame { } return frame } + +func protoName(proto uint8) string { + switch proto { + case tunnet.ProtoTCP: + return "tcp" + case tunnet.ProtoUDP: + return "udp" + case tunnet.ProtoICMP: + return "icmp" + default: + return strconv.Itoa(int(proto)) + } +} diff --git a/sandboxd/pool/template.go b/sandboxd/pool/template.go index f7139ace7..e9f9a9b0a 100644 --- a/sandboxd/pool/template.go +++ b/sandboxd/pool/template.go @@ -51,6 +51,9 @@ func (m *Manager) Promote(ctx context.Context, id string, cred Cred, template, t if !sb.Key.Capturable() { return types.PoolKey{}, "", ErrNoEgressFork } + if sb.TunOff { + return types.PoolKey{}, "", ErrTunOffPromote + } if err := m.diskErr(); err != nil { return types.PoolKey{}, "", fmt.Errorf("promote %s: %w", sb.ID, err) } diff --git a/sandboxd/pool/tun.go b/sandboxd/pool/tun.go new file mode 100644 index 000000000..3284e8c68 --- /dev/null +++ b/sandboxd/pool/tun.go @@ -0,0 +1,233 @@ +package pool + +import ( + "context" + "encoding/json/v2" + "fmt" + "maps" + "net/netip" + "slices" + + "github.com/projecteru2/core/log" + + "github.com/cocoonstack/sandbox/sandboxd/archives" + "github.com/cocoonstack/sandbox/sandboxd/tunnet" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +// tunSeeder is a node-local address store that learns which addresses the loaded claims hold. +type tunSeeder interface { + Seed(held map[string]netip.Addr) +} + +// tunState is the part of a claim SetTun changes. +type tunState struct { + off bool + profile string + addr netip.Addr +} + +// SetTun updates a claim's tun state without waking a hibernated or archived claim. +func (m *Manager) SetTun(ctx context.Context, id string, enabled *bool, profile *string) error { + sb, ok := m.byID(id) + if !ok { + return ErrUnknownSandbox + } + locked := sb.Transition.TryLock() + if locked { + defer sb.Transition.Unlock() + } + prev, next, assign, err := m.planTun(sb, locked, enabled, profile) + if err != nil { + return err + } + if assign { + if next.addr, err = m.tunAddrs.Assign(ctx, id); err != nil { + return fmt.Errorf("set tun %s: %w", id, err) + } + } + if next == prev { + if m.tunRowed(sb) { + return m.rewriteArchiveRow(ctx, sb, locked) + } + return nil + } + sock, rowed, err := m.commitTun(ctx, sb, locked, prev, next, assign) + if err != nil { + return err + } + switch { + case next.off: + m.out.DisarmTun(ctx, id, sock) + case sock != "": + m.out.RearmTun(ctx, sb) + } + if rowed { + if err := m.rewriteArchiveRow(ctx, sb, locked); err != nil { + return fmt.Errorf("set tun %s: %w", id, err) + } + } + return nil +} + +// TunStats snapshots the tun router's counters; false when the node has no tun block. +func (m *Manager) TunStats() (tunnet.Stats, bool) { + if m.tunRouter == nil { + return tunnet.Stats{}, false + } + return m.tunRouter.Stats(), true +} + +// assignTun gives each new claim of a tun pool its overlay address; a failure leaves that claim without tun, and a mesh never falls back to a local address. +func (m *Manager) assignTun(ctx context.Context, sbs []*types.Sandbox) { + if m.tunAddrs == nil { + return + } + v := m.view.Load().out + for _, sb := range sbs { + if !v.Tun(sb.PolicyKey()) { + continue + } + addr, err := m.tunAddrs.Assign(ctx, sb.ID) + if err != nil { + log.WithFunc("pool.assignTun").Warnf(ctx, "claim %s runs without tun: %v", sb.ID, err) + continue + } + sb.TunAddr = addr + } +} + +func (m *Manager) seedTun() { + seeder, ok := m.tunAddrs.(tunSeeder) + if !ok { + return + } + held := map[string]netip.Addr{} + m.mu.Lock() + for id, sb := range m.claimed { + if sb.TunAddr.IsValid() { + held[id] = sb.TunAddr + } + } + m.mu.Unlock() + seeder.Seed(held) +} + +func (m *Manager) releaseTun(ctx context.Context, id string, addr netip.Addr) { + if m.tunAddrs == nil || !addr.IsValid() { + return + } + if err := m.tunAddrs.Release(context.WithoutCancel(ctx), id, addr); err != nil { + log.WithFunc("pool.releaseTun").Warnf(ctx, "release overlay address %s of %s: %v", addr, id, err) + } +} + +func (m *Manager) releaseRowTun(ctx context.Context, rec archives.Record) { + if m.tunAddrs == nil { + return + } + sb := &types.Sandbox{} + if err := json.Unmarshal(rec.Claim, sb); err != nil { + log.WithFunc("pool.releaseRowTun").Warnf(ctx, "decode archive row %s: %v", rec.ID, err) + return + } + m.releaseTun(ctx, rec.ID, sb.TunAddr) +} + +// foldTunBytes copies every armed claim's live tun byte count into its record, so the next commit persists it. +func (m *Manager) foldTunBytes() { + if m.tunRouter == nil { + return + } + m.mu.Lock() + sbs := slices.Collect(maps.Values(m.claimed)) + m.mu.Unlock() + for _, sb := range sbs { + if used, ok := m.out.TunUsed(sb.ID); ok { + m.setTunBytes(sb, used) + } + } +} + +func (m *Manager) tunOn(sb *types.Sandbox) bool { + m.mu.Lock() + defer m.mu.Unlock() + return !sb.TunOff +} + +func (m *Manager) tunRowed(sb *types.Sandbox) bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.rowFollows(sb) || sb.ArchivePending != "" +} + +func (m *Manager) planTun(sb *types.Sandbox, locked bool, enabled *bool, profile *string) (prev, next tunState, assign bool, err error) { + m.mu.Lock() + defer m.mu.Unlock() + switch { + case m.claimed[sb.ID] != sb: + return prev, next, false, ErrUnknownSandbox + case m.pausedForMoveLocked(sb): + return prev, next, false, ErrEvacuating + } + prev = tunState{off: sb.TunOff, profile: sb.TunProfile, addr: sb.TunAddr} + next = prev + if enabled != nil { + next.off = !*enabled + } + if profile != nil { + next.profile = *profile + } + // only a disable runs beside a transition, which reads the rest of the tun state + if !locked && (!next.off || next.profile != prev.profile) { + return prev, next, false, ErrPaused + } + assign = !next.off && !prev.addr.IsValid() && m.tunAddrs != nil && m.view.Load().out.Tun(sb.PolicyKey()) + return prev, next, assign, nil +} + +// commitTun persists next on sb and returns the running guest's socket, empty when it is not running or a capture holds it. +func (m *Manager) commitTun( + ctx context.Context, sb *types.Sandbox, locked bool, prev, next tunState, assign bool, +) (sock string, rowed bool, err error) { + m.mu.Lock() + if m.claimed[sb.ID] != sb { + m.mu.Unlock() + if assign { + m.releaseTun(ctx, sb.ID, next.addr) + } + return "", false, ErrUnknownSandbox + } + sb.TunOff, sb.TunProfile, sb.TunAddr = next.off, next.profile, next.addr + if locked && runningShaped(sb) { + sock = sb.VsockSocket + } + rowed = m.rowFollows(sb) || sb.ArchivePending != "" + js := m.store.set(sb) + m.mu.Unlock() + if err = m.store.commit(js); err == nil { + return sock, rowed, nil + } + m.mu.Lock() + var rb claimSnapshot + undone := m.claimed[sb.ID] == sb && sb.TunOff == next.off && sb.TunProfile == next.profile && sb.TunAddr == next.addr + if undone { + sb.TunOff, sb.TunProfile, sb.TunAddr = prev.off, prev.profile, prev.addr + rb = m.store.set(sb) + } + m.mu.Unlock() + m.store.recommit(ctx, rb) + if undone && assign { + m.releaseTun(ctx, sb.ID, next.addr) + } + return "", false, fmt.Errorf("set tun %s: persist claims: %w", sb.ID, err) +} + +func (m *Manager) setTunBytes(sb *types.Sandbox, used int64) { + m.mu.Lock() + defer m.mu.Unlock() + sb.TunBytes = used + if m.claimed[sb.ID] == sb { + m.store.set(sb) + } +} diff --git a/sandboxd/pool/tun_test.go b/sandboxd/pool/tun_test.go new file mode 100644 index 000000000..4303fe69c --- /dev/null +++ b/sandboxd/pool/tun_test.go @@ -0,0 +1,401 @@ +package pool + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "net/netip" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/cocoonstack/sandbox/sandboxd/config" + "github.com/cocoonstack/sandbox/sandboxd/egress" + "github.com/cocoonstack/sandbox/sandboxd/engine" + "github.com/cocoonstack/sandbox/sandboxd/outbound/outboundtest" + "github.com/cocoonstack/sandbox/sandboxd/tunnet" + "github.com/cocoonstack/sandbox/sandboxd/types" +) + +var anyPolicy = &egress.Policy{Allow: []egress.Rule{{Host: "*"}}} + +func TestATunPoolClaimGetsAnAddressAndAPacketDoor(t *testing.T) { + eng, addrs := newFakeEngine(), &fakeAddrs{} + plain := tplKey("plain") + m := tunManager(t, eng, addrs, config.PoolSpec{PoolKey: testKey, Tun: true, Egress: anyPolicy}, config.PoolSpec{PoolKey: plain, Egress: anyPolicy}) + sb := mustClaim(t, m, testKey) + if !sb.TunAddr.IsValid() { + t.Fatal("a claim of a tun pool has no overlay address") + } + if !tunHello(t, sb) { + t.Error("the packet door of a tun claim refused the guest or answered off") + } + waitFor(t, func() bool { return slices.Equal(tunVerbsOf(eng), []string{"up " + sb.VsockSocket}) }) + if got := journaled(t, m)[sb.ID].TunAddr; got != sb.TunAddr { + t.Errorf("journaled address %s, want %s", got, sb.TunAddr) + } + other := mustClaim(t, m, plain) + if other.TunAddr.IsValid() { + t.Errorf("a claim of a pool without tun got the address %s", other.TunAddr) + } + if doorOpen(other) { + t.Error("a claim of a pool without tun has a packet door") + } + if got := tunVerbsOf(eng); len(got) != 1 { + t.Errorf("tun verbs %q, want none for the claim without tun", got) + } +} + +func TestReleaseFreesTheOverlayAddress(t *testing.T) { + eng, addrs := newFakeEngine(), &fakeAddrs{} + m := tunManager(t, eng, addrs, config.PoolSpec{PoolKey: testKey, Tun: true, Egress: anyPolicy}) + sb := mustClaim(t, m, testKey) + if err := m.Release(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatalf("release: %v", err) + } + if got, want := addrs.releasedList(), []string{sb.ID + " " + sb.TunAddr.String()}; !slices.Equal(got, want) { + t.Errorf("released %q, want %q", got, want) + } + if doorOpen(sb) { + t.Error("the packet door outlived the release") + } +} + +func TestAForkGetsItsOwnOverlayAddress(t *testing.T) { + eng, addrs := newFakeEngine(), &fakeAddrs{} + m := tunManager(t, eng, addrs, config.PoolSpec{PoolKey: testKey, Tun: true, Egress: anyPolicy}) + parent := mustClaim(t, m, testKey) + children, err := m.Fork(t.Context(), parent.ID, Cred{Token: parent.Token}, 2, time.Hour, "", "") + if err != nil { + t.Fatalf("fork: %v", err) + } + seen := map[netip.Addr]bool{parent.TunAddr: true} + for _, c := range children { + if !c.TunAddr.IsValid() || seen[c.TunAddr] { + t.Errorf("child %s has address %s, want its own", c.ID, c.TunAddr) + } + seen[c.TunAddr] = true + if !tunHello(t, c) { + t.Errorf("child %s has no live packet door", c.ID) + } + } +} + +func TestABranchOfADisabledClaimStaysDisabled(t *testing.T) { + eng, addrs := newFakeEngine(), &fakeAddrs{} + m := tunManager(t, eng, addrs, config.PoolSpec{PoolKey: testKey, Tun: true, Egress: anyPolicy}) + sb := mustClaim(t, m, testKey) + if err := m.SetTun(t.Context(), sb.ID, new(false), new("res")); err != nil { + t.Fatalf("set tun: %v", err) + } + ck, err := m.Checkpoint(t.Context(), sb.ID, Cred{Token: sb.Token}, "", "") + if err != nil { + t.Fatalf("checkpoint: %v", err) + } + branch, err := m.ClaimCheckpoint(t.Context(), ck.ID, ClaimOptions{TTL: time.Hour}) + if err != nil { + t.Fatalf("claim checkpoint: %v", err) + } + if !branch.TunOff || branch.TunProfile != "res" || doorOpen(branch) { + t.Errorf("branch tun_off %v profile %q door %v, want the source's switch kept", branch.TunOff, branch.TunProfile, doorOpen(branch)) + } + if _, _, err = m.Promote(t.Context(), sb.ID, Cred{Token: sb.Token}, "tpl", ""); !errors.Is(err, ErrTunOffPromote) { + t.Errorf("promote of a disabled claim: %v, want refused", err) + } +} + +func TestAFailedAssignLeavesTheClaimWithoutTun(t *testing.T) { + eng, addrs := newFakeEngine(), &fakeAddrs{err: errors.New("tun address store: connection refused")} + m := tunManager(t, eng, addrs, config.PoolSpec{PoolKey: testKey, Tun: true, Egress: anyPolicy}) + sb := mustClaim(t, m, testKey) + if sb.TunAddr.IsValid() { + t.Errorf("claim holds %s after the address store failed", sb.TunAddr) + } + if doorOpen(sb) { + t.Error("a claim without an address has a packet door") + } + if got := tunVerbsOf(eng); len(got) != 0 { + t.Errorf("tun verbs %q, want none", got) + } +} + +func TestSetTunRewritesAHibernatedClaimWithoutWakingIt(t *testing.T) { + eng, addrs := newFakeEngine(), &fakeAddrs{} + m := tunManager(t, eng, addrs, config.PoolSpec{PoolKey: testKey, Tun: true, Egress: anyPolicy}) + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + if err := m.SetTun(t.Context(), sb.ID, new(false), new("res")); err != nil { + t.Fatalf("set tun: %v", err) + } + if hibernated(m) != 1 { + t.Error("set tun woke the hibernated claim") + } + if got := journaled(t, m)[sb.ID]; !got.TunOff || got.TunProfile != "res" || got.TunAddr != sb.TunAddr { + t.Errorf("journaled tun_off %v profile %q address %s, want off, res and the address kept", got.TunOff, got.TunProfile, got.TunAddr) + } + if err := m.Wake(t.Context(), sb.ID, Cred{Token: sb.Token}); err != nil { + t.Fatalf("wake: %v", err) + } + if doorOpen(sb) { + t.Error("a disabled claim got its packet door back at the wake") + } +} + +func TestDisablingARunningClaimRevokesItsPacketPath(t *testing.T) { + eng, addrs := newFakeEngine(), &fakeAddrs{} + m := tunManager(t, eng, addrs, config.PoolSpec{PoolKey: testKey, Tun: true, Egress: anyPolicy}) + sb := mustClaim(t, m, testKey) + stream := openStream(t, sb) + if err := m.SetTun(t.Context(), sb.ID, new(false), nil); err != nil { + t.Fatalf("disable: %v", err) + } + _ = stream.SetReadDeadline(time.Now().Add(2 * time.Second)) + if _, err := stream.Read(make([]byte, 1)); !errors.Is(err, io.EOF) { + t.Errorf("the established stream read %v, want EOF", err) + } + if doorOpen(sb) { + t.Error("the packet door still accepts after the disable") + } + waitFor(t, func() bool { + return slices.Equal(tunVerbsOf(eng), []string{"up " + sb.VsockSocket, "down " + sb.VsockSocket}) + }) + if err := m.SetTun(t.Context(), sb.ID, new(true), nil); err != nil { + t.Fatalf("enable: %v", err) + } + if !tunHello(t, sb) { + t.Error("the re-enabled claim has no live packet door") + } +} + +func TestADisableTheWakeMissedStillTurnsTheGuestOff(t *testing.T) { + eng, addrs := newFakeEngine(), &fakeAddrs{} + m := tunManager(t, eng, addrs, config.PoolSpec{PoolKey: testKey, Tun: true, Egress: anyPolicy}) + sb := mustClaim(t, m, testKey) + m.mu.Lock() + sb.TunOff = true + m.mu.Unlock() + if tunHello(t, sb) { + t.Error("a claim disabled after its door was bound still answered on") + } +} + +func TestOnlyADisableRunsBesideATransition(t *testing.T) { + eng, addrs := newFakeEngine(), &fakeAddrs{} + m := tunManager(t, eng, addrs, config.PoolSpec{PoolKey: testKey, Tun: true, Egress: anyPolicy}) + sb := mustClaim(t, m, testKey) + hibernateClaim(t, m, sb) + sb.Transition.Lock() + for _, req := range []struct { + enabled *bool + profile *string + }{{new(true), nil}, {nil, new("res")}} { + if err := m.SetTun(t.Context(), sb.ID, req.enabled, req.profile); !errors.Is(err, ErrPaused) { + t.Errorf("set tun %v %v during a transition: %v, want ErrPaused", req.enabled, req.profile, err) + } + } + if err := m.SetTun(t.Context(), sb.ID, new(false), nil); err != nil { + t.Errorf("disable during a transition: %v", err) + } + sb.Transition.Unlock() +} + +func TestASetTunResendRewritesTheArchiveRowAFailedWriteMissed(t *testing.T) { + c := newFailoverCluster(t) + a := c.node(t, "a", t.TempDir(), keepAnHour) + sb := mustClaim(t, a, testKey) + mustArchive(t, a, sb) + reg := &hookRegistry{Registry: c.reg, failPut: true} + a.archives = reg + if err := a.SetTun(t.Context(), sb.ID, new(false), nil); err == nil { + t.Fatal("disable with the registry down succeeded") + } + reg.failPut = false + if err := a.SetTun(t.Context(), sb.ID, new(false), nil); err != nil { + t.Fatalf("resend: %v", err) + } + if !rowClaim(t, c, sb).TunOff { + t.Error("the archive row still has the claim on after the resend") + } +} + +func TestFlushClaimsPersistsTheLiveTunBytes(t *testing.T) { + eng, addrs := newFakeEngine(), &fakeAddrs{} + m := tunManager(t, eng, addrs, config.PoolSpec{PoolKey: testKey, Tun: true, Egress: anyPolicy}) + sb := mustClaim(t, m, testKey) + stream := openStream(t, sb) + pkt := udpFrame(netip.MustParseAddr("93.184.216.34")) + if _, err := stream.Write(binary.LittleEndian.AppendUint32(nil, uint32(len(pkt)))); err != nil { + t.Fatal(err) + } + if _, err := stream.Write(pkt); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { used, _ := m.out.TunUsed(sb.ID); return used > 0 }) + if err := m.FlushClaims(); err != nil { + t.Fatalf("flush: %v", err) + } + if got := journaled(t, m)[sb.ID].TunBytes; got != int64(len(pkt)-tunnet.VnetHdrLen) { + t.Errorf("journaled tun bytes %d, want the %d sent", got, len(pkt)-tunnet.VnetHdrLen) + } +} + +func TestATunFlowEndWritesOneAuditRecord(t *testing.T) { + dir := t.TempDir() + cfg := &config.Config{DataDir: dir, AuditLog: true} + m, err := NewManager(t.Context(), cfg, newFakeEngine(), testSecrets(t), testBackends(t, cfg)) + if err != nil { + t.Fatalf("manager: %v", err) + } + start := time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC) + m.recordTunFlow(t.Context(), "sb_t", "acme", "", tunnet.FlowRecord{ + Proto: tunnet.ProtoUDP, Dst: netip.MustParseAddr("93.184.216.34"), Port: 443, Allowed: true, + Up: 1200, Down: 64000, Start: start, End: start.Add(time.Minute), + }) + raw, err := os.ReadFile(filepath.Join(dir, "audit.jsonl")) + if err != nil { + t.Fatalf("read audit journal: %v", err) + } + want := `{"t":"2026-11-05T12:01:00Z","id":"sb_t","op":"tun_flow","path":"tun","tenant":"acme","proto":"udp","dst_addr":"93.184.216.34:443",` + + `"decision":"allow","sent":1200,"received":64000,"start":"2026-11-05T12:00:00Z","end":"2026-11-05T12:01:00Z","profile":"default"}` + if got := strings.TrimSpace(string(raw)); got != want { + t.Errorf("audit record\n%s\nwant\n%s", got, want) + } +} + +func tunManager(t *testing.T, eng *fakeEngine, addrs tunnet.Addresses, pools ...config.PoolSpec) *Manager { + t.Helper() + eng.sockRoot = outboundtest.SockRoot(t) + tun := &config.TunConfig{Subnet: "240.0.0.0/4", Listen: "127.0.0.1:4789", VNI: 1, Port: 4789, MTU: 1450, Exits: map[string]config.TunExit{"default": {VTEP: "127.0.0.2", MAC: "02:aa:00:00:00:01"}}} + cfg := &config.Config{DataDir: t.TempDir(), Tun: tun, Pools: pools} + b := testBackends(t, cfg) + b.TunRouter, b.TunAddrs = tunnet.New(fakeUnderlay{}, tun.Overlay()), addrs + m, err := NewManager(t.Context(), cfg, eng, testSecrets(t), b) + if err != nil { + t.Fatalf("manager: %v", err) + } + t.Cleanup(func() { settleCaptures(m) }) + return m +} + +// tunHello sends a guest hello on sb's packet door and reports whether the host answered on. +func tunHello(t *testing.T, sb *types.Sandbox) bool { + t.Helper() + conn, err := net.Dial("unix", engine.TunSocketPath(sb.VsockSocket)) + if err != nil { + return false + } + defer func() { _ = conn.Close() }() + return helloOn(t, conn) +} + +// openStream returns a packet stream the host turned on. +func openStream(t *testing.T, sb *types.Sandbox) net.Conn { + t.Helper() + conn, err := net.Dial("unix", engine.TunSocketPath(sb.VsockSocket)) + if err != nil { + t.Fatalf("dial the packet door: %v", err) + } + t.Cleanup(func() { _ = conn.Close() }) + if !helloOn(t, conn) { + t.Fatal("the host answered off") + } + return conn +} + +func helloOn(t *testing.T, conn net.Conn) bool { + t.Helper() + hello := tunnet.Hello{Version: tunnet.Version}.Encode() + if _, err := conn.Write(hello[:]); err != nil { + t.Fatalf("write hello: %v", err) + } + _ = conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + got, err := tunnet.ReadHello(conn) + if err != nil { + t.Fatalf("read host hello: %v", err) + } + _ = conn.SetReadDeadline(time.Time{}) + return got.Flags&tunnet.FlagOn != 0 +} + +func doorOpen(sb *types.Sandbox) bool { + conn, err := net.Dial("unix", engine.TunSocketPath(sb.VsockSocket)) + if err != nil { + return false + } + _ = conn.Close() + return true +} + +// udpFrame is a guest UDP datagram to dst with a virtio-net header and no checksum. +func udpFrame(dst netip.Addr) []byte { + f := make([]byte, tunnet.VnetHdrLen+20+8+16) + ip := f[tunnet.VnetHdrLen:] + ip[0], ip[8], ip[9] = 0x45, 64, tunnet.ProtoUDP + binary.BigEndian.PutUint16(ip[2:], uint16(len(ip))) + s, d := tunnet.GuestAddr.As4(), dst.As4() + copy(ip[12:], s[:]) + copy(ip[16:], d[:]) + binary.BigEndian.PutUint16(ip[20:], 40000) + binary.BigEndian.PutUint16(ip[22:], 443) + binary.BigEndian.PutUint16(ip[24:], 24) + return f +} + +func tunVerbsOf(eng *fakeEngine) []string { + eng.mu.Lock() + defer eng.mu.Unlock() + return slices.Clone(eng.tunVerbs) +} + +type fakeAddrs struct { + mu sync.Mutex + err error + next int + held map[string]netip.Addr + released []string +} + +func (f *fakeAddrs) Assign(_ context.Context, id string) (netip.Addr, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.err != nil { + return netip.Addr{}, f.err + } + if addr, ok := f.held[id]; ok { + return addr, nil + } + if f.held == nil { + f.held = map[string]netip.Addr{} + } + f.next++ + addr := tunnet.HostAt(netip.MustParsePrefix("240.0.0.0/4"), uint64(f.next)) + f.held[id] = addr + return addr, nil +} + +func (f *fakeAddrs) Release(_ context.Context, id string, addr netip.Addr) error { + f.mu.Lock() + defer f.mu.Unlock() + delete(f.held, id) + f.released = append(f.released, fmt.Sprintf("%s %s", id, addr)) + return nil +} + +func (f *fakeAddrs) releasedList() []string { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.released) +} + +type fakeUnderlay struct{} + +func (fakeUnderlay) Send([]byte, int, netip.AddrPort, uint64) error { return nil } + +func (fakeUnderlay) Recv(func([]byte, netip.AddrPort)) error { return net.ErrClosed } diff --git a/sandboxd/server/metrics.go b/sandboxd/server/metrics.go index ec6f6a85e..d299ef44d 100644 --- a/sandboxd/server/metrics.go +++ b/sandboxd/server/metrics.go @@ -8,6 +8,7 @@ import ( "slices" "github.com/cocoonstack/sandbox/sandboxd/pool" + "github.com/cocoonstack/sandbox/sandboxd/tunnet" ) func (s *Server) handleMetrics(w http.ResponseWriter, _ *http.Request) { @@ -95,6 +96,27 @@ func (s *Server) handleMetrics(w http.ResponseWriter, _ *http.Request) { metric(row.name, "counter", row.help) _, _ = fmt.Fprintf(w, "sandboxd_%s %d\n", row.name, row.value) } + if st, ok := s.mgr.TunStats(); ok { + writeTunStats(w, metric, st) + } +} + +func writeTunStats(w io.Writer, metric func(name, kind, help string), st tunnet.Stats) { + metric("tun_frames_total", "counter", "packet frames on the claims' vsock streams, by direction from the guest") + _, _ = fmt.Fprintf(w, "sandboxd_tun_frames_total{dir=\"out\"} %d\n", st.FramesIn) + _, _ = fmt.Fprintf(w, "sandboxd_tun_frames_total{dir=\"in\"} %d\n", st.FramesOut) + metric("tun_datagrams_total", "counter", "VXLAN datagrams on the underlay, by direction from the guest") + _, _ = fmt.Fprintf(w, "sandboxd_tun_datagrams_total{dir=\"out\"} %d\n", st.DgramsOut) + _, _ = fmt.Fprintf(w, "sandboxd_tun_datagrams_total{dir=\"in\"} %d\n", st.DgramsIn) + metric("tun_arp_total", "counter", "ARP replies to exits and gratuitous ARPs sent") + _, _ = fmt.Fprintf(w, "sandboxd_tun_arp_total{kind=\"answered\"} %d\n", st.ARPAnswered) + _, _ = fmt.Fprintf(w, "sandboxd_tun_arp_total{kind=\"announced\"} %d\n", st.ARPSent) + metric("tun_drops_total", "counter", "packets the tun path did not carry, by reason") + for i, n := range st.Drops { + if reason := tunnet.Drop(i).String(); reason != "" { + _, _ = fmt.Fprintf(w, "sandboxd_tun_drops_total{reason=%q} %d\n", reason, n) + } + } } func writeClaimLatency(w io.Writer, tier string, latency pool.ClaimLatency) { diff --git a/sandboxd/server/server.go b/sandboxd/server/server.go index ca0ae9bbb..a78d4cefb 100644 --- a/sandboxd/server/server.go +++ b/sandboxd/server/server.go @@ -24,6 +24,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/pool" "github.com/cocoonstack/sandbox/sandboxd/store/peer" "github.com/cocoonstack/sandbox/sandboxd/tenants" + "github.com/cocoonstack/sandbox/sandboxd/tunnet" "github.com/cocoonstack/sandbox/sandboxd/types" "github.com/cocoonstack/sandbox/sandboxd/usage" ) @@ -67,6 +68,7 @@ var poolErrHTTP = []struct { {pool.ErrNoEgress, http.StatusConflict, ""}, {pool.ErrNoEgressHibernate, http.StatusConflict, ""}, {pool.ErrNoEgressFork, http.StatusConflict, ""}, + {pool.ErrTunOffPromote, http.StatusConflict, ""}, {pool.ErrVolumeCapture, http.StatusConflict, ""}, {pool.ErrVolumeBusy, http.StatusConflict, ""}, {pool.ErrVolumeRegistrationConflict, http.StatusConflict, pool.ErrVolumeRegistrationConflict.Error()}, @@ -116,6 +118,8 @@ type Manager interface { SetEnv(ctx context.Context, id string, env types.Env, tenant string) error PatchEnv(ctx context.Context, id string, patch types.EnvPatch, tenant string) error Env(id, tenant string) (types.Env, error) + SetTun(ctx context.Context, id string, enabled *bool, profile *string) error + TunStats() (tunnet.Stats, bool) Audit(ctx context.Context, id string, line []byte) AuditEnabled() bool ClaimCheckpoint(ctx context.Context, ckptID string, o pool.ClaimOptions) (*types.Sandbox, error) @@ -227,6 +231,12 @@ type SandboxEnvPatch struct { Env types.EnvPatch `json:"env"` } +// SandboxTun is the wire body of PUT /v1/sandboxes/{id}/tun; an omitted field keeps the claim's value, and an empty profile means "default". +type SandboxTun struct { + Enabled *bool `json:"enabled,omitempty"` + Profile *string `json:"profile,omitempty"` +} + // TenantsRequest is the wire body of PUT /v1/tenants: the whole tenant set, an entry without a token keeping that tenant's. type TenantsRequest struct { Tenants []tenants.TenantSpec `json:"tenants"` @@ -324,6 +334,7 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("GET /v1/sandboxes/{id}", s.requireRoot(s.handleSandbox)) mux.HandleFunc("GET /v1/sandboxes/{id}/stats", s.requireRoot(s.handleSandboxStats)) mux.HandleFunc("PUT /v1/sandboxes/{id}/instance-metadata", s.requireRoot(s.handleInstanceMetadata)) + mux.HandleFunc("PUT /v1/sandboxes/{id}/tun", s.requireRoot(s.handleSetTun)) mux.HandleFunc("GET /v1/sandboxes/{id}/env", s.requireToken(s.handleGetEnv)) mux.HandleFunc("PUT /v1/sandboxes/{id}/env", s.requireToken(s.handleSetEnv)) mux.HandleFunc("PATCH /v1/sandboxes/{id}/env", s.requireToken(s.handlePatchEnv)) @@ -605,6 +616,20 @@ func (s *Server) handlePatchEnv(w http.ResponseWriter, r *http.Request) { writeResult(w, r, "env", id, "patch env failed", err, noContent(w)) } +func (s *Server) handleSetTun(w http.ResponseWriter, r *http.Request) { + req, ok := decodeBodyStrict[SandboxTun](w, r) + if !ok { + return + } + if p := req.Profile; p != nil && *p != "" && !types.NameRe.MatchString(*p) { + writeErr(w, http.StatusBadRequest, fmt.Sprintf("profile %q must match %s", *p, types.NameRe)) + return + } + id := r.PathValue("id") + err := s.mgr.SetTun(r.Context(), id, req.Enabled, req.Profile) + writeResult(w, r, "tun", id, "set tun failed", err, noContent(w)) +} + func (s *Server) handleSandboxVerb(verb string, do func(ctx context.Context, id string, cred pool.Cred) error) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { token, ok := sandboxToken(w, r) diff --git a/sandboxd/server/server_test.go b/sandboxd/server/server_test.go index fbcd18e83..b629cb317 100644 --- a/sandboxd/server/server_test.go +++ b/sandboxd/server/server_test.go @@ -22,6 +22,7 @@ import ( "github.com/cocoonstack/sandbox/sandboxd/pool" "github.com/cocoonstack/sandbox/sandboxd/store/peer" "github.com/cocoonstack/sandbox/sandboxd/tenants" + "github.com/cocoonstack/sandbox/sandboxd/tunnet" "github.com/cocoonstack/sandbox/sandboxd/types" ) @@ -2209,6 +2210,46 @@ func TestExpireActionThreadsThroughEveryDecoder(t *testing.T) { } } +func TestTunVerbIsRootOnlyAndPassesOnlyTheNamedFields(t *testing.T) { + var gotEnabled *bool + var gotProfile *string + mgr := &fakeManager{setTun: func(id string, enabled *bool, profile *string) error { + if id != "sb_1" { + return pool.ErrUnknownSandbox + } + gotEnabled, gotProfile = enabled, profile + return nil + }} + ts := newTenantTestServer(t, "sekret", []tenants.TenantSpec{{Name: "acme", Token: "acme-tok"}}, mgr, nil) + put := func(token, id, body string) int { + t.Helper() + return statusOf(t, http.MethodPut, ts.URL+"/v1/sandboxes/"+id+"/tun", token, body) + } + if code := put("sekret", "sb_1", `{"enabled": false}`); code != http.StatusNoContent { + t.Fatalf("status %d, want 204", code) + } + if gotEnabled == nil || *gotEnabled || gotProfile != nil { + t.Errorf("manager got enabled %v profile %v, want false and no profile", gotEnabled, gotProfile) + } + if code := put("sekret", "sb_1", `{"profile": "res"}`); code != http.StatusNoContent || gotEnabled != nil || gotProfile == nil || *gotProfile != "res" { + t.Errorf("profile only: status %d, enabled %v profile %v", code, gotEnabled, gotProfile) + } + for _, tt := range []struct { + name, token, id, body string + want int + }{ + {"tenant token", "acme-tok", "sb_1", `{"enabled": true}`, http.StatusForbidden}, + {"no token", "", "sb_1", `{"enabled": true}`, http.StatusUnauthorized}, + {"bad profile", "sekret", "sb_1", `{"profile": "two words"}`, http.StatusBadRequest}, + {"unknown field", "sekret", "sb_1", `{"quota": 1}`, http.StatusBadRequest}, + {"unknown sandbox", "sekret", "sb_2", `{"enabled": true}`, http.StatusNotFound}, + } { + if code := put(tt.token, tt.id, tt.body); code != tt.want { + t.Errorf("%s: status %d, want %d", tt.name, code, tt.want) + } + } +} + type ownerReply struct { OwnerAddr string `json:"owner_addr"` } @@ -2270,6 +2311,7 @@ type fakeManager struct { deleteCheckpoint idActionFunc setPools func(pools []config.PoolSpec) error setInstanceMetadata func(id string, doc []byte) error + setTun func(id string, enabled *bool, profile *string) error infoPools []pool.PoolInfo infoTemplates []pool.TemplateInfo labeled []string @@ -2568,6 +2610,17 @@ func (f *fakeManager) SetInstanceMetadata(_ context.Context, id string, doc []by return f.setInstanceMetadata(id, doc) } +func (f *fakeManager) SetTun(_ context.Context, id string, enabled *bool, profile *string) error { + if f.setTun == nil { + return nil + } + return f.setTun(id, enabled, profile) +} + +func (f *fakeManager) TunStats() (tunnet.Stats, bool) { + return tunnet.Stats{}, false +} + func (f *fakeManager) Wake(_ context.Context, id string, cred pool.Cred) error { if f.wake == nil { return nil diff --git a/sandboxd/types/types.go b/sandboxd/types/types.go index 82cf05505..8e740c041 100644 --- a/sandboxd/types/types.go +++ b/sandboxd/types/types.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "iter" + "net/netip" "regexp" "slices" "strings" @@ -287,6 +288,14 @@ type Sandbox struct { VsockSocket string `json:"vsock_socket,omitempty"` // TAP is the egress-lane NIC's host tap; empty on the none lane. TAP string `json:"tap,omitempty"` + // TunAddr is the claim's overlay address in the advanced network mode; zero when its pool has no tun. + TunAddr netip.Addr `json:"tun_addr,omitzero"` + // TunOff disables the claim's advanced network mode; only the root sets it. + TunOff bool `json:"tun_off,omitzero"` + // TunProfile labels the claim's exit; empty means "default". + TunProfile string `json:"tun_profile,omitempty"` + // TunBytes counts the claim's tun bytes both ways against its quota. + TunBytes int64 `json:"tun_bytes,omitzero"` // HibernateSnap names the memory snapshot while the VM is hibernated; empty means running. HibernateSnap string `json:"hibernate_snap,omitempty"` // PendingSnap is the journaled intent of a hibernate in flight, cleared by its commit. @@ -420,6 +429,9 @@ type Checkpoint struct { // PolicySource and NoEgress carry the source sandbox's egress to a branch. PolicySource PoolKey `json:"policy_source,omitzero"` NoEgress bool `json:"no_egress,omitzero"` + // TunOff and TunProfile carry the operator's tun switch and exit label to a branch, so a branch cannot turn tun back on. + TunOff bool `json:"tun_off,omitzero"` + TunProfile string `json:"tun_profile,omitempty"` // GuestEnv marks a capture whose guest holds an env file a branch must replace. GuestEnv bool `json:"guest_env,omitzero"` From 2fbdb2055e98fcaa3861c79c01758bfef4522827 Mon Sep 17 00:00:00 2001 From: CMGS Date: Fri, 6 Nov 2026 01:25:33 +0800 Subject: [PATCH 03/10] docs: the advanced network mode's host side, exits and configuration --- docs/advanced-network.md | 160 ++++++++++++++++++++++++++++++++++----- docs/deploy.md | 16 ++-- docs/egress.md | 4 +- docs/sandboxd-api.md | 53 ++++++++++++- 4 files changed, 203 insertions(+), 30 deletions(-) diff --git a/docs/advanced-network.md b/docs/advanced-network.md index e32880dc7..3a781aa9d 100644 --- a/docs/advanced-network.md +++ b/docs/advanced-network.md @@ -114,26 +114,146 @@ lost; TCP retransmits. ## Host side -The host side is sandboxd; its design is -[#471](https://github.com/cocoonstack/sandbox/issues/471). Whatever it does -with the packets, it owes the guest this contract: - -- **One stream per claim.** The packet stream arrives on the claim's own - vsock socket, so the stream, not any address in a packet, names the claim. - Every guest uses `100.64.0.2`; the host must not trust a source address the - guest wrote and must deliver only the claim's own return traffic on its - stream, with the destination set back to `100.64.0.2`. -- **Enforcement on the host.** The host answers `ON` only for a claim that is - allowed packet mode, and answers off, closes the stream or refuses the dial - otherwise; revocation closes the stream. Nothing depends on the guest - obeying `tun_down`. -- **Checksums after a rewrite.** A host that changes an address updates the - IPv4 header checksum and the L4 checksum incrementally. For a frame with - `VIRTIO_NET_HDR_F_NEEDS_CSUM` the L4 checksum sits at - `csum_start + csum_offset` and holds the uncomplemented pseudo-header sum; - a UDP checksum of 0 stays 0. With GSO this happens once per frame. -- **Offloads it offered.** The host sends only GSO types the guest's hello - offered and segments the rest itself. +sandboxd needs no TUN, no kernel routes and no NAT rules on the node. It +carries each claim's packets over VXLAN, on an ordinary UDP socket, to an +**exit**: plain Linux configuration on the same host or on a gateway machine. +The design is [#471](https://github.com/cocoonstack/sandbox/issues/471). + +``` +guest -> tun0 -> silkd --vsock 2051--> sandboxd --VXLAN--> exit (vxlan0, forward, NAT) -> uplink +``` + +- **Overlay address**: each tun-enabled claim gets a unique address from + `tun.subnet`, kept across hibernate, archive and move and freed at release. + A single node allocates it in memory and keeps it in `claims.json`, with no + disk write on the claim path; each release is appended to + `/tun-freed.log` so its reuse window outlives a restart. A node + with a `mesh` must have `meta_store.cell` and allocates it in the cell's + table, one round trip per claim, and stays off (proxy mode only) while the + meta store is unreachable. A freed address is reused after + `tun.reuse_after_seconds`, so the subnet's size over that window bounds the + sustained claim rate: a `/8` with the 5-day default carries about 38 new tun + claims a second across the cell. The claim's MAC is `02:00:` followed by the + address bytes. +- **Outbound**: the source address becomes the claim's overlay address + whatever the guest wrote; a GSO frame is cut into wire-sized packets with + full checksums; each gets an Ethernet header (claim MAC to the exit's MAC) + and a VXLAN header and leaves for the exit picked by the claim's profile + label (`default` when it has none). The outer source port follows a hash of + the inner 5-tuple. +- **Inbound**: a VXLAN packet from an exit's VTEP whose inner destination is + a claim on this node goes back to `100.64.0.2` and onto that claim's stream; + anything else is dropped. In-order TCP segments of one flow whose checksums + verify are merged into one GSO frame for the guest. An ICMP error gets the + quoted packet's source rewritten to `100.64.0.2` too, so path MTU discovery + and port-unreachable reach the guest's socket. +- **ARP**: sandboxd answers ARP from the exits for its own claims only, + announces a claim with a gratuitous ARP when it is armed (claim, wake, + move-in), after a reload of `tun.exits` and every 45 seconds, and never + forwards a frame the guest built. +- **Policy**: a flow table keyed by 5-tuple; the first packet of a flow is + judged by `EvalFlow` (bare `host: "*"` rules only) and the internal-address + guard, which also refuses the overlay subnet and `100.64.0.0/30`. + Per-claim byte, packet and new-flow rates, a flow cap and a byte quota apply + before any further work on a packet; a GSO frame is charged for every packet + and header it puts on the wire. The limits and flows belong to the claim, so + a guest that reconnects its stream starts with what it had used. +- **Untrusted input**: frames, virtio-net headers and IPv4 headers are bounds + checked; non-IPv4, IP options, fragments, a packet or GSO segment larger + than `tun.mtu`, a GSO type that does not match the protocol and a partial + checksum outside the L4 checksum field are dropped and counted per reason. +- **Enforcement**: the packet port answers `ON` only for an armed, + tun-enabled claim; disabling a claim ends its stream, drops its flows and + stops its ARP answers. The claim switch and the profile label change only + with the operator token, through + [`PUT /v1/sandboxes/{id}/tun`](sandboxd-api.md#put-v1sandboxesidtun). +- **Audit and usage**: with `audit_log` on, each flow writes one + `op:"tun_flow"` record when it ends ([fields](deploy.md#configuration)); with + `egress_usage_bytes` on, each allowed flow also appends an `egress_bytes` + usage event with its IP bytes both ways, the destination address and port, + and `upstream` `tun/