diff --git a/.github/contributors.yaml b/.github/contributors.yaml index 7f7307a3d..728bbe3d8 100644 --- a/.github/contributors.yaml +++ b/.github/contributors.yaml @@ -128,3 +128,6 @@ users: OdysseasKalaitsidis: name: Odysseas Kalaitsidis email: odysseaskalaitsides@gmail.com + alimx07: + name: Ali Mohamed + email: amx746@gmail.com diff --git a/.github/linters/urunc-dict.txt b/.github/linters/urunc-dict.txt index 933cca0b1..4e35e0e68 100644 --- a/.github/linters/urunc-dict.txt +++ b/.github/linters/urunc-dict.txt @@ -433,3 +433,15 @@ hyperlight Hyperlight Odysseas Kalaitsidis +lhlog +dnat +PREROUTING +resolv +ndots +clsact +Sysctls +localnet +skbedit +ptype +pedit +nslookup diff --git a/Makefile b/Makefile index b0329d2dc..a5105e518 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,7 @@ URUNC_SRC += $(wildcard $(CURDIR)/pkg/unikontainers/unikernels/*.go) URUNC_SRC += $(wildcard $(CURDIR)/pkg/unikontainers/types/*.go) URUNC_SRC += $(wildcard $(CURDIR)/pkg/unikontainers/initrd/*.go) URUNC_SRC += $(wildcard $(CURDIR)/pkg/network/*.go) +URUNC_SRC += $(wildcard $(CURDIR)/pkg/network/localhost/*.go) SHIM_SRC := $(wildcard $(CURDIR)/cmd/containerd-shim-urunc-v2/*.go) SHIM_SRC += $(wildcard $(CURDIR)/pkg/containerd-shim/*.go) SHIM_SRC += $(wildcard $(CURDIR)/pkg/containerd-shim/containerd/*.go) @@ -233,7 +234,7 @@ test: unittest e2etest ## unittest Run all unit tests .PHONY: unittest -unittest: test_unikontainers test_metrics test_network test_hypervisors test_unikernels +unittest: test_unikontainers test_metrics test_network test_localhost test_hypervisors test_unikernels ## e2etest Run all end-to-end tests .PHONY: e2etest @@ -257,6 +258,12 @@ test_network: @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./pkg/network -v @echo " " +## test_localhost Run unit tests for network/localhost package +test_localhost: + @echo "Unit testing in pkg/network/localhost" + @GOFLAGS=$(TEST_FLAGS) $(GO) test $(TEST_OPTS) ./pkg/network/localhost -v + @echo " " + ## test_hypervisors Run unit tests for hypervisors package test_hypervisors: @echo "Unit testing in hypervisors" diff --git a/internal/constants/network_constants.go b/internal/constants/network_constants.go index de8a30ccb..640fe8574 100644 --- a/internal/constants/network_constants.go +++ b/internal/constants/network_constants.go @@ -18,6 +18,7 @@ const ( StaticNetworkTapIP = "172.16.1.1" StaticNetworkUnikernelIP = "172.16.1.2" // TODO: Experiment with DynamicNetworkTapIP starting from 172.16.X.1 - DynamicNetworkTapIP = "172.16.X.2" - QueueProxyRedirectIP = "172.16.1.2" + DynamicNetworkTapIP = "172.16.X.2" + QueueProxyRedirectIP = "172.16.1.2" + LocalhostDNSResolverIP = "192.168.100.100" ) diff --git a/pkg/network/localhost/docker.go b/pkg/network/localhost/docker.go new file mode 100644 index 000000000..9d7b69af2 --- /dev/null +++ b/pkg/network/localhost/docker.go @@ -0,0 +1,112 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package localhost + +import ( + "errors" + "fmt" + "net" + "os/exec" + "strings" + "syscall" + + "github.com/vishvananda/netlink" +) + +func dockerRules(f *Forwarder) error { + + tcpPort, udpPort, err := dockerDNSPorts(f.LoIP) + if err != nil { + return err + } + lhlog.Debugf("Docker DNS resolver: tcp/%s udp/%s", tcpPort, udpPort) + + if err := dnat(f.VirtIP, "udp", net.JoinHostPort(f.LoIP.String(), udpPort)); err != nil { + return err + } + if err := dnat(f.VirtIP, "tcp", net.JoinHostPort(f.LoIP.String(), tcpPort)); err != nil { + return err + } + lhlog.Debug("Applied Docker DNAT rules") + + return nil +} + +func dockerDNSPorts(loIP net.IP) (string, string, error) { + var tcpPort, udpPort string + + // syscall.AF_INET : IPv4 + tcp, err := netlink.SocketDiagTCPInfo(syscall.AF_INET) + if err != nil { + return "", "", fmt.Errorf("could not list TCP sockets in namespace: %w", err) + } + for _, s := range tcp { + if s.InetDiagMsg.ID.Source.String() == loIP.String() && s.InetDiagMsg.ID.Destination.String() == "0.0.0.0" && int(s.InetDiagMsg.State) == netlink.TCP_LISTEN { + tcpPort = fmt.Sprintf("%d", s.InetDiagMsg.ID.SourcePort) + break + } + } + udp, err := netlink.SocketDiagUDPInfo(syscall.AF_INET) + if err != nil { + return "", "", fmt.Errorf("could not list UDP sockets in namespace: %w", err) + } + for _, s := range udp { + + // As udp has no connection states as tcp. by default (which is our case) it should have tcp_close (7) state to read it as "no peer association" + // TODO: is there any other states that could make this condition fails ? + if s.InetDiagMsg.ID.Source.String() == loIP.String() && s.InetDiagMsg.ID.Destination.String() == "0.0.0.0" && int(s.InetDiagMsg.State) == netlink.TCP_CLOSE { + udpPort = fmt.Sprintf("%d", s.InetDiagMsg.ID.SourcePort) + break + } + } + if tcpPort == "" || udpPort == "" { + return "", "", fmt.Errorf("could not find docker embedded resolver ports for %s (tcp=%q udp=%q)", loIP, tcpPort, udpPort) + } + return tcpPort, udpPort, nil +} + +// ClearRules removes DNAT rules dockerRules could have added. +func clearRules() error { + + // This will be run from kill() so no memory state could be used. + // we list all IPtable rules from scratch. + + // iptables-save output example: + // -A PREROUTING -d 192.168.100.100/32 -i lo -p udp -j DNAT --to-destination 127.0.0.11:60269 + ipt, err := exec.LookPath("iptables-save") + if err != nil { + return err + } + out, err := exec.Command(ipt, "-t", "nat").Output() //nolint:gosec + if err != nil { + return fmt.Errorf("iptables-save failed: %w", err) + } + + var retErr error + for line := range strings.SplitSeq(string(out), "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || fields[0] != "-A" || fields[1] != "PREROUTING" { + continue + } + if !strings.Contains(line, "-i lo") || !strings.Contains(line, "-j DNAT") { + continue + } + args := append([]string{"-t", "nat", "-D"}, fields[1:]...) + if err := ipTablesExec(args); err != nil { + retErr = errors.Join(retErr, err) + } + } + return retErr +} diff --git a/pkg/network/localhost/local_dns.go b/pkg/network/localhost/local_dns.go new file mode 100644 index 000000000..325260a9e --- /dev/null +++ b/pkg/network/localhost/local_dns.go @@ -0,0 +1,332 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package localhost + +import ( + "bytes" + "errors" + "fmt" + "net" + "os" + "os/exec" + "strings" + + "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" +) + +var lhlog = logrus.WithField("subsystem", "network") + +// rules installs the custom part of the forwarding for some scenario. +type rules func(*Forwarder) error + +type Forwarder struct { + VirtIP net.IP // virtual resolver IP the guest dials + LoIP net.IP // loopback IP the real resolver listens on + ResolvConf string // host path of the container's resolv.conf + custom rules // scenario-specific extra rules, picked by Detect +} + +func Detect(resolvConf string, virtIP string) (*Forwarder, error) { + if resolvConf == "" { + return nil, nil + } + data, err := os.ReadFile(resolvConf) + if err != nil { + return nil, err + } + loIP := loNameserver(string(data)) + if loIP == nil { + return nil, nil + } + + ip := net.ParseIP(virtIP) + if ip == nil { + return nil, fmt.Errorf("invalid virtual resolver IP %s", virtIP) + } + + f := &Forwarder{ + VirtIP: ip, + LoIP: loIP, + ResolvConf: resolvConf, + + // now by default, if we detect localhost nameserver we apply dockerRules + // which is the only case we cover. + custom: dockerRules, + } + return f, nil +} + +// RewriteResolvConf points the loopback nameservers in the container's +// resolv.conf at the virtIP. +func (f *Forwarder) RewriteResolvConf() error { + data, err := os.ReadFile(f.ResolvConf) + if err != nil { + return fmt.Errorf("failed to read %s: %w", f.ResolvConf, err) + } + + lines := strings.Split(string(data), "\n") + out := make([]string, 0, len(lines)) + + // changed is a flag for detect if we rewrite any nameserver + changed := false + for _, line := range lines { + fields := strings.Fields(line) + + // if we have `resolv.conf` as: + // nameserver 127.0.0.10 + // nameserver 127.0.0.20 + // nameserver 8.8.8.8 + // ------------------- + // our `loNameserver` function below will detect + // and set our `loIP` as `127.0.0.10` (first loopback nameserver), + // which will be the one in TC rules and Iptables one. + // Knowing that we should only rewrite the first nameserver in our `resolv.conf`, + // so final result will be: + // nameserver virtIP + // nameserver 127.0.0.20 + // nameserver 8.8.8.8 + // ``` + if !changed && len(fields) >= 2 && fields[0] == "nameserver" { + if ip := net.ParseIP(fields[1]); ip != nil && ip.IsLoopback() { + out = append(out, "nameserver "+f.VirtIP.String()) + changed = true + continue + } + } + out = append(out, line) + } + + if err := os.WriteFile(f.ResolvConf, []byte(strings.Join(out, "\n")), 0o644); err != nil { //nolint: gosec + return fmt.Errorf("failed to write %s: %w", f.ResolvConf, err) + } + return nil +} + +// loNameserver returns the first loopback nameserver in resolv.conf data, if any. +func loNameserver(data string) net.IP { + for line := range strings.SplitSeq(data, "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == "nameserver" { + if ip := net.ParseIP(fields[1]); ip != nil && ip.IsLoopback() { + return ip + } + } + } + return nil +} + +// Apply installs the host side plumbing in the current netns: +// the general tc +// Kernel flags required +// the scenario's custom rules +func (f *Forwarder) Apply(tap netlink.Link, eth netlink.Link) error { + lo, err := netlink.LinkByName("lo") + if err != nil { + return err + } + + if err := addClsact(lo); err != nil { + return fmt.Errorf("addClsact(lo) failed: %w", err) + } + if err := tapToLo(tap, lo, f.VirtIP); err != nil { + return fmt.Errorf("tapToLo(%s) failed: %w", tap.Attrs().Name, err) + } + + // The guest reuses the container's MAC, so replies must be addressed to it. + if err := loToTap(lo, tap, f.VirtIP, eth.Attrs().HardwareAddr); err != nil { + return fmt.Errorf("loToTap(%s) failed: %w", tap.Attrs().Name, err) + } + lhlog.Debug("Applied tc redirects between tap and lo") + + err = setSysctls(map[string]string{ + "/proc/sys/net/ipv4/conf/lo/route_localnet": "1", + fmt.Sprintf("/proc/sys/net/ipv4/conf/%s/accept_local", tap.Attrs().Name): "1", + "/proc/sys/net/ipv4/conf/lo/accept_local": "1", + }) + if err != nil { + return fmt.Errorf("failed to set required kernel flags for forwarding: %w", err) + } + lhlog.Debug("Applied kernel flags required for packet forwarding") + + return f.custom(f) +} + +func dnat(virtIP net.IP, proto string, dst string) error { + args := []string{ + "-t", "nat", "-A", "PREROUTING", + "-i", "lo", + "-d", virtIP.String(), + "-p", proto, + "-j", "DNAT", + "--to-destination", dst, + "--wait", "1", + } + if err := ipTablesExec(args); err != nil { + return err + } + return nil +} + +func addClsact(link netlink.Link) error { + clsact := &netlink.Clsact{ + QdiscAttrs: netlink.QdiscAttrs{ + LinkIndex: link.Attrs().Index, + Parent: netlink.HANDLE_CLSACT, + }, + } + return netlink.QdiscAdd(clsact) +} + +func tapToLo(tap netlink.Link, lo netlink.Link, virtIP net.IP) error { + + // tc filter add dev ingress protocol ip pref x \ + // flower dst_ip \ + // action skbedit ptype host pipe \ + // action mirred ingress redirect dev lo + return netlink.FilterAdd(&netlink.Flower{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: tap.Attrs().Index, + Parent: netlink.MakeHandle(0xffff, 0), + Priority: 100, + Protocol: unix.ETH_P_IP, + }, + DestIP: virtIP, + Actions: []netlink.Action{ + &netlink.SkbEditAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_PIPE, + }, + PType: uint16Ptr(unix.PACKET_HOST), + }, &netlink.MirredAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_STOLEN, + }, + MirredAction: netlink.TCA_INGRESS_REDIR, + Ifindex: lo.Attrs().Index, + }, + }, + }) +} + +func loToTap(lo netlink.Link, tap netlink.Link, virtIP net.IP, guestMAC net.HardwareAddr) error { + + // tc filter add dev lo egress protocol ip pref x \ + // flower src_ip \ + // action pedit ex munge eth dst set pipe \ + // action mirred egress redirect dev + return netlink.FilterAdd(&netlink.Flower{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: lo.Attrs().Index, + Parent: netlink.HANDLE_MIN_EGRESS, + Priority: 100, + Protocol: unix.ETH_P_IP, + }, + SrcIP: virtIP, + Actions: []netlink.Action{ + &netlink.PeditAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_PIPE, + }, + DstMacAddr: guestMAC, + }, + &netlink.MirredAction{ + ActionAttrs: netlink.ActionAttrs{ + Action: netlink.TC_ACT_STOLEN, + }, + MirredAction: netlink.TCA_EGRESS_REDIR, + Ifindex: tap.Attrs().Index, + }, + }, + }) +} + +func setSysctls(flags map[string]string) error { + for flag, value := range flags { + file, err := os.OpenFile(flag, os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("failed to open %s: %w", flag, err) + } + defer file.Close() + + _, err = file.WriteString(value) + if err != nil { + return fmt.Errorf("failed to set flag %s to %s: %w", flag, value, err) + } + } + return nil +} + +// Cleanup removes the host side plumbing installed by Apply: +// - Del TC link +// - Del rules applied (e.g. iptables rules) +func Cleanup() error { + var retErr error + + lo, err := netlink.LinkByName("lo") + if err != nil { + return fmt.Errorf("LinkByName(lo) failed: %w", err) + } + if err := deleteClsact(lo); err != nil { + retErr = errors.Join(retErr, fmt.Errorf("failed to delete clsact qdisc on lo: %w", err)) + } + if err := clearRules(); err != nil { + retErr = errors.Join(retErr, fmt.Errorf("failed to flush DNAT rules: %w", err)) + } + return retErr +} + +func deleteClsact(link netlink.Link) error { + qdiscs, err := netlink.QdiscList(link) + if err != nil { + return fmt.Errorf("QdiscList(%s) failed: %w", link.Attrs().Name, err) + } + for _, q := range qdiscs { + if _, ok := q.(*netlink.Clsact); ok { + if err := netlink.QdiscDel(q); err != nil { + return fmt.Errorf("QdiscDel(clsact on %s) failed: %w", link.Attrs().Name, err) + } + } + } + return nil +} + +func uint16Ptr(v uint16) *uint16 { + return &v +} + +func ipTablesExec(args []string) error { + var stdout, stderr bytes.Buffer + + ipt, err := exec.LookPath("iptables") + if err != nil { + return err + } + args = append([]string{ipt}, args...) + cmd := exec.Cmd{ + Path: ipt, + Args: args, + Stdout: &stdout, + Stderr: &stderr, + } + if err := cmd.Run(); err != nil { + if _, ok := err.(*exec.ExitError); ok { + return fmt.Errorf("iptables command %s failed: %s", cmd.String(), stderr.String()) + } + return err + } + return nil +} diff --git a/pkg/network/localhost/local_dns_test.go b/pkg/network/localhost/local_dns_test.go new file mode 100644 index 000000000..204ad67de --- /dev/null +++ b/pkg/network/localhost/local_dns_test.go @@ -0,0 +1,123 @@ +// Copyright (c) 2023-2026, Nubificus LTD +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package localhost + +import ( + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func writeTemp(t *testing.T, dir, content string) string { + t.Helper() + require.NoError(t, os.MkdirAll(dir, 0o755)) + path := filepath.Join(dir, "resolv.conf") + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) //nolint: gosec + return path +} + +func TestDetect(t *testing.T) { + tmp := t.TempDir() + + tests := []struct { + name string + dir string // subdirectory holding resolv.conf + content string + virtIP string + wantNil bool + wantLoIP string + wantVirtIP string + }{ + { + name: "docker embedded resolver", + dir: "var/lib/docker/containers/abc", + content: "nameserver 127.0.0.11\noptions ndots:0\n", + virtIP: "192.168.100.100", + wantLoIP: "127.0.0.11", + wantVirtIP: "192.168.100.100", + }, + { + name: "loopback after public nameserver", + dir: "mixed", + content: "nameserver 8.8.8.8\nnameserver 127.0.0.11\n", + virtIP: "192.168.100.100", + wantLoIP: "127.0.0.11", + wantVirtIP: "192.168.100.100", + }, + { + name: "no loopback nameserver", + dir: "plain", + content: "search example.com\nnameserver 8.8.8.8\n", + virtIP: "192.168.100.100", + wantNil: true, + }, + { + name: "garbage lines are ignored", + dir: "garbage", + content: "# a comment\nnameserver\nnot-an-ip\n", + virtIP: "192.168.100.100", + wantNil: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := writeTemp(t, filepath.Join(tmp, tt.dir), tt.content) + fwd, err := Detect(path, tt.virtIP) + require.NoError(t, err) + if tt.wantNil { + assert.Nil(t, fwd, "Detect() should not return a Forwarder") + return + } + require.NotNil(t, fwd, "Detect() should return a Forwarder") + assert.Equal(t, tt.wantLoIP, fwd.LoIP.String()) + assert.Equal(t, tt.wantVirtIP, fwd.VirtIP.String()) + assert.Equal(t, path, fwd.ResolvConf) + assert.Equal(t, reflect.ValueOf(dockerRules).Pointer(), reflect.ValueOf(fwd.custom).Pointer(), "custom rules should be dockerRules") + }) + } + + t.Run("empty path", func(t *testing.T) { + fwd, err := Detect("", "192.168.100.100") + assert.NoError(t, err) + assert.Nil(t, fwd) + }) + + t.Run("missing file", func(t *testing.T) { + fwd, err := Detect(filepath.Join(tmp, "does/not/exist"), "192.168.100.100") + assert.Error(t, err) + assert.Nil(t, fwd) + }) +} + +func TestRewriteResolvConf(t *testing.T) { + tmp := t.TempDir() + content := "search example.com\nnameserver 127.0.0.11\nnameserver 8.8.8.8\noptions ndots:0\n" + path := writeTemp(t, filepath.Join(tmp, "var/lib/docker/containers/abc"), content) + + fwd, err := Detect(path, "192.168.100.100") + require.NoError(t, err) + require.NotNil(t, fwd) + require.NoError(t, fwd.RewriteResolvConf()) + + got, err := os.ReadFile(path) + require.NoError(t, err) + want := "search example.com\nnameserver 192.168.100.100\nnameserver 8.8.8.8\noptions ndots:0\n" + assert.Equal(t, want, string(got), "only loopback nameservers should be rewritten") +} diff --git a/pkg/network/network.go b/pkg/network/network.go index cbc73c707..207f1c326 100644 --- a/pkg/network/network.go +++ b/pkg/network/network.go @@ -25,6 +25,8 @@ import ( "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" + + "github.com/urunc-dev/urunc/pkg/network/localhost" ) const ( @@ -36,6 +38,9 @@ var netlog = logrus.WithField("subsystem", "network") type UnikernelNetworkInfo struct { TapDevice string EthDevice Interface + // DNSServer is the virtual resolver IP the guest should be configured + // with, set only when localhost DNS forwarding was applied. + DNSServer string } type Manager interface { NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkInfo, error) @@ -50,12 +55,12 @@ type Interface struct { MTU int } -func NewNetworkManager(networkType string) (Manager, error) { +func NewNetworkManager(networkType string, fwd *localhost.Forwarder) (Manager, error) { switch networkType { case "static": return &StaticNetwork{}, nil case "dynamic": - return &DynamicNetwork{}, nil + return &DynamicNetwork{DNSForwarder: fwd}, nil default: return nil, fmt.Errorf("network manager %s not supported", networkType) diff --git a/pkg/network/network_dynamic.go b/pkg/network/network_dynamic.go index 1d6a2237b..379f88bb5 100644 --- a/pkg/network/network_dynamic.go +++ b/pkg/network/network_dynamic.go @@ -18,9 +18,14 @@ import ( "fmt" "strconv" "strings" + + "github.com/urunc-dev/urunc/pkg/network/localhost" ) type DynamicNetwork struct { + // DNSForwarder when set, is applied on the tap/veth the dynamic NetworkSetup creates + // so the guest DNS queries get redirected to the host loopback resolver. + DNSForwarder *localhost.Forwarder } // NetworkSetup checks if any tap device is available in the current netns. If it is, it assumes a running unikernel @@ -62,8 +67,27 @@ func (n DynamicNetwork) NetworkSetup(uid uint32, gid uint32) (*UnikernelNetworkI return nil, fmt.Errorf("getInterfaceInfo(%s) failed: %w", redirectLink.Attrs().Name, err) } - return &UnikernelNetworkInfo{ + info := &UnikernelNetworkInfo{ TapDevice: newTapDevice.Attrs().Name, EthDevice: ifInfo, - }, nil + } + + // DNS forwarding is a good feature, but the unikernel can still run without it. + // so we fail open, failure here is only logged and DNSServer stays empty. + if n.DNSForwarder != nil { + if err := n.DNSForwarder.Apply(newTapDevice, redirectLink); err != nil { + netlog.Warnf("failed to apply localhost forwarding rules: %v", err) + return info, nil + } + if err := n.DNSForwarder.RewriteResolvConf(); err != nil { + netlog.Warnf("failed to rewrite container's resolv.conf: %v", err) + // TODO: If we failed to rewrite localhost nameserver. we should do + // cleanup for the applied rules, as no benefit from them now + return info, nil + } + info.DNSServer = n.DNSForwarder.VirtIP.String() + netlog.Debugf("localhost forwarding applied: virtual resolvIP %s", info.DNSServer) + } + + return info, nil } diff --git a/pkg/network/network_test.go b/pkg/network/network_test.go index da8c1d3d3..409a3a7cd 100644 --- a/pkg/network/network_test.go +++ b/pkg/network/network_test.go @@ -46,7 +46,7 @@ func TestNewNetworkManager(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - got, err := NewNetworkManager(tt.networkType) + got, err := NewNetworkManager(tt.networkType, nil) if tt.expectedErr { assert.Error(t, err, "NewNetworkManager() should return an error") } else { diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index f5d668b0d..954220175 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -45,12 +45,13 @@ type VMM interface { } type NetDevParams struct { - IP string // The veth device IP - Mask string // The veth device mask - Gateway string // The veth device gateway - MAC string // The MAC address of the guest network device - TapDev string // The tap device name - MTU int // The MTU value of the tap device + IP string // The veth device IP + Mask string // The veth device mask + Gateway string // The veth device gateway + MAC string // The MAC address of the guest network device + TapDev string // The tap device name + MTU int // The MTU value of the tap device + DNSServer string // The resolver IP when localhost forwarding is active, empty otherwise } type BlockDevParams struct { diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index ad0bf2897..7bbbf0247 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -30,6 +30,7 @@ import ( "syscall" "github.com/urunc-dev/urunc/pkg/network" + "github.com/urunc-dev/urunc/pkg/network/localhost" "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" "github.com/urunc-dev/urunc/pkg/unikontainers/types" "github.com/urunc-dev/urunc/pkg/unikontainers/unikernels" @@ -259,7 +260,13 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { networkType := u.getNetworkType() uniklog.WithField("network type", networkType).Debug("Retrieved network type") netArgs := types.NetDevParams{} - netManager, err := network.NewNetworkManager(networkType) + + fwd, err := localhost.Detect(resolvConfSource(u.Spec.Mounts), u.UruncCfg.Network.DNSResolverIP) + if err != nil { + uniklog.Warnf("failed to detect loopback resolver: %v", err) + } + + netManager, err := network.NewNetworkManager(networkType, fwd) if err != nil { return netArgs, fmt.Errorf("failed to create network manager for %s type: %v", networkType, err) } @@ -282,8 +289,9 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) { // virtual ethernet interface inside the namespace netArgs.MAC = networkInfo.EthDevice.MAC netArgs.MTU = networkInfo.EthDevice.MTU + // Empty unless dynamic networking applied localhost DNS forwarding. + netArgs.DNSServer = networkInfo.DNSServer } - return netArgs, nil } @@ -831,6 +839,9 @@ func (u *Unikontainer) Kill() error { return err } + if err := localhost.Cleanup(); err != nil { + uniklog.Errorf("failed to cleanup localhost forwarding rules: %v", err) + } err = network.CleanupAllUruncTaps() if err != nil { uniklog.Errorf("failed to cleanup tap devices: %v", err) diff --git a/pkg/unikontainers/urunc_config.go b/pkg/unikontainers/urunc_config.go index 22573f43c..7f8cec526 100644 --- a/pkg/unikontainers/urunc_config.go +++ b/pkg/unikontainers/urunc_config.go @@ -19,6 +19,7 @@ import ( "strings" "github.com/BurntSushi/toml" + "github.com/urunc-dev/urunc/internal/constants" "github.com/urunc-dev/urunc/pkg/unikontainers/types" ) @@ -34,9 +35,14 @@ type UruncTimestamps struct { Destination string `toml:"destination"` // Used to specify a file for timestamps } +type UruncNetwork struct { + DNSResolverIP string `toml:"dns_resolver_ip"` +} + type UruncConfig struct { Log UruncLog `toml:"log"` Timestamps UruncTimestamps `toml:"timestamps"` + Network UruncNetwork `toml:"network"` Monitors map[string]types.MonitorConfig `toml:"monitors"` ExtraBins map[string]types.ExtraBinConfig `toml:"extra_binaries"` } @@ -83,6 +89,12 @@ const ( defaultMonitorVCPUs uint = 1 ) +func defaultNetworkConfig() UruncNetwork { + return UruncNetwork{ + DNSResolverIP: constants.LocalhostDNSResolverIP, + } +} + func defaultMonitorsConfig() map[string]types.MonitorConfig { return map[string]types.MonitorConfig{ "qemu": {DefaultMemoryMB: defaultMonitorMemoryMB, DefaultVCPUs: defaultMonitorVCPUs}, @@ -103,6 +115,7 @@ func defaultUruncConfig() *UruncConfig { return &UruncConfig{ Log: defaultLogConfig(), Timestamps: defaultTimestampsConfig(), + Network: defaultNetworkConfig(), Monitors: defaultMonitorsConfig(), ExtraBins: defaultExtraBinConfig(), } @@ -138,6 +151,9 @@ func (p *UruncConfig) Map() map[string]string { // them to this map. this map will be used to save the rest of the urunc config to state.json cfgMap := make(map[string]string) + if p.Network.DNSResolverIP != "" { + cfgMap["urunc_config.network.dns_resolver_ip"] = p.Network.DNSResolverIP + } for hv, hvCfg := range p.Monitors { prefix := "urunc_config.monitors." + hv + "." cfgMap[prefix+"default_memory_mb"] = strconv.FormatUint(uint64(hvCfg.DefaultMemoryMB), 10) @@ -158,10 +174,14 @@ func UruncConfigFromMap(cfgMap map[string]string) *UruncConfig { // since log and timestamps are loaded at the start of urunc, we will not be reading // them from this map. this map will be used to parse the rest of the urunc config from state.json cfg := &UruncConfig{ + Network: defaultNetworkConfig(), Monitors: defaultMonitorsConfig(), ExtraBins: defaultExtraBinConfig(), } + if val, ok := cfgMap["urunc_config.network.dns_resolver_ip"]; ok && val != "" { + cfg.Network.DNSResolverIP = val + } for key, val := range cfgMap { if !strings.HasPrefix(key, "urunc_config.monitors.") { continue diff --git a/pkg/unikontainers/urunc_config_test.go b/pkg/unikontainers/urunc_config_test.go index a51a3eb96..17952219f 100644 --- a/pkg/unikontainers/urunc_config_test.go +++ b/pkg/unikontainers/urunc_config_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/urunc-dev/urunc/internal/constants" "github.com/urunc-dev/urunc/pkg/unikontainers/types" ) @@ -33,6 +34,7 @@ const ( testHvtMemoryKey = "urunc_config.monitors.hvt.default_memory_mb" testVirtiofsdPathKey = "urunc_config.extra_binaries.virtiofsd.path" testVirtiofsdOptsKey = "urunc_config.extra_binaries.virtiofsd.options" + testDNSResolverIPKey = "urunc_config.network.dns_resolver_ip" testVirtiofsdDefOpts = "--cache always --sandbox none" testBinOpts = "opt1 opt2" testQemuBinaryPath = "/usr/bin/qemu" @@ -50,6 +52,19 @@ func TestUruncConfigFromMap(t *testing.T) { assert.NotNil(t, config) assert.Equal(t, defaultMonitorsConfig(), config.Monitors) assert.Equal(t, defaultExtraBinConfig(), config.ExtraBins) + assert.Equal(t, defaultNetworkConfig(), config.Network) + }) + + t.Run("network dns resolver ip is picked up", func(t *testing.T) { + t.Parallel() + cfgMap := map[string]string{ + testDNSResolverIPKey: "192.168.150.150", + } + + config := UruncConfigFromMap(cfgMap) + + assert.NotNil(t, config) + assert.Equal(t, "192.168.150.150", config.Network.DNSResolverIP) }) t.Run("single monitor with all fields", func(t *testing.T) { @@ -383,6 +398,7 @@ func TestUruncConfigMap(t *testing.T) { "urunc_config.monitors.firecracker.binary_path", testVirtiofsdPathKey, testVirtiofsdOptsKey, + testDNSResolverIPKey, } for _, key := range expectedKeys { @@ -395,6 +411,7 @@ func TestUruncConfigMap(t *testing.T) { assert.Equal(t, "", cfgMap[testQemuBinaryKey]) assert.Equal(t, "/usr/libexec/virtiofsd", cfgMap[testVirtiofsdPathKey]) assert.Equal(t, testVirtiofsdDefOpts, cfgMap[testVirtiofsdOptsKey]) + assert.Equal(t, constants.LocalhostDNSResolverIP, cfgMap[testDNSResolverIPKey]) }) t.Run("custom config produces expected map", func(t *testing.T) { diff --git a/pkg/unikontainers/utils.go b/pkg/unikontainers/utils.go index 90b9500cf..642336602 100644 --- a/pkg/unikontainers/utils.go +++ b/pkg/unikontainers/utils.go @@ -378,3 +378,14 @@ func executeHook(hook specs.Hook, state []byte) error { return nil } + +// resolvConfSource returns the host file bind-mounted over the container's +// /etc/resolv.conf, or "" when there is no such mount. +func resolvConfSource(mounts []specs.Mount) string { + for _, m := range mounts { + if m.Destination == "/etc/resolv.conf" { + return m.Source + } + } + return "" +} diff --git a/tests/e2e/common.go b/tests/e2e/common.go index 7953d71c1..fe3f43650 100644 --- a/tests/e2e/common.go +++ b/tests/e2e/common.go @@ -32,12 +32,14 @@ type testTool interface { setContainerID(string) createPod() (string, error) createContainer() (string, error) + createNetwork() (string, error) startContainer(bool) (string, error) runContainer(bool) (string, error) stopContainer() error stopPod() error rmContainer() error rmPod() error + rmNetwork() error logContainer() (string, error) searchContainer(string) (bool, error) searchPod(string) (bool, error) @@ -63,6 +65,7 @@ type containerTestArgs struct { Memory string Cli string Volumes []containerVolume + Network string StaticNet bool SideContainers []string Skippable bool @@ -90,6 +93,9 @@ func commonNewContainerCmd(a containerTestArgs) string { if a.Memory != "" { cmdBase += fmt.Sprintf("-m %s ", a.Memory) } + if a.Network != "" { + cmdBase += fmt.Sprintf("--network %s ", a.Network) + } if a.UID != 0 && a.GID != 0 { cmdBase += fmt.Sprintf("-u %d:%d ", a.UID, a.GID) } @@ -158,6 +164,17 @@ func commonRmImage(tool string, image string) error { return nil } +func commonNetworkCreate(tool string, network string) (string, error) { + cmdBase := tool + " network create " + network + return commonCmdExec(cmdBase) +} + +func commonNetworkRm(tool string, network string) error { + cmdBase := tool + " network rm " + network + _, err := commonCmdExec(cmdBase) + return err +} + func commonCreate(tool string, cntrArgs containerTestArgs) (output string, err error) { cmdBase := tool + " create " cmdBase += commonNewContainerCmd(cntrArgs) diff --git a/tests/e2e/crictl.go b/tests/e2e/crictl.go index 05ed0f05c..9b5226275 100644 --- a/tests/e2e/crictl.go +++ b/tests/e2e/crictl.go @@ -201,6 +201,11 @@ func (i *crictlInfo) createContainer() (string, error) { return commonCmdExec(cmdBase) } +func (i *crictlInfo) createNetwork() (string, error) { + // Not supported by crictl + return "", errToolDoesNotSupport +} + func (i *crictlInfo) startContainer(bool) (string, error) { cmdBase := crictlName cmdBase += " start " @@ -298,6 +303,11 @@ func (i *crictlInfo) rmPod() error { return nil } +func (i *crictlInfo) rmNetwork() error { + // Not supported by crictl + return errToolDoesNotSupport +} + func (i *crictlInfo) logContainer() (string, error) { return commonLogs(crictlName, i.containerID) } diff --git a/tests/e2e/ctr.go b/tests/e2e/ctr.go index b510259dd..7f685763c 100644 --- a/tests/e2e/ctr.go +++ b/tests/e2e/ctr.go @@ -97,6 +97,11 @@ func (i *ctrInfo) createContainer() (string, error) { return commonCmdExec(cmdBase) } +func (i *ctrInfo) createNetwork() (string, error) { + // Not supported by ctr + return "", errToolDoesNotSupport +} + // nolint:unused func (i *ctrInfo) startPod() (string, error) { // Not supported by ctr @@ -166,6 +171,11 @@ func (i *ctrInfo) rmPod() error { return errToolDoesNotSupport } +func (i *ctrInfo) rmNetwork() error { + // Not supported by ctr + return errToolDoesNotSupport +} + func (i *ctrInfo) logContainer() (string, error) { // Not supported by ctr // TODO: We need to fix this diff --git a/tests/e2e/docker.go b/tests/e2e/docker.go index 9c0982a9c..0496d9d19 100644 --- a/tests/e2e/docker.go +++ b/tests/e2e/docker.go @@ -66,6 +66,10 @@ func (i *dockerInfo) createContainer() (string, error) { return commonCreate(dockerName, i.testArgs) } +func (i *dockerInfo) createNetwork() (string, error) { + return commonNetworkCreate(dockerName, i.testArgs.Network) +} + // nolint:unused func (i *dockerInfo) startPod() (string, error) { // Not supported by docker @@ -108,6 +112,10 @@ func (i *dockerInfo) rmPod() error { return errToolDoesNotSupport } +func (i *dockerInfo) rmNetwork() error { + return commonNetworkRm(dockerName, i.testArgs.Network) +} + func (i *dockerInfo) logContainer() (string, error) { return commonLogs(dockerName, i.containerID) } diff --git a/tests/e2e/docker_test.go b/tests/e2e/docker_test.go index 4d61ba07f..20e94ac15 100644 --- a/tests/e2e/docker_test.go +++ b/tests/e2e/docker_test.go @@ -27,7 +27,6 @@ var _ = Describe("Docker", Ordered, ContinueOnFailure, func() { images := getTestImages(cases) err := pullAllImages(testDocker, images) Expect(err).NotTo(HaveOccurred(), "Failed to pull docker images") - DeferCleanup(func() { removeAllImages(testDocker, images) }) @@ -43,12 +42,25 @@ var _ = Describe("Docker", Ordered, ContinueOnFailure, func() { } }) - DescribeTable("unikernel containers", - func(tc containerTestArgs) { - skipMissingVolumes(tc) - tool = newDockerTool(tc) - runDetachedTest(tool, tc) - }, - toTableEntries(dockerTestCases()), - ) + Context("foreground containers", func() { + DescribeTable("unikernel containers", + func(tc containerTestArgs) { + skipMissingVolumes(tc) + tool = newDockerTool(tc) + runForegroundTest(tool, tc) + }, + toTableEntries(selectTestCases(dockerTestCases(), false)), + ) + }) + + Context("detached containers", func() { + DescribeTable("unikernel containers", + func(tc containerTestArgs) { + skipMissingVolumes(tc) + tool = newDockerTool(tc) + runDetachedTest(tool, tc) + }, + toTableEntries(selectTestCases(dockerTestCases(), true)), + ) + }) }) diff --git a/tests/e2e/nerdctl.go b/tests/e2e/nerdctl.go index 024e83791..5ebd5704e 100644 --- a/tests/e2e/nerdctl.go +++ b/tests/e2e/nerdctl.go @@ -66,6 +66,10 @@ func (i *nerdctlInfo) createContainer() (string, error) { return commonCreate(nerdctlName, i.testArgs) } +func (i *nerdctlInfo) createNetwork() (string, error) { + return commonNetworkCreate(nerdctlName, i.testArgs.Network) +} + // nolint:unused func (i *nerdctlInfo) startPod() (string, error) { // Not supported by nerdctl @@ -108,6 +112,9 @@ func (i *nerdctlInfo) rmPod() error { return errToolDoesNotSupport } +func (i *nerdctlInfo) rmNetwork() error { + return commonNetworkRm(nerdctlName, i.testArgs.Network) +} func (i *nerdctlInfo) logContainer() (string, error) { return commonLogs(nerdctlName, i.containerID) } diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go index 03d7db82e..f893a4f55 100644 --- a/tests/e2e/suite_test.go +++ b/tests/e2e/suite_test.go @@ -97,6 +97,12 @@ func captureContainerLogs(tool testTool) { // runDetachedTest runs a container in detached mode: create, start, and // verify via TestFunc. func runDetachedTest(tool testTool, tc containerTestArgs) { + if tc.Network != "" { + By("Creating user-defined network") + output, err := tool.createNetwork() + Expect(err).NotTo(HaveOccurred(), "Failed to create network %s: %s", tc.Network, output) + } + By("Creating container") cID, err := tool.createContainer() Expect(err).NotTo(HaveOccurred(), "Failed to create container: %s", cID) @@ -120,6 +126,12 @@ func runDetachedTest(tool testTool, tc containerTestArgs) { GinkgoLogr.Error(err, "Failed to verify removal") } } + if tc.Network != "" { + By("Removing user-defined network") + if err := tool.rmNetwork(); err != nil { + GinkgoLogr.Error(err, "Failed to remove network "+tc.Network) + } + } }) By("Starting container") @@ -137,6 +149,12 @@ func runDetachedTest(tool testTool, tc containerTestArgs) { func runForegroundTest(tool testTool, tc containerTestArgs) { tool.setContainerID(tc.Name) + if tc.Network != "" { + By("Creating user-defined network") + output, err := tool.createNetwork() + Expect(err).NotTo(HaveOccurred(), "Failed to create network %s: %s", tc.Network, output) + } + DeferCleanup(func() { if tool.getContainerID() != "" { By("Cleaning up container") @@ -144,6 +162,12 @@ func runForegroundTest(tool testTool, tc containerTestArgs) { GinkgoLogr.Error(err, "Container cleanup failed") } } + if tc.Network != "" { + By("Removing user-defined network") + if err := tool.rmNetwork(); err != nil { + GinkgoLogr.Error(err, "Failed to remove network "+tc.Network) + } + } }) By("Running container") diff --git a/tests/e2e/test_cases.go b/tests/e2e/test_cases.go index 3b90a381e..a80cccaca 100644 --- a/tests/e2e/test_cases.go +++ b/tests/e2e/test_cases.go @@ -1241,5 +1241,23 @@ func dockerTestCases() []containerTestArgs { Skippable: false, TestFunc: namespaceTest, }, + { + Image: "harbor.nbfc.io/nubificus/urunc/busybox-qemu-linux-raw:latest", + Name: "Qemu-linux-docker-user-defined-network-external", + Devmapper: false, + Seccomp: true, + UID: 0, + GID: 0, + Groups: []int64{}, + Memory: "", + Cli: "/bin/nslookup google-public-dns-a.google.com", + Volumes: []containerVolume{}, + Network: "urunc-dns-test", + StaticNet: false, + SideContainers: []string{}, + Skippable: false, + ExpectOut: "Address: 8.8.8.8", + TestFunc: matchTest, + }, } }