coder/tailnet/test/integration/network.go

//go:build linux
// +build linux

package integration

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"testing"

	"github.com/stretchr/testify/require"
	"github.com/tailscale/netlink"
	"golang.org/x/xerrors"

	"cdr.dev/slog"

	"github.com/coder/coder/v2/cryptorand"
)

type TestNetworking struct {
	// ServerListenAddr is the IP address and port that the server listens on,
	// passed to StartServer.
	ServerListenAddr string
	// ServerAccessURLClient1 is the hostname and port that the first client
	// uses to access the server.
	ServerAccessURLClient1 string
	// ServerAccessURLClient2 is the hostname and port that the second client
	// uses to access the server.
	ServerAccessURLClient2 string

	// Networking settings for each subprocess.
	ProcessServer  TestNetworkingProcess
	ProcessClient1 TestNetworkingProcess
	ProcessClient2 TestNetworkingProcess
}

type TestNetworkingProcess struct {
	// NetNS to enter. If nil, the current network namespace is used.
	NetNS *os.File
}

// SetupNetworkingLoopback creates a network namespace with a loopback interface
// for all tests to share. This is the simplest networking setup. The network
// namespace only exists for isolation on the host and doesn't serve any routing
// purpose.
func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking {
	netNSName := "codertest_netns_"
	randStr, err := cryptorand.String(4)
	require.NoError(t, err, "generate random string for netns name")
	netNSName += randStr

	// Create a single network namespace for all tests so we can have an
	// isolated loopback interface.
	netNSFile := createNetNS(t, netNSName)

	var (
		listenAddr = "127.0.0.1:8080"
		process    = TestNetworkingProcess{
			NetNS: netNSFile,
		}
	)
	return TestNetworking{
		ServerListenAddr:       listenAddr,
		ServerAccessURLClient1: "http://" + listenAddr,
		ServerAccessURLClient2: "http://" + listenAddr,
		ProcessServer:          process,
		ProcessClient1:         process,
		ProcessClient2:         process,
	}
}

// SetupNetworkingEasyNAT creates a network namespace with a router that NATs
// packets between two clients and a server.
// See createFakeRouter for the full topology.
// NAT is achieved through a single iptables masquerade rule.
func SetupNetworkingEasyNAT(t *testing.T, _ slog.Logger) TestNetworking {
	router := createFakeRouter(t)

	// Set up iptables masquerade rules to allow the router to NAT packets
	// between the Three Kingdoms.
	_, err := commandInNetNS(router.RouterNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output()
	require.NoError(t, wrapExitErr(err), "enable IP forwarding in router NetNS")
	_, err = commandInNetNS(router.RouterNetNS, "iptables", []string{
		"-t", "nat",
		"-A", "POSTROUTING",
		// Every interface except loopback.
		"!", "-o", "lo",
		"-j", "MASQUERADE",
	}).Output()
	require.NoError(t, wrapExitErr(err), "add iptables masquerade rule")

	return router.Net
}

type fakeRouter struct {
	Net TestNetworking

	RouterNetNS *os.File
	RouterVeths struct {
		Server  string
		Client1 string
		Client2 string
	}
	ServerNetNS  *os.File
	ServerVeth   string
	Client1NetNS *os.File
	Client1Veth  string
	Client2NetNS *os.File
	Client2Veth  string
}

// fakeRouter creates multiple namespaces with veth pairs between them with
// the following topology:
//
// namespaces:
//   - router
//   - server
//   - client1
//   - client2
//
// veth pairs:
//   - router-server  (10.0.1.1) <-> server-router  (10.0.1.2)
//   - router-client1 (10.0.2.1) <-> client1-router (10.0.2.2)
//   - router-client2 (10.0.3.1) <-> client2-router (10.0.3.2)
//
// No iptables rules are created, so packets will not be forwarded out of the
// box. Routes are created between all namespaces based on the veth pairs,
// however.
func createFakeRouter(t *testing.T) fakeRouter {
	t.Helper()
	const (
		routerServerPrefix  = "10.0.1."
		routerServerIP      = routerServerPrefix + "1"
		serverIP            = routerServerPrefix + "2"
		routerClient1Prefix = "10.0.2."
		routerClient1IP     = routerClient1Prefix + "1"
		client1IP           = routerClient1Prefix + "2"
		routerClient2Prefix = "10.0.3."
		routerClient2IP     = routerClient2Prefix + "1"
		client2IP           = routerClient2Prefix + "2"
	)

	prefix := uniqNetName(t) + "_"
	router := fakeRouter{}
	router.RouterVeths.Server = prefix + "r-s"
	router.RouterVeths.Client1 = prefix + "r-c1"
	router.RouterVeths.Client2 = prefix + "r-c2"
	router.ServerVeth = prefix + "s-r"
	router.Client1Veth = prefix + "c1-r"
	router.Client2Veth = prefix + "c2-r"

	// Create namespaces.
	router.RouterNetNS = createNetNS(t, prefix+"r")
	serverNS := createNetNS(t, prefix+"s")
	client1NS := createNetNS(t, prefix+"c1")
	client2NS := createNetNS(t, prefix+"c2")

	vethPairs := []struct {
		parentName string
		peerName   string
		parentNS   *os.File
		peerNS     *os.File
		parentIP   string
		peerIP     string
	}{
		{
			parentName: router.RouterVeths.Server,
			peerName:   router.ServerVeth,
			parentNS:   router.RouterNetNS,
			peerNS:     serverNS,
			parentIP:   routerServerIP,
			peerIP:     serverIP,
		},
		{
			parentName: router.RouterVeths.Client1,
			peerName:   router.Client1Veth,
			parentNS:   router.RouterNetNS,
			peerNS:     client1NS,
			parentIP:   routerClient1IP,
			peerIP:     client1IP,
		},
		{
			parentName: router.RouterVeths.Client2,
			peerName:   router.Client2Veth,
			parentNS:   router.RouterNetNS,
			peerNS:     client2NS,
			parentIP:   routerClient2IP,
			peerIP:     client2IP,
		},
	}

	for _, vethPair := range vethPairs {
		err := createVethPair(vethPair.parentName, vethPair.peerName)
		require.NoErrorf(t, err, "create veth pair %q <-> %q", vethPair.parentName, vethPair.peerName)

		// Move the veth interfaces to the respective network namespaces.
		err = setVethNetNS(vethPair.parentName, int(vethPair.parentNS.Fd()))
		require.NoErrorf(t, err, "set veth %q to NetNS", vethPair.parentName)
		err = setVethNetNS(vethPair.peerName, int(vethPair.peerNS.Fd()))
		require.NoErrorf(t, err, "set veth %q to NetNS", vethPair.peerName)

		// Set IP addresses on the interfaces.
		err = setInterfaceIP(vethPair.parentNS, vethPair.parentName, vethPair.parentIP)
		require.NoErrorf(t, err, "set IP %q on interface %q", vethPair.parentIP, vethPair.parentName)
		err = setInterfaceIP(vethPair.peerNS, vethPair.peerName, vethPair.peerIP)
		require.NoErrorf(t, err, "set IP %q on interface %q", vethPair.peerIP, vethPair.peerName)

		// Bring up both interfaces.
		err = setInterfaceUp(vethPair.parentNS, vethPair.parentName)
		require.NoErrorf(t, err, "bring up interface %q", vethPair.parentName)
		err = setInterfaceUp(vethPair.peerNS, vethPair.peerName)
		require.NoErrorf(t, err, "bring up interface %q", vethPair.parentName)

		// We don't need to add a route from parent to peer since the kernel
		// already adds a default route for the /24. We DO need to add a default
		// route from peer to parent, however.
		err = addRouteInNetNS(vethPair.peerNS, []string{"default", "via", vethPair.parentIP, "dev", vethPair.peerName})
		require.NoErrorf(t, err, "add peer default route to %q", vethPair.peerName)
	}

	router.Net = TestNetworking{
		ServerListenAddr:       serverIP + ":8080",
		ServerAccessURLClient1: "http://" + serverIP + ":8080",
		ServerAccessURLClient2: "http://" + serverIP + ":8080",
		ProcessServer: TestNetworkingProcess{
			NetNS: serverNS,
		},
		ProcessClient1: TestNetworkingProcess{
			NetNS: client1NS,
		},
		ProcessClient2: TestNetworkingProcess{
			NetNS: client2NS,
		},
	}
	return router
}

func uniqNetName(t *testing.T) string {
	t.Helper()
	netNSName := "cdr_"
	randStr, err := cryptorand.String(3)
	require.NoError(t, err, "generate random string for netns name")
	netNSName += randStr
	return netNSName
}

// createNetNS creates a new network namespace with the given name. The returned
// file is a file descriptor to the network namespace.
// Note: all cleanup is handled for you, you do not need to call Close on the
// returned file.
func createNetNS(t *testing.T, name string) *os.File {
	// We use ip-netns here because it handles the process of creating a
	// disowned netns for us.
	// The only way to create a network namespace is by calling unshare(2) or
	// clone(2) with the CLONE_NEWNET flag, and as soon as the last process in a
	// network namespace exits, the namespace is destroyed.
	// However, if you create a bind mount of /proc/$PID/ns/net to a file, it
	// will keep the namespace alive until the mount is removed.
	// ip-netns does this for us. Without it, we would have to fork anyways.
	// Later, we will use nsenter to enter this network namespace.
	_, err := exec.Command("ip", "netns", "add", name).Output()
	require.NoError(t, wrapExitErr(err), "create network namespace via ip-netns")
	t.Cleanup(func() {
		_, _ = exec.Command("ip", "netns", "delete", name).Output()
	})

	// Open /run/netns/$name to get a file descriptor to the network namespace.
	path := fmt.Sprintf("/run/netns/%s", name)
	file, err := os.OpenFile(path, os.O_RDONLY, 0)
	require.NoError(t, err, "open network namespace file")
	t.Cleanup(func() {
		_ = file.Close()
	})

	// Exec "ip link set lo up" in the namespace to bring up loopback
	// networking.
	//nolint:gosec
	_, err = exec.Command("ip", "-netns", name, "link", "set", "lo", "up").Output()
	require.NoError(t, wrapExitErr(err), "bring up loopback interface in network namespace")

	return file
}

// createVethPair creates a veth pair with the given names.
func createVethPair(parentVethName, peerVethName string) error {
	vethLinkAttrs := netlink.NewLinkAttrs()
	vethLinkAttrs.Name = parentVethName
	veth := &netlink.Veth{
		LinkAttrs: vethLinkAttrs,
		PeerName:  peerVethName,
	}

	err := netlink.LinkAdd(veth)
	if err != nil {
		return xerrors.Errorf("LinkAdd(name: %q, peerName: %q): %w", parentVethName, peerVethName, err)
	}

	return nil
}

// setVethNetNS moves the veth interface to the specified network namespace.
func setVethNetNS(vethName string, netNSFd int) error {
	veth, err := netlink.LinkByName(vethName)
	if err != nil {
		return xerrors.Errorf("LinkByName(%q): %w", vethName, err)
	}

	err = netlink.LinkSetNsFd(veth, netNSFd)
	if err != nil {
		return xerrors.Errorf("LinkSetNsFd(%q, %v): %w", vethName, netNSFd, err)
	}

	return nil
}

// setInterfaceIP sets the IP address on the given interface. It automatically
// adds a /24 subnet mask.
func setInterfaceIP(netNS *os.File, ifaceName, ip string) error {
	_, err := commandInNetNS(netNS, "ip", []string{"addr", "add", ip + "/24", "dev", ifaceName}).Output()
	if err != nil {
		return xerrors.Errorf("set IP %q on interface %q in netns: %w", ip, ifaceName, wrapExitErr(err))
	}

	return nil
}

// setInterfaceUp brings the given interface up.
func setInterfaceUp(netNS *os.File, ifaceName string) error {
	_, err := commandInNetNS(netNS, "ip", []string{"link", "set", ifaceName, "up"}).Output()
	if err != nil {
		return xerrors.Errorf("bring up interface %q in netns: %w", ifaceName, wrapExitErr(err))
	}

	return nil
}

// addRouteInNetNS adds a route to the given network namespace.
func addRouteInNetNS(netNS *os.File, route []string) error {
	_, err := commandInNetNS(netNS, "ip", append([]string{"route", "add"}, route...)).Output()
	if err != nil {
		return xerrors.Errorf("add route %q in netns: %w", route, wrapExitErr(err))
	}

	return nil
}

func commandInNetNS(netNS *os.File, bin string, args []string) *exec.Cmd {
	//nolint:gosec
	cmd := exec.Command("nsenter", append([]string{"--net=/proc/self/fd/3", bin}, args...)...)
	cmd.ExtraFiles = []*os.File{netNS}
	return cmd
}

func wrapExitErr(err error) error {
	if err == nil {
		return nil
	}

	var exitErr *exec.ExitError
	if xerrors.As(err, &exitErr) {
		return xerrors.Errorf("output: %s\n\n%w", bytes.TrimSpace(exitErr.Stderr), exitErr)
	}
	return err
}