chore: improve tailnet integration test (#18124)

Refactors tailnet integration test and adds UDP echo tests with different MTU related to #15523

I still haven't gotten to the bottom of what's causing the issue (the added test case I expected to fail actually succeeds), but these integration test improvements are generally useful.

also:
 * consolidates networking setup with easy and hard NAT
 * consolidates client setup
 * makes Client2 act like an agent at the tailnet layer, so it will send ReadyForHandshake and speed up the tunnel establishment
 * adds support for logging tunneled packets
 * adds support for dumping outer (underlay) IP traffic
 * adds support for adjusting veth MTU
 * adds support for IPv6 in the outer (underlay) network topology

tailnet/test/integration/integration.go

@@ -28,8 +28,10 @@ import (
 	"golang.org/x/xerrors"
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
+	"tailscale.com/net/packet"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
+	"tailscale.com/wgengine/capture"
 
 	"cdr.dev/slog"
 	"github.com/coder/coder/v2/coderd/httpapi"
@@ -54,6 +56,7 @@ type Client struct {
 	ID             uuid.UUID
 	ListenPort     uint16
 	ShouldRunTests bool
+	TunnelSrc      bool
 }
 
 var Client1 = Client{
@@ -61,6 +64,7 @@ var Client1 = Client{
 	ID:             uuid.MustParse("00000000-0000-0000-0000-000000000001"),
 	ListenPort:     client1Port,
 	ShouldRunTests: true,
+	TunnelSrc:      true,
 }
 
 var Client2 = Client{
@@ -68,21 +72,20 @@ var Client2 = Client{
 	ID:             uuid.MustParse("00000000-0000-0000-0000-000000000002"),
 	ListenPort:     client2Port,
 	ShouldRunTests: false,
+	TunnelSrc:      false,
 }
 
 type TestTopology struct {
 	Name string
-	// SetupNetworking creates interfaces and network namespaces for the test.
+
-	// The most simple implementation is NetworkSetupDefault, which only creates
+	NetworkingProvider NetworkingProvider
-	// a network namespace shared for all tests.
-	SetupNetworking func(t *testing.T, logger slog.Logger) TestNetworking
 
 	// Server is the server starter for the test. It is executed in the server
 	// subprocess.
 	Server ServerStarter
-	// StartClient gets called in each client subprocess. It's expected to
+	// ClientStarter.StartClient gets called in each client subprocess. It's expected to
 	// create the tailnet.Conn and ensure connectivity to it's peer.
-	StartClient func(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me Client, peer Client) *tailnet.Conn
+	ClientStarter ClientStarter
 
 	// RunTests is the main test function. It's called in each of the client
 	// subprocesses. If tests can only run once, they should check the client ID
@@ -97,6 +100,17 @@ type ServerStarter interface {
 	StartServer(t *testing.T, logger slog.Logger, listenAddr string)
 }
 
+type NetworkingProvider interface {
+	// SetupNetworking creates interfaces and network namespaces for the test.
+	// The most simple implementation is NetworkSetupDefault, which only creates
+	// a network namespace shared for all tests.
+	SetupNetworking(t *testing.T, logger slog.Logger) TestNetworking
+}
+
+type ClientStarter interface {
+	StartClient(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me Client, peer Client) *tailnet.Conn
+}
+
 type SimpleServerOptions struct {
 	// FailUpgradeDERP will make the DERP server fail to handle the initial DERP
 	// upgrade in a way that causes the client to fallback to
@@ -369,77 +383,107 @@ http {
 	_, _ = ExecBackground(t, "server.nginx", nil, "nginx", []string{"-c", cfgPath})
 }
 
-// StartClientDERP creates a client connection to the server for coordination
+type BasicClientStarter struct {
-// and creates a tailnet.Conn which will only use DERP to connect to the peer.
+	BlockEndpoints      bool
-func StartClientDERP(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn {
+	DERPForceWebsockets bool
-	return startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{
+	// WaitForConnection means wait for (any) peer connection before returning from StartClient
-		Addresses:           []netip.Prefix{tailnet.TailscaleServicePrefix.PrefixFromUUID(me.ID)},
+	WaitForConnection bool
-		DERPMap:             derpMap,
+	// WaitForConnection means wait for a direct peer connection before returning from StartClient
-		BlockEndpoints:      true,
+	WaitForDirect bool
-		Logger:              logger,
+	// Service is a network service (e.g. an echo server) to start on the client. If Wait* is set, the service is
-		DERPForceWebSockets: false,
+	// started prior to waiting.
-		ListenPort:          me.ListenPort,
+	Service    NetworkService
-		// These tests don't have internet connection, so we need to force
+	LogPackets bool
-		// magicsock to do anything.
-		ForceNetworkUp: true,
-	})
 }
 
-// StartClientDERPWebSockets does the same thing as StartClientDERP but will
+type NetworkService interface {
-// only use DERP WebSocket fallback.
+	StartService(t *testing.T, logger slog.Logger, conn *tailnet.Conn)
-func StartClientDERPWebSockets(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn {
-	return startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{
-		Addresses:           []netip.Prefix{tailnet.TailscaleServicePrefix.PrefixFromUUID(me.ID)},
-		DERPMap:             derpMap,
-		BlockEndpoints:      true,
-		Logger:              logger,
-		DERPForceWebSockets: true,
-		ListenPort:          me.ListenPort,
-		// These tests don't have internet connection, so we need to force
-		// magicsock to do anything.
-		ForceNetworkUp: true,
-	})
 }
 
-// StartClientDirect does the same thing as StartClientDERP but disables
+func (b BasicClientStarter) StartClient(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn {
-// BlockEndpoints (which enables Direct connections), and waits for a direct
+	var hook capture.Callback
-// connection to be established between the two peers.
+	if b.LogPackets {
-func StartClientDirect(t *testing.T, logger slog.Logger, serverURL *url.URL, derpMap *tailcfg.DERPMap, me, peer Client) *tailnet.Conn {
+		pktLogger := packetLogger{logger}
+		hook = pktLogger.LogPacket
+	}
 	conn := startClientOptions(t, logger, serverURL, me, peer, &tailnet.Options{
 		Addresses:           []netip.Prefix{tailnet.TailscaleServicePrefix.PrefixFromUUID(me.ID)},
 		DERPMap:             derpMap,
-		BlockEndpoints:      false,
+		BlockEndpoints:      b.BlockEndpoints,
 		Logger:              logger,
-		DERPForceWebSockets: true,
+		DERPForceWebSockets: b.DERPForceWebsockets,
 		ListenPort:          me.ListenPort,
 		// These tests don't have internet connection, so we need to force
 		// magicsock to do anything.
 		ForceNetworkUp: true,
+		CaptureHook:    hook,
 	})
 
-	// Wait for direct connection to be established.
+	if b.Service != nil {
-	peerIP := tailnet.TailscaleServicePrefix.AddrFromUUID(peer.ID)
+		b.Service.StartService(t, logger, conn)
-	require.Eventually(t, func() bool {
+	}
-		t.Log("attempting ping to peer to judge direct connection")
+
-		ctx := testutil.Context(t, testutil.WaitShort)
+	if b.WaitForConnection || b.WaitForDirect {
-		_, p2p, pong, err := conn.Ping(ctx, peerIP)
+		// Wait for connection to be established.
-		if err != nil {
+		peerIP := tailnet.TailscaleServicePrefix.AddrFromUUID(peer.ID)
-			t.Logf("ping failed: %v", err)
+		require.Eventually(t, func() bool {
-			return false
+			t.Log("attempting ping to peer to judge direct connection")
-		}
+			ctx := testutil.Context(t, testutil.WaitShort)
-		if !p2p {
+			_, p2p, pong, err := conn.Ping(ctx, peerIP)
-			t.Log("ping succeeded, but not direct yet")
+			if err != nil {
-			return false
+				t.Logf("ping failed: %v", err)
-		}
+				return false
-		t.Logf("ping succeeded, direct connection established via %s", pong.Endpoint)
+			}
-		return true
+			if !p2p && b.WaitForDirect {
-	}, testutil.WaitLong, testutil.IntervalMedium)
+				t.Log("ping succeeded, but not direct yet")
+				return false
+			}
+			t.Logf("ping succeeded, p2p=%t, endpoint=%s", p2p, pong.Endpoint)
+			return true
+		}, testutil.WaitLong, testutil.IntervalMedium)
+	}
 
 	return conn
 }
 
-type ClientStarter struct {
+const EchoPort = 2381
-	Options *tailnet.Options
+
+type UDPEchoService struct{}
+
+func (UDPEchoService) StartService(t *testing.T, logger slog.Logger, _ *tailnet.Conn) {
+	// tailnet doesn't handle UDP connections "in-process" the way we do for TCP, so we need to listen in the OS,
+	// and tailnet will forward packets.
+	l, err := net.ListenUDP("udp", &net.UDPAddr{
+		IP:   net.IPv6zero, // all interfaces
+		Port: EchoPort,
+	})
+	require.NoError(t, err)
+	logger.Info(context.Background(), "started UDPEcho server")
+	t.Cleanup(func() {
+		lCloseErr := l.Close()
+		if lCloseErr != nil {
+			t.Logf("error closing UDPEcho listener: %v", lCloseErr)
+		}
+	})
+	go func() {
+		buf := make([]byte, 1500)
+		for {
+			n, remote, readErr := l.ReadFromUDP(buf)
+			if readErr != nil {
+				logger.Info(context.Background(), "error reading UDPEcho listener", slog.Error(readErr))
+				return
+			}
+			logger.Info(context.Background(), "received UDPEcho packet",
+				slog.F("len", n), slog.F("remote", remote))
+			n, writeErr := l.WriteToUDP(buf[:n], remote)
+			if writeErr != nil {
+				logger.Info(context.Background(), "error writing UDPEcho listener", slog.Error(writeErr))
+				return
+			}
+			logger.Info(context.Background(), "wrote UDPEcho packet",
+				slog.F("len", n), slog.F("remote", remote))
+		}
+	}()
 }
 
 func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, me, peer Client, options *tailnet.Options) *tailnet.Conn {
@@ -467,9 +511,16 @@ func startClientOptions(t *testing.T, logger slog.Logger, serverURL *url.URL, me
 		_ = conn.Close()
 	})
 
-	ctrl := tailnet.NewTunnelSrcCoordController(logger, conn)
+	var coordination tailnet.CloserWaiter
-	ctrl.AddDestination(peer.ID)
+	if me.TunnelSrc {
-	coordination := ctrl.New(coord)
+		ctrl := tailnet.NewTunnelSrcCoordController(logger, conn)
+		ctrl.AddDestination(peer.ID)
+		coordination = ctrl.New(coord)
+	} else {
+		// use the "Agent" controller so that we act as a tunnel destination and send "ReadyForHandshake" acks.
+		ctrl := tailnet.NewAgentCoordinationController(logger, conn)
+		coordination = ctrl.New(coord)
+	}
 	t.Cleanup(func() {
 		cctx, cancel := context.WithTimeout(context.Background(), testutil.WaitShort)
 		defer cancel()
@@ -492,11 +543,17 @@ func basicDERPMap(serverURLStr string) (*tailcfg.DERPMap, error) {
 	}
 
 	hostname := serverURL.Hostname()
-	ipv4 := ""
+	ipv4 := "none"
+	ipv6 := "none"
 	ip, err := netip.ParseAddr(hostname)
 	if err == nil {
 		hostname = ""
-		ipv4 = ip.String()
+		if ip.Is4() {
+			ipv4 = ip.String()
+		}
+		if ip.Is6() {
+			ipv6 = ip.String()
+		}
 	}
 
 	return &tailcfg.DERPMap{
@@ -511,7 +568,7 @@ func basicDERPMap(serverURLStr string) (*tailcfg.DERPMap, error) {
 						RegionID:         1,
 						HostName:         hostname,
 						IPv4:             ipv4,
-						IPv6:             "none",
+						IPv6:             ipv6,
 						DERPPort:         port,
 						STUNPort:         -1,
 						ForceHTTP:        true,
@@ -648,3 +705,35 @@ func (w *testWriter) Flush() {
 	}
 	w.capturedLines = nil
 }
+
+type packetLogger struct {
+	l slog.Logger
+}
+
+func (p packetLogger) LogPacket(path capture.Path, when time.Time, pkt []byte, _ packet.CaptureMeta) {
+	q := new(packet.Parsed)
+	q.Decode(pkt)
+	p.l.Info(context.Background(), "Packet",
+		slog.F("path", pathString(path)),
+		slog.F("when", when),
+		slog.F("decode", q.String()),
+		slog.F("len", len(pkt)),
+	)
+}
+
+func pathString(path capture.Path) string {
+	switch path {
+	case capture.FromLocal:
+		return "Local"
+	case capture.FromPeer:
+		return "Peer"
+	case capture.SynthesizedToLocal:
+		return "SynthesizedToLocal"
+	case capture.SynthesizedToPeer:
+		return "SynthesizedToPeer"
+	case capture.PathDisco:
+		return "Disco"
+	default:
+		return "<<UNKNOWN>>"
+	}
+}

tailnet/test/integration/integration_test.go

@@ -76,70 +76,90 @@ func TestMain(m *testing.M) {
 var topologies = []integration.TestTopology{
 	{
 		// Test that DERP over loopback works.
-		Name:            "BasicLoopbackDERP",
+		Name:               "BasicLoopbackDERP",
-		SetupNetworking: integration.SetupNetworkingLoopback,
+		NetworkingProvider: integration.NetworkingLoopback{},
-		Server:          integration.SimpleServerOptions{},
+		Server:             integration.SimpleServerOptions{},
-		StartClient:     integration.StartClientDERP,
+		ClientStarter:      integration.BasicClientStarter{BlockEndpoints: true},
-		RunTests:        integration.TestSuite,
+		RunTests:           integration.TestSuite,
 	},
 	{
 		// Test that DERP over "easy" NAT works. The server, client 1 and client
 		// 2 are on different networks with their own routers, which are joined
 		// by a bridge.
-		Name:            "EasyNATDERP",
+		Name:               "EasyNATDERP",
-		SetupNetworking: integration.SetupNetworkingEasyNAT,
+		NetworkingProvider: integration.NetworkingNAT{StunCount: 0, Client1Hard: false, Client2Hard: false},
-		Server:          integration.SimpleServerOptions{},
+		Server:             integration.SimpleServerOptions{},
-		StartClient:     integration.StartClientDERP,
+		ClientStarter:      integration.BasicClientStarter{BlockEndpoints: true},
-		RunTests:        integration.TestSuite,
+		RunTests:           integration.TestSuite,
 	},
 	{
 		// Test that direct over "easy" NAT works with IP/ports grabbed from
 		// STUN.
-		Name:            "EasyNATDirect",
+		Name:               "EasyNATDirect",
-		SetupNetworking: integration.SetupNetworkingEasyNATWithSTUN,
+		NetworkingProvider: integration.NetworkingNAT{StunCount: 1, Client1Hard: false, Client2Hard: false},
-		Server:          integration.SimpleServerOptions{},
+		Server:             integration.SimpleServerOptions{},
-		StartClient:     integration.StartClientDirect,
+		ClientStarter:      integration.BasicClientStarter{WaitForDirect: true},
-		RunTests:        integration.TestSuite,
+		RunTests:           integration.TestSuite,
 	},
 	{
 		// Test that direct over hard NAT <=> easy NAT works.
-		Name:            "HardNATEasyNATDirect",
+		Name:               "HardNATEasyNATDirect",
-		SetupNetworking: integration.SetupNetworkingHardNATEasyNATDirect,
+		NetworkingProvider: integration.NetworkingNAT{StunCount: 2, Client1Hard: true, Client2Hard: false},
-		Server:          integration.SimpleServerOptions{},
+		Server:             integration.SimpleServerOptions{},
-		StartClient:     integration.StartClientDirect,
+		ClientStarter:      integration.BasicClientStarter{WaitForDirect: true},
-		RunTests:        integration.TestSuite,
+		RunTests:           integration.TestSuite,
+	},
+	{
+		// Test that direct over normal MTU works.
+		Name:               "DirectMTU1500",
+		NetworkingProvider: integration.TriangleNetwork{InterClientMTU: 1500},
+		Server:             integration.SimpleServerOptions{},
+		ClientStarter: integration.BasicClientStarter{
+			WaitForDirect: true,
+			Service:       integration.UDPEchoService{},
+			LogPackets:    true,
+		},
+		RunTests: integration.TestBigUDP,
+	},
+	{
+		// Test that small MTU works.
+		Name:               "MTU1280",
+		NetworkingProvider: integration.TriangleNetwork{InterClientMTU: 1280},
+		Server:             integration.SimpleServerOptions{},
+		ClientStarter:      integration.BasicClientStarter{Service: integration.UDPEchoService{}, LogPackets: true},
+		RunTests:           integration.TestBigUDP,
 	},
 	{
 		// Test that DERP over WebSocket (as well as DERPForceWebSockets works).
 		// This does not test the actual DERP failure detection code and
 		// automatic fallback.
-		Name:            "DERPForceWebSockets",
+		Name:               "DERPForceWebSockets",
-		SetupNetworking: integration.SetupNetworkingEasyNAT,
+		NetworkingProvider: integration.NetworkingNAT{StunCount: 0, Client1Hard: false, Client2Hard: false},
 		Server: integration.SimpleServerOptions{
 			FailUpgradeDERP:   false,
 			DERPWebsocketOnly: true,
 		},
-		StartClient: integration.StartClientDERPWebSockets,
+		ClientStarter: integration.BasicClientStarter{BlockEndpoints: true, DERPForceWebsockets: true},
-		RunTests:    integration.TestSuite,
+		RunTests:      integration.TestSuite,
 	},
 	{
 		// Test that falling back to DERP over WebSocket works.
-		Name:            "DERPFallbackWebSockets",
+		Name:               "DERPFallbackWebSockets",
-		SetupNetworking: integration.SetupNetworkingEasyNAT,
+		NetworkingProvider: integration.NetworkingNAT{StunCount: 0, Client1Hard: false, Client2Hard: false},
 		Server: integration.SimpleServerOptions{
 			FailUpgradeDERP:   true,
 			DERPWebsocketOnly: false,
 		},
 		// Use a basic client that will try `Upgrade: derp` first.
-		StartClient: integration.StartClientDERP,
+		ClientStarter: integration.BasicClientStarter{BlockEndpoints: true},
-		RunTests:    integration.TestSuite,
+		RunTests:      integration.TestSuite,
 	},
 	{
-		Name:            "BasicLoopbackDERPNGINX",
+		Name:               "BasicLoopbackDERPNGINX",
-		SetupNetworking: integration.SetupNetworkingLoopback,
+		NetworkingProvider: integration.NetworkingLoopback{},
-		Server:          integration.NGINXServerOptions{},
+		Server:             integration.NGINXServerOptions{},
-		StartClient:     integration.StartClientDERP,
+		ClientStarter:      integration.BasicClientStarter{BlockEndpoints: true},
-		RunTests:        integration.TestSuite,
+		RunTests:           integration.TestSuite,
 	},
 }
 
@@ -151,7 +171,6 @@ func TestIntegration(t *testing.T) {
 	}
 
 	for _, topo := range topologies {
-		topo := topo
 		t.Run(topo.Name, func(t *testing.T) {
 			// These can run in parallel because every test should be in an
 			// isolated NetNS.
@@ -166,7 +185,11 @@ func TestIntegration(t *testing.T) {
 			}
 
 			log := testutil.Logger(t)
-			networking := topo.SetupNetworking(t, log)
+			networking := topo.NetworkingProvider.SetupNetworking(t, log)
+
+			tempDir := t.TempDir()
+			// useful for debugging:
+			// networking.Client1.Process.CapturePackets(t, "client1", tempDir)
 
 			// Useful for debugging network namespaces by avoiding cleanup.
 			// t.Cleanup(func() {
@@ -181,7 +204,6 @@ func TestIntegration(t *testing.T) {
 			}
 
 			// Write the DERP maps to a file.
-			tempDir := t.TempDir()
 			client1DERPMapPath := filepath.Join(tempDir, "client1-derp-map.json")
 			client1DERPMap, err := networking.Client1.ResolveDERPMap()
 			require.NoError(t, err, "resolve client 1 DERP map")
@@ -270,7 +292,7 @@ func handleTestSubprocess(t *testing.T) {
 
 			waitForServerAvailable(t, serverURL)
 
-			conn := topo.StartClient(t, logger, serverURL, &derpMap, me, peer)
+			conn := topo.ClientStarter.StartClient(t, logger, serverURL, &derpMap, me, peer)
 
 			if me.ShouldRunTests {
 				// Wait for connectivity.

tailnet/test/integration/network.go

@@ -5,9 +5,11 @@ package integration
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"os"
 	"os/exec"
+	"path"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -71,11 +73,21 @@ type TestNetworkingProcess struct {
 	NetNS *os.File
 }
 
-// SetupNetworkingLoopback creates a network namespace with a loopback interface
+func (p TestNetworkingProcess) CapturePackets(t *testing.T, name, dir string) {
+	dumpfile := path.Join(dir, name+".pcap")
+	_, _ = ExecBackground(t, name+".pcap", p.NetNS, "tcpdump", []string{
+		"-i", "any",
+		"-w", dumpfile,
+	})
+}
+
+// NetworkingLoopback creates a network namespace with a loopback interface
 // for all tests to share. This is the simplest networking setup. The network
 // namespace only exists for isolation on the host and doesn't serve any routing
 // purpose.
-func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking {
+type NetworkingLoopback struct{}
+
+func (NetworkingLoopback) SetupNetworking(t *testing.T, _ slog.Logger) TestNetworking {
 	// Create a single network namespace for all tests so we can have an
 	// isolated loopback interface.
 	netNSFile := createNetNS(t, uniqNetName(t))
@@ -102,91 +114,25 @@ func SetupNetworkingLoopback(t *testing.T, _ slog.Logger) TestNetworking {
 	}
 }
 
-func easyNAT(t *testing.T) fakeInternet {
+// NetworkingNAT creates a fake internet and sets up "NAT"
-	internet := createFakeInternet(t)
+// forwarding rules, either easy or hard.
-
-	_, err := commandInNetNS(internet.BridgeNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output()
-	require.NoError(t, wrapExitErr(err), "enable IP forwarding in bridge NetNS")
-
-	// Set up iptables masquerade rules to allow each router to NAT packets.
-	leaves := []struct {
-		fakeRouterLeaf
-		clientPort int
-		natPort    int
-	}{
-		{internet.Client1, client1Port, client1RouterPort},
-		{internet.Client2, client2Port, client2RouterPort},
-	}
-	for _, leaf := range leaves {
-		_, err := commandInNetNS(leaf.RouterNetNS, "sysctl", []string{"-w", "net.ipv4.ip_forward=1"}).Output()
-		require.NoError(t, wrapExitErr(err), "enable IP forwarding in router NetNS")
-
-		// All non-UDP traffic should use regular masquerade e.g. for HTTP.
-		_, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{
-			"-t", "nat",
-			"-A", "POSTROUTING",
-			// Every interface except loopback.
-			"!", "-o", "lo",
-			// Every protocol except UDP.
-			"!", "-p", "udp",
-			"-j", "MASQUERADE",
-		}).Output()
-		require.NoError(t, wrapExitErr(err), "add iptables non-UDP masquerade rule")
-
-		// Outgoing traffic should get NATed to the router's IP.
-		_, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{
-			"-t", "nat",
-			"-A", "POSTROUTING",
-			"-p", "udp",
-			"--sport", fmt.Sprint(leaf.clientPort),
-			"-j", "SNAT",
-			"--to-source", fmt.Sprintf("%s:%d", leaf.RouterIP, leaf.natPort),
-		}).Output()
-		require.NoError(t, wrapExitErr(err), "add iptables SNAT rule")
-
-		// Incoming traffic should be forwarded to the client's IP.
-		_, err = commandInNetNS(leaf.RouterNetNS, "iptables", []string{
-			"-t", "nat",
-			"-A", "PREROUTING",
-			"-p", "udp",
-			"--dport", fmt.Sprint(leaf.natPort),
-			"-j", "DNAT",
-			"--to-destination", fmt.Sprintf("%s:%d", leaf.ClientIP, leaf.clientPort),
-		}).Output()
-		require.NoError(t, wrapExitErr(err), "add iptables DNAT rule")
-	}
-
-	return internet
-}
-
-// SetupNetworkingEasyNAT creates a fake internet and sets up "easy NAT"
-// forwarding rules.
 // See createFakeInternet.
 // NAT is achieved through a single iptables masquerade rule.
-func SetupNetworkingEasyNAT(t *testing.T, _ slog.Logger) TestNetworking {
+type NetworkingNAT struct {
-	return easyNAT(t).Net
+	StunCount   int
+	Client1Hard bool
+	Client2Hard bool
 }
 
-// SetupNetworkingEasyNATWithSTUN does the same as SetupNetworkingEasyNAT, but
+// SetupNetworking creates a fake internet with multiple STUN servers and sets up
-// also creates a namespace and bridge address for a STUN server.
+// NAT forwarding rules. Client NATs are controlled by the switches ClientXHard, which if true, sets up hard
-func SetupNetworkingEasyNATWithSTUN(t *testing.T, _ slog.Logger) TestNetworking {
+// nat.
-	internet := easyNAT(t)
+func (n NetworkingNAT) SetupNetworking(t *testing.T, l slog.Logger) TestNetworking {
-	internet.Net.STUNs = []TestNetworkingSTUN{
+	logger := l.Named("setup-networking").Leveled(slog.LevelDebug)
-		prepareSTUNServer(t, &internet, 0),
-	}
-
-	return internet.Net
-}
-
-// hardNAT creates a fake internet with multiple STUN servers and sets up "hard
-// NAT" forwarding rules. If bothHard is false, only the first client will have
-// hard NAT rules, and the second client will have easy NAT rules.
-//
-//nolint:revive
-func hardNAT(t *testing.T, stunCount int, bothHard bool) fakeInternet {
 	internet := createFakeInternet(t)
-	internet.Net.STUNs = make([]TestNetworkingSTUN, stunCount)
+	logger.Debug(context.Background(), "preparing STUN", slog.F("stun_count", n.StunCount))
-	for i := 0; i < stunCount; i++ {
+	internet.Net.STUNs = make([]TestNetworkingSTUN, n.StunCount)
+	for i := 0; i < n.StunCount; i++ {
 		internet.Net.STUNs[i] = prepareSTUNServer(t, &internet, i)
 	}
 
@@ -202,8 +148,14 @@ func hardNAT(t *testing.T, stunCount int, bothHard bool) fakeInternet {
 		natStartPortSTUN int
 	}{
 		{
-			fakeRouterLeaf:   internet.Client1,
+			fakeRouterLeaf: internet.Client1,
-			peerIP:           internet.Client2.RouterIP,
+			// If peerIP is empty, we do easy NAT (even for STUN)
+			peerIP: func() string {
+				if n.Client1Hard {
+					return internet.Client2.RouterIP
+				}
+				return ""
+			}(),
 			clientPort:       client1Port,
 			natPortPeer:      client1RouterPort,
 			natStartPortSTUN: client1RouterPortSTUN,
@@ -212,7 +164,7 @@ func hardNAT(t *testing.T, stunCount int, bothHard bool) fakeInternet {
 			fakeRouterLeaf: internet.Client2,
 			// If peerIP is empty, we do easy NAT (even for STUN)
 			peerIP: func() string {
-				if bothHard {
+				if n.Client2Hard {
 					return internet.Client1.RouterIP
 				}
 				return ""
@@ -235,6 +187,9 @@ func hardNAT(t *testing.T, stunCount int, bothHard bool) fakeInternet {
 		// NAT from this client to each STUN server. Only do this if we're doing
 		// hard NAT, as the rule above will also touch STUN traffic in easy NAT.
 		if leaf.peerIP != "" {
+			logger.Debug(context.Background(), "creating NAT to STUN",
+				slog.F("client_ip", leaf.ClientIP), slog.F("peer_ip", leaf.peerIP),
+			)
 			for i, stun := range internet.Net.STUNs {
 				natPort := leaf.natStartPortSTUN + i
 				iptablesNAT(t, leaf.RouterNetNS, leaf.ClientIP, leaf.clientPort, leaf.RouterIP, natPort, stun.IP)
@@ -242,11 +197,7 @@ func hardNAT(t *testing.T, stunCount int, bothHard bool) fakeInternet {
 		}
 	}
 
-	return internet
+	return internet.Net
-}
-
-func SetupNetworkingHardNATEasyNATDirect(t *testing.T, _ slog.Logger) TestNetworking {
-	return hardNAT(t, 2, false).Net
 }
 
 type vethPair struct {
@@ -438,6 +389,149 @@ func createFakeInternet(t *testing.T) fakeInternet {
 	return router
 }
 
+type TriangleNetwork struct {
+	InterClientMTU int
+}
+
+type fakeTriangleNetwork struct {
+	NamePrefix             string
+	ServerNetNS            *os.File
+	Client1NetNS           *os.File
+	Client2NetNS           *os.File
+	ServerClient1VethPair  vethPair
+	ServerClient2VethPair  vethPair
+	Client1Client2VethPair vethPair
+}
+
+// SetupNetworking creates multiple namespaces with veth pairs between them
+// with the following topology:
+// .
+// .     ┌────────────────────────────────────────────┐
+// .     │                  Server                    │
+// .     └─────┬───────────────────────────────────┬──┘
+// .           │fdac:38fa:ffff:2::3                │fdac:38fa:ffff:3::3
+// .       veth│                               veth│
+// .           │fdac:38fa:ffff:2::1                │fdac:38fa:ffff:3::2
+// .   ┌───────┴──────┐                      ┌─────┴───────┐
+// .   │              │   fdac:38fa:ffff:1::2│             │
+// .   │  Client 1    ├──────────────────────┤  Client 2   │
+// .   │              │fdac:38fa:ffff:1::1   │             │
+// .   └──────────────┘                      └─────────────┘
+func (n TriangleNetwork) SetupNetworking(t *testing.T, l slog.Logger) TestNetworking {
+	logger := l.Named("setup-networking").Leveled(slog.LevelDebug)
+	t.Helper()
+	var (
+		namePrefix = uniqNetName(t) + "_"
+		network    = fakeTriangleNetwork{
+			NamePrefix: namePrefix,
+		}
+		// Unique Local Address prefix
+		ula = "fdac:38fa:ffff:"
+	)
+
+	// Create three network namespaces for server, client1, and client2
+	network.ServerNetNS = createNetNS(t, namePrefix+"server")
+	network.Client1NetNS = createNetNS(t, namePrefix+"client1")
+	network.Client2NetNS = createNetNS(t, namePrefix+"client2")
+
+	// Create veth pair between server and client1
+	network.ServerClient1VethPair = vethPair{
+		Outer: namePrefix + "s-1",
+		Inner: namePrefix + "1-s",
+	}
+	err := createVethPair(network.ServerClient1VethPair.Outer, network.ServerClient1VethPair.Inner)
+	require.NoErrorf(t, err, "create veth pair %q <-> %q",
+		network.ServerClient1VethPair.Outer, network.ServerClient1VethPair.Inner)
+
+	// Move server-client1 veth ends to their respective namespaces
+	err = setVethNetNS(network.ServerClient1VethPair.Outer, int(network.ServerNetNS.Fd()))
+	require.NoErrorf(t, err, "set veth %q to server NetNS", network.ServerClient1VethPair.Outer)
+	err = setVethNetNS(network.ServerClient1VethPair.Inner, int(network.Client1NetNS.Fd()))
+	require.NoErrorf(t, err, "set veth %q to client1 NetNS", network.ServerClient1VethPair.Inner)
+
+	// Create veth pair between server and client2
+	network.ServerClient2VethPair = vethPair{
+		Outer: namePrefix + "s-2",
+		Inner: namePrefix + "2-s",
+	}
+	err = createVethPair(network.ServerClient2VethPair.Outer, network.ServerClient2VethPair.Inner)
+	require.NoErrorf(t, err, "create veth pair %q <-> %q",
+		network.ServerClient2VethPair.Outer, network.ServerClient2VethPair.Inner)
+
+	// Move server-client2 veth ends to their respective namespaces
+	err = setVethNetNS(network.ServerClient2VethPair.Outer, int(network.ServerNetNS.Fd()))
+	require.NoErrorf(t, err, "set veth %q to server NetNS", network.ServerClient2VethPair.Outer)
+	err = setVethNetNS(network.ServerClient2VethPair.Inner, int(network.Client2NetNS.Fd()))
+	require.NoErrorf(t, err, "set veth %q to client2 NetNS", network.ServerClient2VethPair.Inner)
+
+	// Create veth pair between client1 and client2
+	network.Client1Client2VethPair = vethPair{
+		Outer: namePrefix + "1-2",
+		Inner: namePrefix + "2-1",
+	}
+	logger.Debug(context.Background(), "creating inter-client link", slog.F("mtu", n.InterClientMTU))
+	err = createVethPair(network.Client1Client2VethPair.Outer, network.Client1Client2VethPair.Inner,
+		withMTU(n.InterClientMTU))
+	require.NoErrorf(t, err, "create veth pair %q <-> %q",
+		network.Client1Client2VethPair.Outer, network.Client1Client2VethPair.Inner)
+
+	// Move client1-client2 veth ends to their respective namespaces
+	err = setVethNetNS(network.Client1Client2VethPair.Outer, int(network.Client1NetNS.Fd()))
+	require.NoErrorf(t, err, "set veth %q to client1 NetNS", network.Client1Client2VethPair.Outer)
+	err = setVethNetNS(network.Client1Client2VethPair.Inner, int(network.Client2NetNS.Fd()))
+	require.NoErrorf(t, err, "set veth %q to client2 NetNS", network.Client1Client2VethPair.Inner)
+
+	// Set IP addresses according to the diagram:
+	err = setInterfaceIP6(network.ServerNetNS, network.ServerClient1VethPair.Outer, ula+"2::3")
+	require.NoErrorf(t, err, "set IP on server-client1 interface")
+	err = setInterfaceIP6(network.ServerNetNS, network.ServerClient2VethPair.Outer, ula+"3::3")
+	require.NoErrorf(t, err, "set IP on server-client2 interface")
+
+	err = setInterfaceIP6(network.Client1NetNS, network.ServerClient1VethPair.Inner, ula+"2::1")
+	require.NoErrorf(t, err, "set IP on client1-server interface")
+	err = setInterfaceIP6(network.Client1NetNS, network.Client1Client2VethPair.Outer, ula+"1::1")
+	require.NoErrorf(t, err, "set IP on client1-client2 interface")
+
+	err = setInterfaceIP6(network.Client2NetNS, network.ServerClient2VethPair.Inner, ula+"3::2")
+	require.NoErrorf(t, err, "set IP on client2-server interface")
+	err = setInterfaceIP6(network.Client2NetNS, network.Client1Client2VethPair.Inner, ula+"1::2")
+	require.NoErrorf(t, err, "set IP on client2-client1 interface")
+
+	// Bring up all interfaces
+	interfaces := []struct {
+		netNS     *os.File
+		ifaceName string
+	}{
+		{network.ServerNetNS, network.ServerClient1VethPair.Outer},
+		{network.ServerNetNS, network.ServerClient2VethPair.Outer},
+		{network.Client1NetNS, network.ServerClient1VethPair.Inner},
+		{network.Client1NetNS, network.Client1Client2VethPair.Outer},
+		{network.Client2NetNS, network.ServerClient2VethPair.Inner},
+		{network.Client2NetNS, network.Client1Client2VethPair.Inner},
+	}
+	for _, iface := range interfaces {
+		err = setInterfaceUp(iface.netNS, iface.ifaceName)
+		require.NoErrorf(t, err, "bring up interface %q", iface.ifaceName)
+		// Note: routes are not needed as we are fully connected, so nothing needs to forward IP to a further
+		// destination.
+	}
+
+	return TestNetworking{
+		Server: TestNetworkingServer{
+			Process:    TestNetworkingProcess{NetNS: network.ServerNetNS},
+			ListenAddr: "[::]:8080", // Server listens on all IPs
+		},
+		Client1: TestNetworkingClient{
+			Process:         TestNetworkingProcess{NetNS: network.Client1NetNS},
+			ServerAccessURL: "http://[" + ula + "2::3]:8080", // Client1 accesses server directly
+		},
+		Client2: TestNetworkingClient{
+			Process:         TestNetworkingProcess{NetNS: network.Client2NetNS},
+			ServerAccessURL: "http://[" + ula + "3::3]:8080", // Client2 accesses server directly
+		},
+	}
+}
+
 func uniqNetName(t *testing.T) string {
 	t.Helper()
 	netNSName := "cdr_"
@@ -522,8 +616,8 @@ func createNetNS(t *testing.T, name string) *os.File {
 	})
 
 	// Open /run/netns/$name to get a file descriptor to the network namespace.
-	path := fmt.Sprintf("/run/netns/%s", name)
+	netnsPath := fmt.Sprintf("/run/netns/%s", name)
-	file, err := os.OpenFile(path, os.O_RDONLY, 0)
+	file, err := os.OpenFile(netnsPath, os.O_RDONLY, 0)
 	require.NoError(t, err, "open network namespace file")
 	t.Cleanup(func() {
 		_ = file.Close()
@@ -568,10 +662,22 @@ func setInterfaceBridge(netNS *os.File, ifaceName, bridgeName string) error {
 	return nil
 }
 
+type linkOption func(attrs netlink.LinkAttrs) netlink.LinkAttrs
+
+func withMTU(mtu int) linkOption {
+	return func(attrs netlink.LinkAttrs) netlink.LinkAttrs {
+		attrs.MTU = mtu
+		return attrs
+	}
+}
+
 // createVethPair creates a veth pair with the given names.
-func createVethPair(parentVethName, peerVethName string) error {
+func createVethPair(parentVethName, peerVethName string, options ...linkOption) error {
 	linkAttrs := netlink.NewLinkAttrs()
 	linkAttrs.Name = parentVethName
+	for _, option := range options {
+		linkAttrs = option(linkAttrs)
+	}
 	veth := &netlink.Veth{
 		LinkAttrs: linkAttrs,
 		PeerName:  peerVethName,
@@ -611,6 +717,17 @@ func setInterfaceIP(netNS *os.File, ifaceName, ip string) error {
 	return nil
 }
 
+// setInterfaceIP6 sets the IPv6 address on the given interface. It automatically
+// adds a /64 subnet mask.
+func setInterfaceIP6(netNS *os.File, ifaceName, ip string) error {
+	_, err := commandInNetNS(netNS, "ip", []string{"addr", "add", ip + "/64", "dev", ifaceName}).Output()
+	if err != nil {
+		return xerrors.Errorf("set IP %q on interface %q in netns: %w", ip, ifaceName, wrapExitErr(err))
+	}
+
+	return nil
+}
+
 // setInterfaceUp brings the given interface up.
 func setInterfaceUp(netNS *os.File, ifaceName string) error {
 	_, err := commandInNetNS(netNS, "ip", []string{"link", "set", ifaceName, "up"}).Output()
@@ -703,7 +820,9 @@ func iptablesMasqueradeNonUDP(t *testing.T, netNS *os.File) {
 // iptablesNAT sets up iptables rules for NAT forwarding. If destIP is
 // specified, the forwarding rule will only apply to traffic to/from that IP
 // (mapvarydest).
-func iptablesNAT(t *testing.T, netNS *os.File, clientIP string, clientPort int, routerIP string, routerPort int, destIP string) {
+func iptablesNAT(
+	t *testing.T, netNS *os.File, clientIP string, clientPort int, routerIP string, routerPort int, destIP string,
+) {
 	t.Helper()
 
 	snatArgs := []string{

tailnet/test/integration/suite.go

@@ -5,6 +5,7 @@ package integration
 
 import (
 	"net/http"
+	"net/netip"
 	"net/url"
 	"testing"
 	"time"
@@ -80,3 +81,40 @@ func TestSuite(t *testing.T, _ slog.Logger, serverURL *url.URL, conn *tailnet.Co
 		require.NoError(t, err, "ping peer after restart")
 	})
 }
+
+func TestBigUDP(t *testing.T, logger slog.Logger, _ *url.URL, conn *tailnet.Conn, _, peer Client) {
+	t.Run("UDPEcho", func(t *testing.T) {
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		peerIP := tailnet.TailscaleServicePrefix.AddrFromUUID(peer.ID)
+		udpConn, err := conn.DialContextUDP(ctx, netip.AddrPortFrom(peerIP, uint16(EchoPort)))
+		require.NoError(t, err)
+		defer udpConn.Close()
+
+		// 1280 max tunnel packet size
+		//  -40
+		//   -8 UDP header
+		// ----------------------------
+		// 1232 data size
+		logger.Info(ctx, "sending UDP test packet")
+		packet := make([]byte, 1232)
+		for i := range packet {
+			packet[i] = byte(i % 256)
+		}
+		err = udpConn.SetWriteDeadline(time.Now().Add(5 * time.Second))
+		require.NoError(t, err)
+		n, err := udpConn.Write(packet)
+		require.NoError(t, err)
+		require.Equal(t, len(packet), n)
+
+		// read the echo
+		logger.Info(ctx, "attempting to read UDP reply")
+		buf := make([]byte, 1280)
+		err = udpConn.SetReadDeadline(time.Now().Add(5 * time.Second))
+		require.NoError(t, err)
+		n, err = udpConn.Read(buf)
+		require.NoError(t, err)
+		require.Equal(t, len(packet), n)
+		require.Equal(t, packet, buf[:n])
+	})
+}