feat: Add Tailscale networking (#3505)

* fix: Add coder user to docker group on installation

This makes for a simpler setup, and reduces the likelihood
a user runs into a strange issue.

* Add wgnet

* Add ping

* Add listening

* Finish refactor to make this work

* Add interface for swapping

* Fix conncache with interface

* chore: update gvisor

* fix tailscale types

* linting

* more linting

* Add coordinator

* Add coordinator tests

* Fix coordination

* It compiles!

* Move all connection negotiation in-memory

* Migrate coordinator to use net.conn

* Add closed func

* Fix close listener func

* Make reconnecting PTY work

* Fix reconnecting PTY

* Update CI to Go 1.19

* Add CLI flags for DERP mapping

* Fix Tailnet test

* Rename ConnCoordinator to TailnetCoordinator

* Remove print statement from workspace agent test

* Refactor wsconncache to use tailnet

* Remove STUN from unit tests

* Add migrate back to dump

* chore: Upgrade to Go 1.19

This is required as part of #3505.

* Fix reconnecting PTY tests

* fix: update wireguard-go to fix devtunnel

* fix migration numbers

* linting

* Return early for status if endpoints are empty

* Update cli/server.go

Co-authored-by: Colin Adler <colin1adler@gmail.com>

* Update cli/server.go

Co-authored-by: Colin Adler <colin1adler@gmail.com>

* Fix frontend entites

* Fix agent bicopy

* Fix race condition for the last node

* Fix down migration

* Fix connection RBAC

* Fix migration numbers

* Fix forwarding TCP to a local port

* Implement ping for tailnet

* Rename to ForceHTTP

* Add external derpmapping

* Expose DERP region names to the API

* Add global option to enable Tailscale networking for web

* Mark DERP flags hidden while testing

* Update DERP map on reconnect

* Add close func to workspace agents

* Fix race condition in upstream dependency

* Fix feature columns race condition

Co-authored-by: Colin Adler <colin1adler@gmail.com>

tailnet/conn.go

@@ -0,0 +1,516 @@
+package tailnet
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/netip"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/google/uuid"
+	"go4.org/netipx"
+	"golang.org/x/xerrors"
+	"gvisor.dev/gvisor/pkg/tcpip/adapters/gonet"
+	"tailscale.com/hostinfo"
+	"tailscale.com/ipn/ipnstate"
+	"tailscale.com/net/dns"
+	"tailscale.com/net/netns"
+	"tailscale.com/net/tsdial"
+	"tailscale.com/net/tstun"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/ipproto"
+	"tailscale.com/types/key"
+	tslogger "tailscale.com/types/logger"
+	"tailscale.com/types/netmap"
+	"tailscale.com/wgengine"
+	"tailscale.com/wgengine/filter"
+	"tailscale.com/wgengine/magicsock"
+	"tailscale.com/wgengine/monitor"
+	"tailscale.com/wgengine/netstack"
+	"tailscale.com/wgengine/router"
+	"tailscale.com/wgengine/wgcfg/nmcfg"
+
+	"github.com/coder/coder/cryptorand"
+
+	"cdr.dev/slog"
+)
+
+func init() {
+	// Globally disable network namespacing.
+	// All networking happens in userspace.
+	netns.SetEnabled(false)
+}
+
+type Options struct {
+	Addresses []netip.Prefix
+	DERPMap   *tailcfg.DERPMap
+
+	Logger slog.Logger
+}
+
+// NewConn constructs a new Wireguard server that will accept connections from the addresses provided.
+func NewConn(options *Options) (*Conn, error) {
+	if options == nil {
+		options = &Options{}
+	}
+	if len(options.Addresses) == 0 {
+		return nil, xerrors.New("At least one IP range must be provided")
+	}
+	if options.DERPMap == nil {
+		return nil, xerrors.New("DERPMap must be provided")
+	}
+	nodePrivateKey := key.NewNode()
+	nodePublicKey := nodePrivateKey.Public()
+
+	netMap := &netmap.NetworkMap{
+		NodeKey:    nodePublicKey,
+		PrivateKey: nodePrivateKey,
+		Addresses:  options.Addresses,
+		PacketFilter: []filter.Match{{
+			// Allow any protocol!
+			IPProto: []ipproto.Proto{ipproto.TCP, ipproto.UDP, ipproto.ICMPv4, ipproto.ICMPv6, ipproto.SCTP},
+			// Allow traffic sourced from anywhere.
+			Srcs: []netip.Prefix{
+				netip.PrefixFrom(netip.AddrFrom4([4]byte{}), 0),
+				netip.PrefixFrom(netip.AddrFrom16([16]byte{}), 0),
+			},
+			// Allow traffic to route anywhere.
+			Dsts: []filter.NetPortRange{
+				{
+					Net: netip.PrefixFrom(netip.AddrFrom4([4]byte{}), 0),
+					Ports: filter.PortRange{
+						First: 0,
+						Last:  65535,
+					},
+				},
+				{
+					Net: netip.PrefixFrom(netip.AddrFrom16([16]byte{}), 0),
+					Ports: filter.PortRange{
+						First: 0,
+						Last:  65535,
+					},
+				},
+			},
+			Caps: []filter.CapMatch{},
+		}},
+	}
+	nodeID, err := cryptorand.Int63()
+	if err != nil {
+		return nil, xerrors.Errorf("generate node id: %w", err)
+	}
+	// This is used by functions below to identify the node via key
+	netMap.SelfNode = &tailcfg.Node{
+		ID:         tailcfg.NodeID(nodeID),
+		Key:        nodePublicKey,
+		Addresses:  options.Addresses,
+		AllowedIPs: options.Addresses,
+	}
+
+	wireguardMonitor, err := monitor.New(Logger(options.Logger.Named("wgmonitor")))
+	if err != nil {
+		return nil, xerrors.Errorf("create wireguard link monitor: %w", err)
+	}
+
+	dialer := &tsdial.Dialer{
+		Logf: Logger(options.Logger),
+	}
+	wireguardEngine, err := wgengine.NewUserspaceEngine(Logger(options.Logger.Named("wgengine")), wgengine.Config{
+		LinkMonitor: wireguardMonitor,
+		Dialer:      dialer,
+	})
+	if err != nil {
+		return nil, xerrors.Errorf("create wgengine: %w", err)
+	}
+	dialer.UseNetstackForIP = func(ip netip.Addr) bool {
+		_, ok := wireguardEngine.PeerForIP(ip)
+		return ok
+	}
+
+	// This is taken from Tailscale:
+	// https://github.com/tailscale/tailscale/blob/0f05b2c13ff0c305aa7a1655fa9c17ed969d65be/tsnet/tsnet.go#L247-L255
+	wireguardInternals, ok := wireguardEngine.(wgengine.InternalsGetter)
+	if !ok {
+		return nil, xerrors.Errorf("wireguard engine isn't the correct type %T", wireguardEngine)
+	}
+	tunDevice, magicConn, dnsManager, ok := wireguardInternals.GetInternals()
+	if !ok {
+		return nil, xerrors.New("failed to get wireguard internals")
+	}
+
+	// Update the keys for the magic connection!
+	err = magicConn.SetPrivateKey(nodePrivateKey)
+	if err != nil {
+		return nil, xerrors.Errorf("set node private key: %w", err)
+	}
+	netMap.SelfNode.DiscoKey = magicConn.DiscoPublicKey()
+
+	netStack, err := netstack.Create(
+		Logger(options.Logger.Named("netstack")), tunDevice, wireguardEngine, magicConn, dialer, dnsManager)
+	if err != nil {
+		return nil, xerrors.Errorf("create netstack: %w", err)
+	}
+	dialer.NetstackDialTCP = func(ctx context.Context, dst netip.AddrPort) (net.Conn, error) {
+		return netStack.DialContextTCP(ctx, dst)
+	}
+	netStack.ProcessLocalIPs = true
+	err = netStack.Start()
+	if err != nil {
+		return nil, xerrors.Errorf("start netstack: %w", err)
+	}
+	wireguardEngine = wgengine.NewWatchdog(wireguardEngine)
+
+	// Update the wireguard configuration to allow traffic to flow.
+	wireguardConfig, err := nmcfg.WGCfg(netMap, Logger(options.Logger.Named("wgconfig")), netmap.AllowSingleHosts, "")
+	if err != nil {
+		return nil, xerrors.Errorf("create wgcfg: %w", err)
+	}
+
+	wireguardRouter := &router.Config{
+		LocalAddrs: wireguardConfig.Addresses,
+	}
+	err = wireguardEngine.Reconfig(wireguardConfig, wireguardRouter, &dns.Config{}, &tailcfg.Debug{})
+	if err != nil {
+		return nil, xerrors.Errorf("reconfig: %w", err)
+	}
+
+	wireguardEngine.SetDERPMap(options.DERPMap)
+	netMapCopy := *netMap
+	wireguardEngine.SetNetworkMap(&netMapCopy)
+
+	localIPSet := netipx.IPSetBuilder{}
+	for _, addr := range netMap.Addresses {
+		localIPSet.AddPrefix(addr)
+	}
+	localIPs, _ := localIPSet.IPSet()
+	logIPSet := netipx.IPSetBuilder{}
+	logIPs, _ := logIPSet.IPSet()
+	wireguardEngine.SetFilter(filter.New(netMap.PacketFilter, localIPs, logIPs, nil, Logger(options.Logger.Named("packet-filter"))))
+	server := &Conn{
+		closed:           make(chan struct{}),
+		logger:           options.Logger,
+		magicConn:        magicConn,
+		dialer:           dialer,
+		listeners:        map[listenKey]*listener{},
+		tunDevice:        tunDevice,
+		netMap:           netMap,
+		netStack:         netStack,
+		wireguardMonitor: wireguardMonitor,
+		wireguardRouter:  wireguardRouter,
+		wireguardEngine:  wireguardEngine,
+	}
+	netStack.ForwardTCPIn = server.forwardTCP
+	return server, nil
+}
+
+// IP generates a new IP with a static service prefix.
+func IP() netip.Addr {
+	// This is Tailscale's ephemeral service prefix.
+	// This can be changed easily later-on, because
+	// all of our nodes are ephemeral.
+	// fd7a:115c:a1e0
+	uid := uuid.New()
+	uid[0] = 0xfd
+	uid[1] = 0x7a
+	uid[2] = 0x11
+	uid[3] = 0x5c
+	uid[4] = 0xa1
+	uid[5] = 0xe0
+	return netip.AddrFrom16(uid)
+}
+
+// Conn is an actively listening Wireguard connection.
+type Conn struct {
+	mutex  sync.Mutex
+	closed chan struct{}
+	logger slog.Logger
+
+	dialer           *tsdial.Dialer
+	tunDevice        *tstun.Wrapper
+	netMap           *netmap.NetworkMap
+	netStack         *netstack.Impl
+	magicConn        *magicsock.Conn
+	wireguardMonitor *monitor.Mon
+	wireguardRouter  *router.Config
+	wireguardEngine  wgengine.Engine
+	listeners        map[listenKey]*listener
+
+	lastMutex sync.Mutex
+	// It's only possible to store these values via status functions,
+	// so the values must be stored for retrieval later on.
+	lastEndpoints     []string
+	lastPreferredDERP int
+	lastDERPLatency   map[string]float64
+}
+
+// SetNodeCallback is triggered when a network change occurs and peer
+// renegotiation may be required. Clients should constantly be emitting
+// node changes.
+func (c *Conn) SetNodeCallback(callback func(node *Node)) {
+	makeNode := func() *Node {
+		return &Node{
+			ID:            c.netMap.SelfNode.ID,
+			Key:           c.netMap.SelfNode.Key,
+			Addresses:     c.netMap.SelfNode.Addresses,
+			AllowedIPs:    c.netMap.SelfNode.AllowedIPs,
+			DiscoKey:      c.magicConn.DiscoPublicKey(),
+			Endpoints:     c.lastEndpoints,
+			PreferredDERP: c.lastPreferredDERP,
+			DERPLatency:   c.lastDERPLatency,
+		}
+	}
+	c.magicConn.SetNetInfoCallback(func(ni *tailcfg.NetInfo) {
+		c.lastMutex.Lock()
+		c.lastPreferredDERP = ni.PreferredDERP
+		c.lastDERPLatency = ni.DERPLatency
+		node := makeNode()
+		c.lastMutex.Unlock()
+		callback(node)
+	})
+	c.wireguardEngine.SetStatusCallback(func(s *wgengine.Status, err error) {
+		if err != nil {
+			return
+		}
+		endpoints := make([]string, 0, len(s.LocalAddrs))
+		for _, addr := range s.LocalAddrs {
+			endpoints = append(endpoints, addr.Addr.String())
+		}
+		c.lastMutex.Lock()
+		c.lastEndpoints = endpoints
+		node := makeNode()
+		c.lastMutex.Unlock()
+		callback(node)
+	})
+}
+
+// SetDERPMap updates the DERPMap of a connection.
+func (c *Conn) SetDERPMap(derpMap *tailcfg.DERPMap) {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	c.wireguardEngine.SetDERPMap(derpMap)
+}
+
+// UpdateNodes connects with a set of peers. This can be constantly updated,
+// and peers will continually be reconnected as necessary.
+func (c *Conn) UpdateNodes(nodes []*Node) error {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	peerMap := map[tailcfg.NodeID]*tailcfg.Node{}
+	status := c.Status()
+	for _, peer := range c.netMap.Peers {
+		if peerStatus, ok := status.Peer[peer.Key]; ok {
+			// Clear out inactive connections!
+			if !peerStatus.Active {
+				continue
+			}
+		}
+		peerMap[peer.ID] = peer
+	}
+	for _, node := range nodes {
+		peerMap[node.ID] = &tailcfg.Node{
+			ID:         node.ID,
+			Key:        node.Key,
+			DiscoKey:   node.DiscoKey,
+			Addresses:  node.Addresses,
+			AllowedIPs: node.AllowedIPs,
+			Endpoints:  node.Endpoints,
+			DERP:       fmt.Sprintf("%s:%d", tailcfg.DerpMagicIP, node.PreferredDERP),
+			Hostinfo:   hostinfo.New().View(),
+		}
+	}
+	c.netMap.Peers = make([]*tailcfg.Node, 0, len(peerMap))
+	for _, peer := range peerMap {
+		c.netMap.Peers = append(c.netMap.Peers, peer)
+	}
+	cfg, err := nmcfg.WGCfg(c.netMap, Logger(c.logger.Named("wgconfig")), netmap.AllowSingleHosts, "")
+	if err != nil {
+		return xerrors.Errorf("update wireguard config: %w", err)
+	}
+	err = c.wireguardEngine.Reconfig(cfg, c.wireguardRouter, &dns.Config{}, &tailcfg.Debug{})
+	if err != nil {
+		return xerrors.Errorf("reconfig: %w", err)
+	}
+	netMapCopy := *c.netMap
+	c.wireguardEngine.SetNetworkMap(&netMapCopy)
+	return nil
+}
+
+// Status returns the current ipnstate of a connection.
+func (c *Conn) Status() *ipnstate.Status {
+	sb := &ipnstate.StatusBuilder{}
+	c.magicConn.UpdateStatus(sb)
+	return sb.Status()
+}
+
+// Ping sends a ping to the Wireguard engine.
+func (c *Conn) Ping(ip netip.Addr, pingType tailcfg.PingType, cb func(*ipnstate.PingResult)) {
+	c.wireguardEngine.Ping(ip, pingType, cb)
+}
+
+// Closed is a channel that ends when the connection has
+// been closed.
+func (c *Conn) Closed() <-chan struct{} {
+	return c.closed
+}
+
+// Close shuts down the Wireguard connection.
+func (c *Conn) Close() error {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	select {
+	case <-c.closed:
+		return nil
+	default:
+	}
+	for _, l := range c.listeners {
+		_ = l.closeNoLock()
+	}
+	close(c.closed)
+	_ = c.dialer.Close()
+	_ = c.magicConn.Close()
+	_ = c.netStack.Close()
+	_ = c.wireguardMonitor.Close()
+	_ = c.tunDevice.Close()
+	c.wireguardEngine.Close()
+	return nil
+}
+
+// This and below is taken _mostly_ verbatim from Tailscale:
+// https://github.com/tailscale/tailscale/blob/c88bd53b1b7b2fcf7ba302f2e53dd1ce8c32dad4/tsnet/tsnet.go#L459-L494
+
+// Listen announces only on the Tailscale network.
+// It will start the server if it has not been started yet.
+func (c *Conn) Listen(network, addr string) (net.Listener, error) {
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, xerrors.Errorf("wgnet: %w", err)
+	}
+	lk := listenKey{network, host, port}
+	ln := &listener{
+		s:    c,
+		key:  lk,
+		addr: addr,
+
+		conn: make(chan net.Conn),
+	}
+	c.mutex.Lock()
+	if c.listeners == nil {
+		c.listeners = map[listenKey]*listener{}
+	}
+	if _, ok := c.listeners[lk]; ok {
+		c.mutex.Unlock()
+		return nil, xerrors.Errorf("wgnet: listener already open for %s, %s", network, addr)
+	}
+	c.listeners[lk] = ln
+	c.mutex.Unlock()
+	return ln, nil
+}
+
+func (c *Conn) DialContextTCP(ctx context.Context, ipp netip.AddrPort) (*gonet.TCPConn, error) {
+	return c.netStack.DialContextTCP(ctx, ipp)
+}
+
+func (c *Conn) DialContextUDP(ctx context.Context, ipp netip.AddrPort) (*gonet.UDPConn, error) {
+	return c.netStack.DialContextUDP(ctx, ipp)
+}
+
+func (c *Conn) forwardTCP(conn net.Conn, port uint16) {
+	c.mutex.Lock()
+	ln, ok := c.listeners[listenKey{"tcp", "", fmt.Sprint(port)}]
+	c.mutex.Unlock()
+	if !ok {
+		c.forwardTCPToLocal(conn, port)
+		return
+	}
+	t := time.NewTimer(time.Second)
+	defer t.Stop()
+	select {
+	case ln.conn <- conn:
+	case <-t.C:
+		_ = conn.Close()
+	}
+}
+
+func (c *Conn) forwardTCPToLocal(conn net.Conn, port uint16) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	defer conn.Close()
+
+	dialAddrStr := net.JoinHostPort("127.0.0.1", strconv.Itoa(int(port)))
+	var stdDialer net.Dialer
+	server, err := stdDialer.DialContext(ctx, "tcp", dialAddrStr)
+	if err != nil {
+		c.logger.Debug(ctx, "dial local port", slog.F("port", port), slog.Error(err))
+		return
+	}
+	defer server.Close()
+
+	connClosed := make(chan error, 2)
+	go func() {
+		_, err := io.Copy(server, conn)
+		connClosed <- err
+	}()
+	go func() {
+		_, err := io.Copy(conn, server)
+		connClosed <- err
+	}()
+	select {
+	case err = <-connClosed:
+	case <-c.closed:
+		return
+	}
+	if err != nil {
+		c.logger.Debug(ctx, "proxy connection closed with error", slog.Error(err))
+	}
+	c.logger.Debug(ctx, "forwarded connection closed", slog.F("local_addr", dialAddrStr))
+}
+
+type listenKey struct {
+	network string
+	host    string
+	port    string
+}
+
+type listener struct {
+	s    *Conn
+	key  listenKey
+	addr string
+	conn chan net.Conn
+}
+
+func (ln *listener) Accept() (net.Conn, error) {
+	c, ok := <-ln.conn
+	if !ok {
+		return nil, xerrors.Errorf("wgnet: %w", net.ErrClosed)
+	}
+	return c, nil
+}
+
+func (ln *listener) Addr() net.Addr { return addr{ln} }
+func (ln *listener) Close() error {
+	ln.s.mutex.Lock()
+	defer ln.s.mutex.Unlock()
+	return ln.closeNoLock()
+}
+
+func (ln *listener) closeNoLock() error {
+	if v, ok := ln.s.listeners[ln.key]; ok && v == ln {
+		delete(ln.s.listeners, ln.key)
+		close(ln.conn)
+	}
+	return nil
+}
+
+type addr struct{ ln *listener }
+
+func (a addr) Network() string { return a.ln.key.network }
+func (a addr) String() string  { return a.ln.addr }
+
+// Logger converts the Tailscale logging function to use slog.
+func Logger(logger slog.Logger) tslogger.Logf {
+	return tslogger.Logf(func(format string, args ...any) {
+		logger.Debug(context.Background(), fmt.Sprintf(format, args...))
+	})
+}

tailnet/conn_test.go

@@ -0,0 +1,83 @@
+package tailnet_test
+
+import (
+	"context"
+	"net/netip"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
+
+	"cdr.dev/slog"
+	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/tailnet"
+	"github.com/coder/coder/tailnet/tailnettest"
+)
+
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m)
+}
+
+func TestTailnet(t *testing.T) {
+	t.Parallel()
+	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+	derpMap := tailnettest.RunDERPAndSTUN(t)
+	t.Run("InstantClose", func(t *testing.T) {
+		t.Parallel()
+		conn, err := tailnet.NewConn(&tailnet.Options{
+			Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)},
+			Logger:    logger.Named("w1"),
+			DERPMap:   derpMap,
+		})
+		require.NoError(t, err)
+		err = conn.Close()
+		require.NoError(t, err)
+	})
+	t.Run("Connect", func(t *testing.T) {
+		t.Parallel()
+		w1IP := tailnet.IP()
+		w1, err := tailnet.NewConn(&tailnet.Options{
+			Addresses: []netip.Prefix{netip.PrefixFrom(w1IP, 128)},
+			Logger:    logger.Named("w1"),
+			DERPMap:   derpMap,
+		})
+		require.NoError(t, err)
+
+		w2, err := tailnet.NewConn(&tailnet.Options{
+			Addresses: []netip.Prefix{netip.PrefixFrom(tailnet.IP(), 128)},
+			Logger:    logger.Named("w2"),
+			DERPMap:   derpMap,
+		})
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			_ = w1.Close()
+			_ = w2.Close()
+		})
+		w1.SetNodeCallback(func(node *tailnet.Node) {
+			w2.UpdateNodes([]*tailnet.Node{node})
+		})
+		w2.SetNodeCallback(func(node *tailnet.Node) {
+			w1.UpdateNodes([]*tailnet.Node{node})
+		})
+
+		conn := make(chan struct{})
+		go func() {
+			listener, err := w1.Listen("tcp", ":35565")
+			assert.NoError(t, err)
+			defer listener.Close()
+			nc, err := listener.Accept()
+			assert.NoError(t, err)
+			_ = nc.Close()
+			conn <- struct{}{}
+		}()
+
+		nc, err := w2.DialContextTCP(context.Background(), netip.AddrPortFrom(w1IP, 35565))
+		require.NoError(t, err)
+		_ = nc.Close()
+		<-conn
+
+		w1.Close()
+		w2.Close()
+	})
+}

tailnet/coordinator.go

@@ -0,0 +1,259 @@
+package tailnet
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"net"
+	"net/netip"
+	"sync"
+
+	"github.com/google/uuid"
+	"golang.org/x/xerrors"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
+)
+
+// Node represents a node in the network.
+type Node struct {
+	ID            tailcfg.NodeID     `json:"id"`
+	Key           key.NodePublic     `json:"key"`
+	DiscoKey      key.DiscoPublic    `json:"disco"`
+	PreferredDERP int                `json:"preferred_derp"`
+	DERPLatency   map[string]float64 `json:"derp_latency"`
+	Addresses     []netip.Prefix     `json:"addresses"`
+	AllowedIPs    []netip.Prefix     `json:"allowed_ips"`
+	Endpoints     []string           `json:"endpoints"`
+}
+
+// ServeCoordinator matches the RW structure of a coordinator to exchange node messages.
+func ServeCoordinator(conn net.Conn, updateNodes func(node []*Node) error) (func(node *Node), <-chan error) {
+	errChan := make(chan error, 3)
+	go func() {
+		decoder := json.NewDecoder(conn)
+		for {
+			var nodes []*Node
+			err := decoder.Decode(&nodes)
+			if err != nil {
+				errChan <- xerrors.Errorf("read: %w", err)
+				return
+			}
+			err = updateNodes(nodes)
+			if err != nil {
+				errChan <- xerrors.Errorf("update nodes: %w", err)
+			}
+		}
+	}()
+
+	return func(node *Node) {
+		data, err := json.Marshal(node)
+		if err != nil {
+			errChan <- xerrors.Errorf("marshal node: %w", err)
+			return
+		}
+		_, err = conn.Write(data)
+		if err != nil {
+			errChan <- xerrors.Errorf("write: %w", err)
+		}
+	}, errChan
+}
+
+// NewCoordinator constructs a new in-memory connection coordinator.
+func NewCoordinator() *Coordinator {
+	return &Coordinator{
+		nodes:                    map[uuid.UUID]*Node{},
+		agentSockets:             map[uuid.UUID]net.Conn{},
+		agentToConnectionSockets: map[uuid.UUID]map[uuid.UUID]net.Conn{},
+	}
+}
+
+// Coordinator exchanges nodes with agents to establish connections.
+// ┌──────────────────┐   ┌────────────────────┐   ┌───────────────────┐   ┌──────────────────┐
+// │tailnet.Coordinate├──►│tailnet.AcceptClient│◄─►│tailnet.AcceptAgent│◄──┤tailnet.Coordinate│
+// └──────────────────┘   └────────────────────┘   └───────────────────┘   └──────────────────┘
+// This coordinator is incompatible with multiple Coder
+// replicas as all node data is in-memory.
+type Coordinator struct {
+	mutex sync.Mutex
+
+	// Maps agent and connection IDs to a node.
+	nodes map[uuid.UUID]*Node
+	// Maps agent ID to an open socket.
+	agentSockets map[uuid.UUID]net.Conn
+	// Maps agent ID to connection ID for sending
+	// new node data as it comes in!
+	agentToConnectionSockets map[uuid.UUID]map[uuid.UUID]net.Conn
+}
+
+// Node returns an in-memory node by ID.
+func (c *Coordinator) Node(id uuid.UUID) *Node {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	node := c.nodes[id]
+	return node
+}
+
+// ServeClient accepts a WebSocket connection that wants to
+// connect to an agent with the specified ID.
+func (c *Coordinator) ServeClient(conn net.Conn, id uuid.UUID, agent uuid.UUID) error {
+	c.mutex.Lock()
+	// When a new connection is requested, we update it with the latest
+	// node of the agent. This allows the connection to establish.
+	node, ok := c.nodes[agent]
+	if ok {
+		data, err := json.Marshal([]*Node{node})
+		if err != nil {
+			c.mutex.Unlock()
+			return xerrors.Errorf("marshal node: %w", err)
+		}
+		_, err = conn.Write(data)
+		if err != nil {
+			c.mutex.Unlock()
+			return xerrors.Errorf("write nodes: %w", err)
+		}
+	}
+	connectionSockets, ok := c.agentToConnectionSockets[agent]
+	if !ok {
+		connectionSockets = map[uuid.UUID]net.Conn{}
+		c.agentToConnectionSockets[agent] = connectionSockets
+	}
+	// Insert this connection into a map so the agent
+	// can publish node updates.
+	connectionSockets[id] = conn
+	c.mutex.Unlock()
+	defer func() {
+		c.mutex.Lock()
+		defer c.mutex.Unlock()
+		// Clean all traces of this connection from the map.
+		delete(c.nodes, id)
+		connectionSockets, ok := c.agentToConnectionSockets[agent]
+		if !ok {
+			return
+		}
+		delete(connectionSockets, id)
+		if len(connectionSockets) != 0 {
+			return
+		}
+		delete(c.agentToConnectionSockets, agent)
+	}()
+
+	decoder := json.NewDecoder(conn)
+	for {
+		var node Node
+		err := decoder.Decode(&node)
+		if errors.Is(err, io.EOF) {
+			return nil
+		}
+		if err != nil {
+			return xerrors.Errorf("read json: %w", err)
+		}
+		c.mutex.Lock()
+		// Update the node of this client in our in-memory map.
+		// If an agent entirely shuts down and reconnects, it
+		// needs to be aware of all clients attempting to
+		// establish connections.
+		c.nodes[id] = &node
+		agentSocket, ok := c.agentSockets[agent]
+		if !ok {
+			c.mutex.Unlock()
+			continue
+		}
+		// Write the new node from this client to the actively
+		// connected agent.
+		data, err := json.Marshal([]*Node{&node})
+		if err != nil {
+			c.mutex.Unlock()
+			return xerrors.Errorf("marshal nodes: %w", err)
+		}
+		_, err = agentSocket.Write(data)
+		if errors.Is(err, io.EOF) {
+			c.mutex.Unlock()
+			return nil
+		}
+		if err != nil {
+			c.mutex.Unlock()
+			return xerrors.Errorf("write json: %w", err)
+		}
+		c.mutex.Unlock()
+	}
+}
+
+// ServeAgent accepts a WebSocket connection to an agent that
+// listens to incoming connections and publishes node updates.
+func (c *Coordinator) ServeAgent(conn net.Conn, id uuid.UUID) error {
+	c.mutex.Lock()
+	sockets, ok := c.agentToConnectionSockets[id]
+	if ok {
+		// Publish all nodes that want to connect to the
+		// desired agent ID.
+		nodes := make([]*Node, 0, len(sockets))
+		for targetID := range sockets {
+			node, ok := c.nodes[targetID]
+			if !ok {
+				continue
+			}
+			nodes = append(nodes, node)
+		}
+		data, err := json.Marshal(nodes)
+		if err != nil {
+			c.mutex.Unlock()
+			return xerrors.Errorf("marshal json: %w", err)
+		}
+		_, err = conn.Write(data)
+		if err != nil {
+			c.mutex.Unlock()
+			return xerrors.Errorf("write nodes: %w", err)
+		}
+	}
+
+	// If an old agent socket is connected, we close it
+	// to avoid any leaks. This shouldn't ever occur because
+	// we expect one agent to be running.
+	oldAgentSocket, ok := c.agentSockets[id]
+	if ok {
+		_ = oldAgentSocket.Close()
+	}
+	c.agentSockets[id] = conn
+	c.mutex.Unlock()
+	defer func() {
+		c.mutex.Lock()
+		defer c.mutex.Unlock()
+		delete(c.agentSockets, id)
+		delete(c.nodes, id)
+	}()
+
+	decoder := json.NewDecoder(conn)
+	for {
+		var node Node
+		err := decoder.Decode(&node)
+		if errors.Is(err, io.EOF) {
+			return nil
+		}
+		if err != nil {
+			return xerrors.Errorf("read json: %w", err)
+		}
+		c.mutex.Lock()
+		c.nodes[id] = &node
+		connectionSockets, ok := c.agentToConnectionSockets[id]
+		if !ok {
+			c.mutex.Unlock()
+			continue
+		}
+		data, err := json.Marshal([]*Node{&node})
+		if err != nil {
+			return xerrors.Errorf("marshal nodes: %w", err)
+		}
+		// Publish the new node to every listening socket.
+		var wg sync.WaitGroup
+		wg.Add(len(connectionSockets))
+		for _, connectionSocket := range connectionSockets {
+			connectionSocket := connectionSocket
+			go func() {
+				_, _ = connectionSocket.Write(data)
+				wg.Done()
+			}()
+		}
+		wg.Wait()
+		c.mutex.Unlock()
+	}
+}

tailnet/coordinator_test.go

@@ -0,0 +1,148 @@
+package tailnet_test
+
+import (
+	"net"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/tailnet"
+	"github.com/coder/coder/testutil"
+)
+
+func TestCoordinator(t *testing.T) {
+	t.Parallel()
+	t.Run("ClientWithoutAgent", func(t *testing.T) {
+		t.Parallel()
+		coordinator := tailnet.NewCoordinator()
+		client, server := net.Pipe()
+		sendNode, errChan := tailnet.ServeCoordinator(client, func(node []*tailnet.Node) error {
+			return nil
+		})
+		id := uuid.New()
+		closeChan := make(chan struct{})
+		go func() {
+			err := coordinator.ServeClient(server, id, uuid.New())
+			assert.NoError(t, err)
+			close(closeChan)
+		}()
+		sendNode(&tailnet.Node{})
+		require.Eventually(t, func() bool {
+			return coordinator.Node(id) != nil
+		}, testutil.WaitShort, testutil.IntervalFast)
+		err := client.Close()
+		require.NoError(t, err)
+		<-errChan
+		<-closeChan
+	})
+
+	t.Run("AgentWithoutClients", func(t *testing.T) {
+		t.Parallel()
+		coordinator := tailnet.NewCoordinator()
+		client, server := net.Pipe()
+		sendNode, errChan := tailnet.ServeCoordinator(client, func(node []*tailnet.Node) error {
+			return nil
+		})
+		id := uuid.New()
+		closeChan := make(chan struct{})
+		go func() {
+			err := coordinator.ServeAgent(server, id)
+			assert.NoError(t, err)
+			close(closeChan)
+		}()
+		sendNode(&tailnet.Node{})
+		require.Eventually(t, func() bool {
+			return coordinator.Node(id) != nil
+		}, testutil.WaitShort, testutil.IntervalFast)
+		err := client.Close()
+		require.NoError(t, err)
+		<-errChan
+		<-closeChan
+	})
+
+	t.Run("AgentWithClient", func(t *testing.T) {
+		t.Parallel()
+		coordinator := tailnet.NewCoordinator()
+
+		agentWS, agentServerWS := net.Pipe()
+		defer agentWS.Close()
+		agentNodeChan := make(chan []*tailnet.Node)
+		sendAgentNode, agentErrChan := tailnet.ServeCoordinator(agentWS, func(nodes []*tailnet.Node) error {
+			agentNodeChan <- nodes
+			return nil
+		})
+		agentID := uuid.New()
+		closeAgentChan := make(chan struct{})
+		go func() {
+			err := coordinator.ServeAgent(agentServerWS, agentID)
+			assert.NoError(t, err)
+			close(closeAgentChan)
+		}()
+		sendAgentNode(&tailnet.Node{})
+		require.Eventually(t, func() bool {
+			return coordinator.Node(agentID) != nil
+		}, testutil.WaitShort, testutil.IntervalFast)
+
+		clientWS, clientServerWS := net.Pipe()
+		defer clientWS.Close()
+		defer clientServerWS.Close()
+		clientNodeChan := make(chan []*tailnet.Node)
+		sendClientNode, clientErrChan := tailnet.ServeCoordinator(clientWS, func(nodes []*tailnet.Node) error {
+			clientNodeChan <- nodes
+			return nil
+		})
+		clientID := uuid.New()
+		closeClientChan := make(chan struct{})
+		go func() {
+			err := coordinator.ServeClient(clientServerWS, clientID, agentID)
+			assert.NoError(t, err)
+			close(closeClientChan)
+		}()
+		agentNodes := <-clientNodeChan
+		require.Len(t, agentNodes, 1)
+		sendClientNode(&tailnet.Node{})
+		clientNodes := <-agentNodeChan
+		require.Len(t, clientNodes, 1)
+
+		// Ensure an update to the agent node reaches the client!
+		sendAgentNode(&tailnet.Node{})
+		agentNodes = <-clientNodeChan
+		require.Len(t, agentNodes, 1)
+
+		// Close the agent WebSocket so a new one can connect.
+		err := agentWS.Close()
+		require.NoError(t, err)
+		<-agentErrChan
+		<-closeAgentChan
+
+		// Create a new agent connection. This is to simulate a reconnect!
+		agentWS, agentServerWS = net.Pipe()
+		defer agentWS.Close()
+		agentNodeChan = make(chan []*tailnet.Node)
+		_, agentErrChan = tailnet.ServeCoordinator(agentWS, func(nodes []*tailnet.Node) error {
+			agentNodeChan <- nodes
+			return nil
+		})
+		closeAgentChan = make(chan struct{})
+		go func() {
+			err := coordinator.ServeAgent(agentServerWS, agentID)
+			assert.NoError(t, err)
+			close(closeAgentChan)
+		}()
+		// Ensure the existing listening client sends it's node immediately!
+		clientNodes = <-agentNodeChan
+		require.Len(t, clientNodes, 1)
+
+		err = agentWS.Close()
+		require.NoError(t, err)
+		<-agentErrChan
+		<-closeAgentChan
+
+		err = clientWS.Close()
+		require.NoError(t, err)
+		<-clientErrChan
+		<-closeClientChan
+	})
+}

tailnet/derpmap.go

@@ -0,0 +1,60 @@
+package tailnet
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"strconv"
+
+	"golang.org/x/xerrors"
+	"tailscale.com/tailcfg"
+)
+
+// NewDERPMap constructs a DERPMap from a set of STUN addresses and optionally a remote
+// URL to fetch a mapping from e.g. https://controlplane.tailscale.com/derpmap/default.
+func NewDERPMap(ctx context.Context, region *tailcfg.DERPRegion, stunAddrs []string, remoteURL string) (*tailcfg.DERPMap, error) {
+	for index, stunAddr := range stunAddrs {
+		host, rawPort, err := net.SplitHostPort(stunAddr)
+		if err != nil {
+			return nil, xerrors.Errorf("split host port for %q: %w", stunAddr, err)
+		}
+		port, err := strconv.Atoi(rawPort)
+		if err != nil {
+			return nil, xerrors.Errorf("parse port for %q: %w", stunAddr, err)
+		}
+		region.Nodes = append(region.Nodes, &tailcfg.DERPNode{
+			Name:     fmt.Sprintf("%dstun%d", region.RegionID, index),
+			RegionID: region.RegionID,
+			HostName: host,
+			STUNOnly: true,
+			STUNPort: port,
+		})
+	}
+
+	derpMap := &tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{},
+	}
+	if remoteURL != "" {
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, remoteURL, nil)
+		if err != nil {
+			return nil, xerrors.Errorf("create request: %w", err)
+		}
+		res, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return nil, xerrors.Errorf("get derpmap: %w", err)
+		}
+		defer res.Body.Close()
+		err = json.NewDecoder(res.Body).Decode(&derpMap)
+		if err != nil {
+			return nil, xerrors.Errorf("fetch derpmap: %w", err)
+		}
+	}
+	_, conflicts := derpMap.Regions[region.RegionID]
+	if conflicts {
+		return nil, xerrors.Errorf("the default region ID conflicts with a remote region from %q", remoteURL)
+	}
+	derpMap.Regions[region.RegionID] = region
+	return derpMap, nil
+}

tailnet/derpmap_test.go

@@ -0,0 +1,60 @@
+package tailnet_test
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"tailscale.com/tailcfg"
+
+	"github.com/coder/coder/tailnet"
+)
+
+func TestNewDERPMap(t *testing.T) {
+	t.Parallel()
+	t.Run("WithoutRemoteURL", func(t *testing.T) {
+		t.Parallel()
+		derpMap, err := tailnet.NewDERPMap(context.Background(), &tailcfg.DERPRegion{
+			RegionID: 1,
+			Nodes:    []*tailcfg.DERPNode{{}},
+		}, []string{"stun.google.com:2345"}, "")
+		require.NoError(t, err)
+		require.Len(t, derpMap.Regions[1].Nodes, 2)
+	})
+	t.Run("RemoteURL", func(t *testing.T) {
+		t.Parallel()
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			data, _ := json.Marshal(&tailcfg.DERPMap{
+				Regions: map[int]*tailcfg.DERPRegion{
+					1: {},
+				},
+			})
+			_, _ = w.Write(data)
+		}))
+		t.Cleanup(server.Close)
+		derpMap, err := tailnet.NewDERPMap(context.Background(), &tailcfg.DERPRegion{
+			RegionID: 2,
+		}, []string{}, server.URL)
+		require.NoError(t, err)
+		require.Len(t, derpMap.Regions, 2)
+	})
+	t.Run("RemoteConflicts", func(t *testing.T) {
+		t.Parallel()
+		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			data, _ := json.Marshal(&tailcfg.DERPMap{
+				Regions: map[int]*tailcfg.DERPRegion{
+					1: {},
+				},
+			})
+			_, _ = w.Write(data)
+		}))
+		t.Cleanup(server.Close)
+		_, err := tailnet.NewDERPMap(context.Background(), &tailcfg.DERPRegion{
+			RegionID: 1,
+		}, []string{}, server.URL)
+		require.Error(t, err)
+	})
+}

tailnet/tailnettest/tailnettest.go

@@ -0,0 +1,63 @@
+package tailnettest
+
+import (
+	"crypto/tls"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"tailscale.com/derp"
+	"tailscale.com/derp/derphttp"
+	"tailscale.com/net/stun/stuntest"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/key"
+	tslogger "tailscale.com/types/logger"
+	"tailscale.com/types/nettype"
+
+	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/tailnet"
+)
+
+// RunDERPAndSTUN creates a DERP mapping for tests.
+func RunDERPAndSTUN(t *testing.T) *tailcfg.DERPMap {
+	logf := tailnet.Logger(slogtest.Make(t, nil))
+	d := derp.NewServer(key.NewNode(), logf)
+	server := httptest.NewUnstartedServer(derphttp.Handler(d))
+	server.Config.ErrorLog = tslogger.StdLogger(logf)
+	server.Config.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
+	server.StartTLS()
+
+	stunAddr, stunCleanup := stuntest.ServeWithPacketListener(t, nettype.Std{})
+	t.Cleanup(func() {
+		server.CloseClientConnections()
+		server.Close()
+		d.Close()
+		stunCleanup()
+	})
+	tcpAddr, ok := server.Listener.Addr().(*net.TCPAddr)
+	if !ok {
+		t.FailNow()
+	}
+
+	return &tailcfg.DERPMap{
+		Regions: map[int]*tailcfg.DERPRegion{
+			1: {
+				RegionID:   1,
+				RegionCode: "test",
+				RegionName: "Test",
+				Nodes: []*tailcfg.DERPNode{
+					{
+						Name:             "t2",
+						RegionID:         1,
+						IPv4:             "127.0.0.1",
+						IPv6:             "none",
+						STUNPort:         stunAddr.Port,
+						DERPPort:         tcpAddr.Port,
+						InsecureForTests: true,
+					},
+				},
+			},
+		},
+	}
+}

tailnet/tailnettest/tailnettest_test.go

@@ -0,0 +1,18 @@
+package tailnettest_test
+
+import (
+	"testing"
+
+	"go.uber.org/goleak"
+
+	"github.com/coder/coder/tailnet/tailnettest"
+)
+
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m)
+}
+
+func TestRunDERPAndSTUN(t *testing.T) {
+	t.Parallel()
+	_ = tailnettest.RunDERPAndSTUN(t)
+}