feat: add Prometheus collector for DERP server expvar metrics (#22583)

This PR does three things:
- Exports derp expvars to the pprof endpoint
- Exports the expvar metrics as prometheus metrics in both coderd and
wsproxy
- Updates our tailscale to a fix I also had to make to avoid a data race
condition

I generated this with mux but I also manually tested that the metrics
were getting properly emitted

enterprise/wsproxy/wsproxy.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/tls"
 	"errors"
+	"expvar"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -42,8 +43,14 @@ import (
 	sharedhttpmw "github.com/coder/coder/v2/httpmw"
 	"github.com/coder/coder/v2/site"
 	"github.com/coder/coder/v2/tailnet"
+	"github.com/coder/coder/v2/tailnet/derpmetrics"
 )
 
+// expDERPOnce guards the global expvar.Publish call for the DERP server.
+// expvar panics on duplicate registration, and tests may create multiple
+// servers in the same process.
+var expDERPOnce sync.Once
+
 type Options struct {
 	Logger      slog.Logger
 	Experiments codersdk.Experiments
@@ -196,6 +203,17 @@ func New(ctx context.Context, opts *Options) (*Server, error) {
 		return nil, xerrors.Errorf("create DERP mesh tls config: %w", err)
 	}
 	derpServer := derp.NewServer(key.NewNode(), tailnet.Logger(opts.Logger.Named("net.derp")))
+	// Publish DERP stats to expvar, available via the pprof
+	// debug server (--pprof-enable) at /debug/vars. This avoids
+	// exposing expvar on the public HTTP router.
+	expDERPOnce.Do(func() {
+		if expvar.Get("derp") == nil {
+			expvar.Publish("derp", derpServer.ExpVar())
+		}
+	})
+	if opts.PrometheusRegistry != nil {
+		opts.PrometheusRegistry.MustRegister(derpmetrics.NewDERPExpvarCollector(derpServer))
+	}
 
 	ctx, cancel := context.WithCancel(context.Background())
 

enterprise/wsproxy/wsproxy_test.go

@@ -1223,3 +1223,55 @@ func createProxyReplicas(ctx context.Context, t *testing.T, opts *createProxyRep
 
 	return proxies
 }
+
+func TestWorkspaceProxyDERPMetrics(t *testing.T) {
+	t.Parallel()
+
+	deploymentValues := coderdtest.DeploymentValues(t)
+	deploymentValues.Experiments = []string{"*"}
+
+	client, closer, api, _ := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
+		Options: &coderdtest.Options{
+			DeploymentValues:         deploymentValues,
+			AppHostname:              "*.primary.test.coder.com",
+			IncludeProvisionerDaemon: true,
+			RealIPConfig: &httpmw.RealIPConfig{
+				TrustedOrigins: []*net.IPNet{{
+					IP:   net.ParseIP("127.0.0.1"),
+					Mask: net.CIDRMask(8, 32),
+				}},
+				TrustedHeaders: []string{
+					"CF-Connecting-IP",
+				},
+			},
+		},
+		LicenseOptions: &coderdenttest.LicenseOptions{
+			Features: license.Features{
+				codersdk.FeatureWorkspaceProxy: 1,
+			},
+		},
+	})
+	t.Cleanup(func() {
+		_ = closer.Close()
+	})
+
+	proxy := coderdenttest.NewWorkspaceProxyReplica(t, api, client, &coderdenttest.ProxyOptions{
+		Name: "metrics-test-proxy",
+	})
+
+	// Gather metrics from the wsproxy's Prometheus registry.
+	metrics, err := proxy.PrometheusRegistry.Gather()
+	require.NoError(t, err)
+
+	names := make(map[string]struct{})
+	for _, m := range metrics {
+		names[m.GetName()] = struct{}{}
+	}
+
+	assert.Contains(t, names, "coder_derp_server_connections",
+		"expected coder_derp_server_connections to be registered")
+	assert.Contains(t, names, "coder_derp_server_bytes_received_total",
+		"expected coder_derp_server_bytes_received_total to be registered")
+	assert.Contains(t, names, "coder_derp_server_packets_dropped_reason_total",
+		"expected coder_derp_server_packets_dropped_reason_total to be registered")
+}