feat: add Prometheus collector for DERP server expvar metrics (#22583)

This PR does three things:
- Exports derp expvars to the pprof endpoint
- Exports the expvar metrics as prometheus metrics in both coderd and
wsproxy
- Updates our tailscale to a fix I also had to make to avoid a data race
condition

I generated this with mux but I also manually tested that the metrics
were getting properly emitted

coderd/coderd.go

@@ -99,6 +99,7 @@ import (
 	"github.com/coder/coder/v2/provisionersdk"
 	"github.com/coder/coder/v2/site"
 	"github.com/coder/coder/v2/tailnet"
+	"github.com/coder/coder/v2/tailnet/derpmetrics"
 	"github.com/coder/quartz"
 	"github.com/coder/serpent"
 )
@@ -899,17 +900,18 @@ func New(options *Options) *API {
 	apiRateLimiter := httpmw.RateLimit(options.APIRateLimit, time.Minute)
 
 	// Register DERP on expvar HTTP handler, which we serve below in the router, c.f. expvar.Handler()
-	// These are the metrics the DERP server exposes.
-	// TODO: export via prometheus
 	expDERPOnce.Do(func() {
 		// We need to do this via a global Once because expvar registry is global and panics if we
 		// register multiple times.  In production there is only one Coderd and one DERP server per
 		// process, but in testing, we create multiple of both, so the Once protects us from
 		// panicking.
-		if options.DERPServer != nil {
+		if options.DERPServer != nil && expvar.Get("derp") == nil {
 			expvar.Publish("derp", api.DERPServer.ExpVar())
 		}
 	})
+	if options.PrometheusRegistry != nil && options.DERPServer != nil {
+		options.PrometheusRegistry.MustRegister(derpmetrics.NewDERPExpvarCollector(options.DERPServer))
+	}
 	cors := httpmw.Cors(options.DeploymentValues.Dangerous.AllowAllCors.Value())
 	prometheusMW := httpmw.Prometheus(options.PrometheusRegistry)
 

coderd/coderd_test.go

@@ -390,3 +390,29 @@ func TestCSRFExempt(t *testing.T) {
 		require.NotContains(t, string(data), "CSRF")
 	})
 }
+
+func TestDERPMetrics(t *testing.T) {
+	t.Parallel()
+
+	_, _, api := coderdtest.NewWithAPI(t, nil)
+
+	require.NotNil(t, api.Options.DERPServer, "DERP server should be configured")
+	require.NotNil(t, api.Options.PrometheusRegistry, "Prometheus registry should be configured")
+
+	// The registry is created internally by coderd. Gather from it
+	// to verify DERP metrics were registered during startup.
+	metrics, err := api.Options.PrometheusRegistry.Gather()
+	require.NoError(t, err)
+
+	names := make(map[string]struct{})
+	for _, m := range metrics {
+		names[m.GetName()] = struct{}{}
+	}
+
+	assert.Contains(t, names, "coder_derp_server_connections",
+		"expected coder_derp_server_connections to be registered")
+	assert.Contains(t, names, "coder_derp_server_bytes_received_total",
+		"expected coder_derp_server_bytes_received_total to be registered")
+	assert.Contains(t, names, "coder_derp_server_packets_dropped_reason_total",
+		"expected coder_derp_server_packets_dropped_reason_total to be registered")
+}