feat: add coderd_api_websocket_probes_total metric (#25012)

Relates to CODAGT-115

Adds metric `coderd_api_websocket_probes_total`. Every successful
heartbeat for a given path will increment the metric.

Comparing this with `coderd_api_concurrent_websockets` will give an
indication of how many websocket connections are open but in a 'wedged'
state (when heartbeats stopped versus when we closed the connection).

coderd/httpmw/prometheus.go

@@ -1,6 +1,7 @@
 package httpmw
 
 import (
+	"context"
 	"net/http"
 	"strconv"
 	"time"
@@ -12,7 +13,63 @@ import (
 	"github.com/coder/coder/v2/coderd/tracing"
 )
 
-func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler {
+// WSMetrics groups all WebSocket-related Prometheus metrics so they
+// can be created once and shared between the HTTP middleware and the
+// WSWatcher probe recorder.
+type WSMetrics struct {
+	Concurrent *prometheus.GaugeVec
+	Durations  *prometheus.HistogramVec
+	Probes     *prometheus.CounterVec
+}
+
+// NewWSMetrics registers and returns WebSocket metrics. The returned
+// struct is safe to pass to both Prometheus() and
+// WSMetrics.RecordProbe.
+func NewWSMetrics(reg prometheus.Registerer) *WSMetrics {
+	factory := promauto.With(reg)
+	return &WSMetrics{
+		Concurrent: factory.NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: "coderd",
+			Subsystem: "api",
+			Name:      "concurrent_websockets",
+			Help:      "The total number of concurrent API websockets.",
+		}, []string{"path"}),
+		Durations: factory.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: "coderd",
+			Subsystem: "api",
+			Name:      "websocket_durations_seconds",
+			Help:      "Websocket duration distribution of requests in seconds.",
+			Buckets: []float64{
+				0.001, // 1ms
+				1,
+				60,           // 1 minute
+				60 * 60,      // 1 hour
+				60 * 60 * 15, // 15 hours
+				60 * 60 * 30, // 30 hours
+			},
+		}, []string{"path"}),
+		Probes: factory.NewCounterVec(prometheus.CounterOpts{
+			Namespace: "coderd",
+			Subsystem: "api",
+			Name:      "websocket_probes_total",
+			Help: "WebSocket liveness probe outcomes by route. " +
+				"Compare rate(...{result=\"ok\"}[1m]) against " +
+				"coderd_api_concurrent_websockets to detect " +
+				"unresponsive WebSocket connections.",
+		}, []string{"path", "result"}),
+	}
+}
+
+// RecordProbe records a single liveness probe outcome. It extracts
+// the HTTP route from ctx via ExtractHTTPRoute.
+func (m *WSMetrics) RecordProbe(ctx context.Context, r httpapi.ProbeResult) {
+	m.Probes.WithLabelValues(ExtractHTTPRoute(ctx), string(r)).Inc()
+}
+
+func Prometheus(register prometheus.Registerer, ws *WSMetrics) func(http.Handler) http.Handler {
+	if ws == nil {
+		panic("developer error: WSMetrics is nil")
+	}
 	factory := promauto.With(register)
 	requestsProcessed := factory.NewCounterVec(prometheus.CounterOpts{
 		Namespace: "coderd",
@@ -26,26 +83,6 @@ func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler
 		Name:      "concurrent_requests",
 		Help:      "The number of concurrent API requests.",
 	}, []string{"method", "path"})
-	websocketsConcurrent := factory.NewGaugeVec(prometheus.GaugeOpts{
-		Namespace: "coderd",
-		Subsystem: "api",
-		Name:      "concurrent_websockets",
-		Help:      "The total number of concurrent API websockets.",
-	}, []string{"path"})
-	websocketsDist := factory.NewHistogramVec(prometheus.HistogramOpts{
-		Namespace: "coderd",
-		Subsystem: "api",
-		Name:      "websocket_durations_seconds",
-		Help:      "Websocket duration distribution of requests in seconds.",
-		Buckets: []float64{
-			0.001, // 1ms
-			1,
-			60,           // 1 minute
-			60 * 60,      // 1 hour
-			60 * 60 * 15, // 15 hours
-			60 * 60 * 30, // 30 hours
-		},
-	}, []string{"path"})
 	requestsDist := factory.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace: "coderd",
 		Subsystem: "api",
@@ -74,10 +111,10 @@ func Prometheus(register prometheus.Registerer) func(http.Handler) http.Handler
 
 			// We want to count WebSockets separately.
 			if httpapi.IsWebsocketUpgrade(r) {
-				websocketsConcurrent.WithLabelValues(path).Inc()
-				defer websocketsConcurrent.WithLabelValues(path).Dec()
+				ws.Concurrent.WithLabelValues(path).Inc()
+				defer ws.Concurrent.WithLabelValues(path).Dec()
 
-				dist = websocketsDist
+				dist = ws.Durations
 			} else {
 				requestsConcurrent.WithLabelValues(method, path).Inc()
 				defer requestsConcurrent.WithLabelValues(method, path).Dec()

coderd/httpmw/prometheus_test.go

@@ -29,7 +29,7 @@ func TestPrometheus(t *testing.T) {
 		req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, chi.NewRouteContext()))
 		res := &tracing.StatusWriter{ResponseWriter: httptest.NewRecorder()}
 		reg := prometheus.NewRegistry()
-		httpmw.HTTPRoute(httpmw.Prometheus(reg)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		httpmw.HTTPRoute(httpmw.Prometheus(reg, httpmw.NewWSMetrics(reg))(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 			w.WriteHeader(http.StatusOK)
 		}))).ServeHTTP(res, req)
 		metrics, err := reg.Gather()
@@ -43,7 +43,7 @@ func TestPrometheus(t *testing.T) {
 		defer cancel()
 
 		reg := prometheus.NewRegistry()
-		promMW := httpmw.Prometheus(reg)
+		promMW := httpmw.Prometheus(reg, httpmw.NewWSMetrics(reg))
 
 		// Create a test handler to simulate a WebSocket connection
 		testHandler := http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
@@ -82,7 +82,7 @@ func TestPrometheus(t *testing.T) {
 	t.Run("UserRoute", func(t *testing.T) {
 		t.Parallel()
 		reg := prometheus.NewRegistry()
-		promMW := httpmw.Prometheus(reg)
+		promMW := httpmw.Prometheus(reg, httpmw.NewWSMetrics(reg))
 
 		r := chi.NewRouter()
 		r.With(httpmw.HTTPRoute).With(promMW).Get("/api/v2/users/{user}", func(w http.ResponseWriter, r *http.Request) {})
@@ -112,7 +112,7 @@ func TestPrometheus(t *testing.T) {
 	t.Run("StaticRoute", func(t *testing.T) {
 		t.Parallel()
 		reg := prometheus.NewRegistry()
-		promMW := httpmw.Prometheus(reg)
+		promMW := httpmw.Prometheus(reg, httpmw.NewWSMetrics(reg))
 
 		r := chi.NewRouter()
 		r.Use(httpmw.HTTPRoute)
@@ -143,7 +143,7 @@ func TestPrometheus(t *testing.T) {
 	t.Run("UnknownRoute", func(t *testing.T) {
 		t.Parallel()
 		reg := prometheus.NewRegistry()
-		promMW := httpmw.Prometheus(reg)
+		promMW := httpmw.Prometheus(reg, httpmw.NewWSMetrics(reg))
 
 		r := chi.NewRouter()
 		r.Use(httpmw.HTTPRoute)
@@ -172,7 +172,7 @@ func TestPrometheus(t *testing.T) {
 	t.Run("Subrouter", func(t *testing.T) {
 		t.Parallel()
 		reg := prometheus.NewRegistry()
-		promMW := httpmw.Prometheus(reg)
+		promMW := httpmw.Prometheus(reg, httpmw.NewWSMetrics(reg))
 
 		r := chi.NewRouter()
 		r.Use(httpmw.HTTPRoute)