chore: add debug logging and recovery to agent api requests (#20785)

This is to debug context timeouts on API requests to the agent.

Because rbac and database cannot be imported in slim, split the logger
middleware into slim and non-slim versions and break out the recovery
middleware.

httpmw/recover.go

@@ -0,0 +1,47 @@
+package httpmw
+
+import (
+	"context"
+	"net/http"
+	"runtime/debug"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/coderd/tracing"
+)
+
+func Recover(log slog.Logger) func(h http.Handler) http.Handler {
+	return func(h http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			defer func() {
+				r := recover()
+
+				// Reverse proxying (among other things) may panic with
+				// http.ErrAbortHandler when the request is aborted. It's not a
+				// real panic so we shouldn't log them.
+				//
+				//nolint:errorlint // this is how the stdlib does the check
+				if r != nil && r != http.ErrAbortHandler {
+					log.Warn(context.Background(),
+						"panic serving http request (recovered)",
+						slog.F("panic", r),
+						slog.F("stack", string(debug.Stack())),
+					)
+
+					var hijacked bool
+					if sw, ok := w.(*tracing.StatusWriter); ok {
+						hijacked = sw.Hijacked
+					}
+
+					// Only try to write errors on
+					// non-hijacked responses.
+					if !hijacked {
+						httpapi.InternalServerError(w, nil)
+					}
+				}
+			}()
+
+			h.ServeHTTP(w, r)
+		})
+	}
+}

httpmw/recover_test.go

@@ -0,0 +1,72 @@
+package httpmw_test
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/coderd/tracing"
+	"github.com/coder/coder/v2/httpmw"
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestRecover(t *testing.T) {
+	t.Parallel()
+
+	handler := func(isPanic, _ bool) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if isPanic {
+				panic("Oh no!")
+			}
+
+			w.WriteHeader(http.StatusOK)
+		})
+	}
+
+	cases := []struct {
+		Name   string
+		Code   int
+		Panic  bool
+		Hijack bool
+	}{
+		{
+			Name:   "OK",
+			Code:   http.StatusOK,
+			Panic:  false,
+			Hijack: false,
+		},
+		{
+			Name:   "Panic",
+			Code:   http.StatusInternalServerError,
+			Panic:  true,
+			Hijack: false,
+		},
+		{
+			Name:   "Hijack",
+			Code:   0,
+			Panic:  true,
+			Hijack: true,
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.Name, func(t *testing.T) {
+			t.Parallel()
+
+			var (
+				log = testutil.Logger(t)
+				r   = httptest.NewRequest("GET", "/", nil)
+				w   = &tracing.StatusWriter{
+					ResponseWriter: httptest.NewRecorder(),
+					Hijacked:       c.Hijack,
+				}
+			)
+
+			httpmw.Recover(log)(handler(c.Panic, c.Hijack)).ServeHTTP(w, r)
+
+			require.Equal(t, c.Code, w.Status)
+		})
+	}
+}