feat: tracing improvements (#4988)

coderd/tracing/httpmw.go

@@ -10,17 +10,31 @@ import (
 	"go.opentelemetry.io/otel/propagation"
 	semconv "go.opentelemetry.io/otel/semconv/v1.11.0"
 	"go.opentelemetry.io/otel/trace"
+
+	"github.com/coder/coder/coderd/httpmw/patternmatcher"
 )
 
 // Middleware adds tracing to http routes.
 func Middleware(tracerProvider trace.TracerProvider) func(http.Handler) http.Handler {
+	// We only want to create spans on the following route patterns, however
+	// we want the middleware to be very high in the middleware stack so it can
+	// capture the entire request.
+	re := patternmatcher.RoutePatterns{
+		"/api",
+		"/api/**",
+		"/@*/*/apps/**",
+		"/%40*/*/apps/**",
+		"/gitauth/*/callback",
+	}.MustCompile()
+
 	var tracer trace.Tracer
 	if tracerProvider != nil {
 		tracer = tracerProvider.Tracer(TracerName)
 	}
+
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
-			if tracer == nil {
+			if tracer == nil || !re.MatchString(r.URL.Path) {
 				next.ServeHTTP(rw, r)
 				return
 			}

coderd/tracing/httpmw_test.go

@@ -0,0 +1,96 @@
+package tracing_test
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+
+	"go.opentelemetry.io/otel/trace"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/coderd/tracing"
+	"github.com/coder/coder/testutil"
+)
+
+type fakeTracer struct {
+	startCalled int64
+}
+
+var _ trace.TracerProvider = &fakeTracer{}
+var _ trace.Tracer = &fakeTracer{}
+
+// Tracer implements trace.TracerProvider.
+func (f *fakeTracer) Tracer(_ string, _ ...trace.TracerOption) trace.Tracer {
+	return f
+}
+
+// Start implements trace.Tracer.
+func (f *fakeTracer) Start(ctx context.Context, _ string, _ ...trace.SpanStartOption) (context.Context, trace.Span) {
+	atomic.AddInt64(&f.startCalled, 1)
+	return ctx, tracing.NoopSpan
+}
+
+func Test_Middleware(t *testing.T) {
+	t.Parallel()
+
+	t.Run("OnlyRunsOnExpectedRoutes", func(t *testing.T) {
+		t.Parallel()
+
+		cases := []struct {
+			path string
+			runs bool
+		}{
+			// Should pass.
+			{"/api", true},
+			{"/api/v0", true},
+			{"/api/v2", true},
+			{"/api/v2/workspaces/", true},
+			{"/api/v2/workspaces", true},
+			{"/@hi/hi/apps/hi", true},
+			{"/@hi/hi/apps/hi/hi", true},
+			{"/@hi/hi/apps/hi/hi", true},
+			{"/%40hi/hi/apps/hi", true},
+			{"/%40hi/hi/apps/hi/hi", true},
+			{"/%40hi/hi/apps/hi/hi", true},
+			{"/gitauth/hi/callback", true},
+
+			// Other routes that should not be collected.
+			{"/index.html", false},
+			{"/static/coder_linux_amd64", false},
+			{"/workspaces", false},
+			{"/templates", false},
+			{"/@hi/hi/terminal", false},
+		}
+
+		for _, c := range cases {
+			c := c
+
+			name := strings.ReplaceAll(strings.TrimPrefix(c.path, "/"), "/", "_")
+			t.Run(name, func(t *testing.T) {
+				t.Parallel()
+
+				fake := &fakeTracer{}
+
+				rw := &tracing.StatusWriter{ResponseWriter: httptest.NewRecorder()}
+				r := httptest.NewRequest("GET", c.path, nil)
+
+				ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+				defer cancel()
+				ctx = context.WithValue(ctx, chi.RouteCtxKey, chi.NewRouteContext())
+				r = r.WithContext(ctx)
+
+				tracing.Middleware(fake)(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+					rw.WriteHeader(http.StatusNoContent)
+				})).ServeHTTP(rw, r)
+
+				didRun := atomic.LoadInt64(&fake.startCalled) == 1
+				require.Equal(t, c.runs, didRun, "expected middleware to run/not run")
+			})
+		}
+	})
+}

coderd/tracing/status_writer.go

@@ -23,6 +23,13 @@ type StatusWriter struct {
 	wroteHeader bool
 }
 
+func StatusWriterMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		sw := &StatusWriter{ResponseWriter: rw}
+		next.ServeHTTP(sw, r)
+	})
+}
+
 func (w *StatusWriter) WriteHeader(status int) {
 	if !w.wroteHeader {
 		w.Status = status

coderd/tracing/util.go

@@ -18,16 +18,7 @@ func init() {
 const TracerName = "coderd"
 
 func FuncName() string {
-	fnpc, _, _, ok := runtime.Caller(1)
-	if !ok {
-		return ""
-	}
-	fn := runtime.FuncForPC(fnpc)
-	name := fn.Name()
-	if i := strings.LastIndex(name, "/"); i > 0 {
-		name = name[i+1:]
-	}
-	return name
+	return FuncNameSkip(1)
 }
 
 func FuncNameSkip(skip int) string {
@@ -42,3 +33,11 @@ func FuncNameSkip(skip int) string {
 	}
 	return name
 }
+
+// RunWithoutSpan runs the given function with the span stripped from the
+// context and replaced with a no-op span. This is useful for avoiding logs
+// being added to span (to save money).
+func RunWithoutSpan(ctx context.Context, fn func(ctx context.Context)) {
+	ctx = trace.ContextWithSpan(ctx, NoopSpan)
+	fn(ctx)
+}