chore: add profiling labels for pprof analysis (#19232)

PProf labels segment the code into groups for determing the source of
cpu/memory profiles. Since the web server and background jobs share a
lot of the same code (eg wsbuilder), it helps to know if the load is
user induced, or background job based.

cli/server.go

@@ -55,6 +55,7 @@ import (
 
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/sloghuman"
+	"github.com/coder/coder/v2/coderd/pproflabel"
 	"github.com/coder/pretty"
 	"github.com/coder/quartz"
 	"github.com/coder/retry"
@@ -1459,14 +1460,14 @@ func newProvisionerDaemon(
 			tracer := coderAPI.TracerProvider.Tracer(tracing.TracerName)
 			terraformClient, terraformServer := drpcsdk.MemTransportPipe()
 			wg.Add(1)
-			go func() {
+			pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceTerraformProvisioner), func(ctx context.Context) {
 				defer wg.Done()
 				<-ctx.Done()
 				_ = terraformClient.Close()
 				_ = terraformServer.Close()
-			}()
+			})
 			wg.Add(1)
-			go func() {
+			pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceTerraformProvisioner), func(ctx context.Context) {
 				defer wg.Done()
 				defer cancel()
 
@@ -1485,7 +1486,7 @@ func newProvisionerDaemon(
 					default:
 					}
 				}
-			}()
+			})
 
 			connector[string(database.ProvisionerTypeTerraform)] = sdkproto.NewDRPCProvisionerClient(terraformClient)
 		default:

coderd/autobuild/lifecycle_executor.go

@@ -20,6 +20,7 @@ import (
 
 	"cdr.dev/slog"
 	"github.com/coder/coder/v2/coderd/files"
+	"github.com/coder/coder/v2/coderd/pproflabel"
 
 	"github.com/coder/coder/v2/coderd/audit"
 	"github.com/coder/coder/v2/coderd/database"
@@ -107,10 +108,10 @@ func (e *Executor) WithStatsChannel(ch chan<- Stats) *Executor {
 // tick from its channel. It will stop when its context is Done, or when
 // its channel is closed.
 func (e *Executor) Run() {
-	go func() {
+	pproflabel.Go(e.ctx, pproflabel.Service(pproflabel.ServiceLifecycles), func(ctx context.Context) {
 		for {
 			select {
-			case <-e.ctx.Done():
+			case <-ctx.Done():
 				return
 			case t, ok := <-e.tick:
 				if !ok {
@@ -120,15 +121,15 @@ func (e *Executor) Run() {
 				e.metrics.autobuildExecutionDuration.Observe(stats.Elapsed.Seconds())
 				if e.statsCh != nil {
 					select {
-					case <-e.ctx.Done():
+					case <-ctx.Done():
 						return
 					case e.statsCh <- stats:
 					}
 				}
-				e.log.Debug(e.ctx, "run stats", slog.F("elapsed", stats.Elapsed), slog.F("transitions", stats.Transitions))
+				e.log.Debug(ctx, "run stats", slog.F("elapsed", stats.Elapsed), slog.F("transitions", stats.Transitions))
 			}
 		}
-	}()
+	})
 }
 
 func (e *Executor) runOnce(t time.Time) Stats {

coderd/coderd.go

@@ -852,6 +852,7 @@ func New(options *Options) *API {
 
 	r.Use(
 		httpmw.Recover(api.Logger),
+		httpmw.WithProfilingLabels,
 		tracing.StatusWriterMiddleware,
 		tracing.Middleware(api.TracerProvider),
 		httpmw.AttachRequestID,

coderd/httpmw/pprof.go

@@ -0,0 +1,30 @@
+package httpmw
+
+import (
+	"context"
+	"net/http"
+	"runtime/pprof"
+
+	"github.com/coder/coder/v2/coderd/pproflabel"
+)
+
+// WithProfilingLabels adds a pprof label to all http request handlers. This is
+// primarily used to determine if load is coming from background jobs, or from
+// http traffic.
+func WithProfilingLabels(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+
+		// Label to differentiate between http and websocket requests. Websocket requests
+		// are assumed to be long-lived and more resource consuming.
+		requestType := "http"
+		if r.Header.Get("Upgrade") == "websocket" {
+			requestType = "websocket"
+		}
+
+		pprof.Do(ctx, pproflabel.Service(pproflabel.ServiceHTTPServer, "request_type", requestType), func(ctx context.Context) {
+			r = r.WithContext(ctx)
+			next.ServeHTTP(rw, r)
+		})
+	})
+}

coderd/pproflabel/pproflabel.go

@@ -0,0 +1,25 @@
+package pproflabel
+
+import (
+	"context"
+	"runtime/pprof"
+)
+
+// Go is just a convince wrapper to set off a labeled goroutine.
+func Go(ctx context.Context, labels pprof.LabelSet, f func(context.Context)) {
+	go pprof.Do(ctx, labels, f)
+}
+
+const (
+	ServiceTag = "service"
+
+	ServiceHTTPServer           = "http-api"
+	ServiceLifecycles           = "lifecycle-executor"
+	ServiceMetricCollector      = "metrics-collector"
+	ServicePrebuildReconciler   = "prebuilds-reconciler"
+	ServiceTerraformProvisioner = "terraform-provisioner"
+)
+
+func Service(name string, pairs ...string) pprof.LabelSet {
+	return pprof.Labels(append([]string{ServiceTag, name}, pairs...)...)
+}

coderd/prometheusmetrics/insights/metricscollector.go

@@ -14,6 +14,7 @@ import (
 	"cdr.dev/slog"
 
 	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/pproflabel"
 	"github.com/coder/coder/v2/coderd/util/slice"
 	"github.com/coder/coder/v2/codersdk"
 )
@@ -158,7 +159,7 @@ func (mc *MetricsCollector) Run(ctx context.Context) (func(), error) {
 		})
 	}
 
-	go func() {
+	pproflabel.Go(ctx, pproflabel.Service(pproflabel.ServiceMetricCollector), func(ctx context.Context) {
 		defer close(done)
 		defer ticker.Stop()
 		for {
@@ -170,7 +171,7 @@ func (mc *MetricsCollector) Run(ctx context.Context) (func(), error) {
 				doTick()
 			}
 		}
-	}()
+	})
 	return func() {
 		closeFunc()
 		<-done

enterprise/coderd/coderd.go

@@ -12,20 +12,20 @@ import (
 	"sync"
 	"time"
 
-	"github.com/coder/quartz"
-
 	"github.com/coder/coder/v2/buildinfo"
 	"github.com/coder/coder/v2/coderd/appearance"
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/entitlements"
 	"github.com/coder/coder/v2/coderd/idpsync"
 	agplportsharing "github.com/coder/coder/v2/coderd/portsharing"
+	"github.com/coder/coder/v2/coderd/pproflabel"
 	agplprebuilds "github.com/coder/coder/v2/coderd/prebuilds"
 	"github.com/coder/coder/v2/coderd/rbac/policy"
 	"github.com/coder/coder/v2/coderd/wsbuilder"
 	"github.com/coder/coder/v2/enterprise/coderd/connectionlog"
 	"github.com/coder/coder/v2/enterprise/coderd/enidpsync"
 	"github.com/coder/coder/v2/enterprise/coderd/portsharing"
+	"github.com/coder/quartz"
 
 	"golang.org/x/xerrors"
 	"tailscale.com/tailcfg"
@@ -903,7 +903,9 @@ func (api *API) updateEntitlements(ctx context.Context) error {
 			}
 
 			api.AGPL.PrebuildsReconciler.Store(&reconciler)
-			go reconciler.Run(context.Background())
+			// TODO: Should this context be the api.ctx context? To cancel when
+			// 	the API (and entire app) is closed via shutdown?
+			pproflabel.Go(context.Background(), pproflabel.Service(pproflabel.ServicePrebuildReconciler), reconciler.Run)
 
 			api.AGPL.PrebuildsClaimer.Store(&claimer)
 		}

enterprise/wsproxy/wsproxy.go

@@ -333,6 +333,7 @@ func New(ctx context.Context, opts *Options) (*Server, error) {
 	r.Use(
 		// TODO: @emyrk Should we standardize these in some other package?
 		httpmw.Recover(s.Logger),
+		httpmw.WithProfilingLabels,
 		tracing.StatusWriterMiddleware,
 		tracing.Middleware(s.TracerProvider),
 		httpmw.AttachRequestID,