feat(coderd): add consolidated /debug/profile endpoint for pprof collection (#22892)

## Summary Adds a new `GET /api/v2/debug/profile` endpoint that collects multiple pprof profiles in a single request and returns them as a tar.gz archive. This allows collecting profiles (including block and mutex) without requiring `CODER_PPROF_ENABLE` to be set, and without restarting `coderd`. Closes #21679 ## What it does The endpoint: - Temporarily enables block and mutex profiling (normally disabled at runtime) - Runs CPU profile and/or trace for a configurable duration (default 10s, max 60s) - Collects snapshot profiles (heap, allocs, block, mutex, goroutine, threadcreate) - Returns a tar.gz archive containing all requested `.prof` files - Uses an atomic bool to prevent concurrent collections (returns 409 Conflict) - Is protected by the existing debug endpoint RBAC (owner-only) **Supported profile types:** cpu, heap, allocs, block, mutex, goroutine, threadcreate, trace **Query parameters:** - `duration`: How long to run timed profiles (default: `10s`, max: `60s`) - `profiles`: Comma-separated list of profile types (default: `cpu,heap,allocs,block,mutex,goroutine`) ## Additional changes - **SDK client method** (`codersdk.Client.DebugCollectProfile`) for easy programmatic access - **`coder support bundle --pprof` integration**: tries the consolidated endpoint first, falls back to individual `/debug/pprof/*` endpoints for older servers - **8 new tests** covering defaults, custom profiles, trace+CPU, validation errors, authorization, and conflict detection
2026-09-24 15:04:27 +08:00 · 2026-03-13 14:09:39 +00:00
parent cc6716c730
commit df2360f56a
8 changed files with 783 additions and 2 deletions
@@ -1,13 +1,20 @@
 package coderd

 import (
+	"archive/tar"
 	"bytes"
+	"compress/gzip"
 	"context"
 	"database/sql"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
+	"runtime"
+	"runtime/pprof"
+	"runtime/trace"
 	"slices"
+	"strings"
 	"time"

 	"github.com/google/uuid"
@@ -330,6 +337,298 @@ func loadDismissedHealthchecks(ctx context.Context, db database.Store, logger sl
 	return dismissedHealthchecks
 }

+// ProfileCollector abstracts the mechanics of collecting pprof/trace
+// data from the Go runtime. Production code uses defaultProfileCollector;
+// tests can substitute a stub to avoid process-global side-effects.
+type ProfileCollector interface {
+	// StartCPUProfile begins CPU profiling, writing to w. It returns
+	// a stop function that must be called to finish profiling.
+	StartCPUProfile(w io.Writer) (stop func(), err error)
+	// StartTrace begins execution tracing, writing to w. It returns
+	// a stop function that must be called to finish tracing.
+	StartTrace(w io.Writer) (stop func(), err error)
+	// LookupProfile writes the named snapshot profile to w.
+	LookupProfile(name string, w io.Writer) error
+	// SetBlockProfileRate enables/disables block profiling.
+	SetBlockProfileRate(rate int)
+	// SetMutexProfileFraction enables/disables mutex profiling.
+	// Returns the previous fraction.
+	SetMutexProfileFraction(rate int) int
+}
+
+// defaultProfileCollector delegates to the real runtime/pprof and
+// runtime/trace packages.
+type defaultProfileCollector struct{}
+
+func (defaultProfileCollector) StartCPUProfile(w io.Writer) (func(), error) {
+	if err := pprof.StartCPUProfile(w); err != nil {
+		return nil, err
+	}
+	return pprof.StopCPUProfile, nil
+}
+
+func (defaultProfileCollector) StartTrace(w io.Writer) (func(), error) {
+	if err := trace.Start(w); err != nil {
+		return nil, err
+	}
+	return trace.Stop, nil
+}
+
+func (defaultProfileCollector) LookupProfile(name string, w io.Writer) error {
+	p := pprof.Lookup(name)
+	if p == nil {
+		return nil
+	}
+	return p.WriteTo(w, 0)
+}
+
+func (defaultProfileCollector) SetBlockProfileRate(rate int) { runtime.SetBlockProfileRate(rate) }
+func (defaultProfileCollector) SetMutexProfileFraction(rate int) int {
+	return runtime.SetMutexProfileFraction(rate)
+}
+
+// defaultProfiles is the set of profiles collected when none are specified.
+var defaultProfiles = []string{"cpu", "heap", "allocs", "block", "mutex", "goroutine"}
+
+// allValidProfiles enumerates every profile name accepted by the endpoint.
+var allValidProfiles = map[string]bool{
+	"cpu":          true,
+	"heap":         true,
+	"allocs":       true,
+	"block":        true,
+	"mutex":        true,
+	"goroutine":    true,
+	"threadcreate": true,
+	"trace":        true,
+}
+
+const (
+	// profileDurationDefault is used when no ?duration is supplied.
+	profileDurationDefault = 10 * time.Second
+	// profileDurationMax prevents callers from asking for arbitrarily long
+	// collections that tie up the runtime-global CPU profiler.
+	profileDurationMax = 60 * time.Second
+)
+
+// @Summary Collect debug profiles
+// @ID collect-debug-profiles
+// @Security CoderSessionToken
+// @Tags Debug
+// @Success 200
+// @Router /debug/profile [post]
+// @x-apidocgen {"skip": true}
+func (api *API) debugCollectProfile(rw http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	// Parse duration.
+	duration := profileDurationDefault
+	if v := r.URL.Query().Get("duration"); v != "" {
+		d, err := time.ParseDuration(v)
+		if err != nil {
+			httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
+				Message: "Invalid duration parameter.",
+				Detail:  err.Error(),
+			})
+			return
+		}
+		if d <= 0 {
+			httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
+				Message: "Duration must be positive.",
+			})
+			return
+		}
+		if d > profileDurationMax {
+			httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
+				Message: fmt.Sprintf("Duration cannot exceed %s.", profileDurationMax),
+			})
+			return
+		}
+		duration = d
+	}
+
+	// Parse requested profiles.
+	profiles := defaultProfiles
+	if v := r.URL.Query().Get("profiles"); v != "" {
+		profiles = strings.Split(v, ",")
+		for _, p := range profiles {
+			if !allValidProfiles[p] {
+				httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
+					Message: fmt.Sprintf("Unknown profile type: %q.", p),
+					Detail:  "Valid types: cpu, heap, allocs, block, mutex, goroutine, threadcreate, trace",
+				})
+				return
+			}
+		}
+	}
+
+	// Only one profile collection can run at a time because the CPU
+	// profiler is process-global.
+	if !api.ProfileCollecting.CompareAndSwap(false, true) {
+		httpapi.Write(ctx, rw, http.StatusConflict, codersdk.Response{
+			Message: "A profile collection is already in progress. Try again later.",
+		})
+		return
+	}
+	defer api.ProfileCollecting.Store(false)
+
+	// Temporarily enable block and mutex profiling so those profiles are
+	// actually populated. Restore previous values when we are done.
+	// SetBlockProfileRate does not return the previous value, so we
+	// simply disable it again after collection (the default is 0).
+	pc := api.ProfileCollector
+	pc.SetBlockProfileRate(1)
+	prevMutexFraction := pc.SetMutexProfileFraction(1)
+	defer pc.SetBlockProfileRate(0)
+	defer pc.SetMutexProfileFraction(prevMutexFraction)
+
+	// Determine which profiles need the timed collection (cpu, trace) vs
+	// instant snapshots.
+	wantCPU := false
+	wantTrace := false
+	for _, p := range profiles {
+		switch p {
+		case "cpu":
+			wantCPU = true
+		case "trace":
+			wantTrace = true
+		}
+	}
+
+	// Collect timed profiles (cpu and/or trace) for the requested
+	// duration. StartCPUProfile and StartTrace each return a stop
+	// function that must be called to finish collection.
+	var cpuBuf, traceBuf bytes.Buffer
+	var stopCPU, stopTrace func()
+	if wantCPU {
+		var err error
+		stopCPU, err = pc.StartCPUProfile(&cpuBuf)
+		if err != nil {
+			httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
+				Message: "Failed to start CPU profile.",
+				Detail:  err.Error(),
+			})
+			return
+		}
+	}
+	if wantTrace {
+		var err error
+		stopTrace, err = pc.StartTrace(&traceBuf)
+		if err != nil {
+			if stopCPU != nil {
+				stopCPU()
+			}
+			httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
+				Message: "Failed to start trace.",
+				Detail:  err.Error(),
+			})
+			return
+		}
+	}
+
+	if wantCPU || wantTrace {
+		timer := api.Clock.NewTimer(duration, "debugCollectProfile")
+		defer timer.Stop()
+		select {
+		case <-ctx.Done():
+			if stopCPU != nil {
+				stopCPU()
+			}
+			if stopTrace != nil {
+				stopTrace()
+			}
+			// Client disconnected; nothing to write.
+			return
+		case <-timer.C:
+		}
+		if stopCPU != nil {
+			stopCPU()
+		}
+		if stopTrace != nil {
+			stopTrace()
+		}
+	}
+
+	// Build the tar.gz archive.
+	var archive bytes.Buffer
+	gzw := gzip.NewWriter(&archive)
+	tw := tar.NewWriter(gzw)
+
+	addFile := func(name string, data []byte) error {
+		hdr := &tar.Header{
+			Name: name,
+			Mode: 0o644,
+			Size: int64(len(data)),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return xerrors.Errorf("write tar header for %s: %w", name, err)
+		}
+		if _, err := tw.Write(data); err != nil {
+			return xerrors.Errorf("write tar data for %s: %w", name, err)
+		}
+		return nil
+	}
+
+	for _, p := range profiles {
+		switch p {
+		case "cpu":
+			if err := addFile("cpu.prof", cpuBuf.Bytes()); err != nil {
+				httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
+					Message: "Failed to write CPU profile to archive.",
+					Detail:  err.Error(),
+				})
+				return
+			}
+		case "trace":
+			if err := addFile("trace.out", traceBuf.Bytes()); err != nil {
+				httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
+					Message: "Failed to write trace to archive.",
+					Detail:  err.Error(),
+				})
+				return
+			}
+		default:
+			// Snapshot profiles: heap, allocs, block, mutex, goroutine,
+			// threadcreate.
+			var buf bytes.Buffer
+			if err := pc.LookupProfile(p, &buf); err != nil {
+				httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
+					Message: fmt.Sprintf("Failed to collect %s profile.", p),
+					Detail:  err.Error(),
+				})
+				return
+			}
+			if err := addFile(p+".prof", buf.Bytes()); err != nil {
+				httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
+					Message: fmt.Sprintf("Failed to write %s profile to archive.", p),
+					Detail:  err.Error(),
+				})
+				return
+			}
+		}
+	}
+
+	if err := tw.Close(); err != nil {
+		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
+			Message: "Failed to finalize tar archive.",
+			Detail:  err.Error(),
+		})
+		return
+	}
+	if err := gzw.Close(); err != nil {
+		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
+			Message: "Failed to finalize gzip archive.",
+			Detail:  err.Error(),
+		})
+		return
+	}
+
+	filename := fmt.Sprintf("coderd-profile-%d.tar.gz", time.Now().Unix())
+	rw.Header().Set("Content-Type", "application/gzip")
+	rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", filename))
+	rw.WriteHeader(http.StatusOK)
+	_, _ = rw.Write(archive.Bytes())
+}
+
 // @Summary Debug pprof index
 // @ID debug-pprof-index
 // @Security CoderSessionToken