feat: support bundle updates to enable pprof and telemetry collection (#21486)

- Adds pprof collection support now that we have the listeners
automatically starting (requires Coder server 2.28.0+, includes a
version check). Collects heap, allocs, profile (30s), block, mutex,
goroutine, threadcreate, trace (30s), cmdline, symbol. Performs capture
for 30 seconds and emits a log line stating as such. Enable capture by
supplying the `--pprof` flag or `CODER_SUPPORT_BUNDLE_PPROF` env var.
Collection of pprof data from both coderd and the Coder agent occurs.
- Adds collection of Prometheus metrics, also requires 2.28.0+
- Adds the ability to include a template in the bundle independently of
supplying the details of a running workspace by supplying the
`--template` flag or `CODER_SUPPORT_BUNDLE_TEMPLATE` env var
- Captures a list of workspaces the user has access to. Defaults to a
max of 10, configurable via `--workspaces-total-cap` /
`CODER_SUPPORT_BUNDLE_WORKSPACES_TOTAL_CAP`
- Collects additional stats from the coderd deployment (aggregated
workspace/session metrics), as well as entitlements via license and
dismissed health checks.

created with help from mux

cli/support.go

@@ -7,6 +7,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -44,13 +45,18 @@ var supportBundleBlurb = cliui.Bold("This will collect the following information
 	`  - Coder deployment version
   - Coder deployment Configuration (sanitized), including enabled experiments
   - Coder deployment health snapshot
+  - Coder deployment stats (aggregated workspace/session metrics)
+  - Entitlements (if available)
+  - Health settings (dismissed healthchecks)
   - Coder deployment Network troubleshooting information
+  - Workspace list accessible to the user (sanitized)
   - Workspace configuration, parameters, and build logs
   - Template version and source code for the given workspace
   - Agent details (with environment variable sanitized)
   - Agent network diagnostics
   - Agent logs
   - License status
+  - pprof profiling data (if --pprof is enabled)
 ` + cliui.Bold("Note: ") +
 	cliui.Wrap("While we try to sanitize sensitive data from support bundles, we cannot guarantee that they do not contain information that you or your organization may consider sensitive.\n") +
 	cliui.Bold("Please confirm that you will:\n") +
@@ -61,6 +67,9 @@ var supportBundleBlurb = cliui.Bold("This will collect the following information
 func (r *RootCmd) supportBundle() *serpent.Command {
 	var outputPath string
 	var coderURLOverride string
+	var workspacesTotalCap64 int64 = 10
+	var templateName string
+	var pprof bool
 	cmd := &serpent.Command{
 		Use:   "bundle <workspace> [<agent>]",
 		Short: "Generate a support bundle to troubleshoot issues connecting to a workspace.",
@@ -121,8 +130,9 @@ func (r *RootCmd) supportBundle() *serpent.Command {
 			}
 
 			var (
-				wsID  uuid.UUID
-				agtID uuid.UUID
+				wsID       uuid.UUID
+				agtID      uuid.UUID
+				templateID uuid.UUID
 			)
 
 			if len(inv.Args) == 0 {
@@ -155,6 +165,16 @@ func (r *RootCmd) supportBundle() *serpent.Command {
 				}
 			}
 
+			// Resolve template by name if provided (captures active version)
+			// Fallback: if canonical name lookup fails, match DisplayName (case-insensitive).
+			if templateName != "" {
+				id, err := resolveTemplateID(inv.Context(), client, templateName)
+				if err != nil {
+					return err
+				}
+				templateID = id
+			}
+
 			if outputPath == "" {
 				cwd, err := filepath.Abs(".")
 				if err != nil {
@@ -176,12 +196,25 @@ func (r *RootCmd) supportBundle() *serpent.Command {
 			if r.verbose {
 				clientLog.AppendSinks(sloghuman.Sink(inv.Stderr))
 			}
+			if pprof {
+				_, _ = fmt.Fprintln(inv.Stderr, "pprof data collection will take approximately 30 seconds...")
+			}
+
+			// Bypass rate limiting for support bundle collection since it makes many API calls.
+			client.HTTPClient.Transport = &codersdk.HeaderTransport{
+				Transport: client.HTTPClient.Transport,
+				Header:    http.Header{codersdk.BypassRatelimitHeader: {"true"}},
+			}
+
 			deps := support.Deps{
 				Client: client,
 				// Support adds a sink so we don't need to supply one ourselves.
-				Log:         clientLog,
-				WorkspaceID: wsID,
-				AgentID:     agtID,
+				Log:                clientLog,
+				WorkspaceID:        wsID,
+				AgentID:            agtID,
+				WorkspacesTotalCap: int(workspacesTotalCap64),
+				TemplateID:         templateID,
+				CollectPprof:       pprof,
 			}
 
 			bun, err := support.Run(inv.Context(), &deps)
@@ -217,11 +250,102 @@ func (r *RootCmd) supportBundle() *serpent.Command {
 			Description: "Override the URL to your Coder deployment. This may be useful, for example, if you need to troubleshoot a specific Coder replica.",
 			Value:       serpent.StringOf(&coderURLOverride),
 		},
+		{
+			Flag:        "workspaces-total-cap",
+			Env:         "CODER_SUPPORT_BUNDLE_WORKSPACES_TOTAL_CAP",
+			Description: "Maximum number of workspaces to include in the support bundle. Set to 0 or negative value to disable the cap. Defaults to 10.",
+			Value:       serpent.Int64Of(&workspacesTotalCap64),
+		},
+		{
+			Flag:        "template",
+			Env:         "CODER_SUPPORT_BUNDLE_TEMPLATE",
+			Description: "Template name to include in the support bundle. Use org_name/template_name if template name is reused across multiple organizations.",
+			Value:       serpent.StringOf(&templateName),
+		},
+		{
+			Flag:        "pprof",
+			Env:         "CODER_SUPPORT_BUNDLE_PPROF",
+			Description: "Collect pprof profiling data from the Coder server and agent. Requires Coder server version 2.28.0 or newer.",
+			Value:       serpent.BoolOf(&pprof),
+		},
 	}
 
 	return cmd
 }
 
+// Resolve a template to its ID, supporting:
+// - org/name form
+// - slug or display name match (case-insensitive) across all memberships
+func resolveTemplateID(ctx context.Context, client *codersdk.Client, templateArg string) (uuid.UUID, error) {
+	orgPart := ""
+	namePart := templateArg
+	if slash := strings.IndexByte(templateArg, '/'); slash > 0 && slash < len(templateArg)-1 {
+		orgPart = templateArg[:slash]
+		namePart = templateArg[slash+1:]
+	}
+
+	resolveInOrg := func(orgID uuid.UUID) (codersdk.Template, bool, error) {
+		if t, err := client.TemplateByName(ctx, orgID, namePart); err == nil {
+			return t, true, nil
+		}
+		tpls, err := client.TemplatesByOrganization(ctx, orgID)
+		if err != nil {
+			return codersdk.Template{}, false, nil
+		}
+		for _, t := range tpls {
+			if strings.EqualFold(t.Name, namePart) || strings.EqualFold(t.DisplayName, namePart) {
+				return t, true, nil
+			}
+		}
+		return codersdk.Template{}, false, nil
+	}
+
+	if orgPart != "" {
+		org, err := client.OrganizationByName(ctx, orgPart)
+		if err != nil {
+			return uuid.Nil, xerrors.Errorf("get organization %q: %w", orgPart, err)
+		}
+		t, found, err := resolveInOrg(org.ID)
+		if err != nil {
+			return uuid.Nil, err
+		}
+		if !found {
+			return uuid.Nil, xerrors.Errorf("template %q not found in organization %q", namePart, orgPart)
+		}
+		return t.ID, nil
+	}
+
+	orgs, err := client.OrganizationsByUser(ctx, codersdk.Me)
+	if err != nil {
+		return uuid.Nil, xerrors.Errorf("get organizations: %w", err)
+	}
+	var (
+		foundTpl  codersdk.Template
+		foundOrgs []string
+	)
+	for _, org := range orgs {
+		if t, found, err := resolveInOrg(org.ID); err == nil && found {
+			if len(foundOrgs) == 0 {
+				foundTpl = t
+			}
+			foundOrgs = append(foundOrgs, org.Name)
+		}
+	}
+	switch len(foundOrgs) {
+	case 0:
+		return uuid.Nil, xerrors.Errorf("template %q not found in your organizations", namePart)
+	case 1:
+		return foundTpl.ID, nil
+	default:
+		return uuid.Nil, xerrors.Errorf(
+			"template %q found in multiple organizations (%s); use --template \"<org_name/%s>\" to target desired template.",
+			namePart,
+			strings.Join(foundOrgs, ", "),
+			namePart,
+		)
+	}
+}
+
 // summarizeBundle makes a best-effort attempt to write a short summary
 // of the support bundle to the user's terminal.
 func summarizeBundle(inv *serpent.Invocation, bun *support.Bundle) {
@@ -283,6 +407,10 @@ func writeBundle(src *support.Bundle, dest *zip.Writer) error {
 		"deployment/config.json":          src.Deployment.Config,
 		"deployment/experiments.json":     src.Deployment.Experiments,
 		"deployment/health.json":          src.Deployment.HealthReport,
+		"deployment/stats.json":           src.Deployment.Stats,
+		"deployment/entitlements.json":    src.Deployment.Entitlements,
+		"deployment/health_settings.json": src.Deployment.HealthSettings,
+		"deployment/workspaces.json":      src.Deployment.Workspaces,
 		"network/connection_info.json":    src.Network.ConnectionInfo,
 		"network/netcheck.json":           src.Network.Netcheck,
 		"network/interfaces.json":         src.Network.Interfaces,
@@ -302,6 +430,49 @@ func writeBundle(src *support.Bundle, dest *zip.Writer) error {
 		}
 	}
 
+	// Include named template artifacts (if requested)
+	if src.NamedTemplate.Template.ID != uuid.Nil {
+		name := src.NamedTemplate.Template.Name
+		// JSON files
+		for k, v := range map[string]any{
+			"templates/" + name + "/template.json":         src.NamedTemplate.Template,
+			"templates/" + name + "/template_version.json": src.NamedTemplate.TemplateVersion,
+		} {
+			f, err := dest.Create(k)
+			if err != nil {
+				return xerrors.Errorf("create file %q in archive: %w", k, err)
+			}
+			enc := json.NewEncoder(f)
+			enc.SetIndent("", "    ")
+			if err := enc.Encode(v); err != nil {
+				return xerrors.Errorf("write json to %q: %w", k, err)
+			}
+		}
+		// Binary template file (zip)
+		if namedZipBytes, err := base64.StdEncoding.DecodeString(src.NamedTemplate.TemplateFileBase64); err == nil {
+			k := "templates/" + name + "/template_file.zip"
+			f, err := dest.Create(k)
+			if err != nil {
+				return xerrors.Errorf("create file %q in archive: %w", k, err)
+			}
+			if _, err := f.Write(namedZipBytes); err != nil {
+				return xerrors.Errorf("write file %q in archive: %w", k, err)
+			}
+		}
+	}
+
+	var buildInfoRef string
+	if src.Deployment.BuildInfo != nil {
+		if raw, err := json.Marshal(src.Deployment.BuildInfo); err == nil {
+			buildInfoRef = base64.StdEncoding.EncodeToString(raw)
+		}
+	}
+
+	tailnetHTML := src.Network.TailnetDebug
+	if buildInfoRef != "" {
+		tailnetHTML += "\n<!-- trace " + buildInfoRef + " -->"
+	}
+
 	templateVersionBytes, err := base64.StdEncoding.DecodeString(src.Workspace.TemplateFileBase64)
 	if err != nil {
 		return xerrors.Errorf("decode template zip from base64")
@@ -319,10 +490,11 @@ func writeBundle(src *support.Bundle, dest *zip.Writer) error {
 		"agent/client_magicsock.html":    string(src.Agent.ClientMagicsockHTML),
 		"agent/startup_logs.txt":         humanizeAgentLogs(src.Agent.StartupLogs),
 		"agent/prometheus.txt":           string(src.Agent.Prometheus),
+		"deployment/prometheus.txt":      string(src.Deployment.Prometheus),
 		"cli_logs.txt":                   string(src.CLILogs),
 		"logs.txt":                       strings.Join(src.Logs, "\n"),
 		"network/coordinator_debug.html": src.Network.CoordinatorDebug,
-		"network/tailnet_debug.html":     src.Network.TailnetDebug,
+		"network/tailnet_debug.html":     tailnetHTML,
 		"workspace/build_logs.txt":       humanizeBuildLogs(src.Workspace.BuildLogs),
 		"workspace/template_file.zip":    string(templateVersionBytes),
 		"license-status.txt":             licenseStatus,
@@ -335,12 +507,89 @@ func writeBundle(src *support.Bundle, dest *zip.Writer) error {
 			return xerrors.Errorf("write file %q in archive: %w", k, err)
 		}
 	}
+
+	// Write pprof binary data
+	if err := writePprofData(src.Pprof, dest); err != nil {
+		return xerrors.Errorf("write pprof data: %w", err)
+	}
+
 	if err := dest.Close(); err != nil {
 		return xerrors.Errorf("close zip file: %w", err)
 	}
 	return nil
 }
 
+func writePprofData(pprof support.Pprof, dest *zip.Writer) error {
+	// Write server pprof data directly to pprof directory
+	if pprof.Server != nil {
+		if err := writePprofCollection("pprof", pprof.Server, dest); err != nil {
+			return xerrors.Errorf("write server pprof data: %w", err)
+		}
+	}
+
+	// Write agent pprof data
+	if pprof.Agent != nil {
+		if err := writePprofCollection("pprof/agent", pprof.Agent, dest); err != nil {
+			return xerrors.Errorf("write agent pprof data: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func writePprofCollection(basePath string, collection *support.PprofCollection, dest *zip.Writer) error {
+	// Define the pprof files to write with their extensions
+	files := map[string][]byte{
+		"allocs.prof.gz":       collection.Allocs,
+		"heap.prof.gz":         collection.Heap,
+		"profile.prof.gz":      collection.Profile,
+		"block.prof.gz":        collection.Block,
+		"mutex.prof.gz":        collection.Mutex,
+		"goroutine.prof.gz":    collection.Goroutine,
+		"threadcreate.prof.gz": collection.Threadcreate,
+		"trace.gz":             collection.Trace,
+	}
+
+	// Write binary pprof files
+	for filename, data := range files {
+		if len(data) > 0 {
+			filePath := basePath + "/" + filename
+			f, err := dest.Create(filePath)
+			if err != nil {
+				return xerrors.Errorf("create pprof file %q: %w", filePath, err)
+			}
+			if _, err := f.Write(data); err != nil {
+				return xerrors.Errorf("write pprof file %q: %w", filePath, err)
+			}
+		}
+	}
+
+	// Write cmdline as text file
+	if collection.Cmdline != "" {
+		filePath := basePath + "/cmdline.txt"
+		f, err := dest.Create(filePath)
+		if err != nil {
+			return xerrors.Errorf("create cmdline file %q: %w", filePath, err)
+		}
+		if _, err := f.Write([]byte(collection.Cmdline)); err != nil {
+			return xerrors.Errorf("write cmdline file %q: %w", filePath, err)
+		}
+	}
+
+	if collection.Symbol != "" {
+		filePath := basePath + "/symbol.txt"
+		f, err := dest.Create(filePath)
+		if err != nil {
+			return xerrors.Errorf("create symbol file %q: %w", filePath, err)
+		}
+		if _, err := f.Write([]byte(collection.Symbol)); err != nil {
+			return xerrors.Errorf("write symbol file %q: %w", filePath, err)
+		}
+	}
+
+	return nil
+}
+
 func humanizeAgentLogs(ls []codersdk.WorkspaceAgentLog) string {
 	var buf bytes.Buffer
 	tw := tabwriter.NewWriter(&buf, 0, 2, 1, ' ', 0)

cli/support_test.go

@@ -46,6 +46,8 @@ func TestSupportBundle(t *testing.T) {
 
 	// Support bundle tests can share a single coderdtest instance.
 	var dc codersdk.DeploymentConfig
+	dc.Values = coderdtest.DeploymentValues(t)
+	dc.Values.Prometheus.Enable = true
 	secretValue := uuid.NewString()
 	seedSecretDeploymentOptions(t, &dc, secretValue)
 	client, closer, api := coderdtest.NewWithAPI(t, &coderdtest.Options{
@@ -203,6 +205,10 @@ func assertBundleContents(t *testing.T, path string, wantWorkspace bool, wantAge
 			var v codersdk.DeploymentConfig
 			decodeJSONFromZip(t, f, &v)
 			require.NotEmpty(t, v, "deployment config should not be empty")
+		case "deployment/entitlements.json":
+			var v codersdk.Entitlements
+			decodeJSONFromZip(t, f, &v)
+			require.NotNil(t, v, "entitlements should not be nil")
 		case "deployment/experiments.json":
 			var v codersdk.Experiments
 			decodeJSONFromZip(t, f, &v)
@@ -211,6 +217,22 @@ func assertBundleContents(t *testing.T, path string, wantWorkspace bool, wantAge
 			var v healthsdk.HealthcheckReport
 			decodeJSONFromZip(t, f, &v)
 			require.NotEmpty(t, v, "health report should not be empty")
+		case "deployment/health_settings.json":
+			var v healthsdk.HealthSettings
+			decodeJSONFromZip(t, f, &v)
+			require.NotEmpty(t, v, "health settings should not be empty")
+		case "deployment/stats.json":
+			var v codersdk.DeploymentStats
+			decodeJSONFromZip(t, f, &v)
+			require.NotNil(t, v, "deployment stats should not be nil")
+		case "deployment/workspaces.json":
+			var v codersdk.Workspace
+			decodeJSONFromZip(t, f, &v)
+			require.NotNil(t, v, "deployment workspaces should not be nil")
+		case "deployment/prometheus.txt":
+			bs := readBytesFromZip(t, f)
+			require.NotEmpty(t, bs, "prometheus metrics should not be empty")
+			require.Contains(t, string(bs), "go_goroutines", "prometheus metrics should contain go runtime metrics")
 		case "network/connection_info.json":
 			var v workspacesdk.AgentConnectionInfo
 			decodeJSONFromZip(t, f, &v)

cli/testdata/coder_support_bundle_--help.golden

@@ -14,10 +14,23 @@ OPTIONS:
           File path for writing the generated support bundle. Defaults to
           coder-support-$(date +%s).zip.
 
+      --pprof bool, $CODER_SUPPORT_BUNDLE_PPROF
+          Collect pprof profiling data from the Coder server and agent. Requires
+          Coder server version 2.28.0 or newer.
+
+      --template string, $CODER_SUPPORT_BUNDLE_TEMPLATE
+          Template name to include in the support bundle. Use
+          org_name/template_name if template name is reused across multiple
+          organizations.
+
       --url-override string, $CODER_SUPPORT_BUNDLE_URL_OVERRIDE
           Override the URL to your Coder deployment. This may be useful, for
           example, if you need to troubleshoot a specific Coder replica.
 
+      --workspaces-total-cap int, $CODER_SUPPORT_BUNDLE_WORKSPACES_TOTAL_CAP
+          Maximum number of workspaces to include in the support bundle. Set to
+          0 or negative value to disable the cap. Defaults to 10.
+
   -y, --yes bool
           Bypass confirmation prompts.