mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: support bundle updates to enable pprof and telemetry collection (#21486)
- Adds pprof collection support now that we have the listeners automatically starting (requires Coder server 2.28.0+, includes a version check). Collects heap, allocs, profile (30s), block, mutex, goroutine, threadcreate, trace (30s), cmdline, symbol. Performs capture for 30 seconds and emits a log line stating as such. Enable capture by supplying the `--pprof` flag or `CODER_SUPPORT_BUNDLE_PPROF` env var. Collection of pprof data from both coderd and the Coder agent occurs. - Adds collection of Prometheus metrics, also requires 2.28.0+ - Adds the ability to include a template in the bundle independently of supplying the details of a running workspace by supplying the `--template` flag or `CODER_SUPPORT_BUNDLE_TEMPLATE` env var - Captures a list of workspaces the user has access to. Defaults to a max of 10, configurable via `--workspaces-total-cap` / `CODER_SUPPORT_BUNDLE_WORKSPACES_TOTAL_CAP` - Collects additional stats from the coderd deployment (aggregated workspace/session metrics), as well as entitlements via license and dismissed health checks. created with help from mux
This commit is contained in:
+567
-13
@@ -2,15 +2,19 @@ package support
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/mod/semver"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/xerrors"
|
||||
"tailscale.com/ipn/ipnstate"
|
||||
@@ -30,20 +34,27 @@ import (
|
||||
// Even though we do attempt to sanitize data, it may still contain
|
||||
// sensitive information and should thus be treated as secret.
|
||||
type Bundle struct {
|
||||
Deployment Deployment `json:"deployment"`
|
||||
Network Network `json:"network"`
|
||||
Workspace Workspace `json:"workspace"`
|
||||
Agent Agent `json:"agent"`
|
||||
Logs []string `json:"logs"`
|
||||
CLILogs []byte `json:"cli_logs"`
|
||||
Deployment Deployment `json:"deployment"`
|
||||
Network Network `json:"network"`
|
||||
Workspace Workspace `json:"workspace"`
|
||||
Agent Agent `json:"agent"`
|
||||
Logs []string `json:"logs"`
|
||||
CLILogs []byte `json:"cli_logs"`
|
||||
NamedTemplate TemplateDump `json:"named_template"`
|
||||
Pprof Pprof `json:"pprof"`
|
||||
}
|
||||
|
||||
type Deployment struct {
|
||||
BuildInfo *codersdk.BuildInfoResponse `json:"build"`
|
||||
Config *codersdk.DeploymentConfig `json:"config"`
|
||||
Experiments codersdk.Experiments `json:"experiments"`
|
||||
HealthReport *healthsdk.HealthcheckReport `json:"health_report"`
|
||||
Licenses []codersdk.License `json:"licenses"`
|
||||
BuildInfo *codersdk.BuildInfoResponse `json:"build"`
|
||||
Config *codersdk.DeploymentConfig `json:"config"`
|
||||
Experiments codersdk.Experiments `json:"experiments"`
|
||||
HealthReport *healthsdk.HealthcheckReport `json:"health_report"`
|
||||
Licenses []codersdk.License `json:"licenses"`
|
||||
Stats *codersdk.DeploymentStats `json:"stats"`
|
||||
Entitlements *codersdk.Entitlements `json:"entitlements"`
|
||||
HealthSettings *healthsdk.HealthSettings `json:"health_settings"`
|
||||
Workspaces *codersdk.WorkspacesResponse `json:"workspaces"`
|
||||
Prometheus []byte `json:"prometheus"`
|
||||
}
|
||||
|
||||
type Network struct {
|
||||
@@ -83,6 +94,32 @@ type Agent struct {
|
||||
StartupLogs []codersdk.WorkspaceAgentLog `json:"startup_logs"`
|
||||
}
|
||||
|
||||
type TemplateDump struct {
|
||||
Template codersdk.Template `json:"template"`
|
||||
TemplateVersion codersdk.TemplateVersion `json:"template_version"`
|
||||
TemplateFileBase64 string `json:"template_file_base64"`
|
||||
}
|
||||
|
||||
type Pprof struct {
|
||||
Server *PprofCollection `json:"server,omitempty"`
|
||||
Agent *PprofCollection `json:"agent,omitempty"`
|
||||
}
|
||||
|
||||
type PprofCollection struct {
|
||||
Heap []byte `json:"heap,omitempty"`
|
||||
Allocs []byte `json:"allocs,omitempty"`
|
||||
Profile []byte `json:"profile,omitempty"`
|
||||
Block []byte `json:"block,omitempty"`
|
||||
Mutex []byte `json:"mutex,omitempty"`
|
||||
Goroutine []byte `json:"goroutine,omitempty"`
|
||||
Threadcreate []byte `json:"threadcreate,omitempty"`
|
||||
Trace []byte `json:"trace,omitempty"`
|
||||
Cmdline string `json:"cmdline,omitempty"`
|
||||
Symbol string `json:"symbol,omitempty"`
|
||||
CollectedAt time.Time `json:"collected_at"`
|
||||
EndpointURL string `json:"endpoint_url"`
|
||||
}
|
||||
|
||||
// Deps is a set of dependencies for discovering information
|
||||
type Deps struct {
|
||||
// Source from which to obtain information.
|
||||
@@ -94,9 +131,17 @@ type Deps struct {
|
||||
// AgentID is the optional agent ID against which to run connection tests.
|
||||
// Defaults to the first agent of the workspace, if not specified.
|
||||
AgentID uuid.UUID
|
||||
// WorkspacesTotalCap limits the TOTAL number of workspaces aggregated into the bundle.
|
||||
// > 0 => cap at this number (default flag value should be 1000 via CLI).
|
||||
// <= 0 => no cap (fetch/keep all available workspaces).
|
||||
WorkspacesTotalCap int
|
||||
// TemplateID optionally specifies a template to capture (active version).
|
||||
TemplateID uuid.UUID
|
||||
// CollectPprof toggles server and agent pprof collection.
|
||||
CollectPprof bool
|
||||
}
|
||||
|
||||
func DeploymentInfo(ctx context.Context, client *codersdk.Client, log slog.Logger) Deployment {
|
||||
func DeploymentInfo(ctx context.Context, client *codersdk.Client, log slog.Logger, workspacesCap int) Deployment {
|
||||
// Note: each goroutine assigns to a different struct field, hence no mutex.
|
||||
var (
|
||||
d Deployment
|
||||
@@ -154,13 +199,157 @@ func DeploymentInfo(ctx context.Context, client *codersdk.Client, log slog.Logge
|
||||
return nil
|
||||
})
|
||||
|
||||
// Deployment stats
|
||||
eg.Go(func() error {
|
||||
stats, err := client.DeploymentStats(ctx)
|
||||
if err != nil {
|
||||
// If unauthorized or forbidden, log and continue
|
||||
if cerr, ok := codersdk.AsError(err); ok && (cerr.StatusCode() == http.StatusForbidden || cerr.StatusCode() == http.StatusUnauthorized || cerr.StatusCode() == http.StatusBadRequest) {
|
||||
log.Warn(ctx, "unable to fetch deployment stats")
|
||||
return nil
|
||||
}
|
||||
return xerrors.Errorf("fetch deployment stats: %w", err)
|
||||
}
|
||||
d.Stats = &stats
|
||||
return nil
|
||||
})
|
||||
|
||||
// Entitlements
|
||||
eg.Go(func() error {
|
||||
ents, err := client.Entitlements(ctx)
|
||||
if err != nil {
|
||||
// Ignore 404 or enterprise-not-enabled
|
||||
if cerr, ok := codersdk.AsError(err); ok && (cerr.StatusCode() == http.StatusNotFound || cerr.StatusCode() == http.StatusForbidden) {
|
||||
log.Warn(ctx, "unable to fetch entitlements")
|
||||
return nil
|
||||
}
|
||||
return xerrors.Errorf("fetch entitlements: %w", err)
|
||||
}
|
||||
d.Entitlements = &ents
|
||||
return nil
|
||||
})
|
||||
|
||||
// Health settings
|
||||
eg.Go(func() error {
|
||||
settings, err := healthsdk.New(client).HealthSettings(ctx)
|
||||
if err != nil {
|
||||
// If not accessible, log and continue
|
||||
if cerr, ok := codersdk.AsError(err); ok && (cerr.StatusCode() == http.StatusForbidden || cerr.StatusCode() == http.StatusUnauthorized) {
|
||||
log.Warn(ctx, "unable to fetch health settings")
|
||||
return nil
|
||||
}
|
||||
return xerrors.Errorf("fetch health settings: %w", err)
|
||||
}
|
||||
d.HealthSettings = &settings
|
||||
return nil
|
||||
})
|
||||
|
||||
// List workspaces (paginated)
|
||||
eg.Go(func() error {
|
||||
var (
|
||||
offset int
|
||||
limit = 200
|
||||
all []codersdk.Workspace
|
||||
count int
|
||||
)
|
||||
capTotal := workspacesCap
|
||||
for {
|
||||
resp, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{Offset: offset, Limit: limit})
|
||||
if err != nil {
|
||||
// Log and continue if forbidden; otherwise return error
|
||||
if cerr, ok := codersdk.AsError(err); ok && (cerr.StatusCode() == http.StatusForbidden || cerr.StatusCode() == http.StatusUnauthorized) {
|
||||
log.Warn(ctx, "unable to list workspaces")
|
||||
break
|
||||
}
|
||||
return xerrors.Errorf("list workspaces: %w", err)
|
||||
}
|
||||
if d.Workspaces == nil {
|
||||
d.Workspaces = &resp
|
||||
}
|
||||
// sanitize env vars on agents in each workspace before appending
|
||||
for i := range resp.Workspaces {
|
||||
ws := &resp.Workspaces[i]
|
||||
for _, res := range ws.LatestBuild.Resources {
|
||||
for _, agt := range res.Agents {
|
||||
// safe to call even if map is nil (range in sanitizeEnv would be empty)
|
||||
sanitizeEnv(agt.EnvironmentVariables)
|
||||
}
|
||||
}
|
||||
}
|
||||
all = append(all, resp.Workspaces...)
|
||||
count = resp.Count
|
||||
// Stop early once we've reached the cap; trim any overflow from the last page.
|
||||
if capTotal > 0 && len(all) >= capTotal {
|
||||
if len(all) > capTotal {
|
||||
all = all[:capTotal]
|
||||
}
|
||||
break
|
||||
}
|
||||
if offset+len(resp.Workspaces) >= count || len(resp.Workspaces) == 0 {
|
||||
break
|
||||
}
|
||||
offset += len(resp.Workspaces)
|
||||
}
|
||||
if d.Workspaces != nil {
|
||||
// Replace with aggregated list
|
||||
d.Workspaces.Workspaces = all
|
||||
// Preserve server-reported total so Run() can log accurate truncation.
|
||||
d.Workspaces.Count = count
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err := eg.Wait(); err != nil {
|
||||
log.Error(ctx, "fetch deployment information", slog.Error(err))
|
||||
}
|
||||
|
||||
if d.Config != nil && d.Config.Values != nil {
|
||||
prometheusCfg := d.Config.Values.Prometheus
|
||||
if prometheusCfg.Enable.Value() {
|
||||
metrics, err := fetchPrometheusMetrics(ctx, client, log)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "fetch coderd prometheus metrics", slog.Error(err))
|
||||
} else {
|
||||
d.Prometheus = metrics
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return d
|
||||
}
|
||||
|
||||
func fetchPrometheusMetrics(ctx context.Context, client *codersdk.Client, log slog.Logger) ([]byte, error) {
|
||||
if client == nil {
|
||||
return nil, xerrors.New("nil client")
|
||||
}
|
||||
|
||||
reqCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
resp, err := client.Request(reqCtx, http.MethodGet, "/api/v2/debug/metrics", nil)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("request metrics: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, xerrors.Errorf("read metrics body: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
log.Debug(ctx, "coderd prometheus metrics fetch non-200",
|
||||
slog.F("status", resp.StatusCode), slog.F("body_len", len(body)))
|
||||
return nil, xerrors.Errorf("unexpected status code %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
trimmed := bytes.TrimSpace(body)
|
||||
if len(trimmed) == 0 {
|
||||
return nil, xerrors.New("empty prometheus metrics response")
|
||||
}
|
||||
return append([]byte(nil), trimmed...), nil
|
||||
}
|
||||
|
||||
func NetworkInfo(ctx context.Context, client *codersdk.Client, log slog.Logger) Network {
|
||||
var (
|
||||
n Network
|
||||
@@ -471,6 +660,234 @@ func connectedAgentInfo(ctx context.Context, client *codersdk.Client, log slog.L
|
||||
return closer
|
||||
}
|
||||
|
||||
func PprofInfo(ctx context.Context, client *codersdk.Client, log slog.Logger) *PprofCollection {
|
||||
if client == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
p PprofCollection
|
||||
eg errgroup.Group
|
||||
)
|
||||
|
||||
if client.URL != nil {
|
||||
if u, err := client.URL.Parse("/api/v2/debug/pprof"); err == nil {
|
||||
p.EndpointURL = u.String()
|
||||
}
|
||||
}
|
||||
if p.EndpointURL == "" {
|
||||
p.EndpointURL = "/api/v2/debug/pprof"
|
||||
}
|
||||
p.CollectedAt = time.Now()
|
||||
|
||||
const basePath = "/api/v2/debug/pprof"
|
||||
endpoints := map[string]func([]byte){
|
||||
"/allocs": func(data []byte) {
|
||||
p.Allocs = compressData(data)
|
||||
},
|
||||
"/heap": func(data []byte) {
|
||||
p.Heap = compressData(data)
|
||||
},
|
||||
"/profile?seconds=30": func(data []byte) {
|
||||
p.Profile = compressData(data)
|
||||
},
|
||||
"/block": func(data []byte) {
|
||||
p.Block = compressData(data)
|
||||
},
|
||||
"/mutex": func(data []byte) {
|
||||
p.Mutex = compressData(data)
|
||||
},
|
||||
"/goroutine": func(data []byte) {
|
||||
p.Goroutine = compressData(data)
|
||||
},
|
||||
"/threadcreate": func(data []byte) {
|
||||
p.Threadcreate = compressData(data)
|
||||
},
|
||||
"/trace?seconds=30": func(data []byte) {
|
||||
p.Trace = compressData(data)
|
||||
},
|
||||
"/cmdline": func(data []byte) {
|
||||
p.Cmdline = string(data)
|
||||
},
|
||||
"/symbol": func(data []byte) {
|
||||
p.Symbol = string(data)
|
||||
},
|
||||
}
|
||||
|
||||
for endpoint, setter := range endpoints {
|
||||
endpoint, setter := endpoint, setter
|
||||
eg.Go(func() error {
|
||||
timeout := 10 * time.Second
|
||||
if strings.Contains(endpoint, "seconds=30") {
|
||||
timeout = 45 * time.Second
|
||||
}
|
||||
|
||||
reqCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
resp, err := client.Request(reqCtx, http.MethodGet, basePath+endpoint, nil)
|
||||
if err != nil {
|
||||
log.Warn(reqCtx, "failed to fetch pprof data", slog.F("endpoint", endpoint), slog.Error(err))
|
||||
return nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
log.Warn(reqCtx, "pprof endpoint returned non-200 status",
|
||||
slog.F("endpoint", endpoint), slog.F("status", resp.StatusCode))
|
||||
return nil
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Warn(reqCtx, "failed to read pprof response", slog.F("endpoint", endpoint), slog.Error(err))
|
||||
return nil
|
||||
}
|
||||
|
||||
setter(data)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if err := eg.Wait(); err != nil {
|
||||
log.Error(ctx, "failed to collect some pprof data", slog.Error(err))
|
||||
}
|
||||
|
||||
return &p
|
||||
}
|
||||
|
||||
func compressData(data []byte) []byte {
|
||||
if len(data) == 0 {
|
||||
return data
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
gz := gzip.NewWriter(&buf)
|
||||
if _, err := gz.Write(data); err != nil {
|
||||
return data // Return uncompressed if compression fails
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
return data
|
||||
}
|
||||
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func PprofInfoFromAgent(ctx context.Context, conn workspacesdk.AgentConn, log slog.Logger) *PprofCollection {
|
||||
if conn == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
p PprofCollection
|
||||
eg errgroup.Group
|
||||
)
|
||||
|
||||
p.EndpointURL = "agent"
|
||||
p.CollectedAt = time.Now()
|
||||
|
||||
// Define agent pprof endpoints - these go through the agent connection
|
||||
endpoints := map[string]func([]byte){
|
||||
"/debug/pprof/allocs": func(data []byte) {
|
||||
p.Allocs = compressData(data)
|
||||
},
|
||||
"/debug/pprof/heap": func(data []byte) {
|
||||
p.Heap = compressData(data)
|
||||
},
|
||||
"/debug/pprof/profile?seconds=30": func(data []byte) {
|
||||
p.Profile = compressData(data)
|
||||
},
|
||||
"/debug/pprof/block": func(data []byte) {
|
||||
p.Block = compressData(data)
|
||||
},
|
||||
"/debug/pprof/mutex": func(data []byte) {
|
||||
p.Mutex = compressData(data)
|
||||
},
|
||||
"/debug/pprof/goroutine": func(data []byte) {
|
||||
p.Goroutine = compressData(data)
|
||||
},
|
||||
"/debug/pprof/threadcreate": func(data []byte) {
|
||||
p.Threadcreate = compressData(data)
|
||||
},
|
||||
"/debug/pprof/trace?seconds=30": func(data []byte) {
|
||||
p.Trace = compressData(data)
|
||||
},
|
||||
"/debug/pprof/cmdline": func(data []byte) {
|
||||
p.Cmdline = string(data)
|
||||
},
|
||||
"/debug/pprof/symbol": func(data []byte) {
|
||||
p.Symbol = string(data)
|
||||
},
|
||||
}
|
||||
|
||||
// Collect each endpoint in parallel
|
||||
for endpoint, setter := range endpoints {
|
||||
endpoint, setter := endpoint, setter // capture loop variables
|
||||
eg.Go(func() error {
|
||||
// Set longer timeout for profile and trace endpoints (they take 30 seconds)
|
||||
timeout := 10 * time.Second
|
||||
if strings.Contains(endpoint, "seconds=30") {
|
||||
timeout = 45 * time.Second
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
// Use the agent's direct HTTP capability
|
||||
// Agent pprof server runs on 127.0.0.1:6060 by default
|
||||
netConn, err := conn.DialContext(ctx, "tcp", "127.0.0.1:6060")
|
||||
if err != nil {
|
||||
log.Warn(ctx, "failed to dial agent pprof endpoint", slog.F("endpoint", endpoint), slog.Error(err))
|
||||
return nil
|
||||
}
|
||||
defer netConn.Close()
|
||||
|
||||
// Create HTTP client using the connection
|
||||
client := &http.Client{
|
||||
Transport: &http.Transport{
|
||||
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
|
||||
return netConn, nil
|
||||
},
|
||||
},
|
||||
Timeout: timeout,
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://127.0.0.1:6060"+endpoint, nil)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "failed to create agent pprof request", slog.F("endpoint", endpoint), slog.Error(err))
|
||||
return nil
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "failed to fetch agent pprof data", slog.F("endpoint", endpoint), slog.Error(err))
|
||||
return nil
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
log.Warn(ctx, "agent pprof endpoint returned non-200 status", slog.F("endpoint", endpoint), slog.F("status", resp.StatusCode))
|
||||
return nil
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Warn(ctx, "failed to read agent pprof response", slog.F("endpoint", endpoint), slog.Error(err))
|
||||
return nil
|
||||
}
|
||||
|
||||
setter(data)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if err := eg.Wait(); err != nil {
|
||||
log.Error(ctx, "failed to collect some agent pprof data", slog.Error(err))
|
||||
}
|
||||
|
||||
return &p
|
||||
}
|
||||
|
||||
// Run generates a support bundle with the given dependencies.
|
||||
func Run(ctx context.Context, d *Deps) (*Bundle, error) {
|
||||
var b Bundle
|
||||
@@ -505,9 +922,28 @@ func Run(ctx context.Context, d *Deps) (*Bundle, error) {
|
||||
}
|
||||
}
|
||||
|
||||
totalCap := d.WorkspacesTotalCap
|
||||
|
||||
var eg errgroup.Group
|
||||
eg.Go(func() error {
|
||||
di := DeploymentInfo(ctx, d.Client, d.Log)
|
||||
di := DeploymentInfo(ctx, d.Client, d.Log, totalCap)
|
||||
|
||||
if di.Workspaces != nil && totalCap > 0 {
|
||||
origTotal := di.Workspaces.Count // server-reported total
|
||||
|
||||
// Ensure at most 'totalCap' are returned (covers non-early-exit path).
|
||||
if len(di.Workspaces.Workspaces) > totalCap {
|
||||
di.Workspaces.Workspaces = di.Workspaces.Workspaces[:totalCap]
|
||||
}
|
||||
// If we returned fewer than the original total, log a truncation.
|
||||
if origTotal > len(di.Workspaces.Workspaces) {
|
||||
di.Workspaces.Count = len(di.Workspaces.Workspaces)
|
||||
d.Log.Warn(ctx, "workspace list truncated",
|
||||
slog.F("cap", totalCap),
|
||||
slog.F("original_total", origTotal),
|
||||
)
|
||||
}
|
||||
}
|
||||
b.Deployment = di
|
||||
return nil
|
||||
})
|
||||
@@ -527,11 +963,129 @@ func Run(ctx context.Context, d *Deps) (*Bundle, error) {
|
||||
return nil
|
||||
})
|
||||
|
||||
// Optional: capture a template's active version and file if TemplateID is set.
|
||||
eg.Go(func() error {
|
||||
if d.TemplateID == uuid.Nil {
|
||||
return nil
|
||||
}
|
||||
var td TemplateDump
|
||||
tpl, err := d.Client.Template(ctx, d.TemplateID)
|
||||
if err != nil {
|
||||
d.Log.Error(ctx, "fetch template", slog.Error(err), slog.F("template_id", d.TemplateID))
|
||||
return nil
|
||||
}
|
||||
td.Template = tpl
|
||||
if tpl.ActiveVersionID == uuid.Nil {
|
||||
d.Log.Error(ctx, "template has nil active version id", slog.F("template_id", tpl.ID))
|
||||
b.NamedTemplate = td
|
||||
return nil
|
||||
}
|
||||
tv, err := d.Client.TemplateVersion(ctx, tpl.ActiveVersionID)
|
||||
if err != nil {
|
||||
d.Log.Error(ctx, "fetch active template version", slog.Error(err), slog.F("active_version_id", tpl.ActiveVersionID))
|
||||
b.NamedTemplate = td
|
||||
return nil
|
||||
}
|
||||
td.TemplateVersion = tv
|
||||
if tv.Job.FileID == uuid.Nil {
|
||||
d.Log.Error(ctx, "template file id is nil", slog.F("template_version_id", tv.ID))
|
||||
b.NamedTemplate = td
|
||||
return nil
|
||||
}
|
||||
raw, ctype, err := d.Client.DownloadWithFormat(ctx, tv.Job.FileID, codersdk.FormatZip)
|
||||
if err != nil || ctype != codersdk.ContentTypeZip {
|
||||
d.Log.Error(ctx, "download template file", slog.Error(err), slog.F("content_type", ctype))
|
||||
b.NamedTemplate = td
|
||||
return nil
|
||||
}
|
||||
td.TemplateFileBase64 = base64.StdEncoding.EncodeToString(raw)
|
||||
b.NamedTemplate = td
|
||||
return nil
|
||||
})
|
||||
|
||||
_ = eg.Wait()
|
||||
|
||||
// Collect pprof data after deployment info is available (need version check).
|
||||
// Pprof endpoints require Coder server version 2.28.0 or newer.
|
||||
if d.CollectPprof {
|
||||
b.Pprof = collectPprof(ctx, d, &b)
|
||||
}
|
||||
|
||||
return &b, nil
|
||||
}
|
||||
|
||||
// minPprofVersion is the minimum Coder server version that supports
|
||||
// the /api/v2/debug/pprof endpoints.
|
||||
const minPprofVersion = "v2.28.0"
|
||||
|
||||
// VersionSupportsPprof checks if the given version supports pprof endpoints.
|
||||
func VersionSupportsPprof(version string) bool {
|
||||
if version == "" {
|
||||
return false
|
||||
}
|
||||
if version[0] != 'v' {
|
||||
version = "v" + version
|
||||
}
|
||||
// For prerelease versions like "v2.28.0-devel+abc123", we compare
|
||||
// the major.minor.patch portion since prereleases of 2.28.0 should
|
||||
// have the pprof feature.
|
||||
canonical := semver.Canonical(version)
|
||||
if idx := strings.Index(canonical, "-"); idx != -1 {
|
||||
canonical = canonical[:idx]
|
||||
}
|
||||
return semver.Compare(canonical, minPprofVersion) >= 0
|
||||
}
|
||||
|
||||
func collectPprof(ctx context.Context, d *Deps, b *Bundle) Pprof {
|
||||
var pprof Pprof
|
||||
|
||||
// Check server version before attempting pprof collection.
|
||||
if b.Deployment.BuildInfo == nil {
|
||||
d.Log.Warn(ctx, "skipping pprof collection: build info not available")
|
||||
return pprof
|
||||
}
|
||||
if !VersionSupportsPprof(b.Deployment.BuildInfo.Version) {
|
||||
d.Log.Warn(ctx, "skipping pprof collection: server version too old",
|
||||
slog.F("version", b.Deployment.BuildInfo.Version),
|
||||
slog.F("min_version", minPprofVersion))
|
||||
return pprof
|
||||
}
|
||||
|
||||
serverPprof := PprofInfo(ctx, d.Client, d.Log)
|
||||
if serverPprof != nil {
|
||||
pprof.Server = serverPprof
|
||||
}
|
||||
|
||||
if d.AgentID != uuid.Nil {
|
||||
conn, err := workspacesdk.New(d.Client).
|
||||
DialAgent(ctx, d.AgentID, &workspacesdk.DialAgentOptions{
|
||||
Logger: d.Log.Named("dial-agent-pprof"),
|
||||
BlockEndpoints: false,
|
||||
})
|
||||
if err != nil {
|
||||
d.Log.Warn(ctx, "failed to dial agent for pprof collection", slog.Error(err))
|
||||
} else {
|
||||
defer func() {
|
||||
if err := conn.Close(); err != nil {
|
||||
d.Log.Error(ctx, "failed to close agent pprof connection", slog.Error(err))
|
||||
}
|
||||
<-conn.TailnetConn().Closed()
|
||||
}()
|
||||
|
||||
if conn.AwaitReachable(ctx) {
|
||||
agentPprof := PprofInfoFromAgent(ctx, conn, d.Log)
|
||||
if agentPprof != nil {
|
||||
pprof.Agent = agentPprof
|
||||
}
|
||||
} else {
|
||||
d.Log.Warn(ctx, "agent not reachable for pprof collection")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return pprof
|
||||
}
|
||||
|
||||
// sanitizeEnv modifies kvs in place and replaces the values all non-empty keys
|
||||
// with the string ***REDACTED***
|
||||
func sanitizeEnv(kvs map[string]string) {
|
||||
|
||||
@@ -3,6 +3,7 @@ package support_test
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -27,6 +28,7 @@ import (
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
"github.com/coder/coder/v2/support"
|
||||
"github.com/coder/coder/v2/testutil"
|
||||
"github.com/coder/serpent"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
@@ -39,6 +41,10 @@ func TestRun(t *testing.T) {
|
||||
t.Run("OK", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
cfg := coderdtest.DeploymentValues(t)
|
||||
promPort := testutil.RandomPort(t)
|
||||
cfg.Prometheus.Enable = serpent.Bool(true)
|
||||
cfg.Prometheus.Address.Host = "127.0.0.1"
|
||||
cfg.Prometheus.Address.Port = fmt.Sprintf("%d", promPort)
|
||||
cfg.Experiments = []string{"foo"}
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
|
||||
@@ -86,8 +92,24 @@ func TestRun(t *testing.T) {
|
||||
assertNotNilNotEmpty(t, bun.Agent.PeerDiagnostics, "agent peer diagnostics should be present")
|
||||
assertNotNilNotEmpty(t, bun.Agent.PingResult, "agent ping result should be present")
|
||||
assertNotNilNotEmpty(t, bun.Agent.Prometheus, "agent prometheus metrics should be present")
|
||||
assertNotNilNotEmpty(t, bun.Deployment.Prometheus, "deployment prometheus metrics should be present")
|
||||
assertNotNilNotEmpty(t, bun.Agent.StartupLogs, "agent startup logs should be present")
|
||||
assertNotNilNotEmpty(t, bun.Logs, "bundle logs should be present")
|
||||
assert.Nil(t, bun.Pprof.Server, "server pprof should not be collected without CollectPprof")
|
||||
assert.Nil(t, bun.Pprof.Agent, "agent pprof should not be collected without CollectPprof")
|
||||
|
||||
// New: deployment health settings should be present
|
||||
assertNotNilNotEmpty(t, bun.Deployment.HealthSettings, "deployment health settings should be present")
|
||||
// New: aggregated workspaces should be present and include created workspace
|
||||
assert.NotNil(t, bun.Deployment.Workspaces, "deployment workspaces should be present")
|
||||
assert.GreaterOrEqual(t, bun.Deployment.Workspaces.Count, 1)
|
||||
for _, aws := range bun.Deployment.Workspaces.Workspaces {
|
||||
for _, res := range aws.LatestBuild.Resources {
|
||||
for _, a := range res.Agents {
|
||||
assertSanitizedEnv(t, a.EnvironmentVariables)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("OK_NoWorkspace", func(t *testing.T) {
|
||||
@@ -120,6 +142,13 @@ func TestRun(t *testing.T) {
|
||||
assert.Empty(t, bun.Workspace.Workspace, "did not expect workspace to be present")
|
||||
assert.Empty(t, bun.Agent, "did not expect agent to be present")
|
||||
assertNotNilNotEmpty(t, bun.Logs, "bundle logs should be present")
|
||||
assert.Nil(t, bun.Pprof.Server, "server pprof should not be collected without CollectPprof")
|
||||
assert.Nil(t, bun.Pprof.Agent, "agent pprof should not be collected without CollectPprof")
|
||||
|
||||
// New: health settings should be present even without workspace context
|
||||
assertNotNilNotEmpty(t, bun.Deployment.HealthSettings, "deployment health settings should be present")
|
||||
// New: aggregated workspaces struct should exist (may be empty)
|
||||
assert.NotNil(t, bun.Deployment.Workspaces)
|
||||
})
|
||||
|
||||
t.Run("NoAuth", func(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package support_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/coder/coder/v2/support"
|
||||
)
|
||||
|
||||
func TestVersionSupportsPprof(t *testing.T) {
|
||||
t.Parallel()
|
||||
tests := []struct {
|
||||
version string
|
||||
want bool
|
||||
}{
|
||||
{"", false},
|
||||
{"v2.27.0", false},
|
||||
{"v2.27.9", false},
|
||||
{"v2.28.0", true},
|
||||
{"v2.28.1", true},
|
||||
{"v2.29.0", true},
|
||||
{"v3.0.0", true},
|
||||
{"2.28.0", true}, // without v prefix
|
||||
{"2.27.0", false}, // without v prefix
|
||||
{"v2.28.0-devel+abc123", true}, // dev version
|
||||
{"v2.27.0-devel+abc123", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.version, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
got := support.VersionSupportsPprof(tt.version)
|
||||
if got != tt.want {
|
||||
t.Errorf("versionSupportsPprof(%q) = %v, want %v", tt.version, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user