opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: expose workspace statuses (with details) as a prometheus metric (#12762)

Browse Source 
Implements #12462
This commit is contained in:
Danny Kopping
2024-04-02 09:57:36 +02:00
committed by GitHub
parent 114830de26
commit 79fb8e43c5
8 changed files with 375 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/prometheusmetrics/prometheusmetrics.go
+65 -20
View File
@@ -24,10 +24,12 @@ import (
"github.com/coder/coder/v2/tailnet"
)
const defaultRefreshRate = time.Minute
// ActiveUsers tracks the number of users that have authenticated within the past hour.
func ActiveUsers(ctx context.Context, registerer prometheus.Registerer, db database.Store, duration time.Duration) (func(), error) {
if duration == 0 {
duration = 5 * time.Minute
duration = defaultRefreshRate
}
gauge := prometheus.NewGauge(prometheus.GaugeOpts{
@@ -72,36 +74,42 @@ func ActiveUsers(ctx context.Context, registerer prometheus.Registerer, db datab
}
// Workspaces tracks the total number of workspaces with labels on status.
func Workspaces(ctx context.Context, registerer prometheus.Registerer, db database.Store, duration time.Duration) (func(), error) {
func Workspaces(ctx context.Context, logger slog.Logger, registerer prometheus.Registerer, db database.Store, duration time.Duration) (func(), error) {
if duration == 0 {
duration = 5 * time.Minute
duration = defaultRefreshRate
}
gauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{
workspaceLatestBuildTotals := prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "coderd",
Subsystem: "api",
Name: "workspace_latest_build_total",
Help: "The latest workspace builds with a status.",
Help: "The current number of workspace builds by status.",
}, []string{"status"})
err := registerer.Register(gauge)
if err != nil {
if err := registerer.Register(workspaceLatestBuildTotals); err != nil {
return nil, err
}
workspaceLatestBuildStatuses := prometheus.NewGaugeVec(prometheus.GaugeOpts{
Namespace: "coderd",
Name: "workspace_latest_build_status",
Help: "The current workspace statuses by template, transition, and owner.",
}, []string{"status", "template_name", "template_version", "workspace_owner", "workspace_transition"})
if err := registerer.Register(workspaceLatestBuildStatuses); err != nil {
return nil, err
}
// This exists so the prometheus metric exports immediately when set.
// It helps with tests so they don't have to wait for a tick.
gauge.WithLabelValues("pending").Set(0)
ctx, cancelFunc := context.WithCancel(ctx)
done := make(chan struct{})
// Use time.Nanosecond to force an initial tick. It will be reset to the
// correct duration after executing once.
ticker := time.NewTicker(time.Nanosecond)
doTick := func() {
defer ticker.Reset(duration)
updateWorkspaceTotals := func() {
builds, err := db.GetLatestWorkspaceBuilds(ctx)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
// clear all series if there are no database entries
workspaceLatestBuildTotals.Reset()
}
logger.Warn(ctx, "failed to load latest workspace builds", slog.Error(err))
return
}
jobIDs := make([]uuid.UUID, 0, len(builds))
@@ -110,16 +118,53 @@ func Workspaces(ctx context.Context, registerer prometheus.Registerer, db databa
}
jobs, err := db.GetProvisionerJobsByIDs(ctx, jobIDs)
if err != nil {
ids := make([]string, 0, len(jobIDs))
for _, id := range jobIDs {
ids = append(ids, id.String())
}
logger.Warn(ctx, "failed to load provisioner jobs", slog.F("ids", ids), slog.Error(err))
return
}
gauge.Reset()
workspaceLatestBuildTotals.Reset()
for _, job := range jobs {
status := codersdk.ProvisionerJobStatus(job.JobStatus)
gauge.WithLabelValues(string(status)).Add(1)
workspaceLatestBuildTotals.WithLabelValues(string(status)).Add(1)
}
}
updateWorkspaceStatuses := func() {
ws, err := db.GetWorkspaces(ctx, database.GetWorkspacesParams{
Deleted: false,
WithSummary: false,
})
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
// clear all series if there are no database entries
workspaceLatestBuildStatuses.Reset()
}
logger.Warn(ctx, "failed to load active workspaces", slog.Error(err))
return
}
workspaceLatestBuildStatuses.Reset()
for _, w := range ws {
workspaceLatestBuildStatuses.WithLabelValues(string(w.LatestBuildStatus), w.TemplateName, w.TemplateVersionName.String, w.Username, string(w.LatestBuildTransition)).Add(1)
}
}
// Use time.Nanosecond to force an initial tick. It will be reset to the
// correct duration after executing once.
ticker := time.NewTicker(time.Nanosecond)
doTick := func() {
defer ticker.Reset(duration)
updateWorkspaceTotals()
updateWorkspaceStatuses()
}
go func() {
defer close(done)
defer ticker.Stop()
@@ -141,7 +186,7 @@ func Workspaces(ctx context.Context, registerer prometheus.Registerer, db databa
// Agents tracks the total number of workspaces with labels on status.
func Agents(ctx context.Context, logger slog.Logger, registerer prometheus.Registerer, db database.Store, coordinator *atomic.Pointer[tailnet.Coordinator], derpMapFn func() *tailcfg.DERPMap, agentInactiveDisconnectTimeout, duration time.Duration) (func(), error) {
if duration == 0 {
duration = 1 * time.Minute
duration = defaultRefreshRate
}
agentsGauge := NewCachedGaugeVec(prometheus.NewGaugeVec(prometheus.GaugeOpts{
@@ -330,7 +375,7 @@ func Agents(ctx context.Context, logger slog.Logger, registerer prometheus.Regis
func AgentStats(ctx context.Context, logger slog.Logger, registerer prometheus.Registerer, db database.Store, initialCreateAfter time.Time, duration time.Duration, aggregateByLabels []string) (func(), error) {
if duration == 0 {
duration = 1 * time.Minute
duration = defaultRefreshRate
}
if len(aggregateByLabels) == 0 {
coderd/prometheusmetrics/prometheusmetrics_test.go
+259 -101
View File
@@ -11,6 +11,7 @@ import (
"testing"
"time"
"github.com/coder/coder/v2/cryptorand"
"github.com/google/uuid"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
@@ -110,89 +111,9 @@ func TestActiveUsers(t *testing.T) {
}
}
func TestWorkspaces(t *testing.T) {
func TestWorkspaceLatestBuildTotals(t *testing.T) {
t.Parallel()
insertRunning := func(db database.Store) database.ProvisionerJob {
job, err := db.InsertProvisionerJob(context.Background(), database.InsertProvisionerJobParams{
ID: uuid.New(),
CreatedAt: dbtime.Now(),
UpdatedAt: dbtime.Now(),
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
Type: database.ProvisionerJobTypeWorkspaceBuild,
})
require.NoError(t, err)
err = db.InsertWorkspaceBuild(context.Background(), database.InsertWorkspaceBuildParams{
ID: uuid.New(),
WorkspaceID: uuid.New(),
JobID: job.ID,
BuildNumber: 1,
Transition: database.WorkspaceTransitionStart,
Reason: database.BuildReasonInitiator,
})
require.NoError(t, err)
// This marks the job as started.
_, err = db.AcquireProvisionerJob(context.Background(), database.AcquireProvisionerJobParams{
OrganizationID: job.OrganizationID,
StartedAt: sql.NullTime{
Time: dbtime.Now(),
Valid: true,
},
Types: []database.ProvisionerType{database.ProvisionerTypeEcho},
})
require.NoError(t, err)
return job
}
insertCanceled := func(db database.Store) {
job := insertRunning(db)
err := db.UpdateProvisionerJobWithCancelByID(context.Background(), database.UpdateProvisionerJobWithCancelByIDParams{
ID: job.ID,
CanceledAt: sql.NullTime{
Time: dbtime.Now(),
Valid: true,
},
})
require.NoError(t, err)
err = db.UpdateProvisionerJobWithCompleteByID(context.Background(), database.UpdateProvisionerJobWithCompleteByIDParams{
ID: job.ID,
CompletedAt: sql.NullTime{
Time: dbtime.Now(),
Valid: true,
},
})
require.NoError(t, err)
}
insertFailed := func(db database.Store) {
job := insertRunning(db)
err := db.UpdateProvisionerJobWithCompleteByID(context.Background(), database.UpdateProvisionerJobWithCompleteByIDParams{
ID: job.ID,
CompletedAt: sql.NullTime{
Time: dbtime.Now(),
Valid: true,
},
Error: sql.NullString{
String: "failed",
Valid: true,
},
})
require.NoError(t, err)
}
insertSuccess := func(db database.Store) {
job := insertRunning(db)
err := db.UpdateProvisionerJobWithCompleteByID(context.Background(), database.UpdateProvisionerJobWithCompleteByIDParams{
ID: job.ID,
CompletedAt: sql.NullTime{
Time: dbtime.Now(),
Valid: true,
},
})
require.NoError(t, err)
}
for _, tc := range []struct {
Name string
Database func() database.Store
@@ -208,13 +129,13 @@ func TestWorkspaces(t *testing.T) {
Name: "Multiple",
Database: func() database.Store {
db := dbmem.New()
insertCanceled(db)
insertFailed(db)
insertFailed(db)
insertSuccess(db)
insertSuccess(db)
insertSuccess(db)
insertRunning(db)
insertCanceled(t, db)
insertFailed(t, db)
insertFailed(t, db)
insertSuccess(t, db)
insertSuccess(t, db)
insertSuccess(t, db)
insertRunning(t, db)
return db
},
Total: 7,
@@ -229,29 +150,32 @@ func TestWorkspaces(t *testing.T) {
t.Run(tc.Name, func(t *testing.T) {
t.Parallel()
registry := prometheus.NewRegistry()
closeFunc, err := prometheusmetrics.Workspaces(context.Background(), registry, tc.Database(), time.Millisecond)
closeFunc, err := prometheusmetrics.Workspaces(context.Background(), slogtest.Make(t, nil).Leveled(slog.LevelWarn), registry, tc.Database(), testutil.IntervalFast)
require.NoError(t, err)
t.Cleanup(closeFunc)
require.Eventually(t, func() bool {
metrics, err := registry.Gather()
assert.NoError(t, err)
if len(metrics) < 1 {
return false
}
sum := 0
for _, metric := range metrics[0].Metric {
count, ok := tc.Status[codersdk.ProvisionerJobStatus(metric.Label[0].GetValue())]
if metric.Gauge.GetValue() == 0 {
for _, m := range metrics {
if m.GetName() != "coderd_api_workspace_latest_build_total" {
continue
}
if !ok {
t.Fail()
for _, metric := range m.Metric {
count, ok := tc.Status[codersdk.ProvisionerJobStatus(metric.Label[0].GetValue())]
if metric.Gauge.GetValue() == 0 {
continue
}
if !ok {
t.Fail()
}
if metric.Gauge.GetValue() != float64(count) {
return false
}
sum += int(metric.Gauge.GetValue())
}
if metric.Gauge.GetValue() != float64(count) {
return false
}
sum += int(metric.Gauge.GetValue())
}
t.Logf("sum %d == total %d", sum, tc.Total)
return sum == tc.Total
@@ -260,6 +184,90 @@ func TestWorkspaces(t *testing.T) {
}
}
func TestWorkspaceLatestBuildStatuses(t *testing.T) {
t.Parallel()
for _, tc := range []struct {
Name string
Database func() database.Store
ExpectedWorkspaces int
ExpectedStatuses map[codersdk.ProvisionerJobStatus]int
}{{
Name: "None",
Database: func() database.Store {
return dbmem.New()
},
ExpectedWorkspaces: 0,
}, {
Name: "Multiple",
Database: func() database.Store {
db := dbmem.New()
insertTemplates(t, db)
insertCanceled(t, db)
insertFailed(t, db)
insertFailed(t, db)
insertSuccess(t, db)
insertSuccess(t, db)
insertSuccess(t, db)
insertRunning(t, db)
return db
},
ExpectedWorkspaces: 7,
ExpectedStatuses: map[codersdk.ProvisionerJobStatus]int{
codersdk.ProvisionerJobCanceled: 1,
codersdk.ProvisionerJobFailed: 2,
codersdk.ProvisionerJobSucceeded: 3,
codersdk.ProvisionerJobRunning: 1,
},
}} {
tc := tc
t.Run(tc.Name, func(t *testing.T) {
t.Parallel()
registry := prometheus.NewRegistry()
closeFunc, err := prometheusmetrics.Workspaces(context.Background(), slogtest.Make(t, nil), registry, tc.Database(), testutil.IntervalFast)
require.NoError(t, err)
t.Cleanup(closeFunc)
require.Eventually(t, func() bool {
metrics, err := registry.Gather()
assert.NoError(t, err)
stMap := map[codersdk.ProvisionerJobStatus]int{}
for _, m := range metrics {
if m.GetName() != "coderd_workspace_latest_build_status" {
continue
}
for _, metric := range m.Metric {
for _, l := range metric.Label {
if l == nil {
continue
}
if l.GetName() == "status" {
status := codersdk.ProvisionerJobStatus(l.GetValue())
stMap[status] += int(metric.Gauge.GetValue())
}
}
}
}
stSum := 0
for st, count := range stMap {
if tc.ExpectedStatuses[st] != count {
return false
}
stSum += count
}
t.Logf("status series = %d, expected == %d", stSum, tc.ExpectedWorkspaces)
return stSum == tc.ExpectedWorkspaces
}, testutil.WaitShort, testutil.IntervalFast)
})
}
}
func TestAgents(t *testing.T) {
t.Parallel()
@@ -601,3 +609,153 @@ func prepareWorkspaceAndAgent(t *testing.T, client *codersdk.Client, user coders
agentClient.SetSessionToken(authToken)
return agentClient
}
var (
templateA = uuid.New()
templateVersionA = uuid.New()
templateB = uuid.New()
templateVersionB = uuid.New()
)
func insertTemplates(t *testing.T, db database.Store) {
require.NoError(t, db.InsertTemplate(context.Background(), database.InsertTemplateParams{
ID: templateA,
Name: "template-a",
Provisioner: database.ProvisionerTypeTerraform,
MaxPortSharingLevel: database.AppSharingLevelAuthenticated,
}))
require.NoError(t, db.InsertTemplateVersion(context.Background(), database.InsertTemplateVersionParams{
ID: templateVersionA,
TemplateID: uuid.NullUUID{UUID: templateA},
Name: "version-1a",
}))
require.NoError(t, db.InsertTemplate(context.Background(), database.InsertTemplateParams{
ID: templateB,
Name: "template-b",
Provisioner: database.ProvisionerTypeTerraform,
MaxPortSharingLevel: database.AppSharingLevelAuthenticated,
}))
require.NoError(t, db.InsertTemplateVersion(context.Background(), database.InsertTemplateVersionParams{
ID: templateVersionB,
TemplateID: uuid.NullUUID{UUID: templateB},
Name: "version-1b",
}))
}
func insertUser(t *testing.T, db database.Store) database.User {
username, err := cryptorand.String(8)
require.NoError(t, err)
user, err := db.InsertUser(context.Background(), database.InsertUserParams{
ID: uuid.New(),
Username: username,
LoginType: database.LoginTypeNone,
})
require.NoError(t, err)
return user
}
func insertRunning(t *testing.T, db database.Store) database.ProvisionerJob {
var template, templateVersion uuid.UUID
rnd, err := cryptorand.Intn(10)
require.NoError(t, err)
if rnd > 5 {
template = templateB
templateVersion = templateVersionB
} else {
template = templateA
templateVersion = templateVersionA
}
workspace, err := db.InsertWorkspace(context.Background(), database.InsertWorkspaceParams{
ID: uuid.New(),
OwnerID: insertUser(t, db).ID,
Name: uuid.NewString(),
TemplateID: template,
AutomaticUpdates: database.AutomaticUpdatesNever,
})
require.NoError(t, err)
job, err := db.InsertProvisionerJob(context.Background(), database.InsertProvisionerJobParams{
ID: uuid.New(),
CreatedAt: dbtime.Now(),
UpdatedAt: dbtime.Now(),
Provisioner: database.ProvisionerTypeEcho,
StorageMethod: database.ProvisionerStorageMethodFile,
Type: database.ProvisionerJobTypeWorkspaceBuild,
})
require.NoError(t, err)
err = db.InsertWorkspaceBuild(context.Background(), database.InsertWorkspaceBuildParams{
ID: uuid.New(),
WorkspaceID: workspace.ID,
JobID: job.ID,
BuildNumber: 1,
Transition: database.WorkspaceTransitionStart,
Reason: database.BuildReasonInitiator,
TemplateVersionID: templateVersion,
})
require.NoError(t, err)
// This marks the job as started.
_, err = db.AcquireProvisionerJob(context.Background(), database.AcquireProvisionerJobParams{
OrganizationID: job.OrganizationID,
StartedAt: sql.NullTime{
Time: dbtime.Now(),
Valid: true,
},
Types: []database.ProvisionerType{database.ProvisionerTypeEcho},
})
require.NoError(t, err)
return job
}
func insertCanceled(t *testing.T, db database.Store) {
job := insertRunning(t, db)
err := db.UpdateProvisionerJobWithCancelByID(context.Background(), database.UpdateProvisionerJobWithCancelByIDParams{
ID: job.ID,
CanceledAt: sql.NullTime{
Time: dbtime.Now(),
Valid: true,
},
})
require.NoError(t, err)
err = db.UpdateProvisionerJobWithCompleteByID(context.Background(), database.UpdateProvisionerJobWithCompleteByIDParams{
ID: job.ID,
CompletedAt: sql.NullTime{
Time: dbtime.Now(),
Valid: true,
},
})
require.NoError(t, err)
}
func insertFailed(t *testing.T, db database.Store) {
job := insertRunning(t, db)
err := db.UpdateProvisionerJobWithCompleteByID(context.Background(), database.UpdateProvisionerJobWithCompleteByIDParams{
ID: job.ID,
CompletedAt: sql.NullTime{
Time: dbtime.Now(),
Valid: true,
},
Error: sql.NullString{
String: "failed",
Valid: true,
},
})
require.NoError(t, err)
}
func insertSuccess(t *testing.T, db database.Store) {
job := insertRunning(t, db)
err := db.UpdateProvisionerJobWithCompleteByID(context.Background(), database.UpdateProvisionerJobWithCompleteByIDParams{
ID: job.ID,
CompletedAt: sql.NullTime{
Time: dbtime.Now(),
Valid: true,
},
})
require.NoError(t, err)
}