opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(coderd): add provisioner_daemons to /debug/health endpoint (#11393)

Browse Source 
Adds a healthcheck for provisioner daemons to /debug/health endpoint.
This commit is contained in:
Cian Johnston
2024-01-08 09:29:04 +00:00
committed by GitHub
parent 31f8fac1b9
commit 04fd96a014
23 changed files with 791 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
coderd/apidoc/docs.go
Generated
+41 -4
View File
@@ -9408,14 +9408,16 @@ const docTemplate = `{
"AccessURL",
"Websocket",
"Database",
"WorkspaceProxy"
"WorkspaceProxy",
"ProvisionerDaemons"
],
"x-enum-varnames": [
"HealthSectionDERP",
"HealthSectionAccessURL",
"HealthSectionWebsocket",
"HealthSectionDatabase",
"HealthSectionWorkspaceProxy"
"HealthSectionWorkspaceProxy",
"HealthSectionProvisionerDaemons"
]
},
"codersdk.HealthSettings": {
@@ -12957,7 +12959,10 @@ const docTemplate = `{
"EACS03",
"EACS04",
"EDERP01",
"EDERP02"
"EDERP02",
"EPD01",
"EPD02",
"EPD03"
],
"x-enum-varnames": [
"CodeUnknown",
@@ -12975,7 +12980,10 @@ const docTemplate = `{
"CodeAccessURLFetch",
"CodeAccessURLNotOK",
"CodeDERPNodeUsesWebsocket",
"CodeDERPOneNodeUnhealthy"
"CodeDERPOneNodeUnhealthy",
"CodeProvisionerDaemonsNoProvisionerDaemons",
"CodeProvisionerDaemonVersionMismatch",
"CodeProvisionerDaemonAPIMajorVersionDeprecated"
]
},
"health.Message": {
@@ -13092,6 +13100,32 @@ const docTemplate = `{
}
}
},
"healthcheck.ProvisionerDaemonsReport": {
"type": "object",
"properties": {
"dismissed": {
"type": "boolean"
},
"error": {
"type": "string"
},
"provisioner_daemons": {
"type": "array",
"items": {
"$ref": "#/definitions/codersdk.ProvisionerDaemon"
}
},
"severity": {
"$ref": "#/definitions/health.Severity"
},
"warnings": {
"type": "array",
"items": {
"$ref": "#/definitions/health.Message"
}
}
}
},
"healthcheck.Report": {
"type": "object",
"properties": {
@@ -13119,6 +13153,9 @@ const docTemplate = `{
"description": "Healthy is true if the report returns no errors.\nDeprecated: use ` + "`" + `Severity` + "`" + ` instead",
"type": "boolean"
},
"provisioner_daemons": {
"$ref": "#/definitions/healthcheck.ProvisionerDaemonsReport"
},
"severity": {
"description": "Severity indicates the status of Coder health.",
"enum": [
coderd/apidoc/swagger.json
Generated
+47 -4
View File
@@ -8440,13 +8440,21 @@
},
"codersdk.HealthSection": {
"type": "string",
"enum": ["DERP", "AccessURL", "Websocket", "Database", "WorkspaceProxy"],
"enum": [
"DERP",
"AccessURL",
"Websocket",
"Database",
"WorkspaceProxy",
"ProvisionerDaemons"
],
"x-enum-varnames": [
"HealthSectionDERP",
"HealthSectionAccessURL",
"HealthSectionWebsocket",
"HealthSectionDatabase",
"HealthSectionWorkspaceProxy"
"HealthSectionWorkspaceProxy",
"HealthSectionProvisionerDaemons"
]
},
"codersdk.HealthSettings": {
@@ -11791,7 +11799,10 @@
"EACS03",
"EACS04",
"EDERP01",
"EDERP02"
"EDERP02",
"EPD01",
"EPD02",
"EPD03"
],
"x-enum-varnames": [
"CodeUnknown",
@@ -11809,7 +11820,10 @@
"CodeAccessURLFetch",
"CodeAccessURLNotOK",
"CodeDERPNodeUsesWebsocket",
"CodeDERPOneNodeUnhealthy"
"CodeDERPOneNodeUnhealthy",
"CodeProvisionerDaemonsNoProvisionerDaemons",
"CodeProvisionerDaemonVersionMismatch",
"CodeProvisionerDaemonAPIMajorVersionDeprecated"
]
},
"health.Message": {
@@ -11910,6 +11924,32 @@
}
}
},
"healthcheck.ProvisionerDaemonsReport": {
"type": "object",
"properties": {
"dismissed": {
"type": "boolean"
},
"error": {
"type": "string"
},
"provisioner_daemons": {
"type": "array",
"items": {
"$ref": "#/definitions/codersdk.ProvisionerDaemon"
}
},
"severity": {
"$ref": "#/definitions/health.Severity"
},
"warnings": {
"type": "array",
"items": {
"$ref": "#/definitions/health.Message"
}
}
}
},
"healthcheck.Report": {
"type": "object",
"properties": {
@@ -11937,6 +11977,9 @@
"description": "Healthy is true if the report returns no errors.\nDeprecated: use `Severity` instead",
"type": "boolean"
},
"provisioner_daemons": {
"$ref": "#/definitions/healthcheck.ProvisionerDaemonsReport"
},
"severity": {
"description": "Severity indicates the status of Coder health.",
"enum": ["ok", "warning", "error"],
coderd/coderd.go
+7 -1
View File
@@ -440,6 +440,12 @@ func New(options *Options) *API {
CurrentVersion: buildinfo.Version(),
WorkspaceProxiesFetchUpdater: *(options.WorkspaceProxiesFetchUpdater).Load(),
},
ProvisionerDaemons: healthcheck.ProvisionerDaemonsReportDeps{
CurrentVersion: buildinfo.Version(),
CurrentAPIMajorVersion: provisionersdk.CurrentMajor,
Store: options.Database,
// TimeNow and StaleInterval set to defaults, see healthcheck/provisioner.go
},
})
}
}
@@ -1188,7 +1194,7 @@ func (api *API) CreateInMemoryProvisionerDaemon(ctx context.Context, name string
Tags: provisionersdk.MutateTags(uuid.Nil, nil),
LastSeenAt: sql.NullTime{Time: dbtime.Now(), Valid: true},
Version: buildinfo.Version(),
APIVersion: provisionersdk.APIVersionCurrent,
APIVersion: provisionersdk.VersionCurrent.String(),
})
if err != nil {
return nil, xerrors.Errorf("failed to create in-memory provisioner daemon: %w", err)
coderd/database/db2sdk/db2sdk.go
+16
View File
@@ -416,3 +416,19 @@ func Apps(dbApps []database.WorkspaceApp, agent database.WorkspaceAgent, ownerNa
}
return apps
}
func ProvisionerDaemon(dbDaemon database.ProvisionerDaemon) codersdk.ProvisionerDaemon {
result := codersdk.ProvisionerDaemon{
ID: dbDaemon.ID,
CreatedAt: dbDaemon.CreatedAt,
LastSeenAt: codersdk.NullTime{NullTime: dbDaemon.LastSeenAt},
Name: dbDaemon.Name,
Tags: dbDaemon.Tags,
Version: dbDaemon.Version,
APIVersion: dbDaemon.APIVersion,
}
for _, provisionerType := range dbDaemon.Provisioners {
result.Provisioners = append(result.Provisioners, codersdk.ProvisionerType(provisionerType))
}
return result
}
coderd/database/dbpurge/dbpurge_test.go
+4 -4
View File
@@ -218,7 +218,7 @@ func TestDeleteOldProvisionerDaemons(t *testing.T) {
CreatedAt: now.Add(-14 * 24 * time.Hour),
LastSeenAt: sql.NullTime{Valid: true, Time: now.Add(-7 * 24 * time.Hour).Add(time.Minute)},
Version: "1.0.0",
APIVersion: provisionersdk.APIVersionCurrent,
APIVersion: provisionersdk.VersionCurrent.String(),
})
require.NoError(t, err)
_, err = db.UpsertProvisionerDaemon(ctx, database.UpsertProvisionerDaemonParams{
@@ -229,7 +229,7 @@ func TestDeleteOldProvisionerDaemons(t *testing.T) {
CreatedAt: now.Add(-8 * 24 * time.Hour),
LastSeenAt: sql.NullTime{Valid: true, Time: now.Add(-8 * 24 * time.Hour).Add(time.Hour)},
Version: "1.0.0",
APIVersion: provisionersdk.APIVersionCurrent,
APIVersion: provisionersdk.VersionCurrent.String(),
})
require.NoError(t, err)
_, err = db.UpsertProvisionerDaemon(ctx, database.UpsertProvisionerDaemonParams{
@@ -242,7 +242,7 @@ func TestDeleteOldProvisionerDaemons(t *testing.T) {
},
CreatedAt: now.Add(-9 * 24 * time.Hour),
Version: "1.0.0",
APIVersion: provisionersdk.APIVersionCurrent,
APIVersion: provisionersdk.VersionCurrent.String(),
})
require.NoError(t, err)
_, err = db.UpsertProvisionerDaemon(ctx, database.UpsertProvisionerDaemonParams{
@@ -256,7 +256,7 @@ func TestDeleteOldProvisionerDaemons(t *testing.T) {
CreatedAt: now.Add(-6 * 24 * time.Hour),
LastSeenAt: sql.NullTime{Valid: true, Time: now.Add(-6 * 24 * time.Hour)},
Version: "1.0.0",
APIVersion: provisionersdk.APIVersionCurrent,
APIVersion: provisionersdk.VersionCurrent.String(),
})
require.NoError(t, err)
coderd/healthcheck/health/model.go
+4
View File
@@ -34,6 +34,10 @@ const (
CodeDERPNodeUsesWebsocket Code = `EDERP01`
CodeDERPOneNodeUnhealthy Code = `EDERP02`
CodeProvisionerDaemonsNoProvisionerDaemons Code = `EPD01`
CodeProvisionerDaemonVersionMismatch Code = `EPD02`
CodeProvisionerDaemonAPIMajorVersionDeprecated Code = `EPD03`
)
// @typescript-generate Severity
coderd/healthcheck/healthcheck.go
+52 -20
View File
@@ -18,6 +18,7 @@ type Checker interface {
Websocket(ctx context.Context, opts *WebsocketReportOptions) WebsocketReport
Database(ctx context.Context, opts *DatabaseReportOptions) DatabaseReport
WorkspaceProxy(ctx context.Context, opts *WorkspaceProxyReportOptions) WorkspaceProxyReport
ProvisionerDaemons(ctx context.Context, opts *ProvisionerDaemonsReportDeps) ProvisionerDaemonsReport
}
// @typescript-generate Report
@@ -32,49 +33,62 @@ type Report struct {
// FailingSections is a list of sections that have failed their healthcheck.
FailingSections []codersdk.HealthSection `json:"failing_sections"`
DERP derphealth.Report `json:"derp"`
AccessURL AccessURLReport `json:"access_url"`
Websocket WebsocketReport `json:"websocket"`
Database DatabaseReport `json:"database"`
WorkspaceProxy WorkspaceProxyReport `json:"workspace_proxy"`
DERP derphealth.Report `json:"derp"`
AccessURL AccessURLReport `json:"access_url"`
Websocket WebsocketReport `json:"websocket"`
Database DatabaseReport `json:"database"`
WorkspaceProxy WorkspaceProxyReport `json:"workspace_proxy"`
ProvisionerDaemons ProvisionerDaemonsReport `json:"provisioner_daemons"`
// The Coder version of the server that the report was generated on.
CoderVersion string `json:"coder_version"`
}
type ReportOptions struct {
AccessURL AccessURLReportOptions
Database DatabaseReportOptions
DerpHealth derphealth.ReportOptions
Websocket WebsocketReportOptions
WorkspaceProxy WorkspaceProxyReportOptions
AccessURL AccessURLReportOptions
Database DatabaseReportOptions
DerpHealth derphealth.ReportOptions
Websocket WebsocketReportOptions
WorkspaceProxy WorkspaceProxyReportOptions
ProvisionerDaemons ProvisionerDaemonsReportDeps
Checker Checker
}
type defaultChecker struct{}
func (defaultChecker) DERP(ctx context.Context, opts *derphealth.ReportOptions) (report derphealth.Report) {
func (defaultChecker) DERP(ctx context.Context, opts *derphealth.ReportOptions) derphealth.Report {
var report derphealth.Report
report.Run(ctx, opts)
return report
}
func (defaultChecker) AccessURL(ctx context.Context, opts *AccessURLReportOptions) (report AccessURLReport) {
func (defaultChecker) AccessURL(ctx context.Context, opts *AccessURLReportOptions) AccessURLReport {
var report AccessURLReport
report.Run(ctx, opts)
return report
}
func (defaultChecker) Websocket(ctx context.Context, opts *WebsocketReportOptions) (report WebsocketReport) {
func (defaultChecker) Websocket(ctx context.Context, opts *WebsocketReportOptions) WebsocketReport {
var report WebsocketReport
report.Run(ctx, opts)
return report
}
func (defaultChecker) Database(ctx context.Context, opts *DatabaseReportOptions) (report DatabaseReport) {
func (defaultChecker) Database(ctx context.Context, opts *DatabaseReportOptions) DatabaseReport {
var report DatabaseReport
report.Run(ctx, opts)
return report
}
func (defaultChecker) WorkspaceProxy(ctx context.Context, opts *WorkspaceProxyReportOptions) (report WorkspaceProxyReport) {
func (defaultChecker) WorkspaceProxy(ctx context.Context, opts *WorkspaceProxyReportOptions) WorkspaceProxyReport {
var report WorkspaceProxyReport
report.Run(ctx, opts)
return report
}
func (defaultChecker) ProvisionerDaemons(ctx context.Context, opts *ProvisionerDaemonsReportDeps) ProvisionerDaemonsReport {
var report ProvisionerDaemonsReport
report.Run(ctx, opts)
return report
}
@@ -149,26 +163,41 @@ func Run(ctx context.Context, opts *ReportOptions) *Report {
report.WorkspaceProxy = opts.Checker.WorkspaceProxy(ctx, &opts.WorkspaceProxy)
}()
wg.Add(1)
go func() {
defer wg.Done()
defer func() {
if err := recover(); err != nil {
report.ProvisionerDaemons.Error = health.Errorf(health.CodeUnknown, "provisioner daemon report panic: %s", err)
}
}()
report.ProvisionerDaemons = opts.Checker.ProvisionerDaemons(ctx, &opts.ProvisionerDaemons)
}()
report.CoderVersion = buildinfo.Version()
wg.Wait()
report.Time = time.Now()
report.FailingSections = []codersdk.HealthSection{}
if !report.DERP.Healthy {
if report.DERP.Severity.Value() > health.SeverityWarning.Value() {
report.FailingSections = append(report.FailingSections, codersdk.HealthSectionDERP)
}
if !report.AccessURL.Healthy {
if report.AccessURL.Severity.Value() > health.SeverityOK.Value() {
report.FailingSections = append(report.FailingSections, codersdk.HealthSectionAccessURL)
}
if !report.Websocket.Healthy {
if report.Websocket.Severity.Value() > health.SeverityWarning.Value() {
report.FailingSections = append(report.FailingSections, codersdk.HealthSectionWebsocket)
}
if !report.Database.Healthy {
if report.Database.Severity.Value() > health.SeverityWarning.Value() {
report.FailingSections = append(report.FailingSections, codersdk.HealthSectionDatabase)
}
if !report.WorkspaceProxy.Healthy {
if report.WorkspaceProxy.Severity.Value() > health.SeverityWarning.Value() {
report.FailingSections = append(report.FailingSections, codersdk.HealthSectionWorkspaceProxy)
}
if report.ProvisionerDaemons.Severity.Value() > health.SeverityWarning.Value() {
report.FailingSections = append(report.FailingSections, codersdk.HealthSectionProvisionerDaemons)
}
report.Healthy = len(report.FailingSections) == 0
@@ -190,6 +219,9 @@ func Run(ctx context.Context, opts *ReportOptions) *Report {
if report.WorkspaceProxy.Severity.Value() > report.Severity.Value() {
report.Severity = report.WorkspaceProxy.Severity
}
if report.ProvisionerDaemons.Severity.Value() > report.Severity.Value() {
report.Severity = report.ProvisionerDaemons.Severity
}
return &report
}
coderd/healthcheck/healthcheck_test.go
+99 -5
View File
@@ -13,11 +13,12 @@ import (
)
type testChecker struct {
DERPReport derphealth.Report
AccessURLReport healthcheck.AccessURLReport
WebsocketReport healthcheck.WebsocketReport
DatabaseReport healthcheck.DatabaseReport
WorkspaceProxyReport healthcheck.WorkspaceProxyReport
DERPReport derphealth.Report
AccessURLReport healthcheck.AccessURLReport
WebsocketReport healthcheck.WebsocketReport
DatabaseReport healthcheck.DatabaseReport
WorkspaceProxyReport healthcheck.WorkspaceProxyReport
ProvisionerDaemonsReport healthcheck.ProvisionerDaemonsReport
}
func (c *testChecker) DERP(context.Context, *derphealth.ReportOptions) derphealth.Report {
@@ -40,6 +41,10 @@ func (c *testChecker) WorkspaceProxy(context.Context, *healthcheck.WorkspaceProx
return c.WorkspaceProxyReport
}
func (c *testChecker) ProvisionerDaemons(context.Context, *healthcheck.ProvisionerDaemonsReportDeps) healthcheck.ProvisionerDaemonsReport {
return c.ProvisionerDaemonsReport
}
func TestHealthcheck(t *testing.T) {
t.Parallel()
@@ -72,6 +77,9 @@ func TestHealthcheck(t *testing.T) {
Healthy: true,
Severity: health.SeverityOK,
},
ProvisionerDaemonsReport: healthcheck.ProvisionerDaemonsReport{
Severity: health.SeverityOK,
},
},
healthy: true,
severity: health.SeverityOK,
@@ -99,6 +107,9 @@ func TestHealthcheck(t *testing.T) {
Healthy: true,
Severity: health.SeverityOK,
},
ProvisionerDaemonsReport: healthcheck.ProvisionerDaemonsReport{
Severity: health.SeverityOK,
},
},
healthy: false,
severity: health.SeverityError,
@@ -127,6 +138,9 @@ func TestHealthcheck(t *testing.T) {
Healthy: true,
Severity: health.SeverityOK,
},
ProvisionerDaemonsReport: healthcheck.ProvisionerDaemonsReport{
Severity: health.SeverityOK,
},
},
healthy: true,
severity: health.SeverityWarning,
@@ -154,6 +168,9 @@ func TestHealthcheck(t *testing.T) {
Healthy: true,
Severity: health.SeverityOK,
},
ProvisionerDaemonsReport: healthcheck.ProvisionerDaemonsReport{
Severity: health.SeverityOK,
},
},
healthy: false,
severity: health.SeverityWarning,
@@ -181,6 +198,9 @@ func TestHealthcheck(t *testing.T) {
Healthy: true,
Severity: health.SeverityOK,
},
ProvisionerDaemonsReport: healthcheck.ProvisionerDaemonsReport{
Severity: health.SeverityOK,
},
},
healthy: false,
severity: health.SeverityError,
@@ -208,6 +228,9 @@ func TestHealthcheck(t *testing.T) {
Healthy: true,
Severity: health.SeverityOK,
},
ProvisionerDaemonsReport: healthcheck.ProvisionerDaemonsReport{
Severity: health.SeverityOK,
},
},
healthy: false,
severity: health.SeverityError,
@@ -235,6 +258,9 @@ func TestHealthcheck(t *testing.T) {
Healthy: false,
Severity: health.SeverityError,
},
ProvisionerDaemonsReport: healthcheck.ProvisionerDaemonsReport{
Severity: health.SeverityOK,
},
},
severity: health.SeverityError,
healthy: false,
@@ -263,6 +289,70 @@ func TestHealthcheck(t *testing.T) {
Warnings: []health.Message{{Message: "foobar", Code: "EFOOBAR"}},
Severity: health.SeverityWarning,
},
ProvisionerDaemonsReport: healthcheck.ProvisionerDaemonsReport{
Severity: health.SeverityOK,
},
},
severity: health.SeverityWarning,
healthy: true,
failingSections: []codersdk.HealthSection{},
}, {
name: "ProvisionerDaemonsFail",
checker: &testChecker{
DERPReport: derphealth.Report{
Healthy: true,
Severity: health.SeverityOK,
},
AccessURLReport: healthcheck.AccessURLReport{
Healthy: true,
Severity: health.SeverityOK,
},
WebsocketReport: healthcheck.WebsocketReport{
Healthy: true,
Severity: health.SeverityOK,
},
DatabaseReport: healthcheck.DatabaseReport{
Healthy: true,
Severity: health.SeverityOK,
},
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
Healthy: true,
Severity: health.SeverityOK,
},
ProvisionerDaemonsReport: healthcheck.ProvisionerDaemonsReport{
Severity: health.SeverityError,
},
},
severity: health.SeverityError,
healthy: false,
failingSections: []codersdk.HealthSection{codersdk.HealthSectionProvisionerDaemons},
}, {
name: "ProvisionerDaemonsWarn",
checker: &testChecker{
DERPReport: derphealth.Report{
Healthy: true,
Severity: health.SeverityOK,
},
AccessURLReport: healthcheck.AccessURLReport{
Healthy: true,
Severity: health.SeverityOK,
},
WebsocketReport: healthcheck.WebsocketReport{
Healthy: true,
Severity: health.SeverityOK,
},
DatabaseReport: healthcheck.DatabaseReport{
Healthy: true,
Severity: health.SeverityOK,
},
WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
Healthy: true,
Severity: health.SeverityOK,
},
ProvisionerDaemonsReport: healthcheck.ProvisionerDaemonsReport{
Severity: health.SeverityWarning,
Warnings: []health.Message{{Message: "foobar", Code: "EFOOBAR"}},
},
},
severity: health.SeverityWarning,
healthy: true,
@@ -291,6 +381,9 @@ func TestHealthcheck(t *testing.T) {
Healthy: false,
Severity: health.SeverityError,
},
ProvisionerDaemonsReport: healthcheck.ProvisionerDaemonsReport{
Severity: health.SeverityError,
},
},
severity: health.SeverityError,
failingSections: []codersdk.HealthSection{
@@ -299,6 +392,7 @@ func TestHealthcheck(t *testing.T) {
codersdk.HealthSectionWebsocket,
codersdk.HealthSectionDatabase,
codersdk.HealthSectionWorkspaceProxy,
codersdk.HealthSectionProvisionerDaemons,
},
}} {
c := c
coderd/healthcheck/provisioner.go
+136
View File
@@ -0,0 +1,136 @@
package healthcheck
import (
"context"
"time"
"golang.org/x/mod/semver"
"github.com/coder/coder/v2/buildinfo"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/db2sdk"
"github.com/coder/coder/v2/coderd/database/dbauthz"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/coderd/healthcheck/health"
"github.com/coder/coder/v2/coderd/provisionerdserver"
"github.com/coder/coder/v2/coderd/util/apiversion"
"github.com/coder/coder/v2/coderd/util/ptr"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/provisionersdk"
)
// @typescript-generate ProvisionerDaemonsReport
type ProvisionerDaemonsReport struct {
Severity health.Severity `json:"severity"`
Warnings []health.Message `json:"warnings"`
Dismissed bool `json:"dismissed"`
Error *string `json:"error"`
ProvisionerDaemons []codersdk.ProvisionerDaemon `json:"provisioner_daemons"`
}
type ProvisionerDaemonsReportDeps struct {
// Required
CurrentVersion string
CurrentAPIMajorVersion int
Store ProvisionerDaemonsStore
// Optional
TimeNow func() time.Time // Defaults to dbtime.Now
StaleInterval time.Duration // Defaults to 3 heartbeats
Dismissed bool
}
type ProvisionerDaemonsStore interface {
GetProvisionerDaemons(ctx context.Context) ([]database.ProvisionerDaemon, error)
}
func (r *ProvisionerDaemonsReport) Run(ctx context.Context, opts *ProvisionerDaemonsReportDeps) {
r.ProvisionerDaemons = make([]codersdk.ProvisionerDaemon, 0)
r.Severity = health.SeverityOK
r.Warnings = make([]health.Message, 0)
r.Dismissed = opts.Dismissed
if opts.TimeNow == nil {
opts.TimeNow = dbtime.Now
}
now := opts.TimeNow()
if opts.StaleInterval == 0 {
opts.StaleInterval = provisionerdserver.DefaultHeartbeatInterval * 3
}
if opts.CurrentVersion == "" {
r.Severity = health.SeverityError
r.Error = ptr.Ref("Developer error: CurrentVersion is empty!")
return
}
if opts.CurrentAPIMajorVersion == 0 {
r.Severity = health.SeverityError
r.Error = ptr.Ref("Developer error: CurrentAPIMajorVersion must be non-zero!")
return
}
if opts.Store == nil {
r.Severity = health.SeverityError
r.Error = ptr.Ref("Developer error: Store is nil!")
return
}
// nolint: gocritic // need an actor to fetch provisioner daemons
daemons, err := opts.Store.GetProvisionerDaemons(dbauthz.AsSystemRestricted(ctx))
if err != nil {
r.Severity = health.SeverityError
r.Error = ptr.Ref("error fetching provisioner daemons: " + err.Error())
return
}
for _, daemon := range daemons {
// Daemon never connected, skip.
if !daemon.LastSeenAt.Valid {
continue
}
// Daemon has gone away, skip.
if now.Sub(daemon.LastSeenAt.Time) > (opts.StaleInterval) {
continue
}
r.ProvisionerDaemons = append(r.ProvisionerDaemons, db2sdk.ProvisionerDaemon(daemon))
// For release versions, just check MAJOR.MINOR and ignore patch.
if !semver.IsValid(daemon.Version) {
if r.Severity.Value() < health.SeverityError.Value() {
r.Severity = health.SeverityError
}
r.Warnings = append(r.Warnings, health.Messagef(health.CodeUnknown, "Provisioner daemon %q reports invalid version %q", opts.CurrentVersion, daemon.Version))
} else if !buildinfo.VersionsMatch(opts.CurrentVersion, daemon.Version) {
if r.Severity.Value() < health.SeverityWarning.Value() {
r.Severity = health.SeverityWarning
}
r.Warnings = append(r.Warnings, health.Messagef(health.CodeProvisionerDaemonVersionMismatch, "Provisioner daemon %q has outdated version %q", daemon.Name, daemon.Version))
}
// Provisioner daemon API version follows different rules; we just want to check the major API version and
// warn about potential later deprecations.
// When we check API versions of connecting provisioner daemons, all active provisioner daemons
// will, by necessity, have a compatible API version.
if maj, _, err := apiversion.Parse(daemon.APIVersion); err != nil {
if r.Severity.Value() < health.SeverityError.Value() {
r.Severity = health.SeverityError
}
r.Warnings = append(r.Warnings, health.Messagef(health.CodeUnknown, "Provisioner daemon %q reports invalid API version: %s", daemon.Name, err.Error()))
} else if maj != opts.CurrentAPIMajorVersion {
if r.Severity.Value() < health.SeverityWarning.Value() {
r.Severity = health.SeverityWarning
}
r.Warnings = append(r.Warnings, health.Messagef(health.CodeProvisionerDaemonAPIMajorVersionDeprecated, "Provisioner daemon %q reports deprecated major API version %d. Consider upgrading!", daemon.Name, provisionersdk.CurrentMajor))
}
}
if len(r.ProvisionerDaemons) == 0 {
r.Severity = health.SeverityError
r.Error = ptr.Ref("No active provisioner daemons found!")
return
}
}
coderd/healthcheck/provisioner_test.go
+191
View File
@@ -0,0 +1,191 @@
package healthcheck_test
import (
"context"
"database/sql"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbmock"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/coderd/healthcheck"
"github.com/coder/coder/v2/coderd/healthcheck/health"
"github.com/coder/coder/v2/provisionersdk"
gomock "go.uber.org/mock/gomock"
)
func TestProvisionerDaemonReport(t *testing.T) {
t.Parallel()
for _, tt := range []struct {
name string
currentVersion string
currentAPIMajorVersion int
provisionerDaemons []database.ProvisionerDaemon
provisionerDaemonsErr error
expectedSeverity health.Severity
expectedWarningCode health.Code
expectedError string
}{
{
name: "current version empty",
currentVersion: "",
expectedSeverity: health.SeverityError,
expectedError: "Developer error: CurrentVersion is empty",
},
{
name: "no daemons",
currentVersion: "v1.2.3",
currentAPIMajorVersion: provisionersdk.CurrentMajor,
expectedSeverity: health.SeverityError,
expectedError: "No active provisioner daemons found!",
},
{
name: "error fetching daemons",
currentVersion: "v1.2.3",
currentAPIMajorVersion: provisionersdk.CurrentMajor,
provisionerDaemonsErr: assert.AnError,
expectedSeverity: health.SeverityError,
expectedError: assert.AnError.Error(),
},
{
name: "one daemon up to date",
currentVersion: "v1.2.3",
currentAPIMajorVersion: provisionersdk.CurrentMajor,
expectedSeverity: health.SeverityOK,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0")},
},
{
name: "one daemon out of date",
currentVersion: "v1.2.3",
currentAPIMajorVersion: provisionersdk.CurrentMajor,
expectedSeverity: health.SeverityWarning,
expectedWarningCode: health.CodeProvisionerDaemonVersionMismatch,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-old", "v1.1.2", "1.0")},
},
{
name: "invalid daemon version",
currentVersion: "v1.2.3",
currentAPIMajorVersion: provisionersdk.CurrentMajor,
expectedSeverity: health.SeverityError,
expectedWarningCode: health.CodeUnknown,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-invalid-version", "invalid", "1.0")},
},
{
name: "invalid daemon api version",
currentVersion: "v1.2.3",
currentAPIMajorVersion: provisionersdk.CurrentMajor,
expectedSeverity: health.SeverityError,
expectedWarningCode: health.CodeUnknown,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-new-minor", "v1.2.3", "invalid")},
},
{
name: "api version backward compat",
currentVersion: "v2.3.4",
currentAPIMajorVersion: 2,
expectedSeverity: health.SeverityWarning,
expectedWarningCode: health.CodeProvisionerDaemonAPIMajorVersionDeprecated,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-old-api", "v2.3.4", "1.0")},
},
{
name: "one up to date, one out of date",
currentVersion: "v1.2.3",
currentAPIMajorVersion: provisionersdk.CurrentMajor,
expectedSeverity: health.SeverityWarning,
expectedWarningCode: health.CodeProvisionerDaemonVersionMismatch,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0"), fakeProvisionerDaemon(t, "pd-old", "v1.1.2", "1.0")},
},
{
name: "one up to date, one newer",
currentVersion: "v1.2.3",
currentAPIMajorVersion: provisionersdk.CurrentMajor,
expectedSeverity: health.SeverityWarning,
expectedWarningCode: health.CodeProvisionerDaemonVersionMismatch,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemon(t, "pd-ok", "v1.2.3", "1.0"), fakeProvisionerDaemon(t, "pd-new", "v2.3.4", "1.0")},
},
{
name: "one up to date, one stale older",
currentVersion: "v2.3.4",
currentAPIMajorVersion: provisionersdk.CurrentMajor,
expectedSeverity: health.SeverityOK,
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemonStale(t, "pd-ok", "v1.2.3", "0.9", dbtime.Now().Add(-5*time.Minute)), fakeProvisionerDaemon(t, "pd-new", "v2.3.4", "1.0")},
},
{
name: "one stale",
currentVersion: "v2.3.4",
currentAPIMajorVersion: provisionersdk.CurrentMajor,
expectedSeverity: health.SeverityError,
expectedError: "No active provisioner daemons found!",
provisionerDaemons: []database.ProvisionerDaemon{fakeProvisionerDaemonStale(t, "pd-ok", "v1.2.3", "0.9", dbtime.Now().Add(-5*time.Minute))},
},
} {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
var rpt healthcheck.ProvisionerDaemonsReport
var deps healthcheck.ProvisionerDaemonsReportDeps
deps.CurrentVersion = tt.currentVersion
deps.CurrentAPIMajorVersion = tt.currentAPIMajorVersion
if tt.currentAPIMajorVersion == 0 {
deps.CurrentAPIMajorVersion = provisionersdk.CurrentMajor
}
now := dbtime.Now()
deps.TimeNow = func() time.Time {
return now
}
ctrl := gomock.NewController(t)
mDB := dbmock.NewMockStore(ctrl)
mDB.EXPECT().GetProvisionerDaemons(gomock.Any()).AnyTimes().Return(tt.provisionerDaemons, tt.provisionerDaemonsErr)
deps.Store = mDB
rpt.Run(context.Background(), &deps)
assert.Equal(t, tt.expectedSeverity, rpt.Severity)
if tt.expectedWarningCode != "" && assert.NotEmpty(t, rpt.Warnings) {
var found bool
for _, w := range rpt.Warnings {
if w.Code == tt.expectedWarningCode {
found = true
break
}
}
assert.True(t, found, "expected warning %s not found in %v", tt.expectedWarningCode, rpt.Warnings)
} else {
assert.Empty(t, rpt.Warnings)
}
if tt.expectedError != "" && assert.NotNil(t, rpt.Error) {
assert.Contains(t, *rpt.Error, tt.expectedError)
}
})
}
}
func fakeProvisionerDaemon(t *testing.T, name, version, apiVersion string) database.ProvisionerDaemon {
t.Helper()
return database.ProvisionerDaemon{
ID: uuid.New(),
Name: name,
CreatedAt: dbtime.Now(),
LastSeenAt: sql.NullTime{Time: dbtime.Now(), Valid: true},
Provisioners: []database.ProvisionerType{database.ProvisionerTypeEcho, database.ProvisionerTypeTerraform},
ReplicaID: uuid.NullUUID{},
Tags: map[string]string{},
Version: version,
APIVersion: apiVersion,
}
}
func fakeProvisionerDaemonStale(t *testing.T, name, version, apiVersion string, lastSeenAt time.Time) database.ProvisionerDaemon {
t.Helper()
d := fakeProvisionerDaemon(t, name, version, apiVersion)
d.LastSeenAt.Valid = true
d.LastSeenAt.Time = lastSeenAt
return d
}
coderd/provisionerdserver/provisionerdserver_test.go
+1 -1
View File
@@ -1786,7 +1786,7 @@ func setup(t *testing.T, ignoreLogErrors bool, ov *overrides) (proto.DRPCProvisi
Tags: database.StringMap{},
LastSeenAt: sql.NullTime{},
Version: buildinfo.Version(),
APIVersion: provisionersdk.APIVersionCurrent,
APIVersion: provisionersdk.VersionCurrent.String(),
})
require.NoError(t, err)
coderd/util/apiversion/apiversion.go
+5
View File
@@ -1,6 +1,7 @@
package apiversion
import (
"fmt"
"strconv"
"strings"
@@ -41,6 +42,10 @@ func (v *APIVersion) WithBackwardCompat(majs ...int) *APIVersion {
// - 1.x is supported,
// - 2.0, 2.1, and 2.2 are supported,
// - 2.3+ is not supported.
func (v *APIVersion) String() string {
return fmt.Sprintf("%d.%d", v.supportedMajor, v.supportedMinor)
}
func (v *APIVersion) Validate(version string) error {
major, minor, err := Parse(version)
if err != nil {