opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: automatically stop workspaces based on failure_ttl (#7989)

Browse Source
This commit is contained in:
Jon Ayers
2023-06-22 00:33:22 -04:00
committed by GitHub
parent d434181941
commit 1b0124ecdb
17 changed files with 421 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files
enterprise/coderd/coderd.go
+1 -1
View File
@@ -402,7 +402,7 @@ func (api *API) updateEntitlements(ctx context.Context) error {
if changed, enabled := featureChanged(codersdk.FeatureAdvancedTemplateScheduling); changed {
if enabled {
store := &enterpriseTemplateScheduleStore{}
store := &EnterpriseTemplateScheduleStore{}
ptr := schedule.TemplateScheduleStore(store)
api.AGPL.TemplateScheduleStore.Store(&ptr)
} else {
enterprise/coderd/provisionerdaemons.go
+4 -4
View File
@@ -310,11 +310,11 @@ func websocketNetConn(ctx context.Context, conn *websocket.Conn, msgType websock
}
}
type enterpriseTemplateScheduleStore struct{}
type EnterpriseTemplateScheduleStore struct{}
var _ schedule.TemplateScheduleStore = &enterpriseTemplateScheduleStore{}
var _ schedule.TemplateScheduleStore = &EnterpriseTemplateScheduleStore{}
func (*enterpriseTemplateScheduleStore) GetTemplateScheduleOptions(ctx context.Context, db database.Store, templateID uuid.UUID) (schedule.TemplateScheduleOptions, error) {
func (*EnterpriseTemplateScheduleStore) GetTemplateScheduleOptions(ctx context.Context, db database.Store, templateID uuid.UUID) (schedule.TemplateScheduleOptions, error) {
tpl, err := db.GetTemplateByID(ctx, templateID)
if err != nil {
return schedule.TemplateScheduleOptions{}, err
@@ -331,7 +331,7 @@ func (*enterpriseTemplateScheduleStore) GetTemplateScheduleOptions(ctx context.C
}, nil
}
func (*enterpriseTemplateScheduleStore) SetTemplateScheduleOptions(ctx context.Context, db database.Store, tpl database.Template, opts schedule.TemplateScheduleOptions) (database.Template, error) {
func (*EnterpriseTemplateScheduleStore) SetTemplateScheduleOptions(ctx context.Context, db database.Store, tpl database.Template, opts schedule.TemplateScheduleOptions) (database.Template, error) {
if int64(opts.DefaultTTL) == tpl.DefaultTTL &&
int64(opts.MaxTTL) == tpl.MaxTTL &&
int64(opts.FailureTTL) == tpl.FailureTTL &&
enterprise/coderd/workspaces_test.go
+156
View File
@@ -10,12 +10,17 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"cdr.dev/slog/sloggers/slogtest"
"github.com/coder/coder/coderd/autobuild"
"github.com/coder/coder/coderd/coderdtest"
"github.com/coder/coder/coderd/database"
"github.com/coder/coder/coderd/util/ptr"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/enterprise/coderd"
"github.com/coder/coder/enterprise/coderd/coderdenttest"
"github.com/coder/coder/enterprise/coderd/license"
"github.com/coder/coder/provisioner/echo"
"github.com/coder/coder/testutil"
)
@@ -74,6 +79,157 @@ func TestCreateWorkspace(t *testing.T) {
})
}
func TestWorkspaceAutobuild(t *testing.T) {
t.Parallel()
t.Run("FailureTTLOK", func(t *testing.T) {
t.Parallel()
var (
ticker = make(chan time.Time)
statCh = make(chan autobuild.Stats)
logger = slogtest.Make(t, &slogtest.Options{
// We ignore errors here since we expect to fail
// builds.
IgnoreErrors: true,
})
failureTTL = time.Millisecond
client = coderdenttest.New(t, &coderdenttest.Options{
Options: &coderdtest.Options{
Logger: &logger,
AutobuildTicker: ticker,
IncludeProvisionerDaemon: true,
AutobuildStats: statCh,
TemplateScheduleStore: &coderd.EnterpriseTemplateScheduleStore{},
},
})
)
user := coderdtest.CreateFirstUser(t, client)
_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
Features: license.Features{
codersdk.FeatureAdvancedTemplateScheduling: 1,
},
})
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionPlan: echo.ProvisionComplete,
ProvisionApply: echo.ProvisionFailed,
})
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
ctr.FailureTTLMillis = ptr.Ref[int64](failureTTL.Milliseconds())
})
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Equal(t, codersdk.WorkspaceStatusFailed, build.Status)
require.Eventually(t,
func() bool {
return database.Now().Sub(*build.Job.CompletedAt) > failureTTL
},
testutil.IntervalMedium, testutil.IntervalFast)
ticker <- time.Now()
stats := <-statCh
// Expect workspace to transition to stopped state for breaching
// failure TTL.
require.Len(t, stats.Transitions, 1)
require.Equal(t, stats.Transitions[ws.ID], database.WorkspaceTransitionStop)
})
t.Run("FailureTTLTooEarly", func(t *testing.T) {
t.Parallel()
var (
ticker = make(chan time.Time)
statCh = make(chan autobuild.Stats)
logger = slogtest.Make(t, &slogtest.Options{
// We ignore errors here since we expect to fail
// builds.
IgnoreErrors: true,
})
failureTTL = time.Minute
client = coderdenttest.New(t, &coderdenttest.Options{
Options: &coderdtest.Options{
Logger: &logger,
AutobuildTicker: ticker,
IncludeProvisionerDaemon: true,
AutobuildStats: statCh,
TemplateScheduleStore: &coderd.EnterpriseTemplateScheduleStore{},
},
})
)
user := coderdtest.CreateFirstUser(t, client)
_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
Features: license.Features{
codersdk.FeatureAdvancedTemplateScheduling: 1,
},
})
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionPlan: echo.ProvisionComplete,
ProvisionApply: echo.ProvisionFailed,
})
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
ctr.FailureTTLMillis = ptr.Ref[int64](failureTTL.Milliseconds())
})
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Equal(t, codersdk.WorkspaceStatusFailed, build.Status)
ticker <- time.Now()
stats := <-statCh
// Expect no transitions since not enough time has elapsed.
require.Len(t, stats.Transitions, 0)
})
t.Run("FailureTTLUnset", func(t *testing.T) {
t.Parallel()
var (
ticker = make(chan time.Time)
statCh = make(chan autobuild.Stats)
logger = slogtest.Make(t, &slogtest.Options{
// We ignore errors here since we expect to fail
// builds.
IgnoreErrors: true,
})
client = coderdenttest.New(t, &coderdenttest.Options{
Options: &coderdtest.Options{
Logger: &logger,
AutobuildTicker: ticker,
IncludeProvisionerDaemon: true,
AutobuildStats: statCh,
TemplateScheduleStore: &coderd.EnterpriseTemplateScheduleStore{},
},
})
)
user := coderdtest.CreateFirstUser(t, client)
_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
Features: license.Features{
codersdk.FeatureAdvancedTemplateScheduling: 1,
},
})
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionPlan: echo.ProvisionComplete,
ProvisionApply: echo.ProvisionFailed,
})
// Create a template without setting a failure_ttl.
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Equal(t, codersdk.WorkspaceStatusFailed, build.Status)
ticker <- time.Now()
stats := <-statCh
// Expect no transitions since the field is unset on the template.
require.Len(t, stats.Transitions, 0)
})
}
func TestWorkspacesFiltering(t *testing.T) {
t.Parallel()