feat: automatically stop workspaces based on failure_ttl (#7989)

2026-09-24 15:04:27 +08:00 · 2023-06-22 00:33:22 -04:00
parent d434181941
commit 1b0124ecdb
17 changed files with 421 additions and 161 deletions
@@ -0,0 +1,3 @@
+// Package autobuild contains logic for scheduling workspace
+// builds in the background.
+package autobuild
@@ -1,4 +1,4 @@
-package executor
+package autobuild

 import (
 	"context"
@@ -13,9 +13,11 @@ import (

 	"cdr.dev/slog"
 	"github.com/coder/coder/coderd/database"
+	"github.com/coder/coder/coderd/database/db2sdk"
 	"github.com/coder/coder/coderd/database/dbauthz"
 	"github.com/coder/coder/coderd/schedule"
 	"github.com/coder/coder/coderd/wsbuilder"
+	"github.com/coder/coder/codersdk"
 )

 // Executor automatically starts or stops workspaces.
@@ -35,8 +37,8 @@ type Stats struct {
 	Error       error
 }

-// New returns a new autobuild executor.
-func New(ctx context.Context, db database.Store, tss *atomic.Pointer[schedule.TemplateScheduleStore], log slog.Logger, tick <-chan time.Time) *Executor {
+// New returns a new wsactions executor.
+func NewExecutor(ctx context.Context, db database.Store, tss *atomic.Pointer[schedule.TemplateScheduleStore], log slog.Logger, tick <-chan time.Time) *Executor {
 	le := &Executor{
 		//nolint:gocritic // Autostart has a limited set of permissions.
 		ctx:                   dbauthz.AsAutostart(ctx),
@@ -108,7 +110,7 @@ func (e *Executor) runOnce(t time.Time) Stats {
 	// NOTE: If a workspace build is created with a given TTL and then the user either
 	//       changes or unsets the TTL, the deadline for the workspace build will not
 	//       have changed. This behavior is as expected per #2229.
-	workspaces, err := e.db.GetWorkspacesEligibleForAutoStartStop(e.ctx, t)
+	workspaces, err := e.db.GetWorkspacesEligibleForTransition(e.ctx, t)
 	if err != nil {
 		e.log.Error(e.ctx, "get workspaces for autostart or autostop", slog.Error(err))
 		return stats
@@ -125,77 +127,56 @@ func (e *Executor) runOnce(t time.Time) Stats {
 		log := e.log.With(slog.F("workspace_id", wsID))

 		eg.Go(func() error {
-			err := e.db.InTx(func(db database.Store) error {
+			err := e.db.InTx(func(tx database.Store) error {
 				// Re-check eligibility since the first check was outside the
 				// transaction and the workspace settings may have changed.
-				ws, err := db.GetWorkspaceByID(e.ctx, wsID)
+				ws, err := tx.GetWorkspaceByID(e.ctx, wsID)
 				if err != nil {
 					log.Error(e.ctx, "get workspace autostart failed", slog.Error(err))
 					return nil
 				}

 				// Determine the workspace state based on its latest build.
-				priorHistory, err := db.GetLatestWorkspaceBuildByWorkspaceID(e.ctx, ws.ID)
+				latestBuild, err := tx.GetLatestWorkspaceBuildByWorkspaceID(e.ctx, ws.ID)
 				if err != nil {
 					log.Warn(e.ctx, "get latest workspace build", slog.Error(err))
 					return nil
 				}
-
-				templateSchedule, err := (*(e.templateScheduleStore.Load())).GetTemplateScheduleOptions(e.ctx, db, ws.TemplateID)
+				templateSchedule, err := (*(e.templateScheduleStore.Load())).GetTemplateScheduleOptions(e.ctx, tx, ws.TemplateID)
 				if err != nil {
 					log.Warn(e.ctx, "get template schedule options", slog.Error(err))
 					return nil
 				}

-				if !isEligibleForAutoStartStop(ws, priorHistory, templateSchedule) {
-					return nil
-				}
-
-				priorJob, err := db.GetProvisionerJobByID(e.ctx, priorHistory.JobID)
+				latestJob, err := tx.GetProvisionerJobByID(e.ctx, latestBuild.JobID)
 				if err != nil {
 					log.Warn(e.ctx, "get last provisioner job for workspace %q: %w", slog.Error(err))
 					return nil
 				}

-				validTransition, nextTransition, err := getNextTransition(ws, priorHistory, priorJob)
+				nextTransition, reason, err := getNextTransition(ws, latestBuild, latestJob, templateSchedule, currentTick)
 				if err != nil {
 					log.Debug(e.ctx, "skipping workspace", slog.Error(err))
 					return nil
 				}

-				if currentTick.Before(nextTransition) {
-					log.Debug(e.ctx, "skipping workspace: too early",
-						slog.F("next_transition_at", nextTransition),
-						slog.F("transition", validTransition),
-						slog.F("current_tick", currentTick),
-					)
-					return nil
-				}
-				builder := wsbuilder.New(ws, validTransition).
-					SetLastWorkspaceBuildInTx(&priorHistory).
-					SetLastWorkspaceBuildJobInTx(&priorJob)
+				builder := wsbuilder.New(ws, nextTransition).
+					SetLastWorkspaceBuildInTx(&latestBuild).
+					SetLastWorkspaceBuildJobInTx(&latestJob).
+					Reason(reason)

-				switch validTransition {
-				case database.WorkspaceTransitionStart:
-					builder = builder.Reason(database.BuildReasonAutostart)
-				case database.WorkspaceTransitionStop:
-					builder = builder.Reason(database.BuildReasonAutostop)
-				default:
-					log.Error(e.ctx, "unsupported transition", slog.F("transition", validTransition))
-					return nil
-				}
-				if _, _, err := builder.Build(e.ctx, db, nil); err != nil {
+				if _, _, err := builder.Build(e.ctx, tx, nil); err != nil {
 					log.Error(e.ctx, "unable to transition workspace",
-						slog.F("transition", validTransition),
+						slog.F("transition", nextTransition),
 						slog.Error(err),
 					)
 					return nil
 				}
 				statsMu.Lock()
-				stats.Transitions[ws.ID] = validTransition
+				stats.Transitions[ws.ID] = nextTransition
 				statsMu.Unlock()

-				log.Info(e.ctx, "scheduling workspace transition", slog.F("transition", validTransition))
+				log.Info(e.ctx, "scheduling workspace transition", slog.F("transition", nextTransition))

 				return nil

@@ -218,53 +199,81 @@ func (e *Executor) runOnce(t time.Time) Stats {
 	return stats
 }

-func isEligibleForAutoStartStop(ws database.Workspace, priorHistory database.WorkspaceBuild, templateSchedule schedule.TemplateScheduleOptions) bool {
-	if ws.Deleted {
-		return false
-	}
-	if templateSchedule.UserAutostartEnabled && ws.AutostartSchedule.Valid && ws.AutostartSchedule.String != "" {
-		return true
-	}
-	// Don't check the template schedule to see whether it allows autostop, this
-	// is done during the build when determining the deadline.
-	if priorHistory.Transition == database.WorkspaceTransitionStart && !priorHistory.Deadline.IsZero() {
-		return true
-	}
-
-	return false
-}
-
 func getNextTransition(
 	ws database.Workspace,
-	priorHistory database.WorkspaceBuild,
-	priorJob database.ProvisionerJob,
+	latestBuild database.WorkspaceBuild,
+	latestJob database.ProvisionerJob,
+	templateSchedule schedule.TemplateScheduleOptions,
+	currentTick time.Time,
 ) (
-	validTransition database.WorkspaceTransition,
-	nextTransition time.Time,
-	err error,
+	database.WorkspaceTransition,
+	database.BuildReason,
+	error,
 ) {
-	if !priorJob.CompletedAt.Valid || priorJob.Error.String != "" {
-		return "", time.Time{}, xerrors.Errorf("last workspace build did not complete successfully")
-	}
-
-	switch priorHistory.Transition {
-	case database.WorkspaceTransitionStart:
-		if priorHistory.Deadline.IsZero() {
-			return "", time.Time{}, xerrors.Errorf("latest workspace build has zero deadline")
-		}
-		// For stopping, do not truncate. This is inconsistent with autostart, but
-		// it ensures we will not stop too early.
-		return database.WorkspaceTransitionStop, priorHistory.Deadline, nil
-	case database.WorkspaceTransitionStop:
-		sched, err := schedule.Weekly(ws.AutostartSchedule.String)
-		if err != nil {
-			return "", time.Time{}, xerrors.Errorf("workspace has invalid autostart schedule: %w", err)
-		}
-		// Round down to the nearest minute, as this is the finest granularity cron supports.
-		// Truncate is probably not necessary here, but doing it anyway to be sure.
-		nextTransition = sched.Next(priorHistory.CreatedAt).Truncate(time.Minute)
-		return database.WorkspaceTransitionStart, nextTransition, nil
+	switch {
+	case isEligibleForAutostop(latestBuild, latestJob, currentTick):
+		return database.WorkspaceTransitionStop, database.BuildReasonAutostop, nil
+	case isEligibleForAutostart(ws, latestBuild, latestJob, templateSchedule, currentTick):
+		return database.WorkspaceTransitionStart, database.BuildReasonAutostart, nil
+	case isEligibleForFailedStop(latestBuild, latestJob, templateSchedule):
+		return database.WorkspaceTransitionStop, database.BuildReasonAutostop, nil
 	default:
-		return "", time.Time{}, xerrors.Errorf("last transition not valid for autostart or autostop")
+		return "", "", xerrors.Errorf("last transition not valid for autostart or autostop")
 	}
 }
+
+// isEligibleForAutostart returns true if the workspace should be autostarted.
+func isEligibleForAutostart(ws database.Workspace, build database.WorkspaceBuild, job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions, currentTick time.Time) bool {
+	// Don't attempt to autostart failed workspaces.
+	if !job.CompletedAt.Valid || job.Error.String != "" {
+		return false
+	}
+
+	// If the last transition for the workspace was not 'stop' then the workspace
+	// cannot be started.
+	if build.Transition != database.WorkspaceTransitionStop {
+		return false
+	}
+
+	// If autostart isn't enabled, or the schedule isn't valid/populated we can't
+	// autostart the workspace.
+	if !templateSchedule.UserAutostartEnabled || !ws.AutostartSchedule.Valid || ws.AutostartSchedule.String == "" {
+		return false
+	}
+
+	sched, err := schedule.Weekly(ws.AutostartSchedule.String)
+	if err != nil {
+		return false
+	}
+	// Round down to the nearest minute, as this is the finest granularity cron supports.
+	// Truncate is probably not necessary here, but doing it anyway to be sure.
+	nextTransition := sched.Next(build.CreatedAt).Truncate(time.Minute)
+
+	return !currentTick.Before(nextTransition)
+}
+
+// isEligibleForAutostart returns true if the workspace should be autostopped.
+func isEligibleForAutostop(build database.WorkspaceBuild, job database.ProvisionerJob, currentTick time.Time) bool {
+	// Don't attempt to autostop failed workspaces.
+	if !job.CompletedAt.Valid || job.Error.String != "" {
+		return false
+	}
+
+	// A workspace must be started in order for it to be auto-stopped.
+	return build.Transition == database.WorkspaceTransitionStart &&
+		!build.Deadline.IsZero() &&
+		// We do not want to stop a workspace prior to it breaching its deadline.
+		!currentTick.Before(build.Deadline)
+}
+
+// isEligibleForFailedStop returns true if the workspace is eligible to be stopped
+// due to a failed build.
+func isEligibleForFailedStop(build database.WorkspaceBuild, job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions) bool {
+	// If the template has specified a failure TLL.
+	return templateSchedule.FailureTTL > 0 &&
+		// And the job resulted in failure.
+		db2sdk.ProvisionerJobStatus(job) == codersdk.ProvisionerJobFailed &&
+		build.Transition == database.WorkspaceTransitionStart &&
+		// And sufficient time has elapsed since the job has completed.
+		job.CompletedAt.Valid && database.Now().Sub(job.CompletedAt.Time) > templateSchedule.FailureTTL
+}
@@ -1,4 +1,4 @@
-package executor_test
+package autobuild_test

 import (
 	"context"
@@ -11,7 +11,9 @@ import (
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"

-	"github.com/coder/coder/coderd/autobuild/executor"
+	"cdr.dev/slog/sloggers/slogtest"
+
+	"github.com/coder/coder/coderd/autobuild"
 	"github.com/coder/coder/coderd/coderdtest"
 	"github.com/coder/coder/coderd/database"
 	"github.com/coder/coder/coderd/schedule"
@@ -19,6 +21,7 @@ import (
 	"github.com/coder/coder/codersdk"
 	"github.com/coder/coder/provisioner/echo"
 	"github.com/coder/coder/provisionersdk/proto"
+	"github.com/coder/coder/testutil"
 )

 func TestExecutorAutostartOK(t *testing.T) {
@@ -27,7 +30,7 @@ func TestExecutorAutostartOK(t *testing.T) {
 	var (
 		sched   = mustSchedule(t, "CRON_TZ=UTC 0 * * * *")
 		tickCh  = make(chan time.Time)
-		statsCh = make(chan executor.Stats)
+		statsCh = make(chan autobuild.Stats)
 		client  = coderdtest.New(t, &coderdtest.Options{
 			AutobuildTicker:          tickCh,
 			IncludeProvisionerDaemon: true,
@@ -66,7 +69,7 @@ func TestExecutorAutostartTemplateUpdated(t *testing.T) {
 		ctx     = context.Background()
 		err     error
 		tickCh  = make(chan time.Time)
-		statsCh = make(chan executor.Stats)
+		statsCh = make(chan autobuild.Stats)
 		client  = coderdtest.New(t, &coderdtest.Options{
 			AutobuildTicker:          tickCh,
 			IncludeProvisionerDaemon: true,
@@ -113,7 +116,7 @@ func TestExecutorAutostartAlreadyRunning(t *testing.T) {
 	var (
 		sched   = mustSchedule(t, "CRON_TZ=UTC 0 * * * *")
 		tickCh  = make(chan time.Time)
-		statsCh = make(chan executor.Stats)
+		statsCh = make(chan autobuild.Stats)
 		client  = coderdtest.New(t, &coderdtest.Options{
 			AutobuildTicker:          tickCh,
 			IncludeProvisionerDaemon: true,
@@ -145,7 +148,7 @@ func TestExecutorAutostartNotEnabled(t *testing.T) {

 	var (
 		tickCh  = make(chan time.Time)
-		statsCh = make(chan executor.Stats)
+		statsCh = make(chan autobuild.Stats)
 		client  = coderdtest.New(t, &coderdtest.Options{
 			AutobuildTicker:          tickCh,
 			IncludeProvisionerDaemon: true,
@@ -180,7 +183,7 @@ func TestExecutorAutostopOK(t *testing.T) {

 	var (
 		tickCh  = make(chan time.Time)
-		statsCh = make(chan executor.Stats)
+		statsCh = make(chan autobuild.Stats)
 		client  = coderdtest.New(t, &coderdtest.Options{
 			AutobuildTicker:          tickCh,
 			IncludeProvisionerDaemon: true,
@@ -216,7 +219,7 @@ func TestExecutorAutostopExtend(t *testing.T) {
 	var (
 		ctx     = context.Background()
 		tickCh  = make(chan time.Time)
-		statsCh = make(chan executor.Stats)
+		statsCh = make(chan autobuild.Stats)
 		client  = coderdtest.New(t, &coderdtest.Options{
 			AutobuildTicker:          tickCh,
 			IncludeProvisionerDaemon: true,
@@ -266,7 +269,7 @@ func TestExecutorAutostopAlreadyStopped(t *testing.T) {

 	var (
 		tickCh  = make(chan time.Time)
-		statsCh = make(chan executor.Stats)
+		statsCh = make(chan autobuild.Stats)
 		client  = coderdtest.New(t, &coderdtest.Options{
 			AutobuildTicker:          tickCh,
 			IncludeProvisionerDaemon: true,
@@ -299,7 +302,7 @@ func TestExecutorAutostopNotEnabled(t *testing.T) {
 	var (
 		ctx     = context.Background()
 		tickCh  = make(chan time.Time)
-		statsCh = make(chan executor.Stats)
+		statsCh = make(chan autobuild.Stats)
 		client  = coderdtest.New(t, &coderdtest.Options{
 			AutobuildTicker:          tickCh,
 			IncludeProvisionerDaemon: true,
@@ -341,7 +344,7 @@ func TestExecutorWorkspaceDeleted(t *testing.T) {
 	var (
 		sched   = mustSchedule(t, "CRON_TZ=UTC 0 * * * *")
 		tickCh  = make(chan time.Time)
-		statsCh = make(chan executor.Stats)
+		statsCh = make(chan autobuild.Stats)
 		client  = coderdtest.New(t, &coderdtest.Options{
 			AutobuildTicker:          tickCh,
 			IncludeProvisionerDaemon: true,
@@ -374,7 +377,7 @@ func TestExecutorWorkspaceAutostartTooEarly(t *testing.T) {
 	var (
 		sched   = mustSchedule(t, "CRON_TZ=UTC 0 * * * *")
 		tickCh  = make(chan time.Time)
-		statsCh = make(chan executor.Stats)
+		statsCh = make(chan autobuild.Stats)
 		client  = coderdtest.New(t, &coderdtest.Options{
 			AutobuildTicker:          tickCh,
 			IncludeProvisionerDaemon: true,
@@ -405,7 +408,7 @@ func TestExecutorWorkspaceAutostopBeforeDeadline(t *testing.T) {

 	var (
 		tickCh  = make(chan time.Time)
-		statsCh = make(chan executor.Stats)
+		statsCh = make(chan autobuild.Stats)
 		client  = coderdtest.New(t, &coderdtest.Options{
 			AutobuildTicker:          tickCh,
 			IncludeProvisionerDaemon: true,
@@ -433,7 +436,7 @@ func TestExecutorWorkspaceAutostopNoWaitChangedMyMind(t *testing.T) {
 	var (
 		ctx     = context.Background()
 		tickCh  = make(chan time.Time)
-		statsCh = make(chan executor.Stats)
+		statsCh = make(chan autobuild.Stats)
 		client  = coderdtest.New(t, &coderdtest.Options{
 			AutobuildTicker:          tickCh,
 			IncludeProvisionerDaemon: true,
@@ -501,8 +504,8 @@ func TestExecutorAutostartMultipleOK(t *testing.T) {
 		sched    = mustSchedule(t, "CRON_TZ=UTC 0 * * * *")
 		tickCh   = make(chan time.Time)
 		tickCh2  = make(chan time.Time)
-		statsCh1 = make(chan executor.Stats)
-		statsCh2 = make(chan executor.Stats)
+		statsCh1 = make(chan autobuild.Stats)
+		statsCh2 = make(chan autobuild.Stats)
 		client   = coderdtest.New(t, &coderdtest.Options{
 			AutobuildTicker:          tickCh,
 			IncludeProvisionerDaemon: true,
@@ -556,7 +559,7 @@ func TestExecutorAutostartWithParameters(t *testing.T) {
 	var (
 		sched   = mustSchedule(t, "CRON_TZ=UTC 0 * * * *")
 		tickCh  = make(chan time.Time)
-		statsCh = make(chan executor.Stats)
+		statsCh = make(chan autobuild.Stats)
 		client  = coderdtest.New(t, &coderdtest.Options{
 			AutobuildTicker:          tickCh,
 			IncludeProvisionerDaemon: true,
@@ -609,7 +612,7 @@ func TestExecutorAutostartTemplateDisabled(t *testing.T) {
 	var (
 		sched   = mustSchedule(t, "CRON_TZ=UTC 0 * * * *")
 		tickCh  = make(chan time.Time)
-		statsCh = make(chan executor.Stats)
+		statsCh = make(chan autobuild.Stats)

 		client = coderdtest.New(t, &coderdtest.Options{
 			AutobuildTicker:          tickCh,
@@ -648,6 +651,60 @@ func TestExecutorAutostartTemplateDisabled(t *testing.T) {
 	assert.Len(t, stats.Transitions, 0)
 }

+// TesetExecutorFailedWorkspace tests that failed workspaces that breach
+// their template failed_ttl threshold trigger a stop job.
+// For enterprise functionality see enterprise/coderd/workspaces_test.go
+func TestExecutorFailedWorkspace(t *testing.T) {
+	t.Parallel()
+
+	// Test that an AGPL TemplateScheduleStore properly disables
+	// functionality.
+	t.Run("OK", func(t *testing.T) {
+		t.Parallel()
+
+		var (
+			ticker = make(chan time.Time)
+			statCh = make(chan autobuild.Stats)
+			logger = slogtest.Make(t, &slogtest.Options{
+				// We ignore errors here since we expect to fail
+				// builds.
+				IgnoreErrors: true,
+			})
+			failureTTL = time.Millisecond
+
+			client = coderdtest.New(t, &coderdtest.Options{
+				Logger:                   &logger,
+				AutobuildTicker:          ticker,
+				IncludeProvisionerDaemon: true,
+				AutobuildStats:           statCh,
+				TemplateScheduleStore:    schedule.NewAGPLTemplateScheduleStore(),
+			})
+		)
+		user := coderdtest.CreateFirstUser(t, client)
+		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
+			Parse:          echo.ParseComplete,
+			ProvisionPlan:  echo.ProvisionComplete,
+			ProvisionApply: echo.ProvisionFailed,
+		})
+		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
+			ctr.FailureTTLMillis = ptr.Ref[int64](failureTTL.Milliseconds())
+		})
+		coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
+		ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
+		build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
+		require.Equal(t, codersdk.WorkspaceStatusFailed, build.Status)
+		require.Eventually(t,
+			func() bool {
+				return database.Now().Sub(*build.Job.CompletedAt) > failureTTL
+			},
+			testutil.IntervalMedium, testutil.IntervalFast)
+		ticker <- time.Now()
+		stats := <-statCh
+		// Expect no transitions since we're using AGPL.
+		require.Len(t, stats.Transitions, 0)
+	})
+}
+
 func mustProvisionWorkspace(t *testing.T, client *codersdk.Client, mut ...func(*codersdk.CreateWorkspaceRequest)) codersdk.Workspace {
 	t.Helper()
 	user := coderdtest.CreateFirstUser(t, client)