diff --git a/coderd/autobuild/lifecycle_executor.go b/coderd/autobuild/lifecycle_executor.go index 5a141ce8cf..8f742a43f5 100644 --- a/coderd/autobuild/lifecycle_executor.go +++ b/coderd/autobuild/lifecycle_executor.go @@ -576,7 +576,7 @@ func getNextTransition( return database.WorkspaceTransitionStop, database.BuildReasonAutostop, nil case isEligibleForAutostart(user, ws, latestBuild, latestJob, templateSchedule, currentTick): return database.WorkspaceTransitionStart, database.BuildReasonAutostart, nil - case isEligibleForFailedStop(latestBuild, latestJob, templateSchedule, currentTick): + case isEligibleForFailedCleanup(latestBuild, latestJob, templateSchedule, currentTick): // Use task-specific reason for AI task workspaces. if ws.TaskID.Valid { return database.WorkspaceTransitionStop, database.BuildReasonTaskAutoPause, nil @@ -688,14 +688,17 @@ func isEligibleForDelete(ws database.Workspace, templateSchedule schedule.Templa return eligible } -// isEligibleForFailedStop returns true if the workspace is eligible to be stopped -// due to a failed build. -func isEligibleForFailedStop(build database.WorkspaceBuild, job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions, currentTick time.Time) bool { - // If the template has specified a failure TLL. +// isEligibleForFailedCleanup returns true if the workspace is eligible to be +// stopped due to a failed build. A failed start is cleaned up by stopping it, +// and a failed stop is retried by issuing another stop. In both cases the +// remediation is a stop build. +func isEligibleForFailedCleanup(build database.WorkspaceBuild, job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions, currentTick time.Time) bool { + // If the template has specified a failure TTL. return templateSchedule.FailureTTL > 0 && // And the job resulted in failure. job.JobStatus == database.ProvisionerJobStatusFailed && - build.Transition == database.WorkspaceTransitionStart && + (build.Transition == database.WorkspaceTransitionStart || + build.Transition == database.WorkspaceTransitionStop) && // And sufficient time has elapsed since the job has completed. job.CompletedAt.Valid && currentTick.Sub(job.CompletedAt.Time) > templateSchedule.FailureTTL diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index bddca21e0c..ba1611db60 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -36236,15 +36236,20 @@ WHERE END ) OR - -- A workspace may be eligible for failed stop if the following are true: + -- A workspace may be eligible for failed cleanup if the following are true: -- * The template has a failure ttl set. - -- * The workspace build was a start transition. + -- * The workspace build was a start or stop transition. A failed start + -- is cleaned up by stopping it; a failed stop is retried by issuing + -- another stop. -- * The provisioner job failed. -- * The provisioner job had completed. -- * The provisioner job has been completed for longer than the failure ttl. ( templates.failure_ttl > 0 AND - workspace_builds.transition = 'start'::workspace_transition AND + ( + workspace_builds.transition = 'start'::workspace_transition OR + workspace_builds.transition = 'stop'::workspace_transition + ) AND provisioner_jobs.job_status = 'failed'::provisioner_job_status AND provisioner_jobs.completed_at IS NOT NULL AND ($1 :: timestamptz) - provisioner_jobs.completed_at > (INTERVAL '1 millisecond' * (templates.failure_ttl / 1000000)) diff --git a/coderd/database/queries/workspaces.sql b/coderd/database/queries/workspaces.sql index 5269ea8fba..c42caef876 100644 --- a/coderd/database/queries/workspaces.sql +++ b/coderd/database/queries/workspaces.sql @@ -786,15 +786,20 @@ WHERE END ) OR - -- A workspace may be eligible for failed stop if the following are true: + -- A workspace may be eligible for failed cleanup if the following are true: -- * The template has a failure ttl set. - -- * The workspace build was a start transition. + -- * The workspace build was a start or stop transition. A failed start + -- is cleaned up by stopping it; a failed stop is retried by issuing + -- another stop. -- * The provisioner job failed. -- * The provisioner job had completed. -- * The provisioner job has been completed for longer than the failure ttl. ( templates.failure_ttl > 0 AND - workspace_builds.transition = 'start'::workspace_transition AND + ( + workspace_builds.transition = 'start'::workspace_transition OR + workspace_builds.transition = 'stop'::workspace_transition + ) AND provisioner_jobs.job_status = 'failed'::provisioner_job_status AND provisioner_jobs.completed_at IS NOT NULL AND (@now :: timestamptz) - provisioner_jobs.completed_at > (INTERVAL '1 millisecond' * (templates.failure_ttl / 1000000)) diff --git a/enterprise/coderd/workspaces_test.go b/enterprise/coderd/workspaces_test.go index ef71a7227e..1915fabe85 100644 --- a/enterprise/coderd/workspaces_test.go +++ b/enterprise/coderd/workspaces_test.go @@ -653,6 +653,72 @@ func TestWorkspaceAutobuild(t *testing.T) { require.Equal(t, stats.Transitions[ws.ID], database.WorkspaceTransitionStop) }) + // FailureTTLStopOK verifies that a workspace whose latest build is a failed + // stop is retried by issuing another stop after the failure TTL elapses. + t.Run("FailureTTLStopOK", func(t *testing.T) { + t.Parallel() + + var ( + ticker = make(chan time.Time) + statCh = make(chan autobuild.Stats) + logger = slogtest.Make(t, &slogtest.Options{ + // We ignore errors here since we expect to fail + // builds. + IgnoreErrors: true, + }) + failureTTL = time.Minute + ) + + client, db, user := coderdenttest.NewWithDatabase(t, &coderdenttest.Options{ + Options: &coderdtest.Options{ + Logger: &logger, + AutobuildTicker: ticker, + IncludeProvisionerDaemon: true, + AutobuildStats: statCh, + TemplateScheduleStore: schedule.NewEnterpriseTemplateScheduleStore(agplUserQuietHoursScheduleStore(), notifications.NewNoopEnqueuer(), logger, nil), + }, + LicenseOptions: &coderdenttest.LicenseOptions{ + Features: license.Features{codersdk.FeatureAdvancedTemplateScheduling: 1}, + }, + }) + + // The start build succeeds, but the stop build fails. This leaves the + // workspace's latest build as a failed stop. + version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{ + Parse: echo.ParseComplete, + ProvisionPlan: echo.PlanComplete, + ProvisionApplyMap: map[proto.WorkspaceTransition][]*proto.Response{ + proto.WorkspaceTransition_START: echo.ApplyComplete, + proto.WorkspaceTransition_STOP: echo.ApplyFailed, + }, + }) + template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) { + ctr.FailureTTLMillis = ptr.Ref[int64](failureTTL.Milliseconds()) + }) + coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID) + ws := coderdtest.CreateWorkspace(t, client, template.ID) + coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, ws.LatestBuild.ID) + + ctx := testutil.Context(t, testutil.WaitLong) + stopBuild, err := client.CreateWorkspaceBuild(ctx, ws.ID, codersdk.CreateWorkspaceBuildRequest{ + Transition: codersdk.WorkspaceTransitionStop, + }) + require.NoError(t, err) + build := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, stopBuild.ID) + require.Equal(t, codersdk.WorkspaceStatusFailed, build.Status) + require.Equal(t, codersdk.WorkspaceTransitionStop, build.Transition) + tickTime := build.Job.CompletedAt.Add(failureTTL * 2) + + p, err := coderdtest.GetProvisionerForTags(db, time.Now(), ws.OrganizationID, nil) + require.NoError(t, err) + coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, tickTime) + ticker <- tickTime + stats := <-statCh + // Expect the workspace to be stopped again for breaching failure TTL. + require.Len(t, stats.Transitions, 1) + require.Equal(t, stats.Transitions[ws.ID], database.WorkspaceTransitionStop) + }) + t.Run("FailureTTLTooEarly", func(t *testing.T) { t.Parallel()