fix: allow lifecycle code path to retry failed stop jobs (#26203)

Co-authored-by: Mux <mux@coder.com>

coderd/autobuild/lifecycle_executor.go

@@ -576,7 +576,7 @@ func getNextTransition(
 		return database.WorkspaceTransitionStop, database.BuildReasonAutostop, nil
 	case isEligibleForAutostart(user, ws, latestBuild, latestJob, templateSchedule, currentTick):
 		return database.WorkspaceTransitionStart, database.BuildReasonAutostart, nil
-	case isEligibleForFailedStop(latestBuild, latestJob, templateSchedule, currentTick):
+	case isEligibleForFailedCleanup(latestBuild, latestJob, templateSchedule, currentTick):
 		// Use task-specific reason for AI task workspaces.
 		if ws.TaskID.Valid {
 			return database.WorkspaceTransitionStop, database.BuildReasonTaskAutoPause, nil
@@ -688,14 +688,17 @@ func isEligibleForDelete(ws database.Workspace, templateSchedule schedule.Templa
 	return eligible
 }
 
-// isEligibleForFailedStop returns true if the workspace is eligible to be stopped
-// due to a failed build.
-func isEligibleForFailedStop(build database.WorkspaceBuild, job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions, currentTick time.Time) bool {
-	// If the template has specified a failure TLL.
+// isEligibleForFailedCleanup returns true if the workspace is eligible to be
+// stopped due to a failed build. A failed start is cleaned up by stopping it,
+// and a failed stop is retried by issuing another stop. In both cases the
+// remediation is a stop build.
+func isEligibleForFailedCleanup(build database.WorkspaceBuild, job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions, currentTick time.Time) bool {
+	// If the template has specified a failure TTL.
 	return templateSchedule.FailureTTL > 0 &&
 		// And the job resulted in failure.
 		job.JobStatus == database.ProvisionerJobStatusFailed &&
-		build.Transition == database.WorkspaceTransitionStart &&
+		(build.Transition == database.WorkspaceTransitionStart ||
+			build.Transition == database.WorkspaceTransitionStop) &&
 		// And sufficient time has elapsed since the job has completed.
 		job.CompletedAt.Valid &&
 		currentTick.Sub(job.CompletedAt.Time) > templateSchedule.FailureTTL

coderd/database/queries.sql.go

@@ -36236,15 +36236,20 @@ WHERE
 			END
 		) OR
 
-		-- A workspace may be eligible for failed stop if the following are true:
+		-- A workspace may be eligible for failed cleanup if the following are true:
 		--   * The template has a failure ttl set.
-		--   * The workspace build was a start transition.
+		--   * The workspace build was a start or stop transition. A failed start
+		--     is cleaned up by stopping it; a failed stop is retried by issuing
+		--     another stop.
 		--   * The provisioner job failed.
 		--   * The provisioner job had completed.
 		--   * The provisioner job has been completed for longer than the failure ttl.
 		(
 			templates.failure_ttl > 0 AND
-			workspace_builds.transition = 'start'::workspace_transition AND
+			(
+				workspace_builds.transition = 'start'::workspace_transition OR
+				workspace_builds.transition = 'stop'::workspace_transition
+			) AND
 			provisioner_jobs.job_status = 'failed'::provisioner_job_status AND
 			provisioner_jobs.completed_at IS NOT NULL AND
 			($1 :: timestamptz) - provisioner_jobs.completed_at > (INTERVAL '1 millisecond' * (templates.failure_ttl / 1000000))

coderd/database/queries/workspaces.sql

@@ -786,15 +786,20 @@ WHERE
 			END
 		) OR
 
-		-- A workspace may be eligible for failed stop if the following are true:
+		-- A workspace may be eligible for failed cleanup if the following are true:
 		--   * The template has a failure ttl set.
-		--   * The workspace build was a start transition.
+		--   * The workspace build was a start or stop transition. A failed start
+		--     is cleaned up by stopping it; a failed stop is retried by issuing
+		--     another stop.
 		--   * The provisioner job failed.
 		--   * The provisioner job had completed.
 		--   * The provisioner job has been completed for longer than the failure ttl.
 		(
 			templates.failure_ttl > 0 AND
-			workspace_builds.transition = 'start'::workspace_transition AND
+			(
+				workspace_builds.transition = 'start'::workspace_transition OR
+				workspace_builds.transition = 'stop'::workspace_transition
+			) AND
 			provisioner_jobs.job_status = 'failed'::provisioner_job_status AND
 			provisioner_jobs.completed_at IS NOT NULL AND
 			(@now :: timestamptz) - provisioner_jobs.completed_at > (INTERVAL '1 millisecond' * (templates.failure_ttl / 1000000))