fix: prevent db deadlock when workspaces go dormant (#10618)

coderd/autobuild/lifecycle_executor.go

@@ -140,42 +140,45 @@ func (e *Executor) runOnce(t time.Time) Stats {
 
 		eg.Go(func() error {
 			var job *database.ProvisionerJob
+			var auditLog *auditParams
 			err := e.db.InTx(func(tx database.Store) error {
 				// Re-check eligibility since the first check was outside the
 				// transaction and the workspace settings may have changed.
 				ws, err := tx.GetWorkspaceByID(e.ctx, wsID)
 				if err != nil {
-					log.Error(e.ctx, "get workspace autostart failed", slog.Error(err))
-					return nil
+					return xerrors.Errorf("get workspace by id: %w", err)
 				}
 
 				// Determine the workspace state based on its latest build.
 				latestBuild, err := tx.GetLatestWorkspaceBuildByWorkspaceID(e.ctx, ws.ID)
 				if err != nil {
-					log.Warn(e.ctx, "get latest workspace build", slog.Error(err))
-					return nil
+					return xerrors.Errorf("get latest workspace build: %w", err)
 				}
+
+				latestJob, err := tx.GetProvisionerJobByID(e.ctx, latestBuild.JobID)
+				if err != nil {
+					return xerrors.Errorf("get latest provisioner job: %w", err)
+				}
+
 				templateSchedule, err := (*(e.templateScheduleStore.Load())).Get(e.ctx, tx, ws.TemplateID)
 				if err != nil {
-					log.Warn(e.ctx, "get template schedule options", slog.Error(err))
-					return nil
+					return xerrors.Errorf("get template scheduling options: %w", err)
 				}
 
 				template, err := tx.GetTemplateByID(e.ctx, ws.TemplateID)
 				if err != nil {
-					log.Warn(e.ctx, "get template by id", slog.Error(err))
+					return xerrors.Errorf("get template by ID: %w", err)
 				}
-				accessControl := (*(e.accessControlStore.Load())).GetTemplateAccessControl(template)
 
-				latestJob, err := tx.GetProvisionerJobByID(e.ctx, latestBuild.JobID)
-				if err != nil {
-					log.Warn(e.ctx, "get last provisioner job for workspace %q: %w", slog.Error(err))
-					return nil
-				}
+				accessControl := (*(e.accessControlStore.Load())).GetTemplateAccessControl(template)
 
 				nextTransition, reason, err := getNextTransition(ws, latestBuild, latestJob, templateSchedule, currentTick)
 				if err != nil {
 					log.Debug(e.ctx, "skipping workspace", slog.Error(err))
+					// err is used to indicate that a workspace is not eligible
+					// so returning nil here is ok although ultimately the distinction
+					// doesn't matter since the transaction is  read-only up to
+					// this point.
 					return nil
 				}
 
@@ -193,17 +196,16 @@ func (e *Executor) runOnce(t time.Time) Stats {
 					}
 
 					build, job, err = builder.Build(e.ctx, tx, nil, audit.WorkspaceBuildBaggage{IP: "127.0.0.1"})
-
 					if err != nil {
 						log.Error(e.ctx, "unable to transition workspace",
 							slog.F("transition", nextTransition),
 							slog.Error(err),
 						)
-						return nil
+						return xerrors.Errorf("build workspace: %w", err)
 					}
 				}
 
-				//  Transition the workspace to dormant if it has breached the template's
+				// Transition the workspace to dormant if it has breached the template's
 				// threshold for inactivity.
 				if reason == database.BuildReasonAutolock {
 					wsOld := ws
@@ -215,21 +217,15 @@ func (e *Executor) runOnce(t time.Time) Stats {
 						},
 					})
 
-					auditBuild(e.ctx, e.log, *e.auditor.Load(), auditParams{
-						Build:   build,
-						Job:     latestJob,
-						Reason:  reason,
-						Old:     wsOld,
-						New:     ws,
-						Success: err == nil,
-					})
-
+					auditLog = &auditParams{
+						Build:  build,
+						Job:    latestJob,
+						Reason: reason,
+						Old:    wsOld,
+						New:    ws,
+					}
 					if err != nil {
-						log.Error(e.ctx, "unable to transition workspace to dormant",
-							slog.F("transition", nextTransition),
-							slog.Error(err),
-						)
-						return nil
+						return xerrors.Errorf("update workspace dormant deleting at: %w", err)
 					}
 
 					log.Info(e.ctx, "dormant workspace",
@@ -267,6 +263,12 @@ func (e *Executor) runOnce(t time.Time) Stats {
 			if err != nil {
 				log.Error(e.ctx, "workspace scheduling failed", slog.Error(err))
 			}
+			if auditLog != nil {
+				// If the transition didn't succeed then updating the workspace
+				// to indicate dormant didn't either.
+				auditLog.Success = err == nil
+				auditBuild(e.ctx, e.log, *e.auditor.Load(), *auditLog)
+			}
 			if job != nil && err == nil {
 				// Note that we can't refactor such that posting the job happens inside wsbuilder because it's called
 				// with an outer transaction like this, and we need to make sure the outer transaction commits before