diff --git a/coderd/database/dbauthz/dbauthz.go b/coderd/database/dbauthz/dbauthz.go index 01d3c4f26a..2081fbf5a8 100644 --- a/coderd/database/dbauthz/dbauthz.go +++ b/coderd/database/dbauthz/dbauthz.go @@ -4933,10 +4933,10 @@ func (q *querier) UpdateOrganizationDeletedByID(ctx context.Context, arg databas return deleteQ(q.log, q.auth, q.db.GetOrganizationByID, deleteF)(ctx, arg.ID) } -func (q *querier) UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg database.UpdatePrebuildProvisionerJobWithCancelParams) ([]uuid.UUID, error) { +func (q *querier) UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg database.UpdatePrebuildProvisionerJobWithCancelParams) ([]database.UpdatePrebuildProvisionerJobWithCancelRow, error) { // Prebuild operation for canceling pending prebuild jobs from non-active template versions if err := q.authorizeContext(ctx, policy.ActionUpdate, rbac.ResourcePrebuiltWorkspace); err != nil { - return []uuid.UUID{}, err + return []database.UpdatePrebuildProvisionerJobWithCancelRow{}, err } return q.db.UpdatePrebuildProvisionerJobWithCancel(ctx, arg) } diff --git a/coderd/database/dbauthz/dbauthz_test.go b/coderd/database/dbauthz/dbauthz_test.go index 8cf622a434..63226271f7 100644 --- a/coderd/database/dbauthz/dbauthz_test.go +++ b/coderd/database/dbauthz/dbauthz_test.go @@ -646,10 +646,13 @@ func (s *MethodTestSuite) TestProvisionerJob() { PresetID: uuid.NullUUID{UUID: uuid.New(), Valid: true}, Now: dbtime.Now(), } - jobIDs := []uuid.UUID{uuid.New(), uuid.New()} + canceledJobs := []database.UpdatePrebuildProvisionerJobWithCancelRow{ + {ID: uuid.New(), WorkspaceID: uuid.New(), TemplateID: uuid.New(), TemplateVersionPresetID: uuid.NullUUID{UUID: uuid.New(), Valid: true}}, + {ID: uuid.New(), WorkspaceID: uuid.New(), TemplateID: uuid.New(), TemplateVersionPresetID: uuid.NullUUID{UUID: uuid.New(), Valid: true}}, + } - dbm.EXPECT().UpdatePrebuildProvisionerJobWithCancel(gomock.Any(), arg).Return(jobIDs, nil).AnyTimes() - check.Args(arg).Asserts(rbac.ResourcePrebuiltWorkspace, policy.ActionUpdate).Returns(jobIDs) + dbm.EXPECT().UpdatePrebuildProvisionerJobWithCancel(gomock.Any(), arg).Return(canceledJobs, nil).AnyTimes() + check.Args(arg).Asserts(rbac.ResourcePrebuiltWorkspace, policy.ActionUpdate).Returns(canceledJobs) })) s.Run("GetProvisionerJobsByIDs", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) { org := testutil.Fake(s.T(), faker, database.Organization{}) diff --git a/coderd/database/dbmetrics/querymetrics.go b/coderd/database/dbmetrics/querymetrics.go index 1bd8fda624..1b36a3fa98 100644 --- a/coderd/database/dbmetrics/querymetrics.go +++ b/coderd/database/dbmetrics/querymetrics.go @@ -3042,7 +3042,7 @@ func (m queryMetricsStore) UpdateOrganizationDeletedByID(ctx context.Context, ar return r0 } -func (m queryMetricsStore) UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg database.UpdatePrebuildProvisionerJobWithCancelParams) ([]uuid.UUID, error) { +func (m queryMetricsStore) UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg database.UpdatePrebuildProvisionerJobWithCancelParams) ([]database.UpdatePrebuildProvisionerJobWithCancelRow, error) { start := time.Now() r0, r1 := m.s.UpdatePrebuildProvisionerJobWithCancel(ctx, arg) m.queryLatencies.WithLabelValues("UpdatePrebuildProvisionerJobWithCancel").Observe(time.Since(start).Seconds()) diff --git a/coderd/database/dbmock/dbmock.go b/coderd/database/dbmock/dbmock.go index 1983092aa5..0ecd0191cb 100644 --- a/coderd/database/dbmock/dbmock.go +++ b/coderd/database/dbmock/dbmock.go @@ -6540,10 +6540,10 @@ func (mr *MockStoreMockRecorder) UpdateOrganizationDeletedByID(ctx, arg any) *go } // UpdatePrebuildProvisionerJobWithCancel mocks base method. -func (m *MockStore) UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg database.UpdatePrebuildProvisionerJobWithCancelParams) ([]uuid.UUID, error) { +func (m *MockStore) UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg database.UpdatePrebuildProvisionerJobWithCancelParams) ([]database.UpdatePrebuildProvisionerJobWithCancelRow, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "UpdatePrebuildProvisionerJobWithCancel", ctx, arg) - ret0, _ := ret[0].([]uuid.UUID) + ret0, _ := ret[0].([]database.UpdatePrebuildProvisionerJobWithCancelRow) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/coderd/database/querier.go b/coderd/database/querier.go index 2b96823028..2aa1cb8650 100644 --- a/coderd/database/querier.go +++ b/coderd/database/querier.go @@ -667,7 +667,7 @@ type sqlcQuerier interface { // Cancels all pending provisioner jobs for prebuilt workspaces on a specific preset from an // inactive template version. // This is an optimization to clean up stale pending jobs. - UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg UpdatePrebuildProvisionerJobWithCancelParams) ([]uuid.UUID, error) + UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg UpdatePrebuildProvisionerJobWithCancelParams) ([]UpdatePrebuildProvisionerJobWithCancelRow, error) UpdatePresetPrebuildStatus(ctx context.Context, arg UpdatePresetPrebuildStatusParams) error UpdateProvisionerDaemonLastSeenAt(ctx context.Context, arg UpdateProvisionerDaemonLastSeenAtParams) error UpdateProvisionerJobByID(ctx context.Context, arg UpdateProvisionerJobByIDParams) error diff --git a/coderd/database/queries.sql.go b/coderd/database/queries.sql.go index 2005f98347..2869316266 100644 --- a/coderd/database/queries.sql.go +++ b/coderd/database/queries.sql.go @@ -8687,12 +8687,8 @@ func (q *sqlQuerier) GetTemplatePresetsWithPrebuilds(ctx context.Context, templa } const updatePrebuildProvisionerJobWithCancel = `-- name: UpdatePrebuildProvisionerJobWithCancel :many -UPDATE provisioner_jobs -SET - canceled_at = $1::timestamptz, - completed_at = $1::timestamptz -WHERE id IN ( - SELECT pj.id +WITH jobs_to_cancel AS ( + SELECT pj.id, w.id AS workspace_id, w.template_id, wpb.template_version_preset_id FROM provisioner_jobs pj INNER JOIN workspace_prebuild_builds wpb ON wpb.job_id = pj.id INNER JOIN workspaces w ON w.id = wpb.workspace_id @@ -8711,7 +8707,13 @@ WHERE id IN ( AND pj.canceled_at IS NULL AND pj.completed_at IS NULL ) -RETURNING id +UPDATE provisioner_jobs +SET + canceled_at = $1::timestamptz, + completed_at = $1::timestamptz +FROM jobs_to_cancel +WHERE provisioner_jobs.id = jobs_to_cancel.id +RETURNING jobs_to_cancel.id, jobs_to_cancel.workspace_id, jobs_to_cancel.template_id, jobs_to_cancel.template_version_preset_id ` type UpdatePrebuildProvisionerJobWithCancelParams struct { @@ -8719,22 +8721,34 @@ type UpdatePrebuildProvisionerJobWithCancelParams struct { PresetID uuid.NullUUID `db:"preset_id" json:"preset_id"` } +type UpdatePrebuildProvisionerJobWithCancelRow struct { + ID uuid.UUID `db:"id" json:"id"` + WorkspaceID uuid.UUID `db:"workspace_id" json:"workspace_id"` + TemplateID uuid.UUID `db:"template_id" json:"template_id"` + TemplateVersionPresetID uuid.NullUUID `db:"template_version_preset_id" json:"template_version_preset_id"` +} + // Cancels all pending provisioner jobs for prebuilt workspaces on a specific preset from an // inactive template version. // This is an optimization to clean up stale pending jobs. -func (q *sqlQuerier) UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg UpdatePrebuildProvisionerJobWithCancelParams) ([]uuid.UUID, error) { +func (q *sqlQuerier) UpdatePrebuildProvisionerJobWithCancel(ctx context.Context, arg UpdatePrebuildProvisionerJobWithCancelParams) ([]UpdatePrebuildProvisionerJobWithCancelRow, error) { rows, err := q.db.QueryContext(ctx, updatePrebuildProvisionerJobWithCancel, arg.Now, arg.PresetID) if err != nil { return nil, err } defer rows.Close() - var items []uuid.UUID + var items []UpdatePrebuildProvisionerJobWithCancelRow for rows.Next() { - var id uuid.UUID - if err := rows.Scan(&id); err != nil { + var i UpdatePrebuildProvisionerJobWithCancelRow + if err := rows.Scan( + &i.ID, + &i.WorkspaceID, + &i.TemplateID, + &i.TemplateVersionPresetID, + ); err != nil { return nil, err } - items = append(items, id) + items = append(items, i) } if err := rows.Close(); err != nil { return nil, err diff --git a/coderd/database/queries/prebuilds.sql b/coderd/database/queries/prebuilds.sql index 6c5520c9da..7c060971ef 100644 --- a/coderd/database/queries/prebuilds.sql +++ b/coderd/database/queries/prebuilds.sql @@ -300,12 +300,8 @@ GROUP BY wpb.template_version_preset_id; -- Cancels all pending provisioner jobs for prebuilt workspaces on a specific preset from an -- inactive template version. -- This is an optimization to clean up stale pending jobs. -UPDATE provisioner_jobs -SET - canceled_at = @now::timestamptz, - completed_at = @now::timestamptz -WHERE id IN ( - SELECT pj.id +WITH jobs_to_cancel AS ( + SELECT pj.id, w.id AS workspace_id, w.template_id, wpb.template_version_preset_id FROM provisioner_jobs pj INNER JOIN workspace_prebuild_builds wpb ON wpb.job_id = pj.id INNER JOIN workspaces w ON w.id = wpb.workspace_id @@ -324,4 +320,10 @@ WHERE id IN ( AND pj.canceled_at IS NULL AND pj.completed_at IS NULL ) -RETURNING id; +UPDATE provisioner_jobs +SET + canceled_at = @now::timestamptz, + completed_at = @now::timestamptz +FROM jobs_to_cancel +WHERE provisioner_jobs.id = jobs_to_cancel.id +RETURNING jobs_to_cancel.id, jobs_to_cancel.workspace_id, jobs_to_cancel.template_id, jobs_to_cancel.template_version_preset_id; diff --git a/enterprise/coderd/prebuilds/reconcile.go b/enterprise/coderd/prebuilds/reconcile.go index ceb16061bd..5e5eec68ab 100644 --- a/enterprise/coderd/prebuilds/reconcile.go +++ b/enterprise/coderd/prebuilds/reconcile.go @@ -57,6 +57,24 @@ type StoreReconciler struct { var _ prebuilds.ReconciliationOrchestrator = &StoreReconciler{} +type DeprovisionMode int + +const ( + DeprovisionModeNormal DeprovisionMode = iota + DeprovisionModeOrphan +) + +func (d DeprovisionMode) String() string { + switch d { + case DeprovisionModeOrphan: + return "orphan" + case DeprovisionModeNormal: + return "normal" + default: + return "unknown" + } +} + func NewStoreReconciler(store database.Store, ps pubsub.Pubsub, fileCache *files.Cache, @@ -642,34 +660,7 @@ func (c *StoreReconciler) executeReconciliationAction(ctx context.Context, logge return multiErr.ErrorOrNil() case prebuilds.ActionTypeCancelPending: - // Cancel pending prebuild jobs from non-active template versions to avoid - // provisioning obsolete workspaces that would immediately be deprovisioned. - // This uses a criteria-based update to ensure only jobs that are still pending - // at execution time are canceled, avoiding race conditions where jobs may have - // transitioned to running status between query and update. - canceledJobs, err := c.store.UpdatePrebuildProvisionerJobWithCancel( - ctx, - database.UpdatePrebuildProvisionerJobWithCancelParams{ - Now: c.clock.Now(), - PresetID: uuid.NullUUID{ - UUID: ps.Preset.ID, - Valid: true, - }, - }) - if err != nil { - logger.Error(ctx, "failed to cancel pending prebuild jobs", - slog.F("template_version_id", ps.Preset.TemplateVersionID.String()), - slog.F("preset_id", ps.Preset.ID), - slog.Error(err)) - return err - } - if len(canceledJobs) > 0 { - logger.Info(ctx, "canceled pending prebuild jobs for inactive version", - slog.F("template_version_id", ps.Preset.TemplateVersionID.String()), - slog.F("preset_id", ps.Preset.ID), - slog.F("count", len(canceledJobs))) - } - return nil + return c.cancelAndOrphanDeletePendingPrebuilds(ctx, ps.Preset.TemplateID, ps.Preset.TemplateVersionID, ps.Preset.ID) default: return xerrors.Errorf("unknown action type: %v", action.ActionType) @@ -717,7 +708,91 @@ func (c *StoreReconciler) createPrebuiltWorkspace(ctx context.Context, prebuiltW c.logger.Info(ctx, "attempting to create prebuild", slog.F("name", name), slog.F("workspace_id", prebuiltWorkspaceID.String()), slog.F("preset_id", presetID.String())) - return c.provision(ctx, db, prebuiltWorkspaceID, template, presetID, database.WorkspaceTransitionStart, workspace) + return c.provision(ctx, db, prebuiltWorkspaceID, template, presetID, database.WorkspaceTransitionStart, workspace, DeprovisionModeNormal) + }, &database.TxOptions{ + Isolation: sql.LevelRepeatableRead, + ReadOnly: false, + }) +} + +// provisionDelete provisions a delete transition for a prebuilt workspace. +// +// If mode is DeprovisionModeOrphan, the builder will not send Terraform state to the provisioner. +// This allows the workspace to be deleted even when no provisioners are available, and is safe +// when no Terraform resources were actually created (e.g., for pending prebuilds that were canceled +// before provisioning started). +// +// IMPORTANT: This function must be called within a database transaction. It does not create its own transaction. +// The caller is responsible for managing the transaction boundary via db.InTx(). +func (c *StoreReconciler) provisionDelete(ctx context.Context, db database.Store, workspaceID uuid.UUID, templateID uuid.UUID, presetID uuid.UUID, mode DeprovisionMode) error { + workspace, err := db.GetWorkspaceByID(ctx, workspaceID) + if err != nil { + return xerrors.Errorf("get workspace by ID: %w", err) + } + + template, err := db.GetTemplateByID(ctx, templateID) + if err != nil { + return xerrors.Errorf("failed to get template: %w", err) + } + + if workspace.OwnerID != database.PrebuildsSystemUserID { + return xerrors.Errorf("prebuilt workspace is not owned by prebuild user anymore, probably it was claimed") + } + + c.logger.Info(ctx, "attempting to delete prebuild", slog.F("orphan", mode.String()), + slog.F("name", workspace.Name), slog.F("workspace_id", workspaceID.String()), slog.F("preset_id", presetID.String())) + + return c.provision(ctx, db, workspaceID, template, presetID, + database.WorkspaceTransitionDelete, workspace, mode) +} + +// cancelAndOrphanDeletePendingPrebuilds cancels pending prebuild jobs from inactive template versions +// and orphan-deletes their associated workspaces. +// +// The cancel operation uses a criteria-based update to ensure only jobs that are still pending at +// execution time are canceled, avoiding race conditions where jobs may have transitioned to running. +// +// Since these jobs were never processed by a provisioner, no Terraform resources were created, +// making it safe to orphan-delete the workspaces (skipping Terraform destroy). +func (c *StoreReconciler) cancelAndOrphanDeletePendingPrebuilds(ctx context.Context, templateID uuid.UUID, templateVersionID uuid.UUID, presetID uuid.UUID) error { + return c.store.InTx(func(db database.Store) error { + canceledJobs, err := db.UpdatePrebuildProvisionerJobWithCancel( + ctx, + database.UpdatePrebuildProvisionerJobWithCancelParams{ + Now: c.clock.Now(), + PresetID: uuid.NullUUID{ + UUID: presetID, + Valid: true, + }, + }) + if err != nil { + c.logger.Error(ctx, "failed to cancel pending prebuild jobs", + slog.F("template_id", templateID.String()), + slog.F("template_version_id", templateVersionID.String()), + slog.F("preset_id", presetID.String()), + slog.Error(err)) + return err + } + + if len(canceledJobs) > 0 { + c.logger.Info(ctx, "canceled pending prebuild jobs for inactive version", + slog.F("template_id", templateID.String()), + slog.F("template_version_id", templateVersionID.String()), + slog.F("preset_id", presetID.String()), + slog.F("count", len(canceledJobs))) + } + + var multiErr multierror.Error + for _, job := range canceledJobs { + err = c.provisionDelete(ctx, db, job.WorkspaceID, job.TemplateID, presetID, DeprovisionModeOrphan) + if err != nil { + c.logger.Error(ctx, "failed to orphan delete canceled prebuild", + slog.F("workspace_id", job.WorkspaceID.String()), slog.Error(err)) + multiErr.Errors = append(multiErr.Errors, err) + } + } + + return multiErr.ErrorOrNil() }, &database.TxOptions{ Isolation: sql.LevelRepeatableRead, ReadOnly: false, @@ -726,24 +801,7 @@ func (c *StoreReconciler) createPrebuiltWorkspace(ctx context.Context, prebuiltW func (c *StoreReconciler) deletePrebuiltWorkspace(ctx context.Context, prebuiltWorkspaceID uuid.UUID, templateID uuid.UUID, presetID uuid.UUID) error { return c.store.InTx(func(db database.Store) error { - workspace, err := db.GetWorkspaceByID(ctx, prebuiltWorkspaceID) - if err != nil { - return xerrors.Errorf("get workspace by ID: %w", err) - } - - template, err := db.GetTemplateByID(ctx, templateID) - if err != nil { - return xerrors.Errorf("failed to get template: %w", err) - } - - if workspace.OwnerID != database.PrebuildsSystemUserID { - return xerrors.Errorf("prebuilt workspace is not owned by prebuild user anymore, probably it was claimed") - } - - c.logger.Info(ctx, "attempting to delete prebuild", - slog.F("workspace_id", prebuiltWorkspaceID.String()), slog.F("preset_id", presetID.String())) - - return c.provision(ctx, db, prebuiltWorkspaceID, template, presetID, database.WorkspaceTransitionDelete, workspace) + return c.provisionDelete(ctx, db, prebuiltWorkspaceID, templateID, presetID, DeprovisionModeNormal) }, &database.TxOptions{ Isolation: sql.LevelRepeatableRead, ReadOnly: false, @@ -758,6 +816,7 @@ func (c *StoreReconciler) provision( presetID uuid.UUID, transition database.WorkspaceTransition, workspace database.Workspace, + mode DeprovisionMode, ) error { tvp, err := db.GetPresetParametersByTemplateVersionID(ctx, template.ActiveVersionID) if err != nil { @@ -795,6 +854,11 @@ func (c *StoreReconciler) provision( builder = builder.RichParameterValues(params) } + // Use orphan mode for deletes when no Terraform resources exist + if transition == database.WorkspaceTransitionDelete && mode == DeprovisionModeOrphan { + builder = builder.Orphan() + } + _, provisionerJob, _, err := builder.Build( ctx, db, diff --git a/enterprise/coderd/prebuilds/reconcile_test.go b/enterprise/coderd/prebuilds/reconcile_test.go index 33b99145d8..4b359ba9df 100644 --- a/enterprise/coderd/prebuilds/reconcile_test.go +++ b/enterprise/coderd/prebuilds/reconcile_test.go @@ -204,7 +204,10 @@ func TestPrebuildReconciliation(t *testing.T) { templateDeleted: []bool{false}, }, { - name: "never attempt to interfere with active builds", + // TODO(ssncferreira): Investigate why the GetRunningPrebuiltWorkspaces query is returning 0 rows. + // When a template version is inactive (templateVersionActive = false), any prebuilds in the + // database.ProvisionerJobStatusRunning state should be deleted. + name: "never attempt to interfere with prebuilds from an active template version", // The workspace builder does not allow scheduling a new build if there is already a build // pending, running, or canceling. As such, we should never attempt to start, stop or delete // such prebuilds. Rather, we should wait for the existing build to complete and reconcile @@ -215,7 +218,7 @@ func TestPrebuildReconciliation(t *testing.T) { database.ProvisionerJobStatusRunning, database.ProvisionerJobStatusCanceling, }, - templateVersionActive: []bool{true, false}, + templateVersionActive: []bool{true}, shouldDeleteOldPrebuild: ptr.To(false), templateDeleted: []bool{false}, }, @@ -2121,16 +2124,16 @@ func TestCancelPendingPrebuilds(t *testing.T) { }, }).SkipCreateTemplate().Do() - var workspace dbfake.WorkspaceResponse + var pendingWorkspace dbfake.WorkspaceResponse if tt.activeTemplateVersion { // Given: a prebuilt workspace, workspace build and respective provisioner job from an // active template version - workspace = tt.setupBuild(t, db, client, + pendingWorkspace = tt.setupBuild(t, db, client, owner.OrganizationID, templateID, activeTemplateVersion.TemplateVersion.ID, activePresetID) } else { // Given: a prebuilt workspace, workspace build and respective provisioner job from a // non-active template version - workspace = tt.setupBuild(t, db, client, + pendingWorkspace = tt.setupBuild(t, db, client, owner.OrganizationID, templateID, nonActiveTemplateVersion.TemplateVersion.ID, nonActivePresetID) } @@ -2145,15 +2148,28 @@ func TestCancelPendingPrebuilds(t *testing.T) { require.NoError(t, reconciler.ReconcileAll(ctx)) if tt.shouldCancel { - // Then: the prebuild related jobs from non-active version should be canceled - cancelledJob, err := db.GetProvisionerJobByID(ctx, workspace.Build.JobID) + // Then: the pending prebuild job from non-active version should be canceled + cancelledJob, err := db.GetProvisionerJobByID(ctx, pendingWorkspace.Build.JobID) require.NoError(t, err) require.Equal(t, clock.Now().UTC(), cancelledJob.CanceledAt.Time.UTC()) require.Equal(t, clock.Now().UTC(), cancelledJob.CompletedAt.Time.UTC()) require.Equal(t, database.ProvisionerJobStatusCanceled, cancelledJob.JobStatus) + + // Then: the workspace should be deleted + deletedWorkspace, err := db.GetWorkspaceByID(ctx, pendingWorkspace.Workspace.ID) + require.NoError(t, err) + require.True(t, deletedWorkspace.Deleted) + latestBuild, err := db.GetLatestWorkspaceBuildByWorkspaceID(ctx, deletedWorkspace.ID) + require.NoError(t, err) + require.Equal(t, database.WorkspaceTransitionDelete, latestBuild.Transition) + deleteJob, err := db.GetProvisionerJobByID(ctx, latestBuild.JobID) + require.NoError(t, err) + require.True(t, deleteJob.CompletedAt.Valid) + require.False(t, deleteJob.WorkerID.Valid) + require.Equal(t, database.ProvisionerJobStatusSucceeded, deleteJob.JobStatus) } else { - // Then: the provisioner job should not be canceled - job, err := db.GetProvisionerJobByID(ctx, workspace.Build.JobID) + // Then: the pending prebuild job should not be canceled + job, err := db.GetProvisionerJobByID(ctx, pendingWorkspace.Build.JobID) require.NoError(t, err) if !tt.previouslyCanceled { require.Zero(t, job.CanceledAt.Time.UTC()) @@ -2162,6 +2178,11 @@ func TestCancelPendingPrebuilds(t *testing.T) { if !tt.previouslyCompleted { require.Zero(t, job.CompletedAt.Time.UTC()) } + + // Then: the workspace should not be deleted + workspace, err := db.GetWorkspaceByID(ctx, pendingWorkspace.Workspace.ID) + require.NoError(t, err) + require.False(t, workspace.Deleted) } }) } @@ -2235,25 +2256,45 @@ func TestCancelPendingPrebuilds(t *testing.T) { return prebuilds } - checkIfJobCanceled := func( + checkIfJobCanceledAndDeleted := func( t *testing.T, clock *quartz.Mock, ctx context.Context, db database.Store, - shouldBeCanceled bool, + shouldBeCanceledAndDeleted bool, prebuilds []dbfake.WorkspaceResponse, ) { for _, prebuild := range prebuilds { - job, err := db.GetProvisionerJobByID(ctx, prebuild.Build.JobID) + pendingJob, err := db.GetProvisionerJobByID(ctx, prebuild.Build.JobID) require.NoError(t, err) - if shouldBeCanceled { - require.Equal(t, database.ProvisionerJobStatusCanceled, job.JobStatus) - require.Equal(t, clock.Now().UTC(), job.CanceledAt.Time.UTC()) - require.Equal(t, clock.Now().UTC(), job.CompletedAt.Time.UTC()) + if shouldBeCanceledAndDeleted { + // Pending job should be canceled + require.Equal(t, database.ProvisionerJobStatusCanceled, pendingJob.JobStatus) + require.Equal(t, clock.Now().UTC(), pendingJob.CanceledAt.Time.UTC()) + require.Equal(t, clock.Now().UTC(), pendingJob.CompletedAt.Time.UTC()) + + // Workspace should be deleted + deletedWorkspace, err := db.GetWorkspaceByID(ctx, prebuild.Workspace.ID) + require.NoError(t, err) + require.True(t, deletedWorkspace.Deleted) + latestBuild, err := db.GetLatestWorkspaceBuildByWorkspaceID(ctx, deletedWorkspace.ID) + require.NoError(t, err) + require.Equal(t, database.WorkspaceTransitionDelete, latestBuild.Transition) + deleteJob, err := db.GetProvisionerJobByID(ctx, latestBuild.JobID) + require.NoError(t, err) + require.True(t, deleteJob.CompletedAt.Valid) + require.False(t, deleteJob.WorkerID.Valid) + require.Equal(t, database.ProvisionerJobStatusSucceeded, deleteJob.JobStatus) } else { - require.NotEqual(t, database.ProvisionerJobStatusCanceled, job.JobStatus) - require.Zero(t, job.CanceledAt.Time.UTC()) + // Pending job should not be canceled + require.NotEqual(t, database.ProvisionerJobStatusCanceled, pendingJob.JobStatus) + require.Zero(t, pendingJob.CanceledAt.Time.UTC()) + + // Workspace should not be deleted + workspace, err := db.GetWorkspaceByID(ctx, prebuild.Workspace.ID) + require.NoError(t, err) + require.False(t, workspace.Deleted) } } } @@ -2309,22 +2350,22 @@ func TestCancelPendingPrebuilds(t *testing.T) { require.NoError(t, reconciler.ReconcileAll(ctx)) // Then: template A version 1 running workspaces should not be canceled - checkIfJobCanceled(t, clock, ctx, db, false, templateAVersion1Running) + checkIfJobCanceledAndDeleted(t, clock, ctx, db, false, templateAVersion1Running) // Then: template A version 1 pending workspaces should be canceled - checkIfJobCanceled(t, clock, ctx, db, true, templateAVersion1Pending) + checkIfJobCanceledAndDeleted(t, clock, ctx, db, true, templateAVersion1Pending) // Then: template A version 2 running and pending workspaces should not be canceled - checkIfJobCanceled(t, clock, ctx, db, false, templateAVersion2Running) - checkIfJobCanceled(t, clock, ctx, db, false, templateAVersion2Pending) + checkIfJobCanceledAndDeleted(t, clock, ctx, db, false, templateAVersion2Running) + checkIfJobCanceledAndDeleted(t, clock, ctx, db, false, templateAVersion2Pending) // Then: template B version 1 running workspaces should not be canceled - checkIfJobCanceled(t, clock, ctx, db, false, templateBVersion1Running) + checkIfJobCanceledAndDeleted(t, clock, ctx, db, false, templateBVersion1Running) // Then: template B version 1 pending workspaces should be canceled - checkIfJobCanceled(t, clock, ctx, db, true, templateBVersion1Pending) + checkIfJobCanceledAndDeleted(t, clock, ctx, db, true, templateBVersion1Pending) // Then: template B version 2 pending workspaces should be canceled - checkIfJobCanceled(t, clock, ctx, db, true, templateBVersion2Pending) + checkIfJobCanceledAndDeleted(t, clock, ctx, db, true, templateBVersion2Pending) // Then: template B version 3 running and pending workspaces should not be canceled - checkIfJobCanceled(t, clock, ctx, db, false, templateBVersion3Running) - checkIfJobCanceled(t, clock, ctx, db, false, templateBVersion3Pending) + checkIfJobCanceledAndDeleted(t, clock, ctx, db, false, templateBVersion3Running) + checkIfJobCanceledAndDeleted(t, clock, ctx, db, false, templateBVersion3Pending) }) }