mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: cancel pending prebuilds from non-active template versions (#20387)
## Description This PR introduces an optimization to automatically cancel pending prebuild-related jobs from non-active template versions in the reconciliation loop. ## Problem Currently, when a template is configured with more prebuild instances than available provisioners, the provisioner queue can become flooded with pending prebuild jobs. This issue is worsened when provisioning/deprovisioning operations take a long time. When the prebuild reconciliation loop generates jobs faster than provisioners can process them, pending jobs accumulate in the queue. Since prebuilt workspaces should always run the latest active template version, pending prebuild jobs from non-active versions become obsolete once a new version is promoted. ## Solution The reconciliation loop cancels pending prebuild-related jobs from non-active template versions that match the following criteria: * Build number: 1 (initial build created by the reconciliation loop) * Job status: `pending` * Not yet picked up by a provisioner (`worker_id` is `NULL`) * Owned by the prebuilds system user * Workspace transition: `start` This prevents the queue from being cluttered with stale prebuild jobs that would provision workspaces on an outdated template version that would consequently need to be deprovisioned. ## Changes * Added new SQL query `CountPendingNonActivePrebuilds` to identify presets with pending jobs from non-active versions * Added new SQL query `UpdatePrebuildProvisionerJobWithCancel` to cancel jobs for a specific preset * New reconciliation action type `ActionTypeCancelPending` handles the cancellation logic * Cancellation is non-blocking: failures to cancel prebuild jobs are logged as errors and don't prevent other reconciliation actions ## Follow-up PR Canceling pending prebuild jobs leaves workspaces in a Canceled state. While no Terraform resources need to be destroyed (since jobs were canceled before provisioning started), these database records should still be cleaned up. This will be addressed in a follow-up PR. Closes: https://github.com/coder/coder/issues/20242
This commit is contained in:
Susana Ferreira
@@ -12,10 +12,13 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/hashicorp/go-multierror"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/coder/quartz"
|
||||
"cdr.dev/slog"
|
||||
|
||||
"github.com/coder/coder/v2/coderd/audit"
|
||||
"github.com/coder/coder/v2/coderd/database"
|
||||
@@ -30,12 +33,7 @@ import (
|
||||
"github.com/coder/coder/v2/coderd/wsbuilder"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
|
||||
|
||||
"cdr.dev/slog"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"golang.org/x/xerrors"
|
||||
"github.com/coder/quartz"
|
||||
)
|
||||
|
||||
type StoreReconciler struct {
|
||||
@@ -412,6 +410,11 @@ func (c *StoreReconciler) SnapshotState(ctx context.Context, store database.Stor
|
||||
return xerrors.Errorf("failed to get prebuilds in progress: %w", err)
|
||||
}
|
||||
|
||||
allPendingPrebuilds, err := db.CountPendingNonActivePrebuilds(ctx)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to get pending prebuilds: %w", err)
|
||||
}
|
||||
|
||||
presetsBackoff, err := db.GetPresetsBackoff(ctx, c.clock.Now().Add(-c.cfg.ReconciliationBackoffLookback.Value()))
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to get backoffs for presets: %w", err)
|
||||
@@ -427,6 +430,7 @@ func (c *StoreReconciler) SnapshotState(ctx context.Context, store database.Stor
|
||||
presetPrebuildSchedules,
|
||||
allRunningPrebuilds,
|
||||
allPrebuildsInProgress,
|
||||
allPendingPrebuilds,
|
||||
presetsBackoff,
|
||||
hardLimitedPresets,
|
||||
c.clock,
|
||||
@@ -581,6 +585,8 @@ func (c *StoreReconciler) executeReconciliationAction(ctx context.Context, logge
|
||||
levelFn = logger.Info
|
||||
case action.ActionType == prebuilds.ActionTypeDelete && len(action.DeleteIDs) > 0:
|
||||
levelFn = logger.Info
|
||||
case action.ActionType == prebuilds.ActionTypeCancelPending:
|
||||
levelFn = logger.Info
|
||||
}
|
||||
|
||||
switch action.ActionType {
|
||||
@@ -635,6 +641,36 @@ func (c *StoreReconciler) executeReconciliationAction(ctx context.Context, logge
|
||||
|
||||
return multiErr.ErrorOrNil()
|
||||
|
||||
case prebuilds.ActionTypeCancelPending:
|
||||
// Cancel pending prebuild jobs from non-active template versions to avoid
|
||||
// provisioning obsolete workspaces that would immediately be deprovisioned.
|
||||
// This uses a criteria-based update to ensure only jobs that are still pending
|
||||
// at execution time are canceled, avoiding race conditions where jobs may have
|
||||
// transitioned to running status between query and update.
|
||||
canceledJobs, err := c.store.UpdatePrebuildProvisionerJobWithCancel(
|
||||
ctx,
|
||||
database.UpdatePrebuildProvisionerJobWithCancelParams{
|
||||
Now: c.clock.Now(),
|
||||
PresetID: uuid.NullUUID{
|
||||
UUID: ps.Preset.ID,
|
||||
Valid: true,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error(ctx, "failed to cancel pending prebuild jobs",
|
||||
slog.F("template_version_id", ps.Preset.TemplateVersionID.String()),
|
||||
slog.F("preset_id", ps.Preset.ID),
|
||||
slog.Error(err))
|
||||
return err
|
||||
}
|
||||
if len(canceledJobs) > 0 {
|
||||
logger.Info(ctx, "canceled pending prebuild jobs for inactive version",
|
||||
slog.F("template_version_id", ps.Preset.TemplateVersionID.String()),
|
||||
slog.F("preset_id", ps.Preset.ID),
|
||||
slog.F("count", len(canceledJobs)))
|
||||
}
|
||||
return nil
|
||||
|
||||
default:
|
||||
return xerrors.Errorf("unknown action type: %v", action.ActionType)
|
||||
}
|
||||
|
||||
@@ -9,36 +9,35 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/coder/coder/v2/coderd/coderdtest"
|
||||
"github.com/coder/coder/v2/coderd/database/dbtime"
|
||||
"github.com/coder/coder/v2/coderd/files"
|
||||
"github.com/coder/coder/v2/coderd/notifications"
|
||||
"github.com/coder/coder/v2/coderd/notifications/notificationstest"
|
||||
"github.com/coder/coder/v2/coderd/util/slice"
|
||||
"github.com/coder/coder/v2/coderd/wsbuilder"
|
||||
sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/xerrors"
|
||||
"tailscale.com/types/ptr"
|
||||
|
||||
"cdr.dev/slog"
|
||||
"cdr.dev/slog/sloggers/slogtest"
|
||||
"github.com/coder/quartz"
|
||||
|
||||
"github.com/coder/serpent"
|
||||
|
||||
"github.com/coder/coder/v2/coderd/coderdtest"
|
||||
"github.com/coder/coder/v2/coderd/database"
|
||||
"github.com/coder/coder/v2/coderd/database/dbfake"
|
||||
"github.com/coder/coder/v2/coderd/database/dbgen"
|
||||
"github.com/coder/coder/v2/coderd/database/dbtestutil"
|
||||
"github.com/coder/coder/v2/coderd/database/dbtime"
|
||||
"github.com/coder/coder/v2/coderd/database/pubsub"
|
||||
"github.com/coder/coder/v2/coderd/files"
|
||||
"github.com/coder/coder/v2/coderd/notifications"
|
||||
"github.com/coder/coder/v2/coderd/notifications/notificationstest"
|
||||
"github.com/coder/coder/v2/coderd/rbac"
|
||||
"github.com/coder/coder/v2/coderd/util/slice"
|
||||
"github.com/coder/coder/v2/coderd/wsbuilder"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
"github.com/coder/coder/v2/enterprise/coderd/prebuilds"
|
||||
sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
|
||||
"github.com/coder/coder/v2/testutil"
|
||||
"github.com/coder/quartz"
|
||||
"github.com/coder/serpent"
|
||||
)
|
||||
|
||||
func TestNoReconciliationActionsIfNoPresets(t *testing.T) {
|
||||
@@ -1783,6 +1782,552 @@ func TestExpiredPrebuildsMultipleActions(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCancelPendingPrebuilds(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("CancelPendingPrebuilds", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
for _, tt := range []struct {
|
||||
name string
|
||||
setupBuild func(
|
||||
t *testing.T,
|
||||
db database.Store,
|
||||
client *codersdk.Client,
|
||||
orgID uuid.UUID,
|
||||
templateID uuid.UUID,
|
||||
templateVersionID uuid.UUID,
|
||||
presetID uuid.NullUUID,
|
||||
) dbfake.WorkspaceResponse
|
||||
activeTemplateVersion bool
|
||||
previouslyCanceled bool
|
||||
previouslyCompleted bool
|
||||
shouldCancel bool
|
||||
}{
|
||||
// Should cancel pending prebuild-related jobs from a non-active template version
|
||||
{
|
||||
name: "CancelsPendingPrebuildJobNonActiveVersion",
|
||||
// Given: a pending prebuild job
|
||||
setupBuild: func(t *testing.T,
|
||||
db database.Store,
|
||||
client *codersdk.Client,
|
||||
orgID uuid.UUID,
|
||||
templateID uuid.UUID,
|
||||
templateVersionID uuid.UUID,
|
||||
presetID uuid.NullUUID,
|
||||
) dbfake.WorkspaceResponse {
|
||||
return dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
|
||||
OwnerID: database.PrebuildsSystemUserID,
|
||||
OrganizationID: orgID,
|
||||
TemplateID: templateID,
|
||||
}).Pending().Seed(database.WorkspaceBuild{
|
||||
InitiatorID: database.PrebuildsSystemUserID,
|
||||
TemplateVersionID: templateVersionID,
|
||||
TemplateVersionPresetID: presetID,
|
||||
}).Do()
|
||||
},
|
||||
activeTemplateVersion: false,
|
||||
previouslyCanceled: false,
|
||||
previouslyCompleted: false,
|
||||
shouldCancel: true,
|
||||
},
|
||||
// Should not cancel pending prebuild-related jobs from an active template version
|
||||
{
|
||||
name: "DoesNotCancelPendingPrebuildJobActiveVersion",
|
||||
// Given: a pending prebuild job
|
||||
setupBuild: func(t *testing.T,
|
||||
db database.Store,
|
||||
client *codersdk.Client,
|
||||
orgID uuid.UUID,
|
||||
templateID uuid.UUID,
|
||||
templateVersionID uuid.UUID,
|
||||
presetID uuid.NullUUID,
|
||||
) dbfake.WorkspaceResponse {
|
||||
return dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
|
||||
OwnerID: database.PrebuildsSystemUserID,
|
||||
OrganizationID: orgID,
|
||||
TemplateID: templateID,
|
||||
}).Pending().Seed(database.WorkspaceBuild{
|
||||
InitiatorID: database.PrebuildsSystemUserID,
|
||||
TemplateVersionID: templateVersionID,
|
||||
TemplateVersionPresetID: presetID,
|
||||
}).Do()
|
||||
},
|
||||
activeTemplateVersion: true,
|
||||
previouslyCanceled: false,
|
||||
previouslyCompleted: false,
|
||||
shouldCancel: false,
|
||||
},
|
||||
// Should not cancel pending prebuild-related jobs associated to a second workspace build
|
||||
{
|
||||
name: "DoesNotCancelPendingPrebuildJobSecondBuild",
|
||||
// Given: a pending prebuild job associated to a second workspace build
|
||||
setupBuild: func(t *testing.T,
|
||||
db database.Store,
|
||||
client *codersdk.Client,
|
||||
orgID uuid.UUID,
|
||||
templateID uuid.UUID,
|
||||
templateVersionID uuid.UUID,
|
||||
presetID uuid.NullUUID,
|
||||
) dbfake.WorkspaceResponse {
|
||||
return dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
|
||||
OwnerID: database.PrebuildsSystemUserID,
|
||||
OrganizationID: orgID,
|
||||
TemplateID: templateID,
|
||||
}).Pending().Seed(database.WorkspaceBuild{
|
||||
InitiatorID: database.PrebuildsSystemUserID,
|
||||
BuildNumber: int32(2),
|
||||
TemplateVersionID: templateVersionID,
|
||||
TemplateVersionPresetID: presetID,
|
||||
}).Do()
|
||||
},
|
||||
activeTemplateVersion: false,
|
||||
previouslyCanceled: false,
|
||||
previouslyCompleted: false,
|
||||
shouldCancel: false,
|
||||
},
|
||||
// Should not cancel pending prebuild-related jobs of a different template
|
||||
{
|
||||
name: "DoesNotCancelPrebuildJobDifferentTemplate",
|
||||
// Given: a pending prebuild job belonging to a different template
|
||||
setupBuild: func(
|
||||
t *testing.T,
|
||||
db database.Store,
|
||||
client *codersdk.Client,
|
||||
orgID uuid.UUID,
|
||||
templateID uuid.UUID,
|
||||
templateVersionID uuid.UUID,
|
||||
presetID uuid.NullUUID,
|
||||
) dbfake.WorkspaceResponse {
|
||||
return dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
|
||||
OwnerID: database.PrebuildsSystemUserID,
|
||||
OrganizationID: orgID,
|
||||
TemplateID: uuid.Nil,
|
||||
}).Pending().Seed(database.WorkspaceBuild{
|
||||
InitiatorID: database.PrebuildsSystemUserID,
|
||||
TemplateVersionID: templateVersionID,
|
||||
TemplateVersionPresetID: presetID,
|
||||
}).Do()
|
||||
},
|
||||
activeTemplateVersion: false,
|
||||
previouslyCanceled: false,
|
||||
previouslyCompleted: false,
|
||||
shouldCancel: false,
|
||||
},
|
||||
// Should not cancel pending user workspace build jobs
|
||||
{
|
||||
name: "DoesNotCancelUserWorkspaceJob",
|
||||
// Given: a pending user workspace build job
|
||||
setupBuild: func(
|
||||
t *testing.T,
|
||||
db database.Store,
|
||||
client *codersdk.Client,
|
||||
orgID uuid.UUID,
|
||||
templateID uuid.UUID,
|
||||
templateVersionID uuid.UUID,
|
||||
presetID uuid.NullUUID,
|
||||
) dbfake.WorkspaceResponse {
|
||||
_, member := coderdtest.CreateAnotherUser(t, client, orgID, rbac.RoleMember())
|
||||
return dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
|
||||
OwnerID: member.ID,
|
||||
OrganizationID: orgID,
|
||||
TemplateID: uuid.Nil,
|
||||
}).Pending().Seed(database.WorkspaceBuild{
|
||||
InitiatorID: member.ID,
|
||||
TemplateVersionID: templateVersionID,
|
||||
TemplateVersionPresetID: presetID,
|
||||
}).Do()
|
||||
},
|
||||
activeTemplateVersion: false,
|
||||
previouslyCanceled: false,
|
||||
previouslyCompleted: false,
|
||||
shouldCancel: false,
|
||||
},
|
||||
// Should not cancel pending prebuild-related jobs with a delete transition
|
||||
{
|
||||
name: "DoesNotCancelPrebuildJobDeleteTransition",
|
||||
// Given: a pending prebuild job with a delete transition
|
||||
setupBuild: func(
|
||||
t *testing.T,
|
||||
db database.Store,
|
||||
client *codersdk.Client,
|
||||
orgID uuid.UUID,
|
||||
templateID uuid.UUID,
|
||||
templateVersionID uuid.UUID,
|
||||
presetID uuid.NullUUID,
|
||||
) dbfake.WorkspaceResponse {
|
||||
return dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
|
||||
OwnerID: database.PrebuildsSystemUserID,
|
||||
OrganizationID: orgID,
|
||||
TemplateID: templateID,
|
||||
}).Pending().Seed(database.WorkspaceBuild{
|
||||
InitiatorID: database.PrebuildsSystemUserID,
|
||||
Transition: database.WorkspaceTransitionDelete,
|
||||
TemplateVersionID: templateVersionID,
|
||||
TemplateVersionPresetID: presetID,
|
||||
}).Do()
|
||||
},
|
||||
activeTemplateVersion: false,
|
||||
previouslyCanceled: false,
|
||||
previouslyCompleted: false,
|
||||
shouldCancel: false,
|
||||
},
|
||||
// Should not cancel prebuild-related jobs already being processed by a provisioner
|
||||
{
|
||||
name: "DoesNotCancelRunningPrebuildJob",
|
||||
// Given: a running prebuild job
|
||||
setupBuild: func(
|
||||
t *testing.T,
|
||||
db database.Store,
|
||||
client *codersdk.Client,
|
||||
orgID uuid.UUID,
|
||||
templateID uuid.UUID,
|
||||
templateVersionID uuid.UUID,
|
||||
presetID uuid.NullUUID,
|
||||
) dbfake.WorkspaceResponse {
|
||||
return dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
|
||||
OwnerID: database.PrebuildsSystemUserID,
|
||||
OrganizationID: orgID,
|
||||
TemplateID: templateID,
|
||||
}).Starting().Seed(database.WorkspaceBuild{
|
||||
InitiatorID: database.PrebuildsSystemUserID,
|
||||
TemplateVersionID: templateVersionID,
|
||||
TemplateVersionPresetID: presetID,
|
||||
}).Do()
|
||||
},
|
||||
activeTemplateVersion: false,
|
||||
previouslyCanceled: false,
|
||||
previouslyCompleted: false,
|
||||
shouldCancel: false,
|
||||
},
|
||||
// Should not cancel already canceled prebuild-related jobs
|
||||
{
|
||||
name: "DoesNotCancelCanceledPrebuildJob",
|
||||
// Given: a canceled prebuild job
|
||||
setupBuild: func(
|
||||
t *testing.T,
|
||||
db database.Store,
|
||||
client *codersdk.Client,
|
||||
orgID uuid.UUID,
|
||||
templateID uuid.UUID,
|
||||
templateVersionID uuid.UUID,
|
||||
presetID uuid.NullUUID,
|
||||
) dbfake.WorkspaceResponse {
|
||||
return dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
|
||||
OwnerID: database.PrebuildsSystemUserID,
|
||||
OrganizationID: orgID,
|
||||
TemplateID: templateID,
|
||||
}).Canceled().Seed(database.WorkspaceBuild{
|
||||
InitiatorID: database.PrebuildsSystemUserID,
|
||||
TemplateVersionID: templateVersionID,
|
||||
TemplateVersionPresetID: presetID,
|
||||
}).Do()
|
||||
},
|
||||
activeTemplateVersion: false,
|
||||
shouldCancel: false,
|
||||
previouslyCanceled: true,
|
||||
previouslyCompleted: true,
|
||||
},
|
||||
// Should not cancel completed prebuild-related jobs
|
||||
{
|
||||
name: "DoesNotCancelCompletedPrebuildJob",
|
||||
// Given: a completed prebuild job
|
||||
setupBuild: func(
|
||||
t *testing.T,
|
||||
db database.Store,
|
||||
client *codersdk.Client,
|
||||
orgID uuid.UUID,
|
||||
templateID uuid.UUID,
|
||||
templateVersionID uuid.UUID,
|
||||
presetID uuid.NullUUID,
|
||||
) dbfake.WorkspaceResponse {
|
||||
return dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
|
||||
OwnerID: database.PrebuildsSystemUserID,
|
||||
OrganizationID: orgID,
|
||||
TemplateID: templateID,
|
||||
}).Seed(database.WorkspaceBuild{
|
||||
InitiatorID: database.PrebuildsSystemUserID,
|
||||
TemplateVersionID: templateVersionID,
|
||||
TemplateVersionPresetID: presetID,
|
||||
}).Do()
|
||||
},
|
||||
activeTemplateVersion: false,
|
||||
shouldCancel: false,
|
||||
previouslyCanceled: false,
|
||||
previouslyCompleted: true,
|
||||
},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Set the clock to Monday, January 1st, 2024 at 8:00 AM UTC to keep the test deterministic
|
||||
clock := quartz.NewMock(t)
|
||||
clock.Set(time.Date(2024, 1, 1, 8, 0, 0, 0, time.UTC))
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
|
||||
defer cancel()
|
||||
|
||||
// Setup
|
||||
db, ps := dbtestutil.NewDB(t)
|
||||
client, _, _ := coderdtest.NewWithAPI(t, &coderdtest.Options{
|
||||
// Explicitly not including provisioner daemons, as we don't want the jobs to be processed
|
||||
// Jobs operations will be simulated via the database model
|
||||
IncludeProvisionerDaemon: false,
|
||||
Database: db,
|
||||
Pubsub: ps,
|
||||
Clock: clock,
|
||||
})
|
||||
fakeEnqueuer := newFakeEnqueuer()
|
||||
registry := prometheus.NewRegistry()
|
||||
cache := files.New(registry, &coderdtest.FakeAuthorizer{})
|
||||
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: false}).Leveled(slog.LevelDebug)
|
||||
reconciler := prebuilds.NewStoreReconciler(db, ps, cache, codersdk.PrebuildsConfig{}, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr())
|
||||
owner := coderdtest.CreateFirstUser(t, client)
|
||||
|
||||
// Given: a template with a version containing a preset with 1 prebuild instance
|
||||
nonActivePresetID := uuid.NullUUID{
|
||||
UUID: uuid.New(),
|
||||
Valid: true,
|
||||
}
|
||||
nonActiveTemplateVersion := dbfake.TemplateVersion(t, db).Seed(database.TemplateVersion{
|
||||
OrganizationID: owner.OrganizationID,
|
||||
CreatedBy: owner.UserID,
|
||||
}).Preset(database.TemplateVersionPreset{
|
||||
ID: nonActivePresetID.UUID,
|
||||
DesiredInstances: sql.NullInt32{
|
||||
Int32: 1,
|
||||
Valid: true,
|
||||
},
|
||||
}).Do()
|
||||
templateID := nonActiveTemplateVersion.Template.ID
|
||||
|
||||
// Given: a new active template version
|
||||
activePresetID := uuid.NullUUID{
|
||||
UUID: uuid.New(),
|
||||
Valid: true,
|
||||
}
|
||||
activeTemplateVersion := dbfake.TemplateVersion(t, db).Seed(database.TemplateVersion{
|
||||
OrganizationID: owner.OrganizationID,
|
||||
CreatedBy: owner.UserID,
|
||||
TemplateID: uuid.NullUUID{
|
||||
UUID: templateID,
|
||||
Valid: true,
|
||||
},
|
||||
}).Preset(database.TemplateVersionPreset{
|
||||
ID: activePresetID.UUID,
|
||||
DesiredInstances: sql.NullInt32{
|
||||
Int32: 1,
|
||||
Valid: true,
|
||||
},
|
||||
}).SkipCreateTemplate().Do()
|
||||
|
||||
var workspace dbfake.WorkspaceResponse
|
||||
if tt.activeTemplateVersion {
|
||||
// Given: a prebuilt workspace, workspace build and respective provisioner job from an
|
||||
// active template version
|
||||
workspace = tt.setupBuild(t, db, client,
|
||||
owner.OrganizationID, templateID, activeTemplateVersion.TemplateVersion.ID, activePresetID)
|
||||
} else {
|
||||
// Given: a prebuilt workspace, workspace build and respective provisioner job from a
|
||||
// non-active template version
|
||||
workspace = tt.setupBuild(t, db, client,
|
||||
owner.OrganizationID, templateID, nonActiveTemplateVersion.TemplateVersion.ID, nonActivePresetID)
|
||||
}
|
||||
|
||||
// Given: the new template version is promoted to active
|
||||
err := db.UpdateTemplateActiveVersionByID(ctx, database.UpdateTemplateActiveVersionByIDParams{
|
||||
ID: templateID,
|
||||
ActiveVersionID: activeTemplateVersion.TemplateVersion.ID,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// When: the reconciliation loop is triggered
|
||||
require.NoError(t, reconciler.ReconcileAll(ctx))
|
||||
|
||||
if tt.shouldCancel {
|
||||
// Then: the prebuild related jobs from non-active version should be canceled
|
||||
cancelledJob, err := db.GetProvisionerJobByID(ctx, workspace.Build.JobID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, clock.Now().UTC(), cancelledJob.CanceledAt.Time.UTC())
|
||||
require.Equal(t, clock.Now().UTC(), cancelledJob.CompletedAt.Time.UTC())
|
||||
require.Equal(t, database.ProvisionerJobStatusCanceled, cancelledJob.JobStatus)
|
||||
} else {
|
||||
// Then: the provisioner job should not be canceled
|
||||
job, err := db.GetProvisionerJobByID(ctx, workspace.Build.JobID)
|
||||
require.NoError(t, err)
|
||||
if !tt.previouslyCanceled {
|
||||
require.Zero(t, job.CanceledAt.Time.UTC())
|
||||
require.NotEqual(t, database.ProvisionerJobStatusCanceled, job.JobStatus)
|
||||
}
|
||||
if !tt.previouslyCompleted {
|
||||
require.Zero(t, job.CompletedAt.Time.UTC())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("CancelPendingPrebuildsMultipleTemplates", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
createTemplateVersionWithPreset := func(
|
||||
t *testing.T,
|
||||
db database.Store,
|
||||
orgID uuid.UUID,
|
||||
userID uuid.UUID,
|
||||
templateID uuid.UUID,
|
||||
prebuiltInstances int32,
|
||||
) (uuid.UUID, uuid.UUID, uuid.UUID) {
|
||||
templatePreset := uuid.NullUUID{
|
||||
UUID: uuid.New(),
|
||||
Valid: true,
|
||||
}
|
||||
templateVersion := dbfake.TemplateVersion(t, db).Seed(database.TemplateVersion{
|
||||
OrganizationID: orgID,
|
||||
CreatedBy: userID,
|
||||
TemplateID: uuid.NullUUID{
|
||||
UUID: templateID,
|
||||
Valid: true,
|
||||
},
|
||||
}).Preset(database.TemplateVersionPreset{
|
||||
ID: templatePreset.UUID,
|
||||
DesiredInstances: sql.NullInt32{
|
||||
Int32: prebuiltInstances,
|
||||
Valid: true,
|
||||
},
|
||||
}).Do()
|
||||
|
||||
return templateVersion.Template.ID, templateVersion.TemplateVersion.ID, templatePreset.UUID
|
||||
}
|
||||
|
||||
setupPrebuilds := func(
|
||||
t *testing.T,
|
||||
db database.Store,
|
||||
orgID uuid.UUID,
|
||||
templateID uuid.UUID,
|
||||
versionID uuid.UUID,
|
||||
presetID uuid.UUID,
|
||||
count int,
|
||||
pending bool,
|
||||
) []dbfake.WorkspaceResponse {
|
||||
prebuilds := make([]dbfake.WorkspaceResponse, count)
|
||||
for i := range count {
|
||||
builder := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
|
||||
OwnerID: database.PrebuildsSystemUserID,
|
||||
OrganizationID: orgID,
|
||||
TemplateID: templateID,
|
||||
})
|
||||
|
||||
if pending {
|
||||
builder = builder.Pending()
|
||||
}
|
||||
|
||||
prebuilds[i] = builder.Seed(database.WorkspaceBuild{
|
||||
InitiatorID: database.PrebuildsSystemUserID,
|
||||
TemplateVersionID: versionID,
|
||||
TemplateVersionPresetID: uuid.NullUUID{
|
||||
UUID: presetID,
|
||||
Valid: true,
|
||||
},
|
||||
}).Do()
|
||||
}
|
||||
|
||||
return prebuilds
|
||||
}
|
||||
|
||||
checkIfJobCanceled := func(
|
||||
t *testing.T,
|
||||
clock *quartz.Mock,
|
||||
ctx context.Context,
|
||||
db database.Store,
|
||||
shouldBeCanceled bool,
|
||||
prebuilds []dbfake.WorkspaceResponse,
|
||||
) {
|
||||
for _, prebuild := range prebuilds {
|
||||
job, err := db.GetProvisionerJobByID(ctx, prebuild.Build.JobID)
|
||||
require.NoError(t, err)
|
||||
|
||||
if shouldBeCanceled {
|
||||
require.Equal(t, database.ProvisionerJobStatusCanceled, job.JobStatus)
|
||||
require.Equal(t, clock.Now().UTC(), job.CanceledAt.Time.UTC())
|
||||
require.Equal(t, clock.Now().UTC(), job.CompletedAt.Time.UTC())
|
||||
} else {
|
||||
require.NotEqual(t, database.ProvisionerJobStatusCanceled, job.JobStatus)
|
||||
require.Zero(t, job.CanceledAt.Time.UTC())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Set the clock to Monday, January 1st, 2024 at 8:00 AM UTC to keep the test deterministic
|
||||
clock := quartz.NewMock(t)
|
||||
clock.Set(time.Date(2024, 1, 1, 8, 0, 0, 0, time.UTC))
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
|
||||
defer cancel()
|
||||
|
||||
// Setup
|
||||
db, ps := dbtestutil.NewDB(t)
|
||||
client, _, _ := coderdtest.NewWithAPI(t, &coderdtest.Options{
|
||||
// Explicitly not including provisioner daemons, as we don't want the jobs to be processed
|
||||
// Jobs operations will be simulated via the database model
|
||||
IncludeProvisionerDaemon: false,
|
||||
Database: db,
|
||||
Pubsub: ps,
|
||||
Clock: clock,
|
||||
})
|
||||
fakeEnqueuer := newFakeEnqueuer()
|
||||
registry := prometheus.NewRegistry()
|
||||
cache := files.New(registry, &coderdtest.FakeAuthorizer{})
|
||||
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: false}).Leveled(slog.LevelDebug)
|
||||
reconciler := prebuilds.NewStoreReconciler(db, ps, cache, codersdk.PrebuildsConfig{}, logger, clock, registry, fakeEnqueuer, newNoopUsageCheckerPtr())
|
||||
owner := coderdtest.CreateFirstUser(t, client)
|
||||
|
||||
// Given: template A with 2 versions
|
||||
// Given: template A version v1: with a preset with 5 instances (2 running, 3 pending)
|
||||
templateAID, templateAVersion1ID, templateAVersion1PresetID := createTemplateVersionWithPreset(t, db, owner.OrganizationID, owner.UserID, uuid.Nil, 5)
|
||||
templateAVersion1Running := setupPrebuilds(t, db, owner.OrganizationID, templateAID, templateAVersion1ID, templateAVersion1PresetID, 2, false)
|
||||
templateAVersion1Pending := setupPrebuilds(t, db, owner.OrganizationID, templateAID, templateAVersion1ID, templateAVersion1PresetID, 3, true)
|
||||
// Given: template A version v2 (active version): with a preset with 2 instances (1 running, 1 pending)
|
||||
_, templateAVersion2ID, templateAVersion2PresetID := createTemplateVersionWithPreset(t, db, owner.OrganizationID, owner.UserID, templateAID, 2)
|
||||
templateAVersion2Running := setupPrebuilds(t, db, owner.OrganizationID, templateAID, templateAVersion2ID, templateAVersion2PresetID, 1, false)
|
||||
templateAVersion2Pending := setupPrebuilds(t, db, owner.OrganizationID, templateAID, templateAVersion2ID, templateAVersion2PresetID, 1, true)
|
||||
|
||||
// Given: template B with 3 versions
|
||||
// Given: template B version v1: with a preset with 3 instances (1 running, 2 pending)
|
||||
templateBID, templateBVersion1ID, templateBVersion1PresetID := createTemplateVersionWithPreset(t, db, owner.OrganizationID, owner.UserID, uuid.Nil, 3)
|
||||
templateBVersion1Running := setupPrebuilds(t, db, owner.OrganizationID, templateBID, templateBVersion1ID, templateBVersion1PresetID, 1, false)
|
||||
templateBVersion1Pending := setupPrebuilds(t, db, owner.OrganizationID, templateBID, templateBVersion1ID, templateBVersion1PresetID, 2, true)
|
||||
// Given: template B version v2: with a preset with 2 instances (2 pending)
|
||||
_, templateBVersion2ID, templateBVersion2PresetID := createTemplateVersionWithPreset(t, db, owner.OrganizationID, owner.UserID, templateBID, 2)
|
||||
templateBVersion2Pending := setupPrebuilds(t, db, owner.OrganizationID, templateBID, templateBVersion2ID, templateBVersion2PresetID, 2, true)
|
||||
// Given: template B version v3 (active version): with a preset with 2 instances (1 running, 1 pending)
|
||||
_, templateBVersion3ID, templateBVersion3PresetID := createTemplateVersionWithPreset(t, db, owner.OrganizationID, owner.UserID, templateBID, 2)
|
||||
templateBVersion3Running := setupPrebuilds(t, db, owner.OrganizationID, templateBID, templateBVersion3ID, templateBVersion3PresetID, 1, false)
|
||||
templateBVersion3Pending := setupPrebuilds(t, db, owner.OrganizationID, templateBID, templateBVersion3ID, templateBVersion3PresetID, 1, true)
|
||||
|
||||
// When: the reconciliation loop is executed
|
||||
require.NoError(t, reconciler.ReconcileAll(ctx))
|
||||
|
||||
// Then: template A version 1 running workspaces should not be canceled
|
||||
checkIfJobCanceled(t, clock, ctx, db, false, templateAVersion1Running)
|
||||
// Then: template A version 1 pending workspaces should be canceled
|
||||
checkIfJobCanceled(t, clock, ctx, db, true, templateAVersion1Pending)
|
||||
// Then: template A version 2 running and pending workspaces should not be canceled
|
||||
checkIfJobCanceled(t, clock, ctx, db, false, templateAVersion2Running)
|
||||
checkIfJobCanceled(t, clock, ctx, db, false, templateAVersion2Pending)
|
||||
|
||||
// Then: template B version 1 running workspaces should not be canceled
|
||||
checkIfJobCanceled(t, clock, ctx, db, false, templateBVersion1Running)
|
||||
// Then: template B version 1 pending workspaces should be canceled
|
||||
checkIfJobCanceled(t, clock, ctx, db, true, templateBVersion1Pending)
|
||||
// Then: template B version 2 pending workspaces should be canceled
|
||||
checkIfJobCanceled(t, clock, ctx, db, true, templateBVersion2Pending)
|
||||
// Then: template B version 3 running and pending workspaces should not be canceled
|
||||
checkIfJobCanceled(t, clock, ctx, db, false, templateBVersion3Running)
|
||||
checkIfJobCanceled(t, clock, ctx, db, false, templateBVersion3Pending)
|
||||
})
|
||||
}
|
||||
|
||||
func newNoopEnqueuer() *notifications.NoopEnqueuer {
|
||||
return notifications.NewNoopEnqueuer()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user