Files
coder/coderd/prebuilds/global_snapshot.go
T
Susana Ferreira f6e86c6fdb feat: cancel pending prebuilds from non-active template versions (#20387)
## Description

This PR introduces an optimization to automatically cancel pending
prebuild-related jobs from non-active template versions in the
reconciliation loop.

## Problem

Currently, when a template is configured with more prebuild instances
than available provisioners, the provisioner queue can become flooded
with pending prebuild jobs. This issue is worsened when
provisioning/deprovisioning operations take a long time.

When the prebuild reconciliation loop generates jobs faster than
provisioners can process them, pending jobs accumulate in the queue.
Since prebuilt workspaces should always run the latest active template
version, pending prebuild jobs from non-active versions become obsolete
once a new version is promoted.

## Solution

The reconciliation loop cancels pending prebuild-related jobs from
non-active template versions that match the following criteria:

* Build number: 1 (initial build created by the reconciliation loop)
* Job status: `pending`
* Not yet picked up by a provisioner (`worker_id` is `NULL`)
* Owned by the prebuilds system user
* Workspace transition: `start`

This prevents the queue from being cluttered with stale prebuild jobs
that would provision workspaces on an outdated template version that
would consequently need to be deprovisioned.

## Changes

* Added new SQL query `CountPendingNonActivePrebuilds` to identify
presets with pending jobs from non-active versions
* Added new SQL query `UpdatePrebuildProvisionerJobWithCancel` to cancel
jobs for a specific preset
* New reconciliation action type `ActionTypeCancelPending` handles the
cancellation logic
* Cancellation is non-blocking: failures to cancel prebuild jobs are
logged as errors and don't prevent other reconciliation actions

## Follow-up PR

Canceling pending prebuild jobs leaves workspaces in a Canceled state.
While no Terraform resources need to be destroyed (since jobs were
canceled before provisioning started), these database records should
still be cleaned up. This will be addressed in a follow-up PR.

Closes: https://github.com/coder/coder/issues/20242
2025-10-24 15:27:49 +01:00

149 lines
4.9 KiB
Go

package prebuilds
import (
"time"
"github.com/google/uuid"
"golang.org/x/xerrors"
"cdr.dev/slog"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/util/slice"
"github.com/coder/quartz"
)
// GlobalSnapshot represents a full point-in-time snapshot of state relating to prebuilds across all templates.
type GlobalSnapshot struct {
Presets []database.GetTemplatePresetsWithPrebuildsRow
PrebuildSchedules []database.TemplateVersionPresetPrebuildSchedule
RunningPrebuilds []database.GetRunningPrebuiltWorkspacesRow
PrebuildsInProgress []database.CountInProgressPrebuildsRow
PendingPrebuilds []database.CountPendingNonActivePrebuildsRow
Backoffs []database.GetPresetsBackoffRow
HardLimitedPresetsMap map[uuid.UUID]database.GetPresetsAtFailureLimitRow
clock quartz.Clock
logger slog.Logger
}
func NewGlobalSnapshot(
presets []database.GetTemplatePresetsWithPrebuildsRow,
prebuildSchedules []database.TemplateVersionPresetPrebuildSchedule,
runningPrebuilds []database.GetRunningPrebuiltWorkspacesRow,
prebuildsInProgress []database.CountInProgressPrebuildsRow,
pendingPrebuilds []database.CountPendingNonActivePrebuildsRow,
backoffs []database.GetPresetsBackoffRow,
hardLimitedPresets []database.GetPresetsAtFailureLimitRow,
clock quartz.Clock,
logger slog.Logger,
) GlobalSnapshot {
hardLimitedPresetsMap := make(map[uuid.UUID]database.GetPresetsAtFailureLimitRow, len(hardLimitedPresets))
for _, preset := range hardLimitedPresets {
hardLimitedPresetsMap[preset.PresetID] = preset
}
return GlobalSnapshot{
Presets: presets,
PrebuildSchedules: prebuildSchedules,
RunningPrebuilds: runningPrebuilds,
PrebuildsInProgress: prebuildsInProgress,
PendingPrebuilds: pendingPrebuilds,
Backoffs: backoffs,
HardLimitedPresetsMap: hardLimitedPresetsMap,
clock: clock,
logger: logger,
}
}
func (s GlobalSnapshot) FilterByPreset(presetID uuid.UUID) (*PresetSnapshot, error) {
preset, found := slice.Find(s.Presets, func(preset database.GetTemplatePresetsWithPrebuildsRow) bool {
return preset.ID == presetID
})
if !found {
return nil, xerrors.Errorf("no preset found with ID %q", presetID)
}
prebuildSchedules := slice.Filter(s.PrebuildSchedules, func(schedule database.TemplateVersionPresetPrebuildSchedule) bool {
return schedule.PresetID == presetID
})
// Only include workspaces that have successfully started
running := slice.Filter(s.RunningPrebuilds, func(prebuild database.GetRunningPrebuiltWorkspacesRow) bool {
if !prebuild.CurrentPresetID.Valid {
return false
}
return prebuild.CurrentPresetID.UUID == preset.ID
})
// Separate running workspaces into non-expired and expired based on the preset's TTL
nonExpired, expired := filterExpiredWorkspaces(preset, running)
// Includes in-progress prebuilds only for active template versions.
// In-progress prebuilds correspond to workspace statuses: 'pending', 'starting', 'stopping', and 'deleting'
inProgress := slice.Filter(s.PrebuildsInProgress, func(prebuild database.CountInProgressPrebuildsRow) bool {
return prebuild.PresetID.UUID == preset.ID
})
// Includes count of pending prebuilds only for non-active template versions
pendingCount := 0
if found, ok := slice.Find(s.PendingPrebuilds, func(prebuild database.CountPendingNonActivePrebuildsRow) bool {
return prebuild.PresetID.UUID == preset.ID
}); ok {
pendingCount = int(found.Count)
}
var backoffPtr *database.GetPresetsBackoffRow
backoff, found := slice.Find(s.Backoffs, func(row database.GetPresetsBackoffRow) bool {
return row.PresetID == preset.ID
})
if found {
backoffPtr = &backoff
}
_, isHardLimited := s.HardLimitedPresetsMap[preset.ID]
presetSnapshot := NewPresetSnapshot(
preset,
prebuildSchedules,
nonExpired,
expired,
inProgress,
pendingCount,
backoffPtr,
isHardLimited,
s.clock,
s.logger,
)
return &presetSnapshot, nil
}
func (s GlobalSnapshot) IsHardLimited(presetID uuid.UUID) bool {
_, isHardLimited := s.HardLimitedPresetsMap[presetID]
return isHardLimited
}
// filterExpiredWorkspaces splits running workspaces into expired and non-expired
// based on the preset's TTL.
// If TTL is missing or zero, all workspaces are considered non-expired.
func filterExpiredWorkspaces(preset database.GetTemplatePresetsWithPrebuildsRow, runningWorkspaces []database.GetRunningPrebuiltWorkspacesRow) (nonExpired []database.GetRunningPrebuiltWorkspacesRow, expired []database.GetRunningPrebuiltWorkspacesRow) {
if !preset.Ttl.Valid {
return runningWorkspaces, expired
}
ttl := time.Duration(preset.Ttl.Int32) * time.Second
if ttl <= 0 {
return runningWorkspaces, expired
}
for _, prebuild := range runningWorkspaces {
if time.Since(prebuild.CreatedAt) > ttl {
expired = append(expired, prebuild)
} else {
nonExpired = append(nonExpired, prebuild)
}
}
return nonExpired, expired
}