diff --git a/enterprise/scaletest/prebuilds/run_test.go b/enterprise/scaletest/prebuilds/run_test.go new file mode 100644 index 0000000000..4334d0c096 --- /dev/null +++ b/enterprise/scaletest/prebuilds/run_test.go @@ -0,0 +1,141 @@ +package prebuilds_test + +import ( + "io" + "strconv" + "sync" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" + + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/enterprise/coderd/coderdenttest" + "github.com/coder/coder/v2/enterprise/coderd/license" + "github.com/coder/coder/v2/scaletest/prebuilds" + "github.com/coder/coder/v2/testutil" + "github.com/coder/quartz" +) + +func TestRun(t *testing.T) { + t.Parallel() + + t.Skip("This test takes several minutes to run, and is intended as a manual regression test") + + ctx := testutil.Context(t, testutil.WaitSuperLong*3) + + client, user := coderdenttest.New(t, &coderdenttest.Options{ + LicenseOptions: &coderdenttest.LicenseOptions{ + Features: license.Features{ + codersdk.FeatureWorkspacePrebuilds: 1, + codersdk.FeatureExternalProvisionerDaemons: 1, + }, + }, + }) + + // This is a real Terraform provisioner + _ = coderdenttest.NewExternalProvisionerDaemonTerraform(t, client, user.OrganizationID, nil) + + numTemplates := 2 + numPresets := 1 + numPresetPrebuilds := 1 + + //nolint:gocritic // It's fine to use the owner user to pause prebuilds + err := client.PutPrebuildsSettings(ctx, codersdk.PrebuildsSettings{ + ReconciliationPaused: true, + }) + require.NoError(t, err) + + setupBarrier := new(sync.WaitGroup) + setupBarrier.Add(numTemplates) + creationBarrier := new(sync.WaitGroup) + creationBarrier.Add(numTemplates) + deletionSetupBarrier := new(sync.WaitGroup) + deletionSetupBarrier.Add(1) + deletionBarrier := new(sync.WaitGroup) + deletionBarrier.Add(numTemplates) + + metrics := prebuilds.NewMetrics(prometheus.NewRegistry()) + + eg, runCtx := errgroup.WithContext(ctx) + + runners := make([]*prebuilds.Runner, 0, numTemplates) + for i := range numTemplates { + cfg := prebuilds.Config{ + OrganizationID: user.OrganizationID, + NumPresets: numPresets, + NumPresetPrebuilds: numPresetPrebuilds, + TemplateVersionJobTimeout: testutil.WaitSuperLong * 2, + PrebuildWorkspaceTimeout: testutil.WaitSuperLong * 2, + Metrics: metrics, + SetupBarrier: setupBarrier, + CreationBarrier: creationBarrier, + DeletionSetupBarrier: deletionSetupBarrier, + DeletionBarrier: deletionBarrier, + Clock: quartz.NewReal(), + } + err := cfg.Validate() + require.NoError(t, err) + + runner := prebuilds.NewRunner(client, cfg) + runners = append(runners, runner) + eg.Go(func() error { + return runner.Run(runCtx, strconv.Itoa(i), io.Discard) + }) + } + + // Wait for all runners to reach the setup barrier (templates created) + setupBarrier.Wait() + + // Resume prebuilds to trigger prebuild creation + err = client.PutPrebuildsSettings(ctx, codersdk.PrebuildsSettings{ + ReconciliationPaused: false, + }) + require.NoError(t, err) + + // Wait for all runners to reach the creation barrier (prebuilds created) + creationBarrier.Wait() + + //nolint:gocritic // Owner user is fine here as we want to view all workspaces + workspaces, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{}) + require.NoError(t, err) + expectedWorkspaces := numTemplates * numPresets * numPresetPrebuilds + require.Equal(t, workspaces.Count, expectedWorkspaces) + + // Pause prebuilds before deletion setup + err = client.PutPrebuildsSettings(ctx, codersdk.PrebuildsSettings{ + ReconciliationPaused: true, + }) + require.NoError(t, err) + + // Signal runners that prebuilds are paused and they can prepare for deletion + deletionSetupBarrier.Done() + + // Wait for all runners to reach the deletion barrier (template versions updated to 0 prebuilds) + deletionBarrier.Wait() + + // Resume prebuilds to trigger prebuild deletion + err = client.PutPrebuildsSettings(ctx, codersdk.PrebuildsSettings{ + ReconciliationPaused: false, + }) + require.NoError(t, err) + + err = eg.Wait() + require.NoError(t, err) + + //nolint:gocritic // Owner user is fine here as we want to view all workspaces + workspaces, err = client.Workspaces(ctx, codersdk.WorkspaceFilter{}) + require.NoError(t, err) + require.Equal(t, workspaces.Count, 0) + + cleanupEg, cleanupCtx := errgroup.WithContext(ctx) + for i, runner := range runners { + cleanupEg.Go(func() error { + return runner.Cleanup(cleanupCtx, strconv.Itoa(i), io.Discard) + }) + } + + err = cleanupEg.Wait() + require.NoError(t, err) +} diff --git a/scaletest/dynamicparameters/template.go b/scaletest/dynamicparameters/template.go index 5faf67e531..dbe4b079b1 100644 --- a/scaletest/dynamicparameters/template.go +++ b/scaletest/dynamicparameters/template.go @@ -1,15 +1,12 @@ package dynamicparameters import ( - "archive/tar" "bytes" "context" _ "embed" "encoding/json" "fmt" "io" - "path/filepath" - "slices" "strings" "text/template" "time" @@ -20,6 +17,7 @@ import ( "cdr.dev/slog" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/cryptorand" + "github.com/coder/coder/v2/scaletest/loadtestutil" "github.com/coder/quartz" ) @@ -89,48 +87,6 @@ func GetModuleFiles() map[string][]byte { } } -func createTarFromFiles(files map[string][]byte) ([]byte, error) { - buf := new(bytes.Buffer) - writer := tar.NewWriter(buf) - dirs := []string{} - for name, content := range files { - // We need to add directories before any files that use them. But, we only need to do this - // once. - dir := filepath.Dir(name) - if dir != "." && !slices.Contains(dirs, dir) { - dirs = append(dirs, dir) - err := writer.WriteHeader(&tar.Header{ - Name: dir, - Mode: 0o755, - Typeflag: tar.TypeDir, - }) - if err != nil { - return nil, err - } - } - - err := writer.WriteHeader(&tar.Header{ - Name: name, - Size: int64(len(content)), - Mode: 0o644, - }) - if err != nil { - return nil, err - } - - _, err = writer.Write(content) - if err != nil { - return nil, err - } - } - // `writer.Close()` function flushes the writer buffer, and adds extra padding to create a legal tarball. - err := writer.Close() - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - func TemplateTarData() ([]byte, error) { mainTF, err := TemplateContent() if err != nil { @@ -144,7 +100,7 @@ func TemplateTarData() ([]byte, error) { for k, v := range moduleFiles { files[k] = v } - tarData, err := createTarFromFiles(files) + tarData, err := loadtestutil.CreateTarFromFiles(files) if err != nil { return nil, xerrors.Errorf("failed to create tarball: %w", err) } diff --git a/scaletest/loadtestutil/files.go b/scaletest/loadtestutil/files.go new file mode 100644 index 0000000000..2890700f4e --- /dev/null +++ b/scaletest/loadtestutil/files.go @@ -0,0 +1,50 @@ +package loadtestutil + +import ( + "archive/tar" + "bytes" + "path/filepath" + "slices" +) + +func CreateTarFromFiles(files map[string][]byte) ([]byte, error) { + buf := new(bytes.Buffer) + writer := tar.NewWriter(buf) + dirs := []string{} + for name, content := range files { + // We need to add directories before any files that use them. But, we only need to do this + // once. + dir := filepath.Dir(name) + if dir != "." && !slices.Contains(dirs, dir) { + dirs = append(dirs, dir) + err := writer.WriteHeader(&tar.Header{ + Name: dir, + Mode: 0o755, + Typeflag: tar.TypeDir, + }) + if err != nil { + return nil, err + } + } + + err := writer.WriteHeader(&tar.Header{ + Name: name, + Size: int64(len(content)), + Mode: 0o644, + }) + if err != nil { + return nil, err + } + + _, err = writer.Write(content) + if err != nil { + return nil, err + } + } + // `writer.Close()` function flushes the writer buffer, and adds extra padding to create a legal tarball. + err := writer.Close() + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} diff --git a/scaletest/prebuilds/config.go b/scaletest/prebuilds/config.go new file mode 100644 index 0000000000..05f1fc48ad --- /dev/null +++ b/scaletest/prebuilds/config.go @@ -0,0 +1,86 @@ +package prebuilds + +import ( + "sync" + "time" + + "github.com/google/uuid" + "golang.org/x/xerrors" + + "github.com/coder/quartz" +) + +type Config struct { + // OrganizationID is the ID of the organization to create the prebuilds in. + OrganizationID uuid.UUID `json:"organization_id"` + // NumPresets is the number of presets the template should have. + NumPresets int `json:"num_presets"` + // NumPresetPrebuilds is the number of prebuilds per preset. + // Total prebuilds = NumPresets * NumPresetPrebuilds + NumPresetPrebuilds int `json:"num_preset_prebuilds"` + + // TemplateVersionJobTimeout is how long to wait for template version + // provisioning jobs to complete. + TemplateVersionJobTimeout time.Duration `json:"template_version_job_timeout"` + + // PrebuildWorkspaceTimeout is how long to wait for all prebuild + // workspaces to be created and completed. + PrebuildWorkspaceTimeout time.Duration `json:"prebuild_workspace_timeout"` + + Metrics *Metrics `json:"-"` + + // SetupBarrier is used to ensure all templates have been created + // before unpausing prebuilds. + SetupBarrier *sync.WaitGroup `json:"-"` + + // CreationBarrier is used to ensure all prebuild creation has completed + // before pausing prebuilds for deletion. + CreationBarrier *sync.WaitGroup `json:"-"` + + // DeletionSetupBarrier is used by the runner owner (CLI/test) to signal when + // prebuilds have been paused, allowing runners to create new template versions + // with 0 prebuilds. Only the owner calls Done(), runners only Wait(). + DeletionSetupBarrier *sync.WaitGroup `json:"-"` + + // DeletionBarrier is used to ensure all templates have been updated + // with 0 prebuilds before resuming prebuilds. + DeletionBarrier *sync.WaitGroup `json:"-"` + + Clock quartz.Clock `json:"-"` +} + +func (c Config) Validate() error { + if c.TemplateVersionJobTimeout <= 0 { + return xerrors.New("template_version_job_timeout must be greater than 0") + } + + if c.PrebuildWorkspaceTimeout <= 0 { + return xerrors.New("prebuild_workspace_timeout must be greater than 0") + } + + if c.SetupBarrier == nil { + return xerrors.New("setup barrier must be set") + } + + if c.CreationBarrier == nil { + return xerrors.New("creation barrier must be set") + } + + if c.DeletionSetupBarrier == nil { + return xerrors.New("deletion setup barrier must be set") + } + + if c.DeletionBarrier == nil { + return xerrors.New("deletion barrier must be set") + } + + if c.Metrics == nil { + return xerrors.New("metrics must be set") + } + + if c.Clock == nil { + return xerrors.New("clock must be set") + } + + return nil +} diff --git a/scaletest/prebuilds/metrics.go b/scaletest/prebuilds/metrics.go new file mode 100644 index 0000000000..553b874e2d --- /dev/null +++ b/scaletest/prebuilds/metrics.go @@ -0,0 +1,125 @@ +package prebuilds + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +type Metrics struct { + PrebuildJobsCreated prometheus.GaugeVec + PrebuildJobsRunning prometheus.GaugeVec + PrebuildJobsFailed prometheus.GaugeVec + PrebuildJobsCompleted prometheus.GaugeVec + + PrebuildDeletionJobsCreated prometheus.GaugeVec + PrebuildDeletionJobsRunning prometheus.GaugeVec + PrebuildDeletionJobsFailed prometheus.GaugeVec + PrebuildDeletionJobsCompleted prometheus.GaugeVec + + PrebuildErrorsTotal prometheus.CounterVec +} + +func NewMetrics(reg prometheus.Registerer) *Metrics { + m := &Metrics{ + PrebuildJobsCreated: *prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "scaletest", + Name: "prebuild_jobs_created", + Help: "Number of prebuild jobs that have been created.", + }, []string{"template_name"}), + PrebuildJobsRunning: *prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "scaletest", + Name: "prebuild_jobs_running", + Help: "Number of prebuild jobs that are currently running.", + }, []string{"template_name"}), + PrebuildJobsFailed: *prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "scaletest", + Name: "prebuild_jobs_failed", + Help: "Number of prebuild jobs that have failed.", + }, []string{"template_name"}), + PrebuildJobsCompleted: *prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "scaletest", + Name: "prebuild_jobs_completed", + Help: "Number of prebuild jobs that have completed successfully.", + }, []string{"template_name"}), + PrebuildDeletionJobsCreated: *prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "scaletest", + Name: "prebuild_deletion_jobs_created", + Help: "Number of prebuild deletion jobs that have been created.", + }, []string{"template_name"}), + PrebuildDeletionJobsRunning: *prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "scaletest", + Name: "prebuild_deletion_jobs_running", + Help: "Number of prebuild deletion jobs that are currently running.", + }, []string{"template_name"}), + PrebuildDeletionJobsFailed: *prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "scaletest", + Name: "prebuild_deletion_jobs_failed", + Help: "Number of prebuild deletion jobs that have failed.", + }, []string{"template_name"}), + PrebuildDeletionJobsCompleted: *prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "coderd", + Subsystem: "scaletest", + Name: "prebuild_deletion_jobs_completed", + Help: "Number of prebuild deletion jobs that have completed successfully.", + }, []string{"template_name"}), + PrebuildErrorsTotal: *prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: "coderd", + Subsystem: "scaletest", + Name: "prebuild_errors_total", + Help: "Total number of prebuild errors", + }, []string{"template_name", "action"}), + } + + reg.MustRegister(m.PrebuildJobsCreated) + reg.MustRegister(m.PrebuildJobsRunning) + reg.MustRegister(m.PrebuildJobsFailed) + reg.MustRegister(m.PrebuildJobsCompleted) + reg.MustRegister(m.PrebuildDeletionJobsCreated) + reg.MustRegister(m.PrebuildDeletionJobsRunning) + reg.MustRegister(m.PrebuildDeletionJobsFailed) + reg.MustRegister(m.PrebuildDeletionJobsCompleted) + reg.MustRegister(m.PrebuildErrorsTotal) + return m +} + +func (m *Metrics) SetJobsCreated(count int, templateName string) { + m.PrebuildJobsCreated.WithLabelValues(templateName).Set(float64(count)) +} + +func (m *Metrics) SetJobsRunning(count int, templateName string) { + m.PrebuildJobsRunning.WithLabelValues(templateName).Set(float64(count)) +} + +func (m *Metrics) SetJobsFailed(count int, templateName string) { + m.PrebuildJobsFailed.WithLabelValues(templateName).Set(float64(count)) +} + +func (m *Metrics) SetJobsCompleted(count int, templateName string) { + m.PrebuildJobsCompleted.WithLabelValues(templateName).Set(float64(count)) +} + +func (m *Metrics) SetDeletionJobsCreated(count int, templateName string) { + m.PrebuildDeletionJobsCreated.WithLabelValues(templateName).Set(float64(count)) +} + +func (m *Metrics) SetDeletionJobsRunning(count int, templateName string) { + m.PrebuildDeletionJobsRunning.WithLabelValues(templateName).Set(float64(count)) +} + +func (m *Metrics) SetDeletionJobsFailed(count int, templateName string) { + m.PrebuildDeletionJobsFailed.WithLabelValues(templateName).Set(float64(count)) +} + +func (m *Metrics) SetDeletionJobsCompleted(count int, templateName string) { + m.PrebuildDeletionJobsCompleted.WithLabelValues(templateName).Set(float64(count)) +} + +func (m *Metrics) AddError(templateName string, action string) { + m.PrebuildErrorsTotal.WithLabelValues(templateName, action).Inc() +} diff --git a/scaletest/prebuilds/run.go b/scaletest/prebuilds/run.go new file mode 100644 index 0000000000..7a62e3638b --- /dev/null +++ b/scaletest/prebuilds/run.go @@ -0,0 +1,343 @@ +package prebuilds + +import ( + "bytes" + "context" + _ "embed" + "html/template" + "io" + "time" + + "golang.org/x/xerrors" + + "github.com/google/uuid" + + "cdr.dev/slog" + "cdr.dev/slog/sloggers/sloghuman" + "github.com/coder/coder/v2/coderd/tracing" + "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/scaletest/harness" + "github.com/coder/coder/v2/scaletest/loadtestutil" +) + +type Runner struct { + client *codersdk.Client + cfg Config + + template codersdk.Template +} + +var ( + _ harness.Runnable = &Runner{} + _ harness.Cleanable = &Runner{} +) + +func NewRunner(client *codersdk.Client, cfg Config) *Runner { + return &Runner{ + client: client, + cfg: cfg, + } +} + +func (r *Runner) Run(ctx context.Context, id string, logs io.Writer) error { + ctx, span := tracing.StartSpan(ctx) + defer span.End() + + reachedSetupBarrier := false + reachedCreationBarrier := false + reachedDeletionBarrier := false + defer func() { + if !reachedSetupBarrier { + r.cfg.SetupBarrier.Done() + } + if !reachedCreationBarrier { + r.cfg.CreationBarrier.Done() + } + if !reachedDeletionBarrier { + r.cfg.DeletionBarrier.Done() + } + }() + + logs = loadtestutil.NewSyncWriter(logs) + logger := slog.Make(sloghuman.Sink(logs)).Leveled(slog.LevelDebug) + r.client.SetLogger(logger) + r.client.SetLogBodies(true) + + templateName := "scaletest-prebuilds-template-" + id + + version, err := r.createTemplateVersion(ctx, uuid.Nil, r.cfg.NumPresets, r.cfg.NumPresetPrebuilds) + if err != nil { + r.cfg.Metrics.AddError(templateName, "create_template_version") + return err + } + + templateReq := codersdk.CreateTemplateRequest{ + Name: templateName, + Description: "`coder exp scaletest prebuilds` template", + VersionID: version.ID, + } + templ, err := r.client.CreateTemplate(ctx, r.cfg.OrganizationID, templateReq) + if err != nil { + r.cfg.Metrics.AddError(templateName, "create_template") + return xerrors.Errorf("create template: %w", err) + } + logger.Info(ctx, "created template", slog.F("template_id", templ.ID)) + + r.template = templ + + logger.Info(ctx, "waiting for all runners to reach setup barrier") + reachedSetupBarrier = true + r.cfg.SetupBarrier.Done() + r.cfg.SetupBarrier.Wait() + logger.Info(ctx, "all runners reached setup barrier, proceeding with prebuild creation test") + + err = r.measureCreation(ctx, logger) + if err != nil { + return err + } + + logger.Info(ctx, "waiting for all runners to reach creation barrier") + reachedCreationBarrier = true + r.cfg.CreationBarrier.Done() + r.cfg.CreationBarrier.Wait() + logger.Info(ctx, "all runners reached creation barrier") + + logger.Info(ctx, "waiting for runner owner to pause prebuilds (deletion setup barrier)") + r.cfg.DeletionSetupBarrier.Wait() + logger.Info(ctx, "prebuilds paused, preparing for deletion") + + // Now prepare for deletion by creating an empty template version + // At this point, prebuilds should be paused by the caller + logger.Info(ctx, "creating empty template version for deletion") + emptyVersion, err := r.createTemplateVersion(ctx, r.template.ID, 0, 0) + if err != nil { + r.cfg.Metrics.AddError(r.template.Name, "create_empty_template_version") + return xerrors.Errorf("create empty template version for deletion: %w", err) + } + + err = r.client.UpdateActiveTemplateVersion(ctx, r.template.ID, codersdk.UpdateActiveTemplateVersion{ + ID: emptyVersion.ID, + }) + if err != nil { + r.cfg.Metrics.AddError(r.template.Name, "update_active_template_version") + return xerrors.Errorf("update active template version to empty for deletion: %w", err) + } + + logger.Info(ctx, "waiting for all runners to reach deletion barrier") + reachedDeletionBarrier = true + r.cfg.DeletionBarrier.Done() + r.cfg.DeletionBarrier.Wait() + logger.Info(ctx, "all runners reached deletion barrier, proceeding with prebuild deletion test") + + err = r.measureDeletion(ctx, logger) + if err != nil { + return err + } + + return nil +} + +func (r *Runner) measureCreation(ctx context.Context, logger slog.Logger) error { + testStartTime := time.Now().UTC() + const workspacesPollInterval = 500 * time.Millisecond + + targetNumWorkspaces := r.cfg.NumPresets * r.cfg.NumPresetPrebuilds + + workspacesCtx, cancel := context.WithTimeout(ctx, r.cfg.PrebuildWorkspaceTimeout) + defer cancel() + + tkr := r.cfg.Clock.TickerFunc(workspacesCtx, workspacesPollInterval, func() error { + workspaces, err := r.client.Workspaces(workspacesCtx, codersdk.WorkspaceFilter{ + Template: r.template.Name, + }) + if err != nil { + return xerrors.Errorf("list workspaces: %w", err) + } + + createdCount := len(workspaces.Workspaces) + runningCount := 0 + failedCount := 0 + succeededCount := 0 + + for _, ws := range workspaces.Workspaces { + switch ws.LatestBuild.Job.Status { + case codersdk.ProvisionerJobRunning: + runningCount++ + case codersdk.ProvisionerJobFailed, codersdk.ProvisionerJobCanceled: + failedCount++ + case codersdk.ProvisionerJobSucceeded: + succeededCount++ + } + } + + r.cfg.Metrics.SetJobsCreated(createdCount, r.template.Name) + r.cfg.Metrics.SetJobsRunning(runningCount, r.template.Name) + r.cfg.Metrics.SetJobsFailed(failedCount, r.template.Name) + r.cfg.Metrics.SetJobsCompleted(succeededCount, r.template.Name) + + if succeededCount >= targetNumWorkspaces { + // All jobs succeeded + return errTickerDone + } + + return nil + }, "waitForPrebuildWorkspaces") + err := tkr.Wait() + if !xerrors.Is(err, errTickerDone) { + r.cfg.Metrics.AddError(r.template.Name, "wait_for_workspaces") + return xerrors.Errorf("wait for workspaces: %w", err) + } + + logger.Info(ctx, "all prebuild workspaces created successfully", slog.F("template_name", r.template.Name), slog.F("duration", time.Since(testStartTime).String())) + return nil +} + +func (r *Runner) measureDeletion(ctx context.Context, logger slog.Logger) error { + deletionStartTime := time.Now().UTC() + const deletionPollInterval = 500 * time.Millisecond + + targetNumWorkspaces := r.cfg.NumPresets * r.cfg.NumPresetPrebuilds + + deletionCtx, cancel := context.WithTimeout(ctx, r.cfg.PrebuildWorkspaceTimeout) + defer cancel() + + tkr := r.cfg.Clock.TickerFunc(deletionCtx, deletionPollInterval, func() error { + workspaces, err := r.client.Workspaces(deletionCtx, codersdk.WorkspaceFilter{ + Template: r.template.Name, + }) + if err != nil { + return xerrors.Errorf("list workspaces: %w", err) + } + + createdCount := 0 + runningCount := 0 + failedCount := 0 + + for _, ws := range workspaces.Workspaces { + if ws.LatestBuild.Transition == codersdk.WorkspaceTransitionDelete { + createdCount++ + switch ws.LatestBuild.Job.Status { + case codersdk.ProvisionerJobRunning: + runningCount++ + case codersdk.ProvisionerJobFailed, codersdk.ProvisionerJobCanceled: + failedCount++ + } + } + } + + completedCount := targetNumWorkspaces - len(workspaces.Workspaces) + createdCount += completedCount + + r.cfg.Metrics.SetDeletionJobsCreated(createdCount, r.template.Name) + r.cfg.Metrics.SetDeletionJobsRunning(runningCount, r.template.Name) + r.cfg.Metrics.SetDeletionJobsFailed(failedCount, r.template.Name) + r.cfg.Metrics.SetDeletionJobsCompleted(completedCount, r.template.Name) + + if len(workspaces.Workspaces) == 0 { + return errTickerDone + } + + return nil + }, "waitForPrebuildWorkspacesDeletion") + err := tkr.Wait() + if !xerrors.Is(err, errTickerDone) { + r.cfg.Metrics.AddError(r.template.Name, "wait_for_workspace_deletion") + return xerrors.Errorf("wait for workspace deletion: %w", err) + } + + logger.Info(ctx, "all prebuild workspaces deleted successfully", slog.F("template_name", r.template.Name), slog.F("duration", time.Since(deletionStartTime).String())) + return nil +} + +func (r *Runner) createTemplateVersion(ctx context.Context, templateID uuid.UUID, numPresets, numPresetPrebuilds int) (codersdk.TemplateVersion, error) { + tarData, err := TemplateTarData(numPresets, numPresetPrebuilds) + if err != nil { + return codersdk.TemplateVersion{}, xerrors.Errorf("create prebuilds template tar: %w", err) + } + uploadResp, err := r.client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(tarData)) + if err != nil { + return codersdk.TemplateVersion{}, xerrors.Errorf("upload prebuilds template tar: %w", err) + } + + versionReq := codersdk.CreateTemplateVersionRequest{ + TemplateID: templateID, + FileID: uploadResp.ID, + Message: "Template version for scaletest prebuilds", + StorageMethod: codersdk.ProvisionerStorageMethodFile, + Provisioner: codersdk.ProvisionerTypeTerraform, + } + version, err := r.client.CreateTemplateVersion(ctx, r.cfg.OrganizationID, versionReq) + if err != nil { + return codersdk.TemplateVersion{}, xerrors.Errorf("create template version: %w", err) + } + if version.MatchedProvisioners != nil && version.MatchedProvisioners.Count == 0 { + return codersdk.TemplateVersion{}, xerrors.Errorf("no provisioners matched for template version") + } + + const pollInterval = 2 * time.Second + versionCtx, cancel := context.WithTimeout(ctx, r.cfg.TemplateVersionJobTimeout) + defer cancel() + + tkr := r.cfg.Clock.TickerFunc(versionCtx, pollInterval, func() error { + version, err := r.client.TemplateVersion(versionCtx, version.ID) + if err != nil { + return xerrors.Errorf("get template version: %w", err) + } + switch version.Job.Status { + case codersdk.ProvisionerJobSucceeded: + return errTickerDone + case codersdk.ProvisionerJobPending, codersdk.ProvisionerJobRunning: + return nil + default: + return xerrors.Errorf("template version provisioning failed: status %s", version.Job.Status) + } + }) + err = tkr.Wait() + if !xerrors.Is(err, errTickerDone) { + return codersdk.TemplateVersion{}, xerrors.Errorf("wait for template version provisioning: %w", err) + } + return version, nil +} + +var errTickerDone = xerrors.New("done") + +func (r *Runner) Cleanup(ctx context.Context, _ string, logs io.Writer) error { + logs = loadtestutil.NewSyncWriter(logs) + logger := slog.Make(sloghuman.Sink(logs)).Leveled(slog.LevelDebug) + + logger.Info(ctx, "deleting template", slog.F("template_name", r.template.Name)) + + err := r.client.DeleteTemplate(ctx, r.template.ID) + if err != nil { + return xerrors.Errorf("delete template: %w", err) + } + + logger.Info(ctx, "template deleted successfully", slog.F("template_name", r.template.Name)) + return nil +} + +//go:embed tf/main.tf.tpl +var templateContent string + +func TemplateTarData(numPresets, numPresetPrebuilds int) ([]byte, error) { + tmpl, err := template.New("prebuilds-template").Parse(templateContent) + if err != nil { + return nil, err + } + result := bytes.Buffer{} + err = tmpl.Execute(&result, map[string]int{ + "NumPresets": numPresets, + "NumPresetPrebuilds": numPresetPrebuilds, + }) + if err != nil { + return nil, err + } + files := map[string][]byte{ + "main.tf": result.Bytes(), + } + tarBytes, err := loadtestutil.CreateTarFromFiles(files) + if err != nil { + return nil, err + } + return tarBytes, nil +} diff --git a/scaletest/prebuilds/tf/main.tf.tpl b/scaletest/prebuilds/tf/main.tf.tpl new file mode 100644 index 0000000000..9465281ac2 --- /dev/null +++ b/scaletest/prebuilds/tf/main.tf.tpl @@ -0,0 +1,18 @@ +terraform { + required_providers { + coder = { + source = "coder/coder" + version = "2.5.3" + } + } +} + +resource "null_resource" "workspace" {} + +data "coder_workspace_preset" "presets" { + count = {{.NumPresets}} + name = "preset-${count.index + 1}" + prebuilds { + instances = {{.NumPresetPrebuilds}} + } +}