From cac6d4ce98094e2d15afae5f6105ab2f3a46cbc1 Mon Sep 17 00:00:00 2001
From: Spike Curtis <spike@coder.com>
Date: Thu, 18 Dec 2025 11:21:35 +0400
Subject: [PATCH] feat: add --max-failures to coder exp scaletest
 create-workspaces (#21315)

Adds `--max-failures` flag to `coder exp scaletest create-workspaces` so that we can tolerate a few failures without failing the command.

When running our scale test infra, we create Kubernetes Jobs to create the initial cluster workspaces, then we have load-generation jobs that depend on them. At high scale, it's kind of expected that some of the requests will fail: even with 99.9% success, you still expect one failure per 1000. It's useful to be able to carry on with the scale test anyway and proceed to traffic generation.
---
 cli/exp_scaletest.go      | 18 +++++++++++++-----
 cli/exp_scaletest_test.go |  1 +
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/cli/exp_scaletest.go b/cli/exp_scaletest.go
index 419b195547..6d1208a16d 100644
--- a/cli/exp_scaletest.go
+++ b/cli/exp_scaletest.go
@@ -640,9 +640,10 @@ func (r *RootCmd) scaletestCleanup() *serpent.Command {
 
 func (r *RootCmd) scaletestCreateWorkspaces() *serpent.Command {
 	var (
-		count    int64
-		retry    int64
-		template string
+		count       int64
+		retry       int64
+		maxFailures int64
+		template    string
 
 		noCleanup bool
 		// TODO: implement this flag
@@ -847,8 +848,8 @@ func (r *RootCmd) scaletestCreateWorkspaces() *serpent.Command {
 				return xerrors.Errorf("cleanup tests: %w", err)
 			}
 
-			if res.TotalFail > 0 {
-				return xerrors.New("load test failed, see above for more details")
+			if res.TotalFail > int(maxFailures) {
+				return xerrors.Errorf("load test failed, %d runs failed (max allowed: %d)", res.TotalFail, maxFailures)
 			}
 
 			return nil
@@ -963,6 +964,13 @@ func (r *RootCmd) scaletestCreateWorkspaces() *serpent.Command {
 			Description: "Use the user logged in on the host machine, instead of creating users.",
 			Value:       serpent.BoolOf(&useHostUser),
 		},
+		{
+			Flag:        "max-failures",
+			Env:         "CODER_SCALETEST_MAX_FAILURES",
+			Default:     "0",
+			Description: "Maximum number of runs that are allowed to fail before the entire test is considered failed. 0 means any failure will cause the test to fail.",
+			Value:       serpent.Int64Of(&maxFailures),
+		},
 	}
 
 	cmd.Options = append(cmd.Options, parameterFlags.cliParameters()...)
diff --git a/cli/exp_scaletest_test.go b/cli/exp_scaletest_test.go
index afcd213fc9..0c5f525e9e 100644
--- a/cli/exp_scaletest_test.go
+++ b/cli/exp_scaletest_test.go
@@ -54,6 +54,7 @@ func TestScaleTestCreateWorkspaces(t *testing.T) {
 		"--output", "json:"+outputFile,
 		"--parameter", "foo=baz",
 		"--rich-parameter-file", "/path/to/some/parameter/file.ext",
+		"--max-failures", "1",
 	)
 	clitest.SetupConfig(t, client, root)
 	pty := ptytest.New(t)