feat: add --max-failures to coder exp scaletest create-workspaces (#21315)

Adds `--max-failures` flag to `coder exp scaletest create-workspaces` so that we can tolerate a few failures without failing the command.

When running our scale test infra, we create Kubernetes Jobs to create the initial cluster workspaces, then we have load-generation jobs that depend on them. At high scale, it's kind of expected that some of the requests will fail: even with 99.9% success, you still expect one failure per 1000. It's useful to be able to carry on with the scale test anyway and proceed to traffic generation.

cli/exp_scaletest.go

@@ -640,9 +640,10 @@ func (r *RootCmd) scaletestCleanup() *serpent.Command {
 
 func (r *RootCmd) scaletestCreateWorkspaces() *serpent.Command {
 	var (
-		count    int64
-		retry    int64
-		template string
+		count       int64
+		retry       int64
+		maxFailures int64
+		template    string
 
 		noCleanup bool
 		// TODO: implement this flag
@@ -847,8 +848,8 @@ func (r *RootCmd) scaletestCreateWorkspaces() *serpent.Command {
 				return xerrors.Errorf("cleanup tests: %w", err)
 			}
 
-			if res.TotalFail > 0 {
-				return xerrors.New("load test failed, see above for more details")
+			if res.TotalFail > int(maxFailures) {
+				return xerrors.Errorf("load test failed, %d runs failed (max allowed: %d)", res.TotalFail, maxFailures)
 			}
 
 			return nil
@@ -963,6 +964,13 @@ func (r *RootCmd) scaletestCreateWorkspaces() *serpent.Command {
 			Description: "Use the user logged in on the host machine, instead of creating users.",
 			Value:       serpent.BoolOf(&useHostUser),
 		},
+		{
+			Flag:        "max-failures",
+			Env:         "CODER_SCALETEST_MAX_FAILURES",
+			Default:     "0",
+			Description: "Maximum number of runs that are allowed to fail before the entire test is considered failed. 0 means any failure will cause the test to fail.",
+			Value:       serpent.Int64Of(&maxFailures),
+		},
 	}
 
 	cmd.Options = append(cmd.Options, parameterFlags.cliParameters()...)

cli/exp_scaletest_test.go

@@ -54,6 +54,7 @@ func TestScaleTestCreateWorkspaces(t *testing.T) {
 		"--output", "json:"+outputFile,
 		"--parameter", "foo=baz",
 		"--rich-parameter-file", "/path/to/some/parameter/file.ext",
+		"--max-failures", "1",
 	)
 	clitest.SetupConfig(t, client, root)
 	pty := ptytest.New(t)