fix: use separate HTTP clients in scale test load generators (#21288)

While scale testing, I noticed that our load generators send basically
all requests to a single Coderd instance.

e.g. 


![image.png](https://app.graphite.com/user-attachments/assets/e259862a-adf1-47e7-a37b-fd14e420058e.png)


This is because our scale test commands create all `Runner`s using the
same codersdk Client, which means they share an underlying HTTP client.
With HTTP/2 a single TCP session can multiplex many different HTTP
requests (including websockets). So, it creates a single TCP connection
to a single coderd, and then sends all the requests down the one TCP
connections.

This PR modifies the `exp scaletest` load generator commands to create
an independent HTTP client per `Runner`. This means that each runner
will create its own TCP connection. This should help spread the load and
make a more realistic test, because in a real deployment, scaled out
load will be coming over different TCP connections.

cli/exp_scaletest.go

@@ -48,6 +48,8 @@ import (
 
 const scaletestTracerName = "coder_scaletest"
 
+var BypassHeader = map[string][]string{codersdk.BypassRatelimitHeader: {"true"}}
+
 func (r *RootCmd) scaletestCmd() *serpent.Command {
 	cmd := &serpent.Command{
 		Use:   "scaletest",
@@ -691,15 +693,6 @@ func (r *RootCmd) scaletestCreateWorkspaces() *serpent.Command {
 				return err
 			}
 
-			client.HTTPClient = &http.Client{
-				Transport: &codersdk.HeaderTransport{
-					Transport: http.DefaultTransport,
-					Header: map[string][]string{
-						codersdk.BypassRatelimitHeader: {"true"},
-					},
-				},
-			}
-
 			if count <= 0 {
 				return xerrors.Errorf("--count is required and must be greater than 0")
 			}
@@ -811,7 +804,13 @@ func (r *RootCmd) scaletestCreateWorkspaces() *serpent.Command {
 					return xerrors.Errorf("validate config: %w", err)
 				}
 
-				var runner harness.Runnable = createworkspaces.NewRunner(client, config)
+				// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
+				// requests being unbalanced among Coder instances.
+				runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
+				if err != nil {
+					return xerrors.Errorf("create runner client: %w", err)
+				}
+				var runner harness.Runnable = createworkspaces.NewRunner(runnerClient, config)
 				if tracingEnabled {
 					runner = &runnableTraceWrapper{
 						tracer:   tracer,
@@ -1019,15 +1018,6 @@ func (r *RootCmd) scaletestWorkspaceUpdates() *serpent.Command {
 				return err
 			}
 
-			client.HTTPClient = &http.Client{
-				Transport: &codersdk.HeaderTransport{
-					Transport: http.DefaultTransport,
-					Header: map[string][]string{
-						codersdk.BypassRatelimitHeader: {"true"},
-					},
-				},
-			}
-
 			if workspaceCount <= 0 {
 				return xerrors.Errorf("--workspace-count must be greater than 0")
 			}
@@ -1166,7 +1156,14 @@ func (r *RootCmd) scaletestWorkspaceUpdates() *serpent.Command {
 			for i, config := range configs {
 				name := fmt.Sprintf("workspaceupdates-%dw", config.WorkspaceCount)
 				id := strconv.Itoa(i)
-				var runner harness.Runnable = workspaceupdates.NewRunner(client, config)
+
+				// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
+				// requests being unbalanced among Coder instances.
+				runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
+				if err != nil {
+					return xerrors.Errorf("create runner client: %w", err)
+				}
+				var runner harness.Runnable = workspaceupdates.NewRunner(runnerClient, config)
 				if tracingEnabled {
 					runner = &runnableTraceWrapper{
 						tracer:   tracer,
@@ -1323,16 +1320,6 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *serpent.Command {
 			prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), prometheusFlags.Address, "prometheus")
 			defer prometheusSrvClose()
 
-			// Bypass rate limiting
-			client.HTTPClient = &http.Client{
-				Transport: &codersdk.HeaderTransport{
-					Transport: http.DefaultTransport,
-					Header: map[string][]string{
-						codersdk.BypassRatelimitHeader: {"true"},
-					},
-				},
-			}
-
 			workspaces, err := targetFlags.getTargetedWorkspaces(ctx, client, me.OrganizationIDs, inv.Stdout)
 			if err != nil {
 				return err
@@ -1429,7 +1416,13 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *serpent.Command {
 				if err := config.Validate(); err != nil {
 					return xerrors.Errorf("validate config: %w", err)
 				}
-				var runner harness.Runnable = workspacetraffic.NewRunner(client, config)
+				// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
+				// requests being unbalanced among Coder instances.
+				runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
+				if err != nil {
+					return xerrors.Errorf("create runner client: %w", err)
+				}
+				var runner harness.Runnable = workspacetraffic.NewRunner(runnerClient, config)
 				if tracingEnabled {
 					runner = &runnableTraceWrapper{
 						tracer:   tracer,
@@ -1617,9 +1610,13 @@ func (r *RootCmd) scaletestDashboard() *serpent.Command {
 					return xerrors.Errorf("create token for user: %w", err)
 				}
 
-				userClient := codersdk.New(client.URL,
-					codersdk.WithSessionToken(userTokResp.Key),
-				)
+				// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
+				// requests being unbalanced among Coder instances.
+				userClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
+				if err != nil {
+					return xerrors.Errorf("create runner client: %w", err)
+				}
+				codersdk.WithSessionToken(userTokResp.Key)(userClient)
 
 				config := dashboard.Config{
 					Interval: interval,
@@ -1766,15 +1763,6 @@ func (r *RootCmd) scaletestAutostart() *serpent.Command {
 				return err
 			}
 
-			client.HTTPClient = &http.Client{
-				Transport: &codersdk.HeaderTransport{
-					Transport: http.DefaultTransport,
-					Header: map[string][]string{
-						codersdk.BypassRatelimitHeader: {"true"},
-					},
-				},
-			}
-
 			if workspaceCount <= 0 {
 				return xerrors.Errorf("--workspace-count must be greater than zero")
 			}
@@ -1840,7 +1828,13 @@ func (r *RootCmd) scaletestAutostart() *serpent.Command {
 				if err := config.Validate(); err != nil {
 					return xerrors.Errorf("validate config: %w", err)
 				}
-				var runner harness.Runnable = autostart.NewRunner(client, config)
+				// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
+				// requests being unbalanced among Coder instances.
+				runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
+				if err != nil {
+					return xerrors.Errorf("create runner client: %w", err)
+				}
+				var runner harness.Runnable = autostart.NewRunner(runnerClient, config)
 				if tracingEnabled {
 					runner = &runnableTraceWrapper{
 						tracer:   tracer,

cli/exp_scaletest_dynamicparameters.go

@@ -4,18 +4,18 @@ package cli
 
 import (
 	"fmt"
-	"net/http"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"golang.org/x/xerrors"
 
+	"github.com/coder/coder/v2/scaletest/loadtestutil"
+
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/sloghuman"
 	"github.com/coder/serpent"
 
-	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/scaletest/dynamicparameters"
 	"github.com/coder/coder/v2/scaletest/harness"
 )
@@ -72,15 +72,6 @@ func (r *RootCmd) scaletestDynamicParameters() *serpent.Command {
 				return err
 			}
 
-			client.HTTPClient = &http.Client{
-				Transport: &codersdk.HeaderTransport{
-					Transport: http.DefaultTransport,
-					Header: map[string][]string{
-						codersdk.BypassRatelimitHeader: {"true"},
-					},
-				},
-			}
-
 			reg := prometheus.NewRegistry()
 			metrics := dynamicparameters.NewMetrics(reg, "concurrent_evaluations")
 
@@ -122,7 +113,13 @@ func (r *RootCmd) scaletestDynamicParameters() *serpent.Command {
 						Metrics:           metrics,
 						MetricLabelValues: []string{fmt.Sprintf("%d", part.ConcurrentEvaluations)},
 					}
-					var runner harness.Runnable = dynamicparameters.NewRunner(client, cfg)
+					// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
+					// requests being unbalanced among Coder instances.
+					runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
+					if err != nil {
+						return xerrors.Errorf("create runner client: %w", err)
+					}
+					var runner harness.Runnable = dynamicparameters.NewRunner(runnerClient, cfg)
 					if tracingEnabled {
 						runner = &runnableTraceWrapper{
 							tracer:   tracer,

cli/exp_scaletest_notifications.go

@@ -18,6 +18,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"golang.org/x/xerrors"
 
+	"github.com/coder/coder/v2/scaletest/loadtestutil"
+
 	"cdr.dev/slog"
 
 	notificationsLib "github.com/coder/coder/v2/coderd/notifications"
@@ -66,15 +68,6 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command {
 				return err
 			}
 
-			client.HTTPClient = &http.Client{
-				Transport: &codersdk.HeaderTransport{
-					Transport: http.DefaultTransport,
-					Header: map[string][]string{
-						codersdk.BypassRatelimitHeader: {"true"},
-					},
-				},
-			}
-
 			if userCount <= 0 {
 				return xerrors.Errorf("--user-count must be greater than 0")
 			}
@@ -206,7 +199,13 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command {
 			for i, config := range configs {
 				id := strconv.Itoa(i)
 				name := fmt.Sprintf("notifications-%s", id)
-				var runner harness.Runnable = notifications.NewRunner(client, config)
+				// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
+				// requests being unbalanced among Coder instances.
+				runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
+				if err != nil {
+					return xerrors.Errorf("create runner client: %w", err)
+				}
+				var runner harness.Runnable = notifications.NewRunner(runnerClient, config)
 				if tracingEnabled {
 					runner = &runnableTraceWrapper{
 						tracer:   tracer,

cli/exp_scaletest_prebuilds.go

@@ -4,7 +4,6 @@ package cli
 
 import (
 	"fmt"
-	"net/http"
 	"os/signal"
 	"strconv"
 	"sync"
@@ -14,6 +13,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"golang.org/x/xerrors"
 
+	"github.com/coder/coder/v2/scaletest/loadtestutil"
+
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/scaletest/harness"
 	"github.com/coder/coder/v2/scaletest/prebuilds"
@@ -56,15 +57,6 @@ func (r *RootCmd) scaletestPrebuilds() *serpent.Command {
 				return err
 			}
 
-			client.HTTPClient = &http.Client{
-				Transport: &codersdk.HeaderTransport{
-					Transport: http.DefaultTransport,
-					Header: map[string][]string{
-						codersdk.BypassRatelimitHeader: {"true"},
-					},
-				},
-			}
-
 			if numTemplates <= 0 {
 				return xerrors.Errorf("--num-templates must be greater than 0")
 			}
@@ -140,7 +132,13 @@ func (r *RootCmd) scaletestPrebuilds() *serpent.Command {
 					return xerrors.Errorf("validate config: %w", err)
 				}
 
-				var runner harness.Runnable = prebuilds.NewRunner(client, cfg)
+				// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
+				// requests being unbalanced among Coder instances.
+				runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
+				if err != nil {
+					return xerrors.Errorf("create runner client: %w", err)
+				}
+				var runner harness.Runnable = prebuilds.NewRunner(runnerClient, cfg)
 				if tracingEnabled {
 					runner = &runnableTraceWrapper{
 						tracer:   tracer,

cli/exp_scaletest_taskstatus.go

@@ -14,6 +14,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"golang.org/x/xerrors"
 
+	"github.com/coder/coder/v2/scaletest/loadtestutil"
+
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/sloghuman"
 	"github.com/coder/serpent"
@@ -143,7 +145,13 @@ After all runners connect, it waits for the baseline duration before triggering
 					return xerrors.Errorf("validate config for runner %d: %w", i, err)
 				}
 
-				var runner harness.Runnable = taskstatus.NewRunner(client, cfg)
+				// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
+				// requests being unbalanced among Coder instances.
+				runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
+				if err != nil {
+					return xerrors.Errorf("create runner client: %w", err)
+				}
+				var runner harness.Runnable = taskstatus.NewRunner(runnerClient, cfg)
 				if tracingEnabled {
 					runner = &runnableTraceWrapper{
 						tracer:   tracer,

scaletest/loadtestutil/client.go

@@ -0,0 +1,45 @@
+package loadtestutil
+
+import (
+	"maps"
+	"net/http"
+
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/codersdk"
+)
+
+// DupClientCopyingHeaders duplicates the Client, but with an independent underlying HTTP transport, so that it will not
+// share connections with the client being duplicated. It copies any headers already on the existing transport as
+// [codersdk.HeaderTransport] and add the headers in the argument.
+func DupClientCopyingHeaders(client *codersdk.Client, header http.Header) (*codersdk.Client, error) {
+	nc := codersdk.New(client.URL, codersdk.WithLogger(client.Logger()))
+	nc.SessionTokenProvider = client.SessionTokenProvider
+	newHeader, t, err := extractHeaderAndInnerTransport(client.HTTPClient.Transport)
+	if err != nil {
+		return nil, xerrors.Errorf("extract headers: %w", err)
+	}
+	maps.Copy(newHeader, header)
+
+	nc.HTTPClient.Transport = &codersdk.HeaderTransport{
+		Transport: t.Clone(),
+		Header:    newHeader,
+	}
+	return nc, nil
+}
+
+func extractHeaderAndInnerTransport(rt http.RoundTripper) (http.Header, *http.Transport, error) {
+	if t, ok := rt.(*http.Transport); ok {
+		// base case
+		return make(http.Header), t, nil
+	}
+	if ht, ok := rt.(*codersdk.HeaderTransport); ok {
+		headers, t, err := extractHeaderAndInnerTransport(ht.Transport)
+		if err != nil {
+			return nil, nil, err
+		}
+		maps.Copy(headers, ht.Header)
+		return headers, t, nil
+	}
+	return nil, nil, xerrors.New("round tripper is neither HeaderTransport nor Transport")
+}

scaletest/loadtestutil/client_test.go

@@ -0,0 +1,50 @@
+package loadtestutil_test
+
+import (
+	"net/http"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/scaletest/loadtestutil"
+)
+
+func TestDupClientCopyingHeaders(t *testing.T) {
+	t.Parallel()
+	httpClient := &http.Client{
+		Transport: &codersdk.HeaderTransport{
+			Transport: &codersdk.HeaderTransport{
+				Transport: http.DefaultTransport,
+				Header: map[string][]string{
+					"X-Coder-Test":  {"foo"},
+					"X-Coder-Test3": {"socks"},
+				},
+			},
+			Header: map[string][]string{
+				"X-Coder-Test":  {"bar"},
+				"X-Coder-Test2": {"baz"},
+			},
+		},
+	}
+	serverURL, err := url.Parse("http://coder.example.com")
+	require.NoError(t, err)
+	sdkClient := codersdk.New(serverURL,
+		codersdk.WithSessionToken("test-token"), codersdk.WithHTTPClient(httpClient))
+
+	dup, err := loadtestutil.DupClientCopyingHeaders(sdkClient, map[string][]string{
+		"X-Coder-Test3": {"clocks"},
+		"X-Coder-Test4": {"bears"},
+	})
+	require.NoError(t, err)
+	require.Equal(t, "http://coder.example.com", dup.URL.String())
+	require.Equal(t, "test-token", dup.SessionToken())
+	ht, ok := dup.HTTPClient.Transport.(*codersdk.HeaderTransport)
+	require.True(t, ok)
+	require.Equal(t, "bar", ht.Header.Get("X-Coder-Test"))
+	require.Equal(t, "baz", ht.Header.Get("X-Coder-Test2"))
+	require.Equal(t, "clocks", ht.Header.Get("X-Coder-Test3"))
+	require.Equal(t, "bears", ht.Header.Get("X-Coder-Test4"))
+	require.NotEqual(t, http.DefaultTransport, ht.Transport)
+}