chore: add prometheus monitoring of workspace traffic generation (#7583)

- Exposes reads/writes from scaletest traffic generation (default: 0.0.0.0:21112)
- Adds self-hosted prometheus with remote_write to loadtest terraform
- Adds convenience script to run a traffic generation test

cli/scaletest.go

@@ -14,9 +14,14 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/xerrors"
 
+	"cdr.dev/slog"
+	"cdr.dev/slog/sloggers/sloghuman"
+
 	"github.com/coder/coder/cli/clibase"
 	"github.com/coder/coder/cli/cliui"
 	"github.com/coder/coder/coderd/httpapi"
@@ -896,8 +901,11 @@ func (r *RootCmd) scaletestCreateWorkspaces() *clibase.Cmd {
 
 func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
 	var (
-		tickInterval    time.Duration
-		bytesPerTick    int64
+		tickInterval               time.Duration
+		bytesPerTick               int64
+		scaletestPrometheusAddress string
+		scaletestPrometheusWait    time.Duration
+
 		client          = &codersdk.Client{}
 		tracingFlags    = &scaletestTracingFlags{}
 		strategy        = &scaletestStrategyFlags{}
@@ -913,6 +921,12 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
 		),
 		Handler: func(inv *clibase.Invocation) error {
 			ctx := inv.Context()
+			reg := prometheus.NewRegistry()
+			metrics := workspacetraffic.NewMetrics(reg, "username", "workspace_name", "agent_name")
+
+			logger := slog.Make(sloghuman.Sink(io.Discard))
+			prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), scaletestPrometheusAddress, "prometheus")
+			defer prometheusSrvClose()
 
 			// Bypass rate limiting
 			client.HTTPClient = &http.Client{
@@ -943,6 +957,9 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
 				_, _ = fmt.Fprintln(inv.Stderr, "\nUploading traces...")
 				if err := closeTracing(ctx); err != nil {
 					_, _ = fmt.Fprintf(inv.Stderr, "\nError uploading traces: %+v\n", err)
+					// Wait for prometheus metrics to be scraped
+					_, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", scaletestPrometheusWait)
+					<-time.After(scaletestPrometheusWait)
 				}
 			}()
 			tracer := tracerProvider.Tracer(scaletestTracerName)
@@ -955,9 +972,10 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
 			th := harness.NewTestHarness(strategy.toStrategy(), cleanupStrategy.toStrategy())
 			for idx, ws := range workspaces {
 				var (
-					agentID uuid.UUID
-					name    = "workspace-traffic"
-					id      = strconv.Itoa(idx)
+					agentID   uuid.UUID
+					agentName string
+					name      = "workspace-traffic"
+					id        = strconv.Itoa(idx)
 				)
 
 				for _, res := range ws.LatestBuild.Resources {
@@ -965,6 +983,7 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
 						continue
 					}
 					agentID = res.Agents[0].ID
+					agentName = res.Agents[0].Name
 				}
 
 				if agentID == uuid.Nil {
@@ -974,16 +993,20 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
 
 				// Setup our workspace agent connection.
 				config := workspacetraffic.Config{
-					AgentID:      agentID,
-					BytesPerTick: bytesPerTick,
-					Duration:     strategy.timeout,
-					TickInterval: tickInterval,
+					AgentID:        agentID,
+					AgentName:      agentName,
+					BytesPerTick:   bytesPerTick,
+					Duration:       strategy.timeout,
+					TickInterval:   tickInterval,
+					WorkspaceName:  ws.Name,
+					WorkspaceOwner: ws.OwnerName,
+					Registry:       reg,
 				}
 
 				if err := config.Validate(); err != nil {
 					return xerrors.Errorf("validate config: %w", err)
 				}
-				var runner harness.Runnable = workspacetraffic.NewRunner(client, config)
+				var runner harness.Runnable = workspacetraffic.NewRunner(client, config, metrics)
 				if tracingEnabled {
 					runner = &runnableTraceWrapper{
 						tracer:   tracer,
@@ -1034,6 +1057,20 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *clibase.Cmd {
 			Description: "How often to send traffic.",
 			Value:       clibase.DurationOf(&tickInterval),
 		},
+		{
+			Flag:        "scaletest-prometheus-address",
+			Env:         "CODER_SCALETEST_PROMETHEUS_ADDRESS",
+			Default:     "0.0.0.0:21112",
+			Description: "Address on which to expose scaletest Prometheus metrics.",
+			Value:       clibase.StringOf(&scaletestPrometheusAddress),
+		},
+		{
+			Flag:        "scaletest-prometheus-wait",
+			Env:         "CODER_SCALETEST_PROMETHEUS_WAIT",
+			Default:     "5s",
+			Description: "How long to wait before exiting in order to allow Prometheus metrics to be scraped.",
+			Value:       clibase.DurationOf(&scaletestPrometheusWait),
+		},
 	}
 
 	tracingFlags.attach(&cmd.Options)

cli/scaletest_test.go

@@ -8,17 +8,12 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/coder/coder/agent"
 	"github.com/coder/coder/cli/clitest"
 	"github.com/coder/coder/coderd/coderdtest"
 	"github.com/coder/coder/codersdk"
-	"github.com/coder/coder/codersdk/agentsdk"
-	"github.com/coder/coder/provisioner/echo"
-	"github.com/coder/coder/provisionersdk/proto"
 	"github.com/coder/coder/pty/ptytest"
 	"github.com/coder/coder/scaletest/harness"
 	"github.com/coder/coder/testutil"
@@ -205,70 +200,28 @@ param3: 1
 	})
 }
 
-// This test pretends to stand up a workspace and run a no-op traffic generation test.
-// It's not a real test, but it's useful for debugging.
-// We do not perform any cleanup.
+// This test just validates that the CLI command accepts its known arguments.
+// A more comprehensive test is performed in workspacetraffic/run_test.go
 func TestScaleTestWorkspaceTraffic(t *testing.T) {
 	t.Parallel()
 
 	ctx, cancelFunc := context.WithTimeout(context.Background(), testutil.WaitMedium)
 	defer cancelFunc()
 
-	client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
-	user := coderdtest.CreateFirstUser(t, client)
-
-	authToken := uuid.NewString()
-	version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
-		Parse:         echo.ParseComplete,
-		ProvisionPlan: echo.ProvisionComplete,
-		ProvisionApply: []*proto.Provision_Response{{
-			Type: &proto.Provision_Response_Complete{
-				Complete: &proto.Provision_Complete{
-					Resources: []*proto.Resource{{
-						Name: "example",
-						Type: "aws_instance",
-						Agents: []*proto.Agent{{
-							Id:   uuid.NewString(),
-							Name: "agent",
-							Auth: &proto.Agent_Token{
-								Token: authToken,
-							},
-							Apps: []*proto.App{},
-						}},
-					}},
-				},
-			},
-		}},
-	})
-	template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
-	coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
-
-	ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID, func(cwr *codersdk.CreateWorkspaceRequest) {
-		cwr.Name = "scaletest-test"
-	})
-	coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
-
-	agentClient := agentsdk.New(client.URL)
-	agentClient.SetSessionToken(authToken)
-	agentCloser := agent.New(agent.Options{
-		Client: agentClient,
-	})
-	t.Cleanup(func() {
-		_ = agentCloser.Close()
-	})
-
-	coderdtest.AwaitWorkspaceAgents(t, client, ws.ID)
+	client := coderdtest.New(t, nil)
+	_ = coderdtest.CreateFirstUser(t, client)
 
 	inv, root := clitest.New(t, "scaletest", "workspace-traffic",
 		"--timeout", "1s",
 		"--bytes-per-tick", "1024",
 		"--tick-interval", "100ms",
+		"--scaletest-prometheus-address", "127.0.0.1:0",
+		"--scaletest-prometheus-wait", "0s",
 	)
 	clitest.SetupConfig(t, client, root)
 	var stdout, stderr bytes.Buffer
 	inv.Stdout = &stdout
 	inv.Stderr = &stderr
 	err := inv.WithContext(ctx).Run()
-	require.NoError(t, err)
-	require.Contains(t, stdout.String(), "Pass:  1")
+	require.ErrorContains(t, err, "no scaletest workspaces exist")
 }

cli/testdata/coder_scaletest_workspace-traffic_--help.golden

@@ -27,6 +27,13 @@ Generate traffic to scaletest workspaces through coderd
           Output format specs in the format "<format>[:<path>]". Not specifying
           a path will default to stdout. Available formats: text, json.
 
+      --scaletest-prometheus-address string, $CODER_SCALETEST_PROMETHEUS_ADDRESS (default: 0.0.0.0:21112)
+          Address on which to expose scaletest Prometheus metrics.
+
+      --scaletest-prometheus-wait duration, $CODER_SCALETEST_PROMETHEUS_WAIT (default: 5s)
+          How long to wait before exiting in order to allow Prometheus metrics
+          to be scraped.
+
       --tick-interval duration, $CODER_SCALETEST_WORKSPACE_TRAFFIC_TICK_INTERVAL (default: 100ms)
           How often to send traffic.