feat: add scaletest Runner for dynamicparameters load gen (#19890)

relates to https://github.com/coder/internal/issues/912

Adds a new scaletest Runner to generate dynamic parameters load.

A later PR will add the CLI command, including creating the template & version.

.github/workflows/typos.toml

@@ -1,5 +1,6 @@
 [default]
 extend-ignore-identifiers-re = ["gho_.*"]
+extend-ignore-re = ["(#|//)\\s*spellchecker:ignore-next-line\\n.*"]
 
 [default.extend-identifiers]
 alog = "alog"

coderd/coderdtest/dynamicparameters.go

@@ -20,6 +20,9 @@ type DynamicParameterTemplateParams struct {
 	Plan           json.RawMessage
 	ModulesArchive []byte
 
+	// ExtraFiles are additional files to include in the template, beyond the MainTF.
+	ExtraFiles map[string][]byte
+
 	// Uses a zip archive instead of a tar
 	Zip bool
 
@@ -36,9 +39,17 @@ type DynamicParameterTemplateParams struct {
 func DynamicParameterTemplate(t *testing.T, client *codersdk.Client, org uuid.UUID, args DynamicParameterTemplateParams) (codersdk.Template, codersdk.TemplateVersion) {
 	t.Helper()
 
-	files := echo.WithExtraFiles(map[string][]byte{
+	// Start with main.tf
+	extraFiles := map[string][]byte{
 		"main.tf": []byte(args.MainTF),
-	})
+	}
+
+	// Add any additional files
+	for name, content := range args.ExtraFiles {
+		extraFiles[name] = content
+	}
+
+	files := echo.WithExtraFiles(extraFiles)
 	files.ProvisionPlan = []*proto.Response{{
 		Type: &proto.Response_Plan{
 			Plan: &proto.PlanComplete{

provisioner/echo/serve.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"text/template"
 
@@ -359,9 +360,26 @@ func TarWithOptions(ctx context.Context, logger slog.Logger, responses *Response
 			}
 		}
 	}
+	dirs := []string{}
 	for name, content := range responses.ExtraFiles {
 		logger.Debug(ctx, "extra file", slog.F("name", name))
 
+		// We need to add directories before any files that use them. But, we only need to do this
+		// once.
+		dir := filepath.Dir(name)
+		if dir != "." && !slices.Contains(dirs, dir) {
+			logger.Debug(ctx, "adding extra file directory", slog.F("dir", dir))
+			dirs = append(dirs, dir)
+			err := writer.WriteHeader(&tar.Header{
+				Name:     dir,
+				Mode:     0o755,
+				Typeflag: tar.TypeDir,
+			})
+			if err != nil {
+				return nil, err
+			}
+		}
+
 		err := writer.WriteHeader(&tar.Header{
 			Name: name,
 			Size: int64(len(content)),

scaletest/dynamicparameters/config.go

@@ -0,0 +1,10 @@
+package dynamicparameters
+
+import "github.com/google/uuid"
+
+type Config struct {
+	TemplateVersion   uuid.UUID `json:"template_version"`
+	SessionToken      string    `json:"session_token"`
+	Metrics           *Metrics  `json:"-"`
+	MetricLabelValues []string  `json:"metric_label_values"`
+}

scaletest/dynamicparameters/metrics.go

@@ -0,0 +1,28 @@
+package dynamicparameters
+
+import "github.com/prometheus/client_golang/prometheus"
+
+type Metrics struct {
+	LatencyInitialResponseSeconds prometheus.HistogramVec
+	LatencyChangeResponseSeconds  prometheus.HistogramVec
+}
+
+func NewMetrics(reg prometheus.Registerer, labelNames ...string) *Metrics {
+	m := &Metrics{
+		LatencyInitialResponseSeconds: *prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: "coderd",
+			Subsystem: "scaletest",
+			Name:      "dynamic_parameters_latency_initial_response_seconds",
+			Help:      "Time in seconds to get the initial dynamic parameters response from start of request.",
+		}, labelNames),
+		LatencyChangeResponseSeconds: *prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: "coderd",
+			Subsystem: "scaletest",
+			Name:      "dynamic_parameters_latency_change_response_seconds",
+			Help:      "Time in seconds to between sending a dynamic parameters change request and receiving the response.",
+		}, labelNames),
+	}
+	reg.MustRegister(m.LatencyInitialResponseSeconds)
+	reg.MustRegister(m.LatencyChangeResponseSeconds)
+	return m
+}

scaletest/dynamicparameters/run.go

@@ -0,0 +1,114 @@
+package dynamicparameters
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"slices"
+	"time"
+
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/scaletest/harness"
+	"github.com/coder/websocket"
+)
+
+type Runner struct {
+	client *codersdk.Client
+	cfg    Config
+}
+
+var _ harness.Runnable = &Runner{}
+
+func NewRunner(client *codersdk.Client, cfg Config) *Runner {
+	clone := codersdk.New(client.URL)
+	clone.HTTPClient = client.HTTPClient
+	clone.SetLogger(client.Logger())
+	clone.SetSessionToken(cfg.SessionToken)
+	return &Runner{
+		client: clone,
+		cfg:    cfg,
+	}
+}
+
+// Run executes the dynamic parameters test, which:
+//
+// 1. connects to the dynamic parameters stream
+// 2. waits for the initial response
+// 3. sends a change request
+// 4. waits for the change response
+// 5. closes the stream
+func (r *Runner) Run(ctx context.Context, _ string, logs io.Writer) (retErr error) {
+	startTime := time.Now()
+	stream, err := r.client.TemplateVersionDynamicParameters(ctx, codersdk.Me, r.cfg.TemplateVersion)
+	if err != nil {
+		return xerrors.Errorf("connect to dynamic parameters stream: %w", err)
+	}
+	defer stream.Close(websocket.StatusNormalClosure)
+	respCh := stream.Chan()
+
+	var initTime time.Time
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case resp, ok := <-respCh:
+		if !ok {
+			return xerrors.Errorf("dynamic parameters stream closed before initial response")
+		}
+		initTime = time.Now()
+		r.cfg.Metrics.LatencyInitialResponseSeconds.
+			WithLabelValues(r.cfg.MetricLabelValues...).
+			Observe(initTime.Sub(startTime).Seconds())
+		_, _ = fmt.Fprintf(logs, "initial response: %+v\n", resp)
+		if !slices.ContainsFunc(resp.Parameters, func(p codersdk.PreviewParameter) bool {
+			return p.Name == "zero"
+		}) {
+			return xerrors.Errorf("missing expected parameter: 'zero'")
+		}
+		if err := checkNoDiagnostics(resp); err != nil {
+			return xerrors.Errorf("unexpected initial response diagnostics: %w", err)
+		}
+	}
+
+	err = stream.Send(codersdk.DynamicParametersRequest{
+		ID: 1,
+		Inputs: map[string]string{
+			"zero": "B",
+		},
+	})
+	if err != nil {
+		return xerrors.Errorf("send change request: %w", err)
+	}
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case resp, ok := <-respCh:
+		if !ok {
+			return xerrors.Errorf("dynamic parameters stream closed before change response")
+		}
+		_, _ = fmt.Fprintf(logs, "change response: %+v\n", resp)
+		r.cfg.Metrics.LatencyChangeResponseSeconds.
+			WithLabelValues(r.cfg.MetricLabelValues...).
+			Observe(time.Since(initTime).Seconds())
+		if resp.ID != 1 {
+			return xerrors.Errorf("unexpected response ID: %d", resp.ID)
+		}
+		if err := checkNoDiagnostics(resp); err != nil {
+			return xerrors.Errorf("unexpected change response diagnostics: %w", err)
+		}
+		return nil
+	}
+}
+
+func checkNoDiagnostics(resp codersdk.DynamicParametersResponse) error {
+	if len(resp.Diagnostics) != 0 {
+		return xerrors.Errorf("unexpected response diagnostics: %v", resp.Diagnostics)
+	}
+	for _, param := range resp.Parameters {
+		if len(param.Diagnostics) != 0 {
+			return xerrors.Errorf("unexpected parameter diagnostics for '%s': %v", param.Name, param.Diagnostics)
+		}
+	}
+	return nil
+}

scaletest/dynamicparameters/run_test.go

@@ -0,0 +1,49 @@
+package dynamicparameters_test
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/require"
+
+	"cdr.dev/slog"
+	"github.com/coder/coder/v2/coderd/coderdtest"
+	"github.com/coder/coder/v2/scaletest/dynamicparameters"
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestRun(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitLong)
+
+	client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerDaemon: true})
+	client.SetLogger(testutil.Logger(t).Leveled(slog.LevelDebug))
+	first := coderdtest.CreateFirstUser(t, client)
+	userClient, _ := coderdtest.CreateAnotherUser(t, client, first.OrganizationID)
+	orgID := first.OrganizationID
+
+	dynamicParametersTerraformSource, err := dynamicparameters.TemplateContent()
+	require.NoError(t, err)
+
+	template, version := coderdtest.DynamicParameterTemplate(t, client, orgID, coderdtest.DynamicParameterTemplateParams{
+		MainTF:         dynamicParametersTerraformSource,
+		Plan:           nil,
+		ModulesArchive: nil,
+		StaticParams:   nil,
+		ExtraFiles:     dynamicparameters.GetModuleFiles(),
+	})
+
+	reg := prometheus.NewRegistry()
+	cfg := dynamicparameters.Config{
+		TemplateVersion:   version.ID,
+		SessionToken:      userClient.SessionToken(),
+		Metrics:           dynamicparameters.NewMetrics(reg, "template", "test_label_name"),
+		MetricLabelValues: []string{template.Name, "test_label_value"},
+	}
+	runner := dynamicparameters.NewRunner(userClient, cfg)
+	var logs strings.Builder
+	err = runner.Run(ctx, t.Name(), &logs)
+	t.Log("Runner logs:\n\n" + logs.String())
+	require.NoError(t, err)
+}

scaletest/dynamicparameters/template.go

@@ -0,0 +1,74 @@
+package dynamicparameters
+
+import (
+	_ "embed"
+	"encoding/json"
+	"strings"
+	"text/template"
+
+	"github.com/coder/coder/v2/cryptorand"
+)
+
+//go:embed tf/main.tf
+var templateContent string
+
+func TemplateContent() (string, error) {
+	randomString, err := cryptorand.String(8)
+	if err != nil {
+		return "", err
+	}
+	tmpl, err := template.New("workspace-template").Parse(templateContent)
+	if err != nil {
+		return "", err
+	}
+	var result strings.Builder
+	err = tmpl.Execute(&result, map[string]string{
+		"RandomString": randomString,
+	})
+	if err != nil {
+		return "", err
+	}
+	return result.String(), nil
+}
+
+//go:embed tf/modules/two/main.tf
+var moduleTwoMainTF string
+
+// GetModuleFiles returns a map of module files to be used with ExtraFiles
+func GetModuleFiles() map[string][]byte {
+	// Create the modules.json that Terraform needs to see the module
+	modulesJSON := struct {
+		Modules []struct {
+			Key    string `json:"Key"`
+			Source string `json:"Source"`
+			Dir    string `json:"Dir"`
+		} `json:"Modules"`
+	}{
+		Modules: []struct {
+			Key    string `json:"Key"`
+			Source string `json:"Source"`
+			Dir    string `json:"Dir"`
+		}{
+			{
+				Key:    "",
+				Source: "",
+				Dir:    ".",
+			},
+			{
+				Key:    "two",
+				Source: "./modules/two",
+				Dir:    "modules/two",
+			},
+		},
+	}
+
+	modulesJSONBytes, err := json.Marshal(modulesJSON)
+	if err != nil {
+		panic(err) // This should never happen with static data
+	}
+
+	return map[string][]byte{
+		"modules/two/main.tf":             []byte(moduleTwoMainTF),
+		".terraform/modules/modules.json": modulesJSONBytes,
+	}
+}

scaletest/dynamicparameters/tf/main.tf

@@ -0,0 +1,120 @@
+# Cache busting string so each copy of the template is unique: {{.RandomString}}
+terraform {
+  required_providers {
+    coder = {
+      source  = "coder/coder"
+      version = "2.5.3"
+    }
+  }
+}
+
+locals {
+  one_options = {
+    "A" = ["AA", "AB"]
+    # spellchecker:ignore-next-line
+    "B" = ["BA", "BB"]
+  }
+
+  three_options = {
+    "AA" = ["AAA", "AAB"]
+    "AB" = ["ABA", "ABB"]
+    # spellchecker:ignore-next-line
+    "BA" = ["BAA", "BAB"]
+    "BB" = ["BBA", "BBB"]
+  }
+
+  username = data.coder_workspace_owner.me.name
+}
+
+data "coder_workspace_owner" "me" {}
+
+data "coder_parameter" "zero" {
+  name         = "zero"
+  display_name = "Root"
+  description  = "Hello ${local.username}, pick your next parameter using this `dropdown` parameter."
+  form_type    = "dropdown"
+  mutable      = true
+  default      = "A"
+
+  option {
+    value = "A"
+    name  = "A"
+  }
+
+  option {
+    value = "B"
+    name  = "B"
+  }
+}
+
+data "coder_parameter" "one" {
+
+  name         = "One"
+  display_name = "Level One"
+  description  = "This is the first level."
+
+  type      = "list(string)"
+  form_type = "multi-select"
+  order     = 2
+  mutable   = true
+  default   = "[\"${local.one_options[data.coder_parameter.zero.value][0]}\"]"
+
+  dynamic "option" {
+    for_each = local.one_options[data.coder_parameter.zero.value]
+    content {
+      name  = option.value
+      value = option.value
+    }
+  }
+}
+
+module "two" {
+  source = "./modules/two"
+
+  one_value = data.coder_parameter.one.value
+}
+
+data "coder_parameter" "three" {
+
+  name         = "Three"
+  display_name = "Level Three"
+  description  = "This is the third level."
+
+  type      = "string"
+  form_type = "radio"
+  order     = 4
+  mutable   = true
+  default   = local.three_options[module.two.two_value][0]
+
+  dynamic "option" {
+    for_each = local.three_options[module.two.two_value]
+    content {
+      name  = option.value
+      value = option.value
+    }
+  }
+}
+
+data "coder_parameter" "four" {
+  name         = "four"
+  display_name = "Level Four"
+  description  = "This is the last level."
+  order        = 5
+
+  type      = "string"
+  form_type = "radio"
+  default   = "a_fake_value_to_satisfy_import"
+
+  option {
+    name  = format("%s-%s", local.username, data.coder_parameter.three.value)
+    value = "a_fake_value_to_satisfy_import"
+  }
+
+  dynamic "option" {
+    for_each = data.coder_workspace_owner.me.rbac_roles
+    content {
+      name  = format("%s-%s", option.value.name, data.coder_parameter.three.value)
+      value = option.value.name
+    }
+  }
+}

scaletest/dynamicparameters/tf/modules/two/main.tf

@@ -0,0 +1,31 @@
+terraform {
+  required_providers {
+    coder = {
+      source  = "coder/coder"
+      version = "2.5.3"
+    }
+  }
+}
+
+variable "one_value" {
+  description = "The value from the 'one' parameter"
+  type        = string
+}
+
+data "coder_parameter" "two" {
+  name         = "Two"
+  display_name = "Level Two"
+  description  = "This is the second level."
+
+  type      = "string"
+  form_type = "textarea"
+  order     = 3
+  mutable   = true
+
+  default = trim(var.one_value, "[\"]")
+}
+
+output "two_value" {
+  description = "The value of the 'two' parameter"
+  value       = data.coder_parameter.two.value
+}