feat(scaletest): switch notification trigger from creating a user to template deletion (#20512)

This PR refactors the notification scale test to use template admins and template deletion as the notification trigger. Additionally, I've added a configurable timeout for SMTP requests.

Previously, notifications were triggered by creating/deleting a user, and notifications were received by users with the owner role. However, because of how many notifications were generated by the runners, we had too many notifications to reliably test notification delivery.

cli/exp_scaletest_notifications.go

@@ -3,6 +3,7 @@
 package cli
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"net/http"
@@ -29,12 +30,13 @@ import (
 
 func (r *RootCmd) scaletestNotifications() *serpent.Command {
 	var (
-		userCount           int64
-		ownerUserPercentage float64
-		notificationTimeout time.Duration
-		dialTimeout         time.Duration
-		noCleanup           bool
-		smtpAPIURL          string
+		userCount               int64
+		templateAdminPercentage float64
+		notificationTimeout     time.Duration
+		smtpRequestTimeout      time.Duration
+		dialTimeout             time.Duration
+		noCleanup               bool
+		smtpAPIURL              string
 
 		tracingFlags = &scaletestTracingFlags{}
 
@@ -77,24 +79,24 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command {
 				return xerrors.Errorf("--user-count must be greater than 0")
 			}
 
-			if ownerUserPercentage < 0 || ownerUserPercentage > 100 {
-				return xerrors.Errorf("--owner-user-percentage must be between 0 and 100")
+			if templateAdminPercentage < 0 || templateAdminPercentage > 100 {
+				return xerrors.Errorf("--template-admin-percentage must be between 0 and 100")
 			}
 
 			if smtpAPIURL != "" && !strings.HasPrefix(smtpAPIURL, "http://") && !strings.HasPrefix(smtpAPIURL, "https://") {
 				return xerrors.Errorf("--smtp-api-url must start with http:// or https://")
 			}
 
-			ownerUserCount := int64(float64(userCount) * ownerUserPercentage / 100)
-			if ownerUserCount == 0 && ownerUserPercentage > 0 {
-				ownerUserCount = 1
+			templateAdminCount := int64(float64(userCount) * templateAdminPercentage / 100)
+			if templateAdminCount == 0 && templateAdminPercentage > 0 {
+				templateAdminCount = 1
 			}
-			regularUserCount := userCount - ownerUserCount
+			regularUserCount := userCount - templateAdminCount
 
 			_, _ = fmt.Fprintf(inv.Stderr, "Distribution plan:\n")
 			_, _ = fmt.Fprintf(inv.Stderr, "  Total users: %d\n", userCount)
-			_, _ = fmt.Fprintf(inv.Stderr, "  Owner users: %d (%.1f%%)\n", ownerUserCount, ownerUserPercentage)
-			_, _ = fmt.Fprintf(inv.Stderr, "  Regular users: %d (%.1f%%)\n", regularUserCount, 100.0-ownerUserPercentage)
+			_, _ = fmt.Fprintf(inv.Stderr, "  Template admins: %d (%.1f%%)\n", templateAdminCount, templateAdminPercentage)
+			_, _ = fmt.Fprintf(inv.Stderr, "  Regular users: %d (%.1f%%)\n", regularUserCount, 100.0-templateAdminPercentage)
 
 			outputs, err := output.parse()
 			if err != nil {
@@ -127,13 +129,12 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command {
 			_, _ = fmt.Fprintln(inv.Stderr, "Creating users...")
 
 			dialBarrier := &sync.WaitGroup{}
-			ownerWatchBarrier := &sync.WaitGroup{}
+			templateAdminWatchBarrier := &sync.WaitGroup{}
 			dialBarrier.Add(int(userCount))
-			ownerWatchBarrier.Add(int(ownerUserCount))
+			templateAdminWatchBarrier.Add(int(templateAdminCount))
 
 			expectedNotificationIDs := map[uuid.UUID]struct{}{
-				notificationsLib.TemplateUserAccountCreated: {},
-				notificationsLib.TemplateUserAccountDeleted: {},
+				notificationsLib.TemplateTemplateDeleted: {},
 			}
 
 			triggerTimes := make(map[uuid.UUID]chan time.Time, len(expectedNotificationIDs))
@@ -142,19 +143,20 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command {
 			}
 
 			configs := make([]notifications.Config, 0, userCount)
-			for range ownerUserCount {
+			for range templateAdminCount {
 				config := notifications.Config{
 					User: createusers.Config{
 						OrganizationID: me.OrganizationIDs[0],
 					},
-					Roles:                    []string{codersdk.RoleOwner},
+					Roles:                    []string{codersdk.RoleTemplateAdmin},
 					NotificationTimeout:      notificationTimeout,
 					DialTimeout:              dialTimeout,
 					DialBarrier:              dialBarrier,
-					ReceivingWatchBarrier:    ownerWatchBarrier,
+					ReceivingWatchBarrier:    templateAdminWatchBarrier,
 					ExpectedNotificationsIDs: expectedNotificationIDs,
 					Metrics:                  metrics,
 					SMTPApiURL:               smtpAPIURL,
+					SMTPRequestTimeout:       smtpRequestTimeout,
 				}
 				if err := config.Validate(); err != nil {
 					return xerrors.Errorf("validate config: %w", err)
@@ -170,9 +172,8 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command {
 					NotificationTimeout:   notificationTimeout,
 					DialTimeout:           dialTimeout,
 					DialBarrier:           dialBarrier,
-					ReceivingWatchBarrier: ownerWatchBarrier,
+					ReceivingWatchBarrier: templateAdminWatchBarrier,
 					Metrics:               metrics,
-					SMTPApiURL:            smtpAPIURL,
 				}
 				if err := config.Validate(); err != nil {
 					return xerrors.Errorf("validate config: %w", err)
@@ -180,7 +181,7 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command {
 				configs = append(configs, config)
 			}
 
-			go triggerUserNotifications(
+			go triggerNotifications(
 				ctx,
 				logger,
 				client,
@@ -261,23 +262,30 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command {
 			Required:      true,
 		},
 		{
-			Flag:        "owner-user-percentage",
-			Env:         "CODER_SCALETEST_NOTIFICATION_OWNER_USER_PERCENTAGE",
+			Flag:        "template-admin-percentage",
+			Env:         "CODER_SCALETEST_NOTIFICATION_TEMPLATE_ADMIN_PERCENTAGE",
 			Default:     "20.0",
-			Description: "Percentage of users to assign Owner role to (0-100).",
-			Value:       serpent.Float64Of(&ownerUserPercentage),
+			Description: "Percentage of users to assign Template Admin role to (0-100).",
+			Value:       serpent.Float64Of(&templateAdminPercentage),
 		},
 		{
 			Flag:        "notification-timeout",
 			Env:         "CODER_SCALETEST_NOTIFICATION_TIMEOUT",
-			Default:     "5m",
+			Default:     "10m",
 			Description: "How long to wait for notifications after triggering.",
 			Value:       serpent.DurationOf(&notificationTimeout),
 		},
+		{
+			Flag:        "smtp-request-timeout",
+			Env:         "CODER_SCALETEST_SMTP_REQUEST_TIMEOUT",
+			Default:     "5m",
+			Description: "Timeout for SMTP requests.",
+			Value:       serpent.DurationOf(&smtpRequestTimeout),
+		},
 		{
 			Flag:        "dial-timeout",
 			Env:         "CODER_SCALETEST_DIAL_TIMEOUT",
-			Default:     "2m",
+			Default:     "10m",
 			Description: "Timeout for dialing the notification websocket endpoint.",
 			Value:       serpent.DurationOf(&dialTimeout),
 		},
@@ -379,9 +387,9 @@ func computeNotificationLatencies(
 	return nil
 }
 
-// triggerUserNotifications waits for all test users to connect,
-// then creates and deletes a test user to trigger notification events for testing.
-func triggerUserNotifications(
+// triggerNotifications waits for all test users to connect,
+// then creates and deletes a test template to trigger notification events for testing.
+func triggerNotifications(
 	ctx context.Context,
 	logger slog.Logger,
 	client *codersdk.Client,
@@ -414,34 +422,49 @@ func triggerUserNotifications(
 		return
 	}
 
-	const (
-		triggerUsername = "scaletest-trigger-user"
-		triggerEmail    = "scaletest-trigger@example.com"
-	)
+	logger.Info(ctx, "creating test template to test notifications")
 
-	logger.Info(ctx, "creating test user to test notifications",
-		slog.F("username", triggerUsername),
-		slog.F("email", triggerEmail),
-		slog.F("org_id", orgID))
+	// Upload empty template file.
+	file, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader([]byte{}))
+	if err != nil {
+		logger.Error(ctx, "upload test template", slog.Error(err))
+		return
+	}
+	logger.Info(ctx, "test template uploaded", slog.F("file_id", file.ID))
 
-	testUser, err := client.CreateUserWithOrgs(ctx, codersdk.CreateUserRequestWithOrgs{
-		OrganizationIDs: []uuid.UUID{orgID},
-		Username:        triggerUsername,
-		Email:           triggerEmail,
-		Password:        "test-password-123",
+	// Create template version.
+	version, err := client.CreateTemplateVersion(ctx, orgID, codersdk.CreateTemplateVersionRequest{
+		StorageMethod: codersdk.ProvisionerStorageMethodFile,
+		FileID:        file.ID,
+		Provisioner:   codersdk.ProvisionerTypeEcho,
 	})
 	if err != nil {
-		logger.Error(ctx, "create test user", slog.Error(err))
+		logger.Error(ctx, "create test template version", slog.Error(err))
 		return
 	}
-	expectedNotifications[notificationsLib.TemplateUserAccountCreated] <- time.Now()
+	logger.Info(ctx, "test template version created", slog.F("template_version_id", version.ID))
 
-	err = client.DeleteUser(ctx, testUser.ID)
+	// Create template.
+	testTemplate, err := client.CreateTemplate(ctx, orgID, codersdk.CreateTemplateRequest{
+		Name:        "scaletest-test-template",
+		Description: "scaletest-test-template",
+		VersionID:   version.ID,
+	})
 	if err != nil {
-		logger.Error(ctx, "delete test user", slog.Error(err))
+		logger.Error(ctx, "create test template", slog.Error(err))
 		return
 	}
-	expectedNotifications[notificationsLib.TemplateUserAccountDeleted] <- time.Now()
-	close(expectedNotifications[notificationsLib.TemplateUserAccountCreated])
-	close(expectedNotifications[notificationsLib.TemplateUserAccountDeleted])
+	logger.Info(ctx, "test template created", slog.F("template_id", testTemplate.ID))
+
+	// Delete template to trigger notification.
+	err = client.DeleteTemplate(ctx, testTemplate.ID)
+	if err != nil {
+		logger.Error(ctx, "delete test template", slog.Error(err))
+		return
+	}
+	logger.Info(ctx, "test template deleted", slog.F("template_id", testTemplate.ID))
+
+	// Record expected notification.
+	expectedNotifications[notificationsLib.TemplateTemplateDeleted] <- time.Now()
+	close(expectedNotifications[notificationsLib.TemplateTemplateDeleted])
 }

scaletest/notifications/config.go

@@ -37,6 +37,9 @@ type Config struct {
 
 	// SMTPApiUrl is the URL of the SMTP mock HTTP API
 	SMTPApiURL string `json:"smtp_api_url"`
+
+	// SMTPRequestTimeout is the timeout for SMTP requests.
+	SMTPRequestTimeout time.Duration `json:"smtp_request_timeout"`
 }
 
 func (c Config) Validate() error {
@@ -61,6 +64,10 @@ func (c Config) Validate() error {
 		return xerrors.New("notification_timeout must be greater than 0")
 	}
 
+	if c.SMTPApiURL != "" && c.SMTPRequestTimeout <= 0 {
+		return xerrors.New("smtp_request_timeout must be set if smtp_api_url is set")
+	}
+
 	if c.DialTimeout <= 0 {
 		return xerrors.New("dial_timeout must be greater than 0")
 	}

scaletest/notifications/metrics.go

@@ -28,6 +28,12 @@ func NewMetrics(reg prometheus.Registerer) *Metrics {
 		Subsystem: "scaletest",
 		Name:      "notification_delivery_latency_seconds",
 		Help:      "Time between notification-creating action and receipt of notification by client",
+		Buckets: []float64{
+			1, 5, 10, 30, 60,
+			120, 180, 240, 300, 360, 420, 480, 540, 600, 660, 720, 780, 840, 900,
+			1200, 1500, 1800, 2100, 2400, 2700, 3000, 3300, 3600, 3900, 4200, 4500,
+			5400, 7200,
+		},
 	}, []string{"notification_id", "notification_type"})
 	errors := prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace: "coderd",

scaletest/notifications/run.go

@@ -299,7 +299,7 @@ func (r *Runner) watchNotificationsSMTP(ctx context.Context, user codersdk.User,
 
 	apiURL := fmt.Sprintf("%s/messages?email=%s", r.cfg.SMTPApiURL, user.Email)
 	httpClient := &http.Client{
-		Timeout: 10 * time.Second,
+		Timeout: r.cfg.SMTPRequestTimeout,
 	}
 
 	const smtpPollInterval = 2 * time.Second

scaletest/notifications/run_test.go

@@ -228,6 +228,7 @@ func TestRunWithSMTP(t *testing.T) {
 			ReceivingWatchBarrier:    receivingWatchBarrier,
 			ExpectedNotificationsIDs: expectedNotificationsIDs,
 			SMTPApiURL:               smtpAPIServer.URL,
+			SMTPRequestTimeout:       testutil.WaitLong,
 		}
 		err := runnerCfg.Validate()
 		require.NoError(t, err)