From 8f78baddb1f656134a030fc24a5a7eb2121318c1 Mon Sep 17 00:00:00 2001 From: Kacper Sawicki Date: Fri, 31 Oct 2025 09:43:06 +0100 Subject: [PATCH] feat(scaletest): switch notification trigger from creating a user to template deletion (#20512) This PR refactors the notification scale test to use template admins and template deletion as the notification trigger. Additionally, I've added a configurable timeout for SMTP requests. Previously, notifications were triggered by creating/deleting a user, and notifications were received by users with the owner role. However, because of how many notifications were generated by the runners, we had too many notifications to reliably test notification delivery. --- cli/exp_scaletest_notifications.go | 129 ++++++++++++++++------------ scaletest/notifications/config.go | 7 ++ scaletest/notifications/metrics.go | 6 ++ scaletest/notifications/run.go | 2 +- scaletest/notifications/run_test.go | 1 + 5 files changed, 91 insertions(+), 54 deletions(-) diff --git a/cli/exp_scaletest_notifications.go b/cli/exp_scaletest_notifications.go index 1ea4785893..0a9eb22d5c 100644 --- a/cli/exp_scaletest_notifications.go +++ b/cli/exp_scaletest_notifications.go @@ -3,6 +3,7 @@ package cli import ( + "bytes" "context" "fmt" "net/http" @@ -29,12 +30,13 @@ import ( func (r *RootCmd) scaletestNotifications() *serpent.Command { var ( - userCount int64 - ownerUserPercentage float64 - notificationTimeout time.Duration - dialTimeout time.Duration - noCleanup bool - smtpAPIURL string + userCount int64 + templateAdminPercentage float64 + notificationTimeout time.Duration + smtpRequestTimeout time.Duration + dialTimeout time.Duration + noCleanup bool + smtpAPIURL string tracingFlags = &scaletestTracingFlags{} @@ -77,24 +79,24 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command { return xerrors.Errorf("--user-count must be greater than 0") } - if ownerUserPercentage < 0 || ownerUserPercentage > 100 { - return xerrors.Errorf("--owner-user-percentage must be between 0 and 100") + if templateAdminPercentage < 0 || templateAdminPercentage > 100 { + return xerrors.Errorf("--template-admin-percentage must be between 0 and 100") } if smtpAPIURL != "" && !strings.HasPrefix(smtpAPIURL, "http://") && !strings.HasPrefix(smtpAPIURL, "https://") { return xerrors.Errorf("--smtp-api-url must start with http:// or https://") } - ownerUserCount := int64(float64(userCount) * ownerUserPercentage / 100) - if ownerUserCount == 0 && ownerUserPercentage > 0 { - ownerUserCount = 1 + templateAdminCount := int64(float64(userCount) * templateAdminPercentage / 100) + if templateAdminCount == 0 && templateAdminPercentage > 0 { + templateAdminCount = 1 } - regularUserCount := userCount - ownerUserCount + regularUserCount := userCount - templateAdminCount _, _ = fmt.Fprintf(inv.Stderr, "Distribution plan:\n") _, _ = fmt.Fprintf(inv.Stderr, " Total users: %d\n", userCount) - _, _ = fmt.Fprintf(inv.Stderr, " Owner users: %d (%.1f%%)\n", ownerUserCount, ownerUserPercentage) - _, _ = fmt.Fprintf(inv.Stderr, " Regular users: %d (%.1f%%)\n", regularUserCount, 100.0-ownerUserPercentage) + _, _ = fmt.Fprintf(inv.Stderr, " Template admins: %d (%.1f%%)\n", templateAdminCount, templateAdminPercentage) + _, _ = fmt.Fprintf(inv.Stderr, " Regular users: %d (%.1f%%)\n", regularUserCount, 100.0-templateAdminPercentage) outputs, err := output.parse() if err != nil { @@ -127,13 +129,12 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command { _, _ = fmt.Fprintln(inv.Stderr, "Creating users...") dialBarrier := &sync.WaitGroup{} - ownerWatchBarrier := &sync.WaitGroup{} + templateAdminWatchBarrier := &sync.WaitGroup{} dialBarrier.Add(int(userCount)) - ownerWatchBarrier.Add(int(ownerUserCount)) + templateAdminWatchBarrier.Add(int(templateAdminCount)) expectedNotificationIDs := map[uuid.UUID]struct{}{ - notificationsLib.TemplateUserAccountCreated: {}, - notificationsLib.TemplateUserAccountDeleted: {}, + notificationsLib.TemplateTemplateDeleted: {}, } triggerTimes := make(map[uuid.UUID]chan time.Time, len(expectedNotificationIDs)) @@ -142,19 +143,20 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command { } configs := make([]notifications.Config, 0, userCount) - for range ownerUserCount { + for range templateAdminCount { config := notifications.Config{ User: createusers.Config{ OrganizationID: me.OrganizationIDs[0], }, - Roles: []string{codersdk.RoleOwner}, + Roles: []string{codersdk.RoleTemplateAdmin}, NotificationTimeout: notificationTimeout, DialTimeout: dialTimeout, DialBarrier: dialBarrier, - ReceivingWatchBarrier: ownerWatchBarrier, + ReceivingWatchBarrier: templateAdminWatchBarrier, ExpectedNotificationsIDs: expectedNotificationIDs, Metrics: metrics, SMTPApiURL: smtpAPIURL, + SMTPRequestTimeout: smtpRequestTimeout, } if err := config.Validate(); err != nil { return xerrors.Errorf("validate config: %w", err) @@ -170,9 +172,8 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command { NotificationTimeout: notificationTimeout, DialTimeout: dialTimeout, DialBarrier: dialBarrier, - ReceivingWatchBarrier: ownerWatchBarrier, + ReceivingWatchBarrier: templateAdminWatchBarrier, Metrics: metrics, - SMTPApiURL: smtpAPIURL, } if err := config.Validate(); err != nil { return xerrors.Errorf("validate config: %w", err) @@ -180,7 +181,7 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command { configs = append(configs, config) } - go triggerUserNotifications( + go triggerNotifications( ctx, logger, client, @@ -261,23 +262,30 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command { Required: true, }, { - Flag: "owner-user-percentage", - Env: "CODER_SCALETEST_NOTIFICATION_OWNER_USER_PERCENTAGE", + Flag: "template-admin-percentage", + Env: "CODER_SCALETEST_NOTIFICATION_TEMPLATE_ADMIN_PERCENTAGE", Default: "20.0", - Description: "Percentage of users to assign Owner role to (0-100).", - Value: serpent.Float64Of(&ownerUserPercentage), + Description: "Percentage of users to assign Template Admin role to (0-100).", + Value: serpent.Float64Of(&templateAdminPercentage), }, { Flag: "notification-timeout", Env: "CODER_SCALETEST_NOTIFICATION_TIMEOUT", - Default: "5m", + Default: "10m", Description: "How long to wait for notifications after triggering.", Value: serpent.DurationOf(¬ificationTimeout), }, + { + Flag: "smtp-request-timeout", + Env: "CODER_SCALETEST_SMTP_REQUEST_TIMEOUT", + Default: "5m", + Description: "Timeout for SMTP requests.", + Value: serpent.DurationOf(&smtpRequestTimeout), + }, { Flag: "dial-timeout", Env: "CODER_SCALETEST_DIAL_TIMEOUT", - Default: "2m", + Default: "10m", Description: "Timeout for dialing the notification websocket endpoint.", Value: serpent.DurationOf(&dialTimeout), }, @@ -379,9 +387,9 @@ func computeNotificationLatencies( return nil } -// triggerUserNotifications waits for all test users to connect, -// then creates and deletes a test user to trigger notification events for testing. -func triggerUserNotifications( +// triggerNotifications waits for all test users to connect, +// then creates and deletes a test template to trigger notification events for testing. +func triggerNotifications( ctx context.Context, logger slog.Logger, client *codersdk.Client, @@ -414,34 +422,49 @@ func triggerUserNotifications( return } - const ( - triggerUsername = "scaletest-trigger-user" - triggerEmail = "scaletest-trigger@example.com" - ) + logger.Info(ctx, "creating test template to test notifications") - logger.Info(ctx, "creating test user to test notifications", - slog.F("username", triggerUsername), - slog.F("email", triggerEmail), - slog.F("org_id", orgID)) + // Upload empty template file. + file, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader([]byte{})) + if err != nil { + logger.Error(ctx, "upload test template", slog.Error(err)) + return + } + logger.Info(ctx, "test template uploaded", slog.F("file_id", file.ID)) - testUser, err := client.CreateUserWithOrgs(ctx, codersdk.CreateUserRequestWithOrgs{ - OrganizationIDs: []uuid.UUID{orgID}, - Username: triggerUsername, - Email: triggerEmail, - Password: "test-password-123", + // Create template version. + version, err := client.CreateTemplateVersion(ctx, orgID, codersdk.CreateTemplateVersionRequest{ + StorageMethod: codersdk.ProvisionerStorageMethodFile, + FileID: file.ID, + Provisioner: codersdk.ProvisionerTypeEcho, }) if err != nil { - logger.Error(ctx, "create test user", slog.Error(err)) + logger.Error(ctx, "create test template version", slog.Error(err)) return } - expectedNotifications[notificationsLib.TemplateUserAccountCreated] <- time.Now() + logger.Info(ctx, "test template version created", slog.F("template_version_id", version.ID)) - err = client.DeleteUser(ctx, testUser.ID) + // Create template. + testTemplate, err := client.CreateTemplate(ctx, orgID, codersdk.CreateTemplateRequest{ + Name: "scaletest-test-template", + Description: "scaletest-test-template", + VersionID: version.ID, + }) if err != nil { - logger.Error(ctx, "delete test user", slog.Error(err)) + logger.Error(ctx, "create test template", slog.Error(err)) return } - expectedNotifications[notificationsLib.TemplateUserAccountDeleted] <- time.Now() - close(expectedNotifications[notificationsLib.TemplateUserAccountCreated]) - close(expectedNotifications[notificationsLib.TemplateUserAccountDeleted]) + logger.Info(ctx, "test template created", slog.F("template_id", testTemplate.ID)) + + // Delete template to trigger notification. + err = client.DeleteTemplate(ctx, testTemplate.ID) + if err != nil { + logger.Error(ctx, "delete test template", slog.Error(err)) + return + } + logger.Info(ctx, "test template deleted", slog.F("template_id", testTemplate.ID)) + + // Record expected notification. + expectedNotifications[notificationsLib.TemplateTemplateDeleted] <- time.Now() + close(expectedNotifications[notificationsLib.TemplateTemplateDeleted]) } diff --git a/scaletest/notifications/config.go b/scaletest/notifications/config.go index ac8daeb9ef..a3953eb540 100644 --- a/scaletest/notifications/config.go +++ b/scaletest/notifications/config.go @@ -37,6 +37,9 @@ type Config struct { // SMTPApiUrl is the URL of the SMTP mock HTTP API SMTPApiURL string `json:"smtp_api_url"` + + // SMTPRequestTimeout is the timeout for SMTP requests. + SMTPRequestTimeout time.Duration `json:"smtp_request_timeout"` } func (c Config) Validate() error { @@ -61,6 +64,10 @@ func (c Config) Validate() error { return xerrors.New("notification_timeout must be greater than 0") } + if c.SMTPApiURL != "" && c.SMTPRequestTimeout <= 0 { + return xerrors.New("smtp_request_timeout must be set if smtp_api_url is set") + } + if c.DialTimeout <= 0 { return xerrors.New("dial_timeout must be greater than 0") } diff --git a/scaletest/notifications/metrics.go b/scaletest/notifications/metrics.go index 0bf3ebad74..6d9c1a03fa 100644 --- a/scaletest/notifications/metrics.go +++ b/scaletest/notifications/metrics.go @@ -28,6 +28,12 @@ func NewMetrics(reg prometheus.Registerer) *Metrics { Subsystem: "scaletest", Name: "notification_delivery_latency_seconds", Help: "Time between notification-creating action and receipt of notification by client", + Buckets: []float64{ + 1, 5, 10, 30, 60, + 120, 180, 240, 300, 360, 420, 480, 540, 600, 660, 720, 780, 840, 900, + 1200, 1500, 1800, 2100, 2400, 2700, 3000, 3300, 3600, 3900, 4200, 4500, + 5400, 7200, + }, }, []string{"notification_id", "notification_type"}) errors := prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "coderd", diff --git a/scaletest/notifications/run.go b/scaletest/notifications/run.go index abe8445746..64ebe71235 100644 --- a/scaletest/notifications/run.go +++ b/scaletest/notifications/run.go @@ -299,7 +299,7 @@ func (r *Runner) watchNotificationsSMTP(ctx context.Context, user codersdk.User, apiURL := fmt.Sprintf("%s/messages?email=%s", r.cfg.SMTPApiURL, user.Email) httpClient := &http.Client{ - Timeout: 10 * time.Second, + Timeout: r.cfg.SMTPRequestTimeout, } const smtpPollInterval = 2 * time.Second diff --git a/scaletest/notifications/run_test.go b/scaletest/notifications/run_test.go index 1e198e9edd..21c89a8714 100644 --- a/scaletest/notifications/run_test.go +++ b/scaletest/notifications/run_test.go @@ -228,6 +228,7 @@ func TestRunWithSMTP(t *testing.T) { ReceivingWatchBarrier: receivingWatchBarrier, ExpectedNotificationsIDs: expectedNotificationsIDs, SMTPApiURL: smtpAPIServer.URL, + SMTPRequestTimeout: testutil.WaitLong, } err := runnerCfg.Validate() require.NoError(t, err)