fix: improve http connection pooling for smtp notifications (#20605)

This change updates how SMTP notifications are polled during scale
tests.

Before, each of the ~2,000 pollers created its own http.Client, which
opened thousands of short-lived TCP connections.
Under heavy load, this ran out of available network ports and caused
errors like `connect: cannot assign requested address`

Now, all pollers share one HTTP connection pool. This prevents port
exhaustion and makes polling faster and more stable.
If a network error happens, the poller will now retry instead of
stopping, so tests keep running until all notifications are received.

The `SMTPRequestTimeout` is now applied per request using a context,
instead of being set on the `http.Client`.

cli/exp_scaletest_notifications.go

@@ -142,6 +142,15 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command {
 				triggerTimes[id] = make(chan time.Time, 1)
 			}
 
+			smtpHTTPTransport := &http.Transport{
+				MaxConnsPerHost:     512,
+				MaxIdleConnsPerHost: 512,
+				IdleConnTimeout:     60 * time.Second,
+			}
+			smtpHTTPClient := &http.Client{
+				Transport: smtpHTTPTransport,
+			}
+
 			configs := make([]notifications.Config, 0, userCount)
 			for range templateAdminCount {
 				config := notifications.Config{
@@ -157,6 +166,7 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command {
 					Metrics:                  metrics,
 					SMTPApiURL:               smtpAPIURL,
 					SMTPRequestTimeout:       smtpRequestTimeout,
+					SMTPHttpClient:           smtpHTTPClient,
 				}
 				if err := config.Validate(); err != nil {
 					return xerrors.Errorf("validate config: %w", err)