feat: add debouncing to provisionerd rpc calls (#5198)

cli/deployment/config.go

@@ -388,6 +388,18 @@ func newConfig() *codersdk.DeploymentConfig {
 				Flag:    "provisioner-daemons",
 				Default: 3,
 			},
+			DaemonPollInterval: &codersdk.DeploymentConfigField[time.Duration]{
+				Name:    "Poll Interval",
+				Usage:   "Time to wait before polling for a new job.",
+				Flag:    "provisioner-daemon-poll-interval",
+				Default: time.Second,
+			},
+			DaemonPollJitter: &codersdk.DeploymentConfigField[time.Duration]{
+				Name:    "Poll Jitter",
+				Usage:   "Random jitter added to the poll interval.",
+				Flag:    "provisioner-daemon-poll-jitter",
+				Default: 100 * time.Millisecond,
+			},
 			ForceCancelInterval: &codersdk.DeploymentConfigField[time.Duration]{
 				Name:    "Force Cancel Interval",
 				Usage:   "Time to force cancel provisioning tasks that are stuck.",

cli/deployment/config_test.go

@@ -2,6 +2,7 @@ package deployment_test
 
 import (
 	"testing"
+	"time"
 
 	"github.com/spf13/pflag"
 	"github.com/stretchr/testify/require"
@@ -25,20 +26,22 @@ func TestConfig(t *testing.T) {
 	}{{
 		Name: "Deployment",
 		Env: map[string]string{
-			"CODER_ADDRESS":              "0.0.0.0:8443",
-			"CODER_ACCESS_URL":           "https://dev.coder.com",
-			"CODER_PG_CONNECTION_URL":    "some-url",
-			"CODER_PPROF_ADDRESS":        "something",
-			"CODER_PPROF_ENABLE":         "true",
-			"CODER_PROMETHEUS_ADDRESS":   "hello-world",
-			"CODER_PROMETHEUS_ENABLE":    "true",
-			"CODER_PROVISIONER_DAEMONS":  "5",
-			"CODER_SECURE_AUTH_COOKIE":   "true",
-			"CODER_SSH_KEYGEN_ALGORITHM": "potato",
-			"CODER_TELEMETRY":            "false",
-			"CODER_TELEMETRY_TRACE":      "false",
-			"CODER_WILDCARD_ACCESS_URL":  "something-wildcard.com",
-			"CODER_UPDATE_CHECK":         "false",
+			"CODER_ADDRESS":                          "0.0.0.0:8443",
+			"CODER_ACCESS_URL":                       "https://dev.coder.com",
+			"CODER_PG_CONNECTION_URL":                "some-url",
+			"CODER_PPROF_ADDRESS":                    "something",
+			"CODER_PPROF_ENABLE":                     "true",
+			"CODER_PROMETHEUS_ADDRESS":               "hello-world",
+			"CODER_PROMETHEUS_ENABLE":                "true",
+			"CODER_PROVISIONER_DAEMONS":              "5",
+			"CODER_PROVISIONER_DAEMON_POLL_INTERVAL": "5s",
+			"CODER_PROVISIONER_DAEMON_POLL_JITTER":   "1s",
+			"CODER_SECURE_AUTH_COOKIE":               "true",
+			"CODER_SSH_KEYGEN_ALGORITHM":             "potato",
+			"CODER_TELEMETRY":                        "false",
+			"CODER_TELEMETRY_TRACE":                  "false",
+			"CODER_WILDCARD_ACCESS_URL":              "something-wildcard.com",
+			"CODER_UPDATE_CHECK":                     "false",
 		},
 		Valid: func(config *codersdk.DeploymentConfig) {
 			require.Equal(t, config.Address.Value, "0.0.0.0:8443")
@@ -49,6 +52,8 @@ func TestConfig(t *testing.T) {
 			require.Equal(t, config.Prometheus.Address.Value, "hello-world")
 			require.Equal(t, config.Prometheus.Enable.Value, true)
 			require.Equal(t, config.Provisioner.Daemons.Value, 5)
+			require.Equal(t, config.Provisioner.DaemonPollInterval.Value, 5*time.Second)
+			require.Equal(t, config.Provisioner.DaemonPollJitter.Value, 1*time.Second)
 			require.Equal(t, config.SecureAuthCookie.Value, true)
 			require.Equal(t, config.SSHKeygenAlgorithm.Value, "potato")
 			require.Equal(t, config.Telemetry.Enable.Value, false)

cli/root_test.go

@@ -22,6 +22,8 @@ import (
 	"github.com/coder/coder/testutil"
 )
 
+// To update the golden files:
+// make update-golden-files
 var updateGoldenFiles = flag.Bool("update", false, "update .golden files")
 
 //nolint:tparallel,paralleltest // These test sets env vars.

cli/server.go

@@ -970,13 +970,16 @@ func newProvisionerDaemon(
 		}()
 		provisioners[string(database.ProvisionerTypeEcho)] = sdkproto.NewDRPCProvisionerClient(echoClient)
 	}
+	debounce := time.Second
 	return provisionerd.New(func(ctx context.Context) (proto.DRPCProvisionerDaemonClient, error) {
 		// This debounces calls to listen every second. Read the comment
 		// in provisionerdserver.go to learn more!
-		return coderAPI.CreateInMemoryProvisionerDaemon(ctx, time.Second)
+		return coderAPI.CreateInMemoryProvisionerDaemon(ctx, debounce)
 	}, &provisionerd.Options{
 		Logger:              logger,
-		PollInterval:        500 * time.Millisecond,
+		JobPollInterval:     cfg.Provisioner.DaemonPollInterval.Value,
+		JobPollJitter:       cfg.Provisioner.DaemonPollJitter.Value,
+		JobPollDebounce:     debounce,
 		UpdateInterval:      500 * time.Millisecond,
 		ForceCancelInterval: cfg.Provisioner.ForceCancelInterval.Value,
 		Provisioners:        provisioners,

cli/testdata/coder_server_--help.golden

@@ -128,6 +128,15 @@ Flags:
       --prometheus-enable                            Serve prometheus metrics on the address
                                                      defined by prometheus address.
                                                      Consumes $CODER_PROMETHEUS_ENABLE
+      --provisioner-daemon-poll-interval duration    Time to wait before polling for a new
+                                                     job.
+                                                     Consumes
+                                                     $CODER_PROVISIONER_DAEMON_POLL_INTERVAL
+                                                     (default 1s)
+      --provisioner-daemon-poll-jitter duration      Random jitter added to the poll interval.
+                                                     Consumes
+                                                     $CODER_PROVISIONER_DAEMON_POLL_JITTER
+                                                     (default 100ms)
       --provisioner-daemons int                      Number of provisioner daemons to create
                                                      on start. If builds are stuck in queued
                                                      state for a long time, consider