feat: add provisioner job queue wait time histogram and jobs enqueued counter (#21869)

This PR adds some metrics to help identify job enqueue rates and
latencies. This work was initiated as a way to help reduce the cost of
the observation/measurement itself for autostart scaletests, which
impacts our ability to identify/reason about the load caused by
autostart. See: https://github.com/coder/internal/issues/1209

I've extended the metrics here to account for regular user initiated
builds, prebuilds, autostarts, etc. IMO there is still the question here
of whether we want to include or need the `transition` label, which is
only present on workspace builds. Including it does lead to an increase
in cardinality, and in the case of the histogram (when not using native
histograms) that's at least a few extra series for every bucket. We
could remove the transition label there but keep it on the counter.

Additionally, the histogram is currently observing latencies for other
jobs, such as template builds/version imports, those do not have a
transition type associated with them.

Tested briefly in a workspace, can see metric values like the following:
-
`coderd_workspace_builds_enqueued_total{build_reason="autostart",provisioner_type="terraform",status="success",transition="start"}
1`
-
`coderd_provisioner_job_queue_wait_seconds_bucket{build_reason="autostart",job_type="workspace_build",provisioner_type="terraform",transition="start",le="0.025"}
1`

---------

Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

enterprise/coderd/coderd.go

@@ -1331,6 +1331,7 @@ func (api *API) setupPrebuilds(featureEnabled bool) (agplprebuilds.Reconciliatio
 		api.AGPL.BuildUsageChecker,
 		api.TracerProvider,
 		int(api.DeploymentValues.PostgresConnMaxOpen.Value()),
+		api.AGPL.WorkspaceBuilderMetrics,
 	)
 	return reconciler, prebuilds.NewEnterpriseClaimer()
 }

enterprise/coderd/prebuilds/claim_test.go

@@ -174,6 +174,7 @@ func TestClaimPrebuild(t *testing.T) {
 					newNoopUsageCheckerPtr(),
 					noop.NewTracerProvider(),
 					10,
+					nil,
 				)
 				var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer()
 				api.AGPL.PrebuildsClaimer.Store(&claimer)

enterprise/coderd/prebuilds/metricscollector_test.go

@@ -204,6 +204,7 @@ func TestMetricsCollector(t *testing.T) {
 										newNoopUsageCheckerPtr(),
 										noop.NewTracerProvider(),
 										10,
+										nil,
 									)
 									ctx := testutil.Context(t, testutil.WaitLong)
 
@@ -344,6 +345,7 @@ func TestMetricsCollector_DuplicateTemplateNames(t *testing.T) {
 		newNoopUsageCheckerPtr(),
 		noop.NewTracerProvider(),
 		10,
+		nil,
 	)
 	ctx := testutil.Context(t, testutil.WaitLong)
 
@@ -500,6 +502,7 @@ func TestMetricsCollector_ReconciliationPausedMetric(t *testing.T) {
 			newNoopUsageCheckerPtr(),
 			noop.NewTracerProvider(),
 			10,
+			nil,
 		)
 		ctx := testutil.Context(t, testutil.WaitLong)
 
@@ -537,6 +540,7 @@ func TestMetricsCollector_ReconciliationPausedMetric(t *testing.T) {
 			newNoopUsageCheckerPtr(),
 			noop.NewTracerProvider(),
 			10,
+			nil,
 		)
 		ctx := testutil.Context(t, testutil.WaitLong)
 
@@ -574,6 +578,7 @@ func TestMetricsCollector_ReconciliationPausedMetric(t *testing.T) {
 			newNoopUsageCheckerPtr(),
 			noop.NewTracerProvider(),
 			10,
+			nil,
 		)
 		ctx := testutil.Context(t, testutil.WaitLong)
 

enterprise/coderd/prebuilds/reconcile.go

@@ -65,7 +65,8 @@ type StoreReconciler struct {
 	// Prebuild state metrics
 	metrics *MetricsCollector
 	// Operational metrics
-	reconciliationDuration prometheus.Histogram
+	reconciliationDuration  prometheus.Histogram
+	workspaceBuilderMetrics *wsbuilder.Metrics
 }
 
 var _ prebuilds.ReconciliationOrchestrator = &StoreReconciler{}
@@ -99,6 +100,7 @@ func NewStoreReconciler(store database.Store,
 	buildUsageChecker *atomic.Pointer[wsbuilder.UsageChecker],
 	tracerProvider trace.TracerProvider,
 	maxDBConnections int,
+	workspaceBuilderMetrics *wsbuilder.Metrics,
 ) *StoreReconciler {
 	reconciliationConcurrency := calculateReconciliationConcurrency(maxDBConnections)
 
@@ -120,6 +122,7 @@ func NewStoreReconciler(store database.Store,
 		done:                      make(chan struct{}, 1),
 		provisionNotifyCh:         make(chan database.ProvisionerJob, 10),
 		reconciliationConcurrency: reconciliationConcurrency,
+		workspaceBuilderMetrics:   workspaceBuilderMetrics,
 	}
 
 	if registerer != nil {
@@ -1052,7 +1055,8 @@ func (c *StoreReconciler) provision(
 	builder := wsbuilder.New(workspace, transition, *c.buildUsageChecker.Load()).
 		Reason(database.BuildReasonInitiator).
 		Initiator(database.PrebuildsSystemUserID).
-		MarkPrebuild()
+		MarkPrebuild().
+		BuildMetrics(c.workspaceBuilderMetrics)
 
 	if transition != database.WorkspaceTransitionDelete {
 		// We don't specify the version for a delete transition,

enterprise/coderd/prebuilds/reconcile_test.go

@@ -61,6 +61,7 @@ func TestNoReconciliationActionsIfNoPresets(t *testing.T) {
 		newNoopUsageCheckerPtr(),
 		noop.NewTracerProvider(),
 		10,
+		nil,
 	)
 
 	// given a template version with no presets
@@ -112,6 +113,7 @@ func TestNoReconciliationActionsIfNoPrebuilds(t *testing.T) {
 		newNoopUsageCheckerPtr(),
 		noop.NewTracerProvider(),
 		10,
+		nil,
 	)
 
 	// given there are presets, but no prebuilds
@@ -450,6 +452,7 @@ func (tc testCase) run(t *testing.T) {
 			newNoopUsageCheckerPtr(),
 			noop.NewTracerProvider(),
 			10,
+			nil,
 		)
 
 		// Run the reconciliation multiple times to ensure idempotency
@@ -527,6 +530,7 @@ func TestMultiplePresetsPerTemplateVersion(t *testing.T) {
 		newNoopUsageCheckerPtr(),
 		noop.NewTracerProvider(),
 		10,
+		nil,
 	)
 
 	ownerID := uuid.New()
@@ -658,6 +662,7 @@ func TestPrebuildScheduling(t *testing.T) {
 				newNoopUsageCheckerPtr(),
 				noop.NewTracerProvider(),
 				10,
+				nil,
 			)
 
 			ownerID := uuid.New()
@@ -767,6 +772,7 @@ func TestInvalidPreset(t *testing.T) {
 		newNoopUsageCheckerPtr(),
 		noop.NewTracerProvider(),
 		10,
+		nil,
 	)
 
 	ownerID := uuid.New()
@@ -837,6 +843,7 @@ func TestDeletionOfPrebuiltWorkspaceWithInvalidPreset(t *testing.T) {
 		newNoopUsageCheckerPtr(),
 		noop.NewTracerProvider(),
 		10,
+		nil,
 	)
 
 	ownerID := uuid.New()
@@ -939,6 +946,7 @@ func TestSkippingHardLimitedPresets(t *testing.T) {
 				newNoopUsageCheckerPtr(),
 				noop.NewTracerProvider(),
 				10,
+				nil,
 			)
 
 			// Set up test environment with a template, version, and preset.
@@ -1090,6 +1098,7 @@ func TestHardLimitedPresetShouldNotBlockDeletion(t *testing.T) {
 				newNoopUsageCheckerPtr(),
 				noop.NewTracerProvider(),
 				10,
+				nil,
 			)
 
 			// Set up test environment with a template, version, and preset.
@@ -1291,6 +1300,7 @@ func TestRunLoop(t *testing.T) {
 		newNoopUsageCheckerPtr(),
 		noop.NewTracerProvider(),
 		10,
+		nil,
 	)
 
 	ownerID := uuid.New()
@@ -1423,6 +1433,7 @@ func TestReconcilerLifecycle(t *testing.T) {
 		newNoopUsageCheckerPtr(),
 		noop.NewTracerProvider(),
 		10,
+		nil,
 	)
 
 	// When: the reconciler is stopped (simulating the prebuilds feature being disabled)
@@ -1438,6 +1449,7 @@ func TestReconcilerLifecycle(t *testing.T) {
 		newNoopUsageCheckerPtr(),
 		noop.NewTracerProvider(),
 		10,
+		nil,
 	)
 
 	// Gracefully stop the reconciliation loop
@@ -1471,6 +1483,7 @@ func TestFailedBuildBackoff(t *testing.T) {
 		newNoopUsageCheckerPtr(),
 		noop.NewTracerProvider(),
 		10,
+		nil,
 	)
 
 	// Given: an active template version with presets and prebuilds configured.
@@ -1595,6 +1608,7 @@ func TestReconciliationLock(t *testing.T) {
 				newNoopEnqueuer(),
 				newNoopUsageCheckerPtr(), noop.NewTracerProvider(),
 				10,
+				nil,
 			)
 			reconciler.WithReconciliationLock(ctx, logger, func(_ context.Context, _ database.Store) error {
 				lockObtained := mutex.TryLock()
@@ -1633,6 +1647,7 @@ func TestTrackResourceReplacement(t *testing.T) {
 		newNoopUsageCheckerPtr(),
 		noop.NewTracerProvider(),
 		10,
+		nil,
 	)
 
 	// Given: a template admin to receive a notification.
@@ -1793,6 +1808,7 @@ func TestExpiredPrebuildsMultipleActions(t *testing.T) {
 				newNoopUsageCheckerPtr(),
 				noop.NewTracerProvider(),
 				10,
+				nil,
 			)
 
 			// Set up test environment with a template, version, and preset
@@ -2258,6 +2274,7 @@ func TestCancelPendingPrebuilds(t *testing.T) {
 					newNoopUsageCheckerPtr(),
 					noop.NewTracerProvider(),
 					10,
+					nil,
 				)
 				owner := coderdtest.CreateFirstUser(t, client)
 
@@ -2503,6 +2520,7 @@ func TestCancelPendingPrebuilds(t *testing.T) {
 			newNoopUsageCheckerPtr(),
 			noop.NewTracerProvider(),
 			10,
+			nil,
 		)
 		owner := coderdtest.CreateFirstUser(t, client)
 
@@ -2576,6 +2594,7 @@ func TestReconciliationStats(t *testing.T) {
 		newNoopUsageCheckerPtr(),
 		noop.NewTracerProvider(),
 		10,
+		nil,
 	)
 	owner := coderdtest.CreateFirstUser(t, client)
 
@@ -3066,6 +3085,7 @@ func TestReconciliationRespectsPauseSetting(t *testing.T) {
 		newNoopUsageCheckerPtr(),
 		noop.NewTracerProvider(),
 		10,
+		nil,
 	)
 
 	// Setup a template with a preset that should create prebuilds
@@ -3172,6 +3192,7 @@ func BenchmarkReconcileAll_NoOps(b *testing.B) {
 				newNoopUsageCheckerPtr(),
 				noop.NewTracerProvider(),
 				maxOpenConns,
+				nil,
 			)
 
 			org := dbgen.Organization(b, db, database.Organization{})
@@ -3283,6 +3304,7 @@ func BenchmarkReconcileAll_ConnectionContention(b *testing.B) {
 					newNoopUsageCheckerPtr(),
 					noop.NewTracerProvider(),
 					maxOpenConns,
+					nil,
 				)
 
 				// Create presets from active template versions that need reconciliation actions
@@ -3402,6 +3424,7 @@ func BenchmarkReconcileAll_Mix(b *testing.B) {
 					newNoopUsageCheckerPtr(),
 					noop.NewTracerProvider(),
 					maxOpenConns,
+					nil,
 				)
 
 				org := dbgen.Organization(b, db, database.Organization{})

enterprise/coderd/workspaces_test.go

@@ -1991,6 +1991,7 @@ func TestPrebuildsAutobuild(t *testing.T) {
 			api.AGPL.BuildUsageChecker,
 			noop.NewTracerProvider(),
 			10,
+			nil,
 		)
 		var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer()
 		api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -2115,6 +2116,7 @@ func TestPrebuildsAutobuild(t *testing.T) {
 			api.AGPL.BuildUsageChecker,
 			noop.NewTracerProvider(),
 			10,
+			nil,
 		)
 		var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer()
 		api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -2239,6 +2241,7 @@ func TestPrebuildsAutobuild(t *testing.T) {
 			api.AGPL.BuildUsageChecker,
 			noop.NewTracerProvider(),
 			10,
+			nil,
 		)
 		var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer()
 		api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -2385,6 +2388,7 @@ func TestPrebuildsAutobuild(t *testing.T) {
 			api.AGPL.BuildUsageChecker,
 			noop.NewTracerProvider(),
 			10,
+			nil,
 		)
 		var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer()
 		api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -2532,6 +2536,7 @@ func TestPrebuildsAutobuild(t *testing.T) {
 			api.AGPL.BuildUsageChecker,
 			noop.NewTracerProvider(),
 			10,
+			nil,
 		)
 		var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer()
 		api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -2979,6 +2984,7 @@ func TestWorkspaceProvisionerdServerMetrics(t *testing.T) {
 		api.AGPL.BuildUsageChecker,
 		noop.NewTracerProvider(),
 		10,
+		nil,
 	)
 	var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer()
 	api.AGPL.PrebuildsClaimer.Store(&claimer)