feat: add provisioner job queue wait time histogram and jobs enqueued counter (#21869)

This PR adds some metrics to help identify job enqueue rates and
latencies. This work was initiated as a way to help reduce the cost of
the observation/measurement itself for autostart scaletests, which
impacts our ability to identify/reason about the load caused by
autostart. See: https://github.com/coder/internal/issues/1209

I've extended the metrics here to account for regular user initiated
builds, prebuilds, autostarts, etc. IMO there is still the question here
of whether we want to include or need the `transition` label, which is
only present on workspace builds. Including it does lead to an increase
in cardinality, and in the case of the histogram (when not using native
histograms) that's at least a few extra series for every bucket. We
could remove the transition label there but keep it on the counter.

Additionally, the histogram is currently observing latencies for other
jobs, such as template builds/version imports, those do not have a
transition type associated with them.

Tested briefly in a workspace, can see metric values like the following:
-
`coderd_workspace_builds_enqueued_total{build_reason="autostart",provisioner_type="terraform",status="success",transition="start"}
1`
-
`coderd_provisioner_job_queue_wait_seconds_bucket{build_reason="autostart",job_type="workspace_build",provisioner_type="terraform",transition="start",le="0.025"}
1`

---------

Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

coderd/autobuild/lifecycle_executor.go

@@ -48,9 +48,10 @@ type Executor struct {
 	tick                  <-chan time.Time
 	statsCh               chan<- Stats
 	// NotificationsEnqueuer handles enqueueing notifications for delivery by SMTP, webhook, etc.
-	notificationsEnqueuer notifications.Enqueuer
-	reg                   prometheus.Registerer
-	experiments           codersdk.Experiments
+	notificationsEnqueuer   notifications.Enqueuer
+	reg                     prometheus.Registerer
+	experiments             codersdk.Experiments
+	workspaceBuilderMetrics *wsbuilder.Metrics
 
 	metrics executorMetrics
 }
@@ -67,23 +68,24 @@ type Stats struct {
 }
 
 // New returns a new wsactions executor.
-func NewExecutor(ctx context.Context, db database.Store, ps pubsub.Pubsub, fc *files.Cache, reg prometheus.Registerer, tss *atomic.Pointer[schedule.TemplateScheduleStore], auditor *atomic.Pointer[audit.Auditor], acs *atomic.Pointer[dbauthz.AccessControlStore], buildUsageChecker *atomic.Pointer[wsbuilder.UsageChecker], log slog.Logger, tick <-chan time.Time, enqueuer notifications.Enqueuer, exp codersdk.Experiments) *Executor {
+func NewExecutor(ctx context.Context, db database.Store, ps pubsub.Pubsub, fc *files.Cache, reg prometheus.Registerer, tss *atomic.Pointer[schedule.TemplateScheduleStore], auditor *atomic.Pointer[audit.Auditor], acs *atomic.Pointer[dbauthz.AccessControlStore], buildUsageChecker *atomic.Pointer[wsbuilder.UsageChecker], log slog.Logger, tick <-chan time.Time, enqueuer notifications.Enqueuer, exp codersdk.Experiments, workspaceBuilderMetrics *wsbuilder.Metrics) *Executor {
 	factory := promauto.With(reg)
 	le := &Executor{
 		//nolint:gocritic // Autostart has a limited set of permissions.
-		ctx:                   dbauthz.AsAutostart(ctx),
-		db:                    db,
-		ps:                    ps,
-		fileCache:             fc,
-		templateScheduleStore: tss,
-		tick:                  tick,
-		log:                   log.Named("autobuild"),
-		auditor:               auditor,
-		accessControlStore:    acs,
-		buildUsageChecker:     buildUsageChecker,
-		notificationsEnqueuer: enqueuer,
-		reg:                   reg,
-		experiments:           exp,
+		ctx:                     dbauthz.AsAutostart(ctx),
+		db:                      db,
+		ps:                      ps,
+		fileCache:               fc,
+		templateScheduleStore:   tss,
+		tick:                    tick,
+		log:                     log.Named("autobuild"),
+		auditor:                 auditor,
+		accessControlStore:      acs,
+		buildUsageChecker:       buildUsageChecker,
+		notificationsEnqueuer:   enqueuer,
+		reg:                     reg,
+		experiments:             exp,
+		workspaceBuilderMetrics: workspaceBuilderMetrics,
 		metrics: executorMetrics{
 			autobuildExecutionDuration: factory.NewHistogram(prometheus.HistogramOpts{
 				Namespace: "coderd",
@@ -335,7 +337,8 @@ func (e *Executor) runOnce(t time.Time) Stats {
 							SetLastWorkspaceBuildInTx(&latestBuild).
 							SetLastWorkspaceBuildJobInTx(&latestJob).
 							Experiments(e.experiments).
-							Reason(reason)
+							Reason(reason).
+							BuildMetrics(e.workspaceBuilderMetrics)
 						log.Debug(e.ctx, "auto building workspace", slog.F("transition", nextTransition))
 						if nextTransition == database.WorkspaceTransitionStart &&
 							useActiveVersion(accessControl, ws) {

coderd/coderd.go

@@ -245,6 +245,7 @@ type Options struct {
 	MetadataBatcherOptions []metadatabatcher.Option
 
 	ProvisionerdServerMetrics *provisionerdserver.Metrics
+	WorkspaceBuilderMetrics   *wsbuilder.Metrics
 
 	// WorkspaceAppAuditSessionTimeout allows changing the timeout for audit
 	// sessions. Raising or lowering this value will directly affect the write

coderd/coderdtest/coderdtest.go

@@ -191,6 +191,7 @@ type Options struct {
 	TelemetryReporter                  telemetry.Reporter
 
 	ProvisionerdServerMetrics *provisionerdserver.Metrics
+	WorkspaceBuilderMetrics   *wsbuilder.Metrics
 	UsageInserter             usage.Inserter
 }
 
@@ -399,6 +400,7 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can
 		options.AutobuildTicker,
 		options.NotificationsEnqueuer,
 		experiments,
+		options.WorkspaceBuilderMetrics,
 	).WithStatsChannel(options.AutobuildStats)
 
 	lifecycleExecutor.Run()
@@ -620,6 +622,7 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can
 			AppEncryptionKeyCache:              options.APIKeyEncryptionCache,
 			OIDCConvertKeyCache:                options.OIDCConvertKeyCache,
 			ProvisionerdServerMetrics:          options.ProvisionerdServerMetrics,
+			WorkspaceBuilderMetrics:            options.WorkspaceBuilderMetrics,
 		}
 }
 

coderd/provisionerdserver/metrics.go

@@ -13,6 +13,7 @@ type Metrics struct {
 	logger                   slog.Logger
 	workspaceCreationTimings *prometheus.HistogramVec
 	workspaceClaimTimings    *prometheus.HistogramVec
+	jobQueueWait             *prometheus.HistogramVec
 }
 
 type WorkspaceTimingType int
@@ -29,6 +30,12 @@ const (
 	workspaceTypePrebuild = "prebuild"
 )
 
+// BuildReasonPrebuild is the build_reason metric label value for prebuild
+// operations. This is distinct from database.BuildReason values since prebuilds
+// use BuildReasonInitiator in the database but we want to track them separately
+// in metrics. This is also used as a label value by the metrics in wsbuilder.
+const BuildReasonPrebuild = workspaceTypePrebuild
+
 type WorkspaceTimingFlags struct {
 	IsPrebuild   bool
 	IsClaim      bool
@@ -90,6 +97,30 @@ func NewMetrics(logger slog.Logger) *Metrics {
 			NativeHistogramZeroThreshold:    0,
 			NativeHistogramMaxZeroThreshold: 0,
 		}, []string{"organization_name", "template_name", "preset_name"}),
+		jobQueueWait: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: "coderd",
+			Name:      "provisioner_job_queue_wait_seconds",
+			Help:      "Time from job creation to acquisition by a provisioner daemon.",
+			Buckets: []float64{
+				0.1,  // 100ms
+				0.5,  // 500ms
+				1,    // 1s
+				5,    // 5s
+				10,   // 10s
+				30,   // 30s
+				60,   // 1m
+				120,  // 2m
+				300,  // 5m
+				600,  // 10m
+				900,  // 15m
+				1800, // 30m
+			},
+			NativeHistogramBucketFactor:     1.1,
+			NativeHistogramMaxBucketNumber:  100,
+			NativeHistogramMinResetDuration: time.Hour,
+			NativeHistogramZeroThreshold:    0,
+			NativeHistogramMaxZeroThreshold: 0,
+		}, []string{"provisioner_type", "job_type", "transition", "build_reason"}),
 	}
 }
 
@@ -97,7 +128,10 @@ func (m *Metrics) Register(reg prometheus.Registerer) error {
 	if err := reg.Register(m.workspaceCreationTimings); err != nil {
 		return err
 	}
-	return reg.Register(m.workspaceClaimTimings)
+	if err := reg.Register(m.workspaceClaimTimings); err != nil {
+		return err
+	}
+	return reg.Register(m.jobQueueWait)
 }
 
 // IsTrackable returns true if the workspace build should be tracked in metrics.
@@ -162,3 +196,9 @@ func (m *Metrics) UpdateWorkspaceTimingsMetrics(
 		// Not a trackable build type (e.g. restart, stop, subsequent builds)
 	}
 }
+
+// ObserveJobQueueWait records the time a provisioner job spent waiting in the queue.
+// For non-workspace-build jobs, transition and buildReason should be empty strings.
+func (m *Metrics) ObserveJobQueueWait(provisionerType, jobType, transition, buildReason string, waitSeconds float64) {
+	m.jobQueueWait.WithLabelValues(provisionerType, jobType, transition, buildReason).Observe(waitSeconds)
+}

coderd/provisionerdserver/provisionerdserver.go

@@ -478,6 +478,10 @@ func (s *server) acquireProtoJob(ctx context.Context, job database.ProvisionerJo
 		TraceMetadata: jobTraceMetadata,
 	}
 
+	// jobTransition and jobBuildReason are used for metrics; only set for workspace builds.
+	var jobTransition string
+	var jobBuildReason string
+
 	switch job.Type {
 	case database.ProvisionerJobTypeWorkspaceBuild:
 		var input WorkspaceProvisionJob
@@ -584,6 +588,15 @@ func (s *server) acquireProtoJob(ctx context.Context, job database.ProvisionerJo
 		if err != nil {
 			return nil, failJob(fmt.Sprintf("convert workspace transition: %s", err))
 		}
+		jobTransition = string(workspaceBuild.Transition)
+		// Prebuilds use BuildReasonInitiator in the database but we want to
+		// track them separately in metrics. Check the initiator ID to detect
+		// prebuild jobs.
+		if job.InitiatorID == database.PrebuildsSystemUserID {
+			jobBuildReason = BuildReasonPrebuild
+		} else {
+			jobBuildReason = string(workspaceBuild.Reason)
+		}
 
 		// A previous workspace build exists
 		var lastWorkspaceBuildParameters []database.WorkspaceBuildParameter
@@ -825,6 +838,12 @@ func (s *server) acquireProtoJob(ctx context.Context, job database.ProvisionerJo
 		return nil, failJob(fmt.Sprintf("payload was too big: %d > %d", protobuf.Size(protoJob), drpcsdk.MaxMessageSize))
 	}
 
+	// Record the time the job spent waiting in the queue.
+	if s.metrics != nil && job.StartedAt.Valid && job.Provisioner.Valid() {
+		queueWaitSeconds := job.StartedAt.Time.Sub(job.CreatedAt).Seconds()
+		s.metrics.ObserveJobQueueWait(string(job.Provisioner), string(job.Type), jobTransition, jobBuildReason, queueWaitSeconds)
+	}
+
 	return protoJob, err
 }
 

coderd/workspacebuilds.go

@@ -382,7 +382,8 @@ func (api *API) postWorkspaceBuildsInternal(
 		LogLevel(string(createBuild.LogLevel)).
 		DeploymentValues(api.Options.DeploymentValues).
 		Experiments(api.Experiments).
-		TemplateVersionPresetID(createBuild.TemplateVersionPresetID)
+		TemplateVersionPresetID(createBuild.TemplateVersionPresetID).
+		BuildMetrics(api.WorkspaceBuilderMetrics)
 
 	if (transition == database.WorkspaceTransitionStart || transition == database.WorkspaceTransitionStop) && createBuild.Reason != "" {
 		builder = builder.Reason(database.BuildReason(createBuild.Reason))

coderd/workspaces.go

@@ -787,7 +787,8 @@ func createWorkspace(
 			ActiveVersion().
 			Experiments(api.Experiments).
 			DeploymentValues(api.DeploymentValues).
-			RichParameterValues(req.RichParameterValues)
+			RichParameterValues(req.RichParameterValues).
+			BuildMetrics(api.WorkspaceBuilderMetrics)
 		if req.TemplateVersionID != uuid.Nil {
 			builder = builder.VersionID(req.TemplateVersionID)
 		}

coderd/workspaces_test.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -21,7 +22,9 @@ import (
 	"github.com/coder/coder/v2/agent/agenttest"
 	"github.com/coder/coder/v2/coderd"
 	"github.com/coder/coder/v2/coderd/audit"
+	"github.com/coder/coder/v2/coderd/autobuild"
 	"github.com/coder/coder/v2/coderd/coderdtest"
+	"github.com/coder/coder/v2/coderd/coderdtest/promhelp"
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/dbauthz"
 	"github.com/coder/coder/v2/coderd/database/dbfake"
@@ -30,6 +33,7 @@ import (
 	"github.com/coder/coder/v2/coderd/database/dbtime"
 	"github.com/coder/coder/v2/coderd/notifications"
 	"github.com/coder/coder/v2/coderd/notifications/notificationstest"
+	"github.com/coder/coder/v2/coderd/provisionerdserver"
 	"github.com/coder/coder/v2/coderd/rbac"
 	"github.com/coder/coder/v2/coderd/rbac/policy"
 	"github.com/coder/coder/v2/coderd/render"
@@ -37,6 +41,7 @@ import (
 	"github.com/coder/coder/v2/coderd/schedule/cron"
 	"github.com/coder/coder/v2/coderd/util/ptr"
 	"github.com/coder/coder/v2/coderd/util/slice"
+	"github.com/coder/coder/v2/coderd/wsbuilder"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/cryptorand"
 	"github.com/coder/coder/v2/provisioner/echo"
@@ -5901,3 +5906,135 @@ func TestWorkspaceCreateWithImplicitPreset(t *testing.T) {
 		require.Equal(t, preset2ID, *ws2.LatestBuild.TemplateVersionPresetID)
 	})
 }
+
+func TestProvisionerJobQueueWaitMetric(t *testing.T) {
+	t.Parallel()
+
+	logger := testutil.Logger(t)
+	reg := prometheus.NewRegistry()
+	metrics := provisionerdserver.NewMetrics(logger)
+	err := metrics.Register(reg)
+	require.NoError(t, err)
+
+	client := coderdtest.New(t, &coderdtest.Options{
+		IncludeProvisionerDaemon:  true,
+		ProvisionerdServerMetrics: metrics,
+	})
+	user := coderdtest.CreateFirstUser(t, client)
+
+	// Create a template version - this triggers a template_version_import job.
+	version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
+	coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+
+	// Check that the queue wait metric was recorded for the template_version_import job.
+	importMetric := promhelp.MetricValue(t, reg, "coderd_provisioner_job_queue_wait_seconds", prometheus.Labels{
+		"provisioner_type": string(database.ProvisionerTypeEcho),
+		"job_type":         string(database.ProvisionerJobTypeTemplateVersionImport),
+		"transition":       "",
+		"build_reason":     "",
+	})
+	require.NotNil(t, importMetric, "import job metric should be recorded")
+	importHistogram := importMetric.GetHistogram()
+	require.NotNil(t, importHistogram)
+	require.Equal(t, uint64(1), importHistogram.GetSampleCount(), "import job should have 1 sample")
+	require.Greater(t, importHistogram.GetSampleSum(), 0.0, "import job queue wait should be non-zero")
+
+	// Create a template and workspace - this triggers a workspace_build job.
+	template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
+	workspace := coderdtest.CreateWorkspace(t, client, template.ID)
+	coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
+
+	// Check that the queue wait metric was recorded for the workspace_build job.
+	buildMetric := promhelp.MetricValue(t, reg, "coderd_provisioner_job_queue_wait_seconds", prometheus.Labels{
+		"provisioner_type": string(database.ProvisionerTypeEcho),
+		"job_type":         string(database.ProvisionerJobTypeWorkspaceBuild),
+		"transition":       string(database.WorkspaceTransitionStart),
+		"build_reason":     string(database.BuildReasonInitiator),
+	})
+	require.NotNil(t, buildMetric, "workspace build job metric should be recorded")
+	buildHistogram := buildMetric.GetHistogram()
+	require.NotNil(t, buildHistogram)
+	require.Equal(t, uint64(1), buildHistogram.GetSampleCount(), "workspace build job should have 1 sample")
+	require.Greater(t, buildHistogram.GetSampleSum(), 0.0, "workspace build job queue wait should be non-zero")
+}
+
+func TestWorkspaceBuildsEnqueuedMetric(t *testing.T) {
+	t.Parallel()
+
+	var (
+		logger  = testutil.Logger(t)
+		reg     = prometheus.NewRegistry()
+		metrics = provisionerdserver.NewMetrics(logger)
+
+		sched   = mustSchedule(t, "CRON_TZ=UTC 0 * * * *")
+		tickCh  = make(chan time.Time)
+		statsCh = make(chan autobuild.Stats)
+	)
+
+	err := metrics.Register(reg)
+	require.NoError(t, err)
+
+	wsBuilderMetrics, err := wsbuilder.NewMetrics(reg)
+	require.NoError(t, err)
+
+	client, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
+		IncludeProvisionerDaemon:  true,
+		ProvisionerdServerMetrics: metrics,
+		WorkspaceBuilderMetrics:   wsBuilderMetrics,
+		AutobuildTicker:           tickCh,
+		AutobuildStats:            statsCh,
+	})
+	user := coderdtest.CreateFirstUser(t, client)
+
+	// Create a template and workspace with autostart schedule.
+	version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
+	coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+	template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
+	workspace := coderdtest.CreateWorkspace(t, client, template.ID, func(cwr *codersdk.CreateWorkspaceRequest) {
+		cwr.AutostartSchedule = ptr.Ref(sched.String())
+	})
+	coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
+
+	// Stop the workspace to prepare for autostart.
+	workspace = coderdtest.MustTransitionWorkspace(t, client, workspace.ID, codersdk.WorkspaceTransitionStart, codersdk.WorkspaceTransitionStop)
+
+	// Trigger an autostart build via the autobuild ticker. This verifies that
+	// autostart builds are recorded with build_reason="autostart".
+	p, err := coderdtest.GetProvisionerForTags(db, time.Now(), workspace.OrganizationID, map[string]string{})
+	require.NoError(t, err)
+
+	go func() {
+		tickTime := sched.Next(workspace.LatestBuild.CreatedAt)
+		coderdtest.UpdateProvisionerLastSeenAt(t, db, p.ID, tickTime)
+		tickCh <- tickTime
+		close(tickCh)
+	}()
+
+	// Wait for the autostart to complete.
+	stats := <-statsCh
+	require.Len(t, stats.Errors, 0)
+	require.Len(t, stats.Transitions, 1)
+	require.Contains(t, stats.Transitions, workspace.ID)
+	require.Equal(t, database.WorkspaceTransitionStart, stats.Transitions[workspace.ID])
+
+	// Verify the workspace was autostarted.
+	workspace = coderdtest.MustWorkspace(t, client, workspace.ID)
+	coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
+	require.Equal(t, codersdk.BuildReasonAutostart, workspace.LatestBuild.Reason)
+
+	// Now check the autostart metric was recorded.
+	autostartCount := promhelp.CounterValue(t, reg, "coderd_workspace_builds_enqueued_total", prometheus.Labels{
+		"provisioner_type": string(database.ProvisionerTypeEcho),
+		"build_reason":     string(database.BuildReasonAutostart),
+		"transition":       string(database.WorkspaceTransitionStart),
+		"status":           wsbuilder.BuildStatusSuccess,
+	})
+	require.Equal(t, 1, autostartCount, "autostart should record 1 enqueue with build_reason=autostart")
+}
+
+func mustSchedule(t *testing.T, s string) *cron.Schedule {
+	t.Helper()
+	sched, err := cron.Weekly(s)
+	require.NoError(t, err)
+	return sched
+}

coderd/wsbuilder/metrics.go

@@ -0,0 +1,42 @@
+package wsbuilder
+
+import "github.com/prometheus/client_golang/prometheus"
+
+// Metrics holds metrics related to workspace build creation.
+type Metrics struct {
+	workspaceBuildsEnqueued *prometheus.CounterVec
+}
+
+// Metric label values for build status.
+const (
+	BuildStatusSuccess = "success"
+	BuildStatusFailed  = "failed"
+)
+
+func NewMetrics(reg prometheus.Registerer) (*Metrics, error) {
+	m := &Metrics{
+		workspaceBuildsEnqueued: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Namespace: "coderd",
+			Name:      "workspace_builds_enqueued_total",
+			Help:      "Total number of workspace build enqueue attempts.",
+		}, []string{"provisioner_type", "build_reason", "transition", "status"}),
+	}
+
+	if reg != nil {
+		if err := reg.Register(m.workspaceBuildsEnqueued); err != nil {
+			return nil, err
+		}
+	}
+
+	return m, nil
+}
+
+// RecordBuildEnqueued records a workspace build enqueue attempt. It determines
+// the status based on whether an error occurred and increments the counter.
+func (m *Metrics) RecordBuildEnqueued(provisionerType, buildReason, transition string, err error) {
+	status := BuildStatusSuccess
+	if err != nil {
+		status = BuildStatusFailed
+	}
+	m.workspaceBuildsEnqueued.WithLabelValues(provisionerType, buildReason, transition, status).Inc()
+}

coderd/wsbuilder/wsbuilder.go

@@ -90,6 +90,8 @@ type Builder struct {
 
 	prebuiltWorkspaceBuildStage  sdkproto.PrebuiltWorkspaceBuildStage
 	verifyNoLegacyParametersOnce bool
+
+	buildMetrics *Metrics
 }
 
 type UsageChecker interface {
@@ -253,6 +255,12 @@ func (b Builder) TemplateVersionPresetID(id uuid.UUID) Builder {
 	return b
 }
 
+func (b Builder) BuildMetrics(m *Metrics) Builder {
+	// nolint: revive
+	b.buildMetrics = m
+	return b
+}
+
 type BuildError struct {
 	// Status is a suitable HTTP status code
 	Status  int
@@ -313,11 +321,34 @@ func (b *Builder) Build(
 		return err
 	})
 	if err != nil {
+		b.recordBuildMetrics(provisionerJob, err)
 		return nil, nil, nil, xerrors.Errorf("build tx: %w", err)
 	}
+	b.recordBuildMetrics(provisionerJob, nil)
 	return workspaceBuild, provisionerJob, provisionerDaemons, nil
 }
 
+// recordBuildMetrics records the workspace build enqueue metric if metrics are
+// configured. It determines the appropriate build reason label, using "prebuild"
+// for prebuild operations instead of the database reason.
+func (b *Builder) recordBuildMetrics(job *database.ProvisionerJob, err error) {
+	if b.buildMetrics == nil {
+		return
+	}
+	if job == nil || !job.Provisioner.Valid() {
+		return
+	}
+
+	// Determine the build reason for metrics. Prebuilds use BuildReasonInitiator
+	// in the database but we want to track them separately in metrics.
+	buildReason := string(b.reason)
+	if b.prebuiltWorkspaceBuildStage == sdkproto.PrebuiltWorkspaceBuildStage_CREATE {
+		buildReason = provisionerdserver.BuildReasonPrebuild
+	}
+
+	b.buildMetrics.RecordBuildEnqueued(string(job.Provisioner), buildReason, string(b.trans), err)
+}
+
 // buildTx contains the business logic of computing a new build.  Attributes of the new database objects are computed
 // in a functional style, rather than imperative, to emphasize the logic of how they are defined.  A simple cache
 // of database-fetched objects is stored on the struct to ensure we only fetch things once, even if they are used in