From bf40d678ec26a7b44e0dcfada44c4a7b2df0af64 Mon Sep 17 00:00:00 2001
From: Ethan <39577870+ethanndickson@users.noreply.github.com>
Date: Tue, 2 Dec 2025 23:10:50 +1100
Subject: [PATCH] fix(cli): close prebuild runner prometheus server last
 (#21053)

## Description

Fixes the prebuilds scaletest command where the prometheus server was
being shut down before waiting for metrics to be scraped.

The issue was the defer order - since defers execute in LIFO (last-in,
first-out) order:

**Before (broken):**
1. Register tracing defer (includes wait for prometheus scrape)
2. Register prometheus server defer

Execution order: prometheus closes first, then wait happens (server
already gone!)

**After (fixed):**
1. Register prometheus server defer
2. Register tracing defer (includes wait for prometheus scrape)

Execution order: wait happens first (server still up), then prometheus
closes.

This matches the pattern used in other scaletest commands.

## Impact

The `coderd_scaletest_prebuild_deletion_jobs_completed` metric (and
potentially others) was always showing 0 because the server shut down
before Prometheus could scrape the final values.

_This PR was generated by [`mux`](https://github.com/coder/mux) and
reviewed by a human._
---
 cli/exp_scaletest.go           | 18 ++++++++++--------
 cli/exp_scaletest_prebuilds.go | 17 +++++++++--------
 2 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/cli/exp_scaletest.go b/cli/exp_scaletest.go
index d8dd10da32..419b195547 100644
--- a/cli/exp_scaletest.go
+++ b/cli/exp_scaletest.go
@@ -1559,6 +1559,15 @@ func (r *RootCmd) scaletestDashboard() *serpent.Command {
 			if err != nil {
 				return xerrors.Errorf("create tracer provider: %w", err)
 			}
+			tracer := tracerProvider.Tracer(scaletestTracerName)
+			outputs, err := output.parse()
+			if err != nil {
+				return xerrors.Errorf("could not parse --output flags")
+			}
+			reg := prometheus.NewRegistry()
+			prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), prometheusFlags.Address, "prometheus")
+			defer prometheusSrvClose()
+
 			defer func() {
 				// Allow time for traces to flush even if command context is
 				// canceled. This is a no-op if tracing is not enabled.
@@ -1570,14 +1579,7 @@ func (r *RootCmd) scaletestDashboard() *serpent.Command {
 				_, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", prometheusFlags.Wait)
 				<-time.After(prometheusFlags.Wait)
 			}()
-			tracer := tracerProvider.Tracer(scaletestTracerName)
-			outputs, err := output.parse()
-			if err != nil {
-				return xerrors.Errorf("could not parse --output flags")
-			}
-			reg := prometheus.NewRegistry()
-			prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), prometheusFlags.Address, "prometheus")
-			defer prometheusSrvClose()
+
 			metrics := dashboard.NewMetrics(reg)
 
 			th := harness.NewTestHarness(strategy.toStrategy(), cleanupStrategy.toStrategy())
diff --git a/cli/exp_scaletest_prebuilds.go b/cli/exp_scaletest_prebuilds.go
index 8e05bacae2..f8cee15514 100644
--- a/cli/exp_scaletest_prebuilds.go
+++ b/cli/exp_scaletest_prebuilds.go
@@ -84,14 +84,6 @@ func (r *RootCmd) scaletestPrebuilds() *serpent.Command {
 			if err != nil {
 				return xerrors.Errorf("create tracer provider: %w", err)
 			}
-			defer func() {
-				_, _ = fmt.Fprintln(inv.Stderr, "\nUploading traces...")
-				if err := closeTracing(ctx); err != nil {
-					_, _ = fmt.Fprintf(inv.Stderr, "\nError uploading traces: %+v\n", err)
-				}
-				_, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", prometheusFlags.Wait)
-				<-time.After(prometheusFlags.Wait)
-			}()
 			tracer := tracerProvider.Tracer(scaletestTracerName)
 
 			reg := prometheus.NewRegistry()
@@ -101,6 +93,15 @@ func (r *RootCmd) scaletestPrebuilds() *serpent.Command {
 			prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), prometheusFlags.Address, "prometheus")
 			defer prometheusSrvClose()
 
+			defer func() {
+				_, _ = fmt.Fprintln(inv.Stderr, "\nUploading traces...")
+				if err := closeTracing(ctx); err != nil {
+					_, _ = fmt.Fprintf(inv.Stderr, "\nError uploading traces: %+v\n", err)
+				}
+				_, _ = fmt.Fprintf(inv.Stderr, "Waiting %s for prometheus metrics to be scraped\n", prometheusFlags.Wait)
+				<-time.After(prometheusFlags.Wait)
+			}()
+
 			err = client.PutPrebuildsSettings(ctx, codersdk.PrebuildsSettings{
 				ReconciliationPaused: true,
 			})