chore: add tracing to prebuilds (#21443)

The implementation for prebuilt workspaces is complex and conversations
regarding edge cases and bugs frequently get bogged down by minutiae,
because it's hard to reason about the behaviour of the system.

To alleviate this, I've introduced otel tracing to the StoreReconciler
(see attached). We can now directly observe the behaviour of the
prebuilds system under load in order to inform our decisions.

Traces are terminated at the boundary between prebuilds and workspace
builder, because of prebuilt workspaces' "fire and forget" philosophy
and to prevent span explosion.

<img width="3024" height="1718" alt="image"
src="https://github.com/user-attachments/assets/f9b207be-8f2c-475e-98a8-46ef70bda446"
/>
This commit is contained in:
Sas Swart
2026-01-07 11:04:40 +02:00
committed by GitHub
parent 6bd2d1c85f
commit 9a0024c45f
7 changed files with 119 additions and 37 deletions
+7
View File
@@ -18,6 +18,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/trace/noop"
"cdr.dev/slog"
"cdr.dev/slog/sloggers/slogtest"
@@ -1986,6 +1987,7 @@ func TestPrebuildsAutobuild(t *testing.T) {
prometheus.NewRegistry(),
notificationsNoop,
api.AGPL.BuildUsageChecker,
noop.NewTracerProvider(),
)
var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(db)
api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -2108,6 +2110,7 @@ func TestPrebuildsAutobuild(t *testing.T) {
prometheus.NewRegistry(),
notificationsNoop,
api.AGPL.BuildUsageChecker,
noop.NewTracerProvider(),
)
var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(db)
api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -2230,6 +2233,7 @@ func TestPrebuildsAutobuild(t *testing.T) {
prometheus.NewRegistry(),
notificationsNoop,
api.AGPL.BuildUsageChecker,
noop.NewTracerProvider(),
)
var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(db)
api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -2374,6 +2378,7 @@ func TestPrebuildsAutobuild(t *testing.T) {
prometheus.NewRegistry(),
notificationsNoop,
api.AGPL.BuildUsageChecker,
noop.NewTracerProvider(),
)
var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(db)
api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -2519,6 +2524,7 @@ func TestPrebuildsAutobuild(t *testing.T) {
prometheus.NewRegistry(),
notificationsNoop,
api.AGPL.BuildUsageChecker,
noop.NewTracerProvider(),
)
var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(db)
api.AGPL.PrebuildsClaimer.Store(&claimer)
@@ -2964,6 +2970,7 @@ func TestWorkspaceProvisionerdServerMetrics(t *testing.T) {
prometheus.NewRegistry(),
notifications.NewNoopEnqueuer(),
api.AGPL.BuildUsageChecker,
noop.NewTracerProvider(),
)
var claimer agplprebuilds.Claimer = prebuilds.NewEnterpriseClaimer(db)
api.AGPL.PrebuildsClaimer.Store(&claimer)