feat: add prebuilds configuration & bootstrapping (#17527)

Closes https://github.com/coder/internal/issues/508

---------

Signed-off-by: Danny Kopping <dannykopping@gmail.com>
Co-authored-by: Cian Johnston <cian@coder.com>

enterprise/coderd/prebuilds/reconcile.go

@@ -38,6 +38,7 @@ type StoreReconciler struct {
 	clock  quartz.Clock
 
 	cancelFn context.CancelCauseFunc
+	running  atomic.Bool
 	stopped  atomic.Bool
 	done     chan struct{}
 }
@@ -61,7 +62,7 @@ func NewStoreReconciler(
 	}
 }
 
-func (c *StoreReconciler) RunLoop(ctx context.Context) {
+func (c *StoreReconciler) Run(ctx context.Context) {
 	reconciliationInterval := c.cfg.ReconciliationInterval.Value()
 	if reconciliationInterval <= 0 { // avoids a panic
 		reconciliationInterval = 5 * time.Minute
@@ -82,6 +83,11 @@ func (c *StoreReconciler) RunLoop(ctx context.Context) {
 	ctx, cancel := context.WithCancelCause(dbauthz.AsPrebuildsOrchestrator(ctx))
 	c.cancelFn = cancel
 
+	// Everything is in place, reconciler can now be considered as running.
+	//
+	// NOTE: without this atomic bool, Stop might race with Run for the c.cancelFn above.
+	c.running.Store(true)
+
 	for {
 		select {
 		// TODO: implement pubsub listener to allow reconciling a specific template imperatively once it has been changed,
@@ -107,16 +113,26 @@ func (c *StoreReconciler) RunLoop(ctx context.Context) {
 }
 
 func (c *StoreReconciler) Stop(ctx context.Context, cause error) {
+	defer c.running.Store(false)
+
 	if cause != nil {
 		c.logger.Error(context.Background(), "stopping reconciler due to an error", slog.Error(cause))
 	} else {
 		c.logger.Info(context.Background(), "gracefully stopping reconciler")
 	}
 
-	if c.isStopped() {
+	// If previously stopped (Swap returns previous value), then short-circuit.
+	//
+	// NOTE: we need to *prospectively* mark this as stopped to prevent Stop being called multiple times and causing problems.
+	if c.stopped.Swap(true) {
 		return
 	}
-	c.stopped.Store(true)
+
+	// If the reconciler is not running, there's nothing else to do.
+	if !c.running.Load() {
+		return
+	}
+
 	if c.cancelFn != nil {
 		c.cancelFn(cause)
 	}
@@ -138,10 +154,6 @@ func (c *StoreReconciler) Stop(ctx context.Context, cause error) {
 	}
 }
 
-func (c *StoreReconciler) isStopped() bool {
-	return c.stopped.Load()
-}
-
 // ReconcileAll will attempt to resolve the desired vs actual state of all templates which have presets with prebuilds configured.
 //
 // NOTE:

enterprise/coderd/prebuilds/reconcile_test.go

@@ -575,7 +575,7 @@ func TestRunLoop(t *testing.T) {
 		t, &slogtest.Options{IgnoreErrors: true},
 	).Leveled(slog.LevelDebug)
 	db, pubSub := dbtestutil.NewDB(t)
-	controller := prebuilds.NewStoreReconciler(db, pubSub, cfg, logger, clock)
+	reconciler := prebuilds.NewStoreReconciler(db, pubSub, cfg, logger, clock)
 
 	ownerID := uuid.New()
 	dbgen.User(t, db, database.User{
@@ -639,7 +639,7 @@ func TestRunLoop(t *testing.T) {
 	// we need to wait until ticker is initialized, and only then use clock.Advance()
 	// otherwise clock.Advance() will be ignored
 	trap := clock.Trap().NewTicker()
-	go controller.RunLoop(ctx)
+	go reconciler.Run(ctx)
 	// wait until ticker is initialized
 	trap.MustWait(ctx).Release()
 	// start 1st iteration of ReconciliationLoop
@@ -681,7 +681,7 @@ func TestRunLoop(t *testing.T) {
 	}, testutil.WaitShort, testutil.IntervalFast)
 
 	// gracefully stop the reconciliation loop
-	controller.Stop(ctx, nil)
+	reconciler.Stop(ctx, nil)
 }
 
 func TestFailedBuildBackoff(t *testing.T) {