feat(coderd): add matched provisioner daemons information to more places (#15688)

- Refactors `checkProvisioners` into `db2sdk.MatchedProvisioners`
- Adds a separate RBAC subject just for reading provisioner daemons
- Adds matched provisioners information to additional endpoints relating to
  workspace builds and templates
-Updates existing unit tests for above endpoints
-Adds API endpoint for matched provisioners of template dry-run job
-Updates CLI to show warning when creating/starting/stopping/deleting
 workspaces for which no provisoners are available

---------

Co-authored-by: Danny Kopping <danny@coder.com>

coderd/apidoc/docs.go

@@ -4851,6 +4851,49 @@ const docTemplate = `{
                 }
             }
         },
+        "/templateversions/{templateversion}/dry-run/{jobID}/matched-provisioners": {
+            "get": {
+                "security": [
+                    {
+                        "CoderSessionToken": []
+                    }
+                ],
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "Templates"
+                ],
+                "summary": "Get template version dry-run matched provisioners",
+                "operationId": "get-template-version-dry-run-matched-provisioners",
+                "parameters": [
+                    {
+                        "type": "string",
+                        "format": "uuid",
+                        "description": "Template version ID",
+                        "name": "templateversion",
+                        "in": "path",
+                        "required": true
+                    },
+                    {
+                        "type": "string",
+                        "format": "uuid",
+                        "description": "Job ID",
+                        "name": "jobID",
+                        "in": "path",
+                        "required": true
+                    }
+                ],
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/codersdk.MatchedProvisioners"
+                        }
+                    }
+                }
+            }
+        },
         "/templateversions/{templateversion}/dry-run/{jobID}/resources": {
             "get": {
                 "security": [
@@ -15068,6 +15111,9 @@ const docTemplate = `{
                 "job": {
                     "$ref": "#/definitions/codersdk.ProvisionerJob"
                 },
+                "matched_provisioners": {
+                    "$ref": "#/definitions/codersdk.MatchedProvisioners"
+                },
                 "max_deadline": {
                     "type": "string",
                     "format": "date-time"

coderd/apidoc/swagger.json

@@ -4275,6 +4275,45 @@
 				}
 			}
 		},
+		"/templateversions/{templateversion}/dry-run/{jobID}/matched-provisioners": {
+			"get": {
+				"security": [
+					{
+						"CoderSessionToken": []
+					}
+				],
+				"produces": ["application/json"],
+				"tags": ["Templates"],
+				"summary": "Get template version dry-run matched provisioners",
+				"operationId": "get-template-version-dry-run-matched-provisioners",
+				"parameters": [
+					{
+						"type": "string",
+						"format": "uuid",
+						"description": "Template version ID",
+						"name": "templateversion",
+						"in": "path",
+						"required": true
+					},
+					{
+						"type": "string",
+						"format": "uuid",
+						"description": "Job ID",
+						"name": "jobID",
+						"in": "path",
+						"required": true
+					}
+				],
+				"responses": {
+					"200": {
+						"description": "OK",
+						"schema": {
+							"$ref": "#/definitions/codersdk.MatchedProvisioners"
+						}
+					}
+				}
+			}
+		},
 		"/templateversions/{templateversion}/dry-run/{jobID}/resources": {
 			"get": {
 				"security": [
@@ -13712,6 +13751,9 @@
 				"job": {
 					"$ref": "#/definitions/codersdk.ProvisionerJob"
 				},
+				"matched_provisioners": {
+					"$ref": "#/definitions/codersdk.MatchedProvisioners"
+				},
 				"max_deadline": {
 					"type": "string",
 					"format": "date-time"

coderd/autobuild/lifecycle_executor.go

@@ -245,7 +245,7 @@ func (e *Executor) runOnce(t time.Time) Stats {
 							}
 						}
 
-						nextBuild, job, err = builder.Build(e.ctx, tx, nil, audit.WorkspaceBuildBaggage{IP: "127.0.0.1"})
+						nextBuild, job, _, err = builder.Build(e.ctx, tx, nil, audit.WorkspaceBuildBaggage{IP: "127.0.0.1"})
 						if err != nil {
 							return xerrors.Errorf("build workspace with transition %q: %w", nextTransition, err)
 						}

coderd/coderd.go

@@ -1055,6 +1055,7 @@ func New(options *Options) *API {
 				r.Get("/{jobID}", api.templateVersionDryRun)
 				r.Get("/{jobID}/resources", api.templateVersionDryRunResources)
 				r.Get("/{jobID}/logs", api.templateVersionDryRunLogs)
+				r.Get("/{jobID}/matched-provisioners", api.templateVersionDryRunMatchedProvisioners)
 				r.Patch("/{jobID}/cancel", api.patchTemplateVersionDryRunCancel)
 			})
 		})

coderd/database/db2sdk/db2sdk.go

@@ -673,3 +673,23 @@ func CryptoKey(key database.CryptoKey) codersdk.CryptoKey {
 		Secret:    key.Secret.String,
 	}
 }
+
+func MatchedProvisioners(provisionerDaemons []database.ProvisionerDaemon, now time.Time, staleInterval time.Duration) codersdk.MatchedProvisioners {
+	minLastSeenAt := now.Add(-staleInterval)
+	mostRecentlySeen := codersdk.NullTime{}
+	var matched codersdk.MatchedProvisioners
+	for _, provisioner := range provisionerDaemons {
+		if !provisioner.LastSeenAt.Valid {
+			continue
+		}
+		matched.Count++
+		if provisioner.LastSeenAt.Time.After(minLastSeenAt) {
+			matched.Available++
+		}
+		if provisioner.LastSeenAt.Time.After(mostRecentlySeen.Time) {
+			matched.MostRecentlySeen.Valid = true
+			matched.MostRecentlySeen.Time = provisioner.LastSeenAt.Time
+		}
+	}
+	return matched
+}

coderd/database/dbauthz/dbauthz.go

@@ -299,7 +299,7 @@ var (
 					rbac.ResourceSystem.Type:                 {policy.WildcardSymbol},
 					rbac.ResourceOrganization.Type:           {policy.ActionCreate, policy.ActionRead},
 					rbac.ResourceOrganizationMember.Type:     {policy.ActionCreate, policy.ActionDelete, policy.ActionRead},
-					rbac.ResourceProvisionerDaemon.Type:      {policy.ActionCreate, policy.ActionUpdate},
+					rbac.ResourceProvisionerDaemon.Type:      {policy.ActionCreate, policy.ActionRead, policy.ActionUpdate},
 					rbac.ResourceProvisionerKeys.Type:        {policy.ActionCreate, policy.ActionRead, policy.ActionDelete},
 					rbac.ResourceUser.Type:                   rbac.ResourceUser.AvailableActions(),
 					rbac.ResourceWorkspaceDormant.Type:       {policy.ActionUpdate, policy.ActionDelete, policy.ActionWorkspaceStop},
@@ -317,6 +317,23 @@ var (
 		}),
 		Scope: rbac.ScopeAll,
 	}.WithCachedASTValue()
+
+	subjectSystemReadProvisionerDaemons = rbac.Subject{
+		FriendlyName: "Provisioner Daemons Reader",
+		ID:           uuid.Nil.String(),
+		Roles: rbac.Roles([]rbac.Role{
+			{
+				Identifier:  rbac.RoleIdentifier{Name: "system-read-provisioner-daemons"},
+				DisplayName: "Coder",
+				Site: rbac.Permissions(map[string][]policy.Action{
+					rbac.ResourceProvisionerDaemon.Type: {policy.ActionRead},
+				}),
+				Org:  map[string][]rbac.Permission{},
+				User: []rbac.Permission{},
+			},
+		}),
+		Scope: rbac.ScopeAll,
+	}.WithCachedASTValue()
 )
 
 // AsProvisionerd returns a context with an actor that has permissions required
@@ -359,6 +376,12 @@ func AsSystemRestricted(ctx context.Context) context.Context {
 	return context.WithValue(ctx, authContextKey{}, subjectSystemRestricted)
 }
 
+// AsSystemReadProvisionerDaemons returns a context with an actor that has permissions
+// to read provisioner daemons.
+func AsSystemReadProvisionerDaemons(ctx context.Context) context.Context {
+	return context.WithValue(ctx, authContextKey{}, subjectSystemReadProvisionerDaemons)
+}
+
 var AsRemoveActor = rbac.Subject{
 	ID: "remove-actor",
 }

coderd/templateversions.go

@@ -10,7 +10,6 @@ import (
 	"fmt"
 	"net/http"
 	"os"
-	"time"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/google/uuid"
@@ -22,6 +21,8 @@ import (
 
 	"github.com/coder/coder/v2/coderd/audit"
 	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/db2sdk"
+	"github.com/coder/coder/v2/coderd/database/dbauthz"
 	"github.com/coder/coder/v2/coderd/database/dbtime"
 	"github.com/coder/coder/v2/coderd/database/provisionerjobs"
 	"github.com/coder/coder/v2/coderd/externalauth"
@@ -32,6 +33,7 @@ import (
 	"github.com/coder/coder/v2/coderd/rbac/policy"
 	"github.com/coder/coder/v2/coderd/render"
 	"github.com/coder/coder/v2/coderd/tracing"
+	"github.com/coder/coder/v2/coderd/util/ptr"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/examples"
 	"github.com/coder/coder/v2/provisioner/terraform/tfparse"
@@ -60,6 +62,22 @@ func (api *API) templateVersion(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	var matchedProvisioners *codersdk.MatchedProvisioners
+	if jobs[0].ProvisionerJob.JobStatus == database.ProvisionerJobStatusPending {
+		// nolint: gocritic // The user hitting this endpoint may not have
+		// permission to read provisioner daemons, but we want to show them
+		// information about the provisioner daemons that are available.
+		provisioners, err := api.Database.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(ctx), database.GetProvisionerDaemonsByOrganizationParams{
+			OrganizationID: jobs[0].ProvisionerJob.OrganizationID,
+			WantTags:       jobs[0].ProvisionerJob.Tags,
+		})
+		if err != nil {
+			api.Logger.Error(ctx, "failed to fetch provisioners for job id", slog.F("job_id", jobs[0].ProvisionerJob.ID), slog.Error(err))
+		} else {
+			matchedProvisioners = ptr.Ref(db2sdk.MatchedProvisioners(provisioners, dbtime.Now(), provisionerdserver.StaleInterval))
+		}
+	}
+
 	schemas, err := api.Database.GetParameterSchemasByJobID(ctx, jobs[0].ProvisionerJob.ID)
 	if errors.Is(err, sql.ErrNoRows) {
 		err = nil
@@ -77,7 +95,7 @@ func (api *API) templateVersion(rw http.ResponseWriter, r *http.Request) {
 		warnings = append(warnings, codersdk.TemplateVersionWarningUnsupportedWorkspaces)
 	}
 
-	httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), nil, warnings))
+	httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), matchedProvisioners, warnings))
 }
 
 // @Summary Patch template version by ID
@@ -173,7 +191,23 @@ func (api *API) patchTemplateVersion(rw http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(updatedTemplateVersion, convertProvisionerJob(jobs[0]), nil, nil))
+	var matchedProvisioners *codersdk.MatchedProvisioners
+	if jobs[0].ProvisionerJob.JobStatus == database.ProvisionerJobStatusPending {
+		// nolint: gocritic // The user hitting this endpoint may not have
+		// permission to read provisioner daemons, but we want to show them
+		// information about the provisioner daemons that are available.
+		provisioners, err := api.Database.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(ctx), database.GetProvisionerDaemonsByOrganizationParams{
+			OrganizationID: jobs[0].ProvisionerJob.OrganizationID,
+			WantTags:       jobs[0].ProvisionerJob.Tags,
+		})
+		if err != nil {
+			api.Logger.Error(ctx, "failed to fetch provisioners for job id", slog.F("job_id", jobs[0].ProvisionerJob.ID), slog.Error(err))
+		} else {
+			matchedProvisioners = ptr.Ref(db2sdk.MatchedProvisioners(provisioners, dbtime.Now(), provisionerdserver.StaleInterval))
+		}
+	}
+
+	httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(updatedTemplateVersion, convertProvisionerJob(jobs[0]), matchedProvisioners, nil))
 }
 
 // @Summary Cancel template version by ID
@@ -546,6 +580,43 @@ func (api *API) templateVersionDryRun(rw http.ResponseWriter, r *http.Request) {
 	httpapi.Write(ctx, rw, http.StatusOK, convertProvisionerJob(job))
 }
 
+// @Summary Get template version dry-run matched provisioners
+// @ID get-template-version-dry-run-matched-provisioners
+// @Security CoderSessionToken
+// @Produce json
+// @Tags Templates
+// @Param templateversion path string true "Template version ID" format(uuid)
+// @Param jobID path string true "Job ID" format(uuid)
+// @Success 200 {object} codersdk.MatchedProvisioners
+// @Router /templateversions/{templateversion}/dry-run/{jobID}/matched-provisioners [get]
+func (api *API) templateVersionDryRunMatchedProvisioners(rw http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	job, ok := api.fetchTemplateVersionDryRunJob(rw, r)
+	if !ok {
+		return
+	}
+
+	// nolint:gocritic // The user may not have permissions to read all
+	// provisioner daemons in the org.
+	daemons, err := api.Database.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(ctx), database.GetProvisionerDaemonsByOrganizationParams{
+		OrganizationID: job.ProvisionerJob.OrganizationID,
+		WantTags:       job.ProvisionerJob.Tags,
+	})
+	if err != nil {
+		if !errors.Is(err, sql.ErrNoRows) {
+			httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
+				Message: "Internal error fetching provisioner daemons by organization.",
+				Detail:  err.Error(),
+			})
+			return
+		}
+		daemons = []database.ProvisionerDaemon{}
+	}
+
+	matchedProvisioners := db2sdk.MatchedProvisioners(daemons, dbtime.Now(), provisionerdserver.StaleInterval)
+	httpapi.Write(ctx, rw, http.StatusOK, matchedProvisioners)
+}
+
 // @Summary Get template version dry-run resources by job ID
 // @ID get-template-version-dry-run-resources-by-job-id
 // @Security CoderSessionToken
@@ -868,8 +939,23 @@ func (api *API) templateVersionByName(rw http.ResponseWriter, r *http.Request) {
 		})
 		return
 	}
+	var matchedProvisioners *codersdk.MatchedProvisioners
+	if jobs[0].ProvisionerJob.JobStatus == database.ProvisionerJobStatusPending {
+		// nolint: gocritic // The user hitting this endpoint may not have
+		// permission to read provisioner daemons, but we want to show them
+		// information about the provisioner daemons that are available.
+		provisioners, err := api.Database.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(ctx), database.GetProvisionerDaemonsByOrganizationParams{
+			OrganizationID: jobs[0].ProvisionerJob.OrganizationID,
+			WantTags:       jobs[0].ProvisionerJob.Tags,
+		})
+		if err != nil {
+			api.Logger.Error(ctx, "failed to fetch provisioners for job id", slog.F("job_id", jobs[0].ProvisionerJob.ID), slog.Error(err))
+		} else {
+			matchedProvisioners = ptr.Ref(db2sdk.MatchedProvisioners(provisioners, dbtime.Now(), provisionerdserver.StaleInterval))
+		}
+	}
 
-	httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), nil, nil))
+	httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), matchedProvisioners, nil))
 }
 
 // @Summary Get template version by organization, template, and name
@@ -934,7 +1020,23 @@ func (api *API) templateVersionByOrganizationTemplateAndName(rw http.ResponseWri
 		return
 	}
 
-	httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), nil, nil))
+	var matchedProvisioners *codersdk.MatchedProvisioners
+	if jobs[0].ProvisionerJob.JobStatus == database.ProvisionerJobStatusPending {
+		// nolint: gocritic // The user hitting this endpoint may not have
+		// permission to read provisioner daemons, but we want to show them
+		// information about the provisioner daemons that are available.
+		provisioners, err := api.Database.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(ctx), database.GetProvisionerDaemonsByOrganizationParams{
+			OrganizationID: jobs[0].ProvisionerJob.OrganizationID,
+			WantTags:       jobs[0].ProvisionerJob.Tags,
+		})
+		if err != nil {
+			api.Logger.Error(ctx, "failed to fetch provisioners for job id", slog.F("job_id", jobs[0].ProvisionerJob.ID), slog.Error(err))
+		} else {
+			matchedProvisioners = ptr.Ref(db2sdk.MatchedProvisioners(provisioners, dbtime.Now(), provisionerdserver.StaleInterval))
+		}
+	}
+
+	httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(templateVersion, convertProvisionerJob(jobs[0]), matchedProvisioners, nil))
 }
 
 // @Summary Get previous template version by organization, template, and name
@@ -1020,7 +1122,23 @@ func (api *API) previousTemplateVersionByOrganizationTemplateAndName(rw http.Res
 		return
 	}
 
-	httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(previousTemplateVersion, convertProvisionerJob(jobs[0]), nil, nil))
+	var matchedProvisioners *codersdk.MatchedProvisioners
+	if jobs[0].ProvisionerJob.JobStatus == database.ProvisionerJobStatusPending {
+		// nolint: gocritic // The user hitting this endpoint may not have
+		// permission to read provisioner daemons, but we want to show them
+		// information about the provisioner daemons that are available.
+		provisioners, err := api.Database.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(ctx), database.GetProvisionerDaemonsByOrganizationParams{
+			OrganizationID: jobs[0].ProvisionerJob.OrganizationID,
+			WantTags:       jobs[0].ProvisionerJob.Tags,
+		})
+		if err != nil {
+			api.Logger.Error(ctx, "failed to fetch provisioners for job id", slog.F("job_id", jobs[0].ProvisionerJob.ID), slog.Error(err))
+		} else {
+			matchedProvisioners = ptr.Ref(db2sdk.MatchedProvisioners(provisioners, dbtime.Now(), provisionerdserver.StaleInterval))
+		}
+	}
+
+	httpapi.Write(ctx, rw, http.StatusOK, convertTemplateVersion(previousTemplateVersion, convertProvisionerJob(jobs[0]), matchedProvisioners, nil))
 }
 
 // @Summary Archive template unused versions by template id
@@ -1513,27 +1631,6 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht
 			return err
 		}
 
-		// Check for eligible provisioners. This allows us to log a message warning deployment administrators
-		// of users submitting jobs for which no provisioners are available.
-		matchedProvisioners, err = checkProvisioners(ctx, tx, organization.ID, tags)
-		if err != nil {
-			api.Logger.Error(ctx, "failed to check eligible provisioner daemons for job", slog.Error(err))
-		} else if matchedProvisioners.Count == 0 {
-			api.Logger.Warn(ctx, "no matching provisioners found for job",
-				slog.F("user_id", apiKey.UserID),
-				slog.F("job_id", jobID),
-				slog.F("job_type", database.ProvisionerJobTypeTemplateVersionImport),
-				slog.F("tags", tags),
-			)
-		} else if matchedProvisioners.Available == 0 {
-			api.Logger.Warn(ctx, "no active provisioners found for job",
-				slog.F("user_id", apiKey.UserID),
-				slog.F("job_id", jobID),
-				slog.F("job_type", database.ProvisionerJobTypeTemplateVersionImport),
-				slog.F("tags", tags),
-			)
-		}
-
 		provisionerJob, err = tx.InsertProvisionerJob(ctx, database.InsertProvisionerJobParams{
 			ID:             jobID,
 			CreatedAt:      dbtime.Now(),
@@ -1559,6 +1656,36 @@ func (api *API) postTemplateVersionsByOrganization(rw http.ResponseWriter, r *ht
 			return err
 		}
 
+		// Check for eligible provisioners. This allows us to return a warning to the user if they
+		// submit a job for which no provisioner is available.
+		// nolint: gocritic // The user hitting this endpoint may not have
+		// permission to read provisioner daemons, but we want to show them
+		// information about the provisioner daemons that are available.
+		eligibleProvisioners, err := tx.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(ctx), database.GetProvisionerDaemonsByOrganizationParams{
+			OrganizationID: organization.ID,
+			WantTags:       provisionerJob.Tags,
+		})
+		if err != nil {
+			// Log the error but do not return any warnings. This is purely advisory and we should not block.
+			api.Logger.Error(ctx, "failed to check eligible provisioner daemons for job", slog.Error(err))
+		}
+		matchedProvisioners = db2sdk.MatchedProvisioners(eligibleProvisioners, provisionerJob.CreatedAt, provisionerdserver.StaleInterval)
+		if matchedProvisioners.Count == 0 {
+			api.Logger.Warn(ctx, "no matching provisioners found for job",
+				slog.F("user_id", apiKey.UserID),
+				slog.F("job_id", jobID),
+				slog.F("job_type", database.ProvisionerJobTypeTemplateVersionImport),
+				slog.F("tags", tags),
+			)
+		} else if matchedProvisioners.Available == 0 {
+			api.Logger.Warn(ctx, "no active provisioners found for job",
+				slog.F("user_id", apiKey.UserID),
+				slog.F("job_id", jobID),
+				slog.F("job_type", database.ProvisionerJobTypeTemplateVersionImport),
+				slog.F("tags", tags),
+			)
+		}
+
 		var templateID uuid.NullUUID
 		if req.TemplateID != uuid.Nil {
 			templateID = uuid.NullUUID{
@@ -1822,34 +1949,3 @@ func (api *API) publishTemplateUpdate(ctx context.Context, templateID uuid.UUID)
 			slog.F("template_id", templateID), slog.Error(err))
 	}
 }
-
-func checkProvisioners(ctx context.Context, store database.Store, orgID uuid.UUID, wantTags map[string]string) (codersdk.MatchedProvisioners, error) {
-	// Check for eligible provisioners. This allows us to return a warning to the user if they
-	// submit a job for which no provisioner is available.
-	eligibleProvisioners, err := store.GetProvisionerDaemonsByOrganization(ctx, database.GetProvisionerDaemonsByOrganizationParams{
-		OrganizationID: orgID,
-		WantTags:       wantTags,
-	})
-	if err != nil {
-		// Log the error but do not return any warnings. This is purely advisory and we should not block.
-		return codersdk.MatchedProvisioners{}, xerrors.Errorf("provisioner daemons by organization: %w", err)
-	}
-
-	staleInterval := time.Now().Add(-provisionerdserver.StaleInterval)
-	mostRecentlySeen := codersdk.NullTime{}
-	var matched codersdk.MatchedProvisioners
-	for _, provisioner := range eligibleProvisioners {
-		if !provisioner.LastSeenAt.Valid {
-			continue
-		}
-		matched.Count++
-		if provisioner.LastSeenAt.Time.After(staleInterval) {
-			matched.Available++
-		}
-		if provisioner.LastSeenAt.Time.After(mostRecentlySeen.Time) {
-			matched.MostRecentlySeen.Valid = true
-			matched.MostRecentlySeen.Time = provisioner.LastSeenAt.Time
-		}
-	}
-	return matched, nil
-}

coderd/templateversions_test.go

@@ -50,6 +50,12 @@ func TestTemplateVersion(t *testing.T) {
 		tv, err := client.TemplateVersion(ctx, version.ID)
 		authz.AssertChecked(t, policy.ActionRead, tv)
 		require.NoError(t, err)
+		if assert.Equal(t, tv.Job.Status, codersdk.ProvisionerJobPending) {
+			assert.NotNil(t, tv.MatchedProvisioners)
+			assert.Zero(t, tv.MatchedProvisioners.Available)
+			assert.Zero(t, tv.MatchedProvisioners.Count)
+			assert.False(t, tv.MatchedProvisioners.MostRecentlySeen.Valid)
+		}
 
 		assert.Equal(t, "bananas", tv.Name)
 		assert.Equal(t, "first try", tv.Message)
@@ -87,8 +93,14 @@ func TestTemplateVersion(t *testing.T) {
 
 		client1, _ := coderdtest.CreateAnotherUser(t, client, user.OrganizationID)
 
-		_, err := client1.TemplateVersion(ctx, version.ID)
+		tv, err := client1.TemplateVersion(ctx, version.ID)
 		require.NoError(t, err)
+		if assert.Equal(t, tv.Job.Status, codersdk.ProvisionerJobPending) {
+			assert.NotNil(t, tv.MatchedProvisioners)
+			assert.Zero(t, tv.MatchedProvisioners.Available)
+			assert.Zero(t, tv.MatchedProvisioners.Count)
+			assert.False(t, tv.MatchedProvisioners.MostRecentlySeen.Valid)
+		}
 	})
 }
 
@@ -158,6 +170,12 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, "bananas", version.Name)
 		require.Equal(t, provisionersdk.ScopeOrganization, version.Job.Tags[provisionersdk.TagScope])
+		if assert.Equal(t, version.Job.Status, codersdk.ProvisionerJobPending) {
+			assert.NotNil(t, version.MatchedProvisioners)
+			assert.Equal(t, version.MatchedProvisioners.Available, 1)
+			assert.Equal(t, version.MatchedProvisioners.Count, 1)
+			assert.True(t, version.MatchedProvisioners.MostRecentlySeen.Valid)
+		}
 
 		require.Len(t, auditor.AuditLogs(), 2)
 		assert.Equal(t, database.AuditActionCreate, auditor.AuditLogs()[1].Action)
@@ -790,8 +808,15 @@ func TestTemplateVersionByName(t *testing.T) {
 		ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
 		defer cancel()
 
-		_, err := client.TemplateVersionByName(ctx, template.ID, version.Name)
+		tv, err := client.TemplateVersionByName(ctx, template.ID, version.Name)
 		require.NoError(t, err)
+
+		if assert.Equal(t, tv.Job.Status, codersdk.ProvisionerJobPending) {
+			assert.NotNil(t, tv.MatchedProvisioners)
+			assert.Zero(t, tv.MatchedProvisioners.Available)
+			assert.Zero(t, tv.MatchedProvisioners.Count)
+			assert.False(t, tv.MatchedProvisioners.MostRecentlySeen.Valid)
+		}
 	})
 }
 
@@ -979,6 +1004,13 @@ func TestTemplateVersionDryRun(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, job.ID, newJob.ID)
 
+		// Check matched provisioners
+		matched, err := client.TemplateVersionDryRunMatchedProvisioners(ctx, version.ID, job.ID)
+		require.NoError(t, err)
+		require.Equal(t, 1, matched.Count)
+		require.Equal(t, 1, matched.Available)
+		require.NotZero(t, matched.MostRecentlySeen.Time)
+
 		// Stream logs
 		logs, closer, err := client.TemplateVersionDryRunLogsAfter(ctx, version.ID, job.ID, 0)
 		require.NoError(t, err)
@@ -1151,6 +1183,49 @@ func TestTemplateVersionDryRun(t *testing.T) {
 			require.Equal(t, http.StatusBadRequest, apiErr.StatusCode())
 		})
 	})
+
+	t.Run("Pending", func(t *testing.T) {
+		t.Parallel()
+		if !dbtestutil.WillUsePostgres() {
+			t.Skip("this test requires postgres")
+		}
+
+		store, ps, db := dbtestutil.NewDBWithSQLDB(t)
+		client, closer := coderdtest.NewWithProvisionerCloser(t, &coderdtest.Options{
+			Database:                 store,
+			Pubsub:                   ps,
+			IncludeProvisionerDaemon: true,
+		})
+		defer closer.Close()
+
+		owner := coderdtest.CreateFirstUser(t, client)
+		version := coderdtest.CreateTemplateVersion(t, client, owner.OrganizationID, &echo.Responses{
+			Parse:          echo.ParseComplete,
+			ProvisionApply: echo.ApplyComplete,
+		})
+		version = coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+		require.Equal(t, codersdk.ProvisionerJobSucceeded, version.Job.Status)
+
+		templateAdmin, _ := coderdtest.CreateAnotherUser(t, client, owner.OrganizationID, rbac.RoleTemplateAdmin())
+		ctx := testutil.Context(t, testutil.WaitShort)
+
+		_, err := db.Exec("DELETE FROM provisioner_daemons")
+		require.NoError(t, err)
+
+		job, err := templateAdmin.CreateTemplateVersionDryRun(ctx, version.ID, codersdk.CreateTemplateVersionDryRunRequest{
+			WorkspaceName:       "test",
+			RichParameterValues: []codersdk.WorkspaceBuildParameter{},
+			UserVariableValues:  []codersdk.VariableValue{},
+		})
+		require.NoError(t, err)
+		require.Equal(t, codersdk.ProvisionerJobPending, job.Status)
+
+		matched, err := templateAdmin.TemplateVersionDryRunMatchedProvisioners(ctx, version.ID, job.ID)
+		require.NoError(t, err)
+		require.Equal(t, 0, matched.Count)
+		require.Equal(t, 0, matched.Available)
+		require.Zero(t, matched.MostRecentlySeen.Time)
+	})
 }
 
 // TestPaginatedTemplateVersions creates a list of template versions and paginate.

coderd/workspacebuilds.go

@@ -27,6 +27,7 @@ import (
 	"github.com/coder/coder/v2/coderd/database/provisionerjobs"
 	"github.com/coder/coder/v2/coderd/httpapi"
 	"github.com/coder/coder/v2/coderd/httpmw"
+	"github.com/coder/coder/v2/coderd/provisionerdserver"
 	"github.com/coder/coder/v2/coderd/rbac"
 	"github.com/coder/coder/v2/coderd/rbac/policy"
 	"github.com/coder/coder/v2/coderd/wsbuilder"
@@ -85,6 +86,7 @@ func (api *API) workspaceBuild(rw http.ResponseWriter, r *http.Request) {
 		data.scripts,
 		data.logSources,
 		data.templateVersions[0],
+		nil,
 	)
 	if err != nil {
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
@@ -289,6 +291,7 @@ func (api *API) workspaceBuildByBuildNumber(rw http.ResponseWriter, r *http.Requ
 		data.scripts,
 		data.logSources,
 		data.templateVersions[0],
+		nil,
 	)
 	if err != nil {
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
@@ -352,7 +355,7 @@ func (api *API) postWorkspaceBuilds(rw http.ResponseWriter, r *http.Request) {
 		builder = builder.State(createBuild.ProvisionerState)
 	}
 
-	workspaceBuild, provisionerJob, err := builder.Build(
+	workspaceBuild, provisionerJob, provisionerDaemons, err := builder.Build(
 		ctx,
 		api.Database,
 		func(action policy.Action, object rbac.Objecter) bool {
@@ -384,12 +387,18 @@ func (api *API) postWorkspaceBuilds(rw http.ResponseWriter, r *http.Request) {
 		})
 		return
 	}
-	err = provisionerjobs.PostJob(api.Pubsub, *provisionerJob)
-	if err != nil {
-		// Client probably doesn't care about this error, so just log it.
-		api.Logger.Error(ctx, "failed to post provisioner job to pubsub", slog.Error(err))
+
+	if provisionerJob != nil {
+		if err := provisionerjobs.PostJob(api.Pubsub, *provisionerJob); err != nil {
+			// Client probably doesn't care about this error, so just log it.
+			api.Logger.Error(ctx, "failed to post provisioner job to pubsub", slog.Error(err))
+		}
 	}
 
+	var matchedProvisioners codersdk.MatchedProvisioners
+	if provisionerJob != nil {
+		matchedProvisioners = db2sdk.MatchedProvisioners(provisionerDaemons, provisionerJob.CreatedAt, provisionerdserver.StaleInterval)
+	}
 	apiBuild, err := api.convertWorkspaceBuild(
 		*workspaceBuild,
 		workspace,
@@ -404,6 +413,7 @@ func (api *API) postWorkspaceBuilds(rw http.ResponseWriter, r *http.Request) {
 		[]database.WorkspaceAgentScript{},
 		[]database.WorkspaceAgentLogSource{},
 		database.TemplateVersion{},
+		&matchedProvisioners,
 	)
 	if err != nil {
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
@@ -804,6 +814,7 @@ func (api *API) convertWorkspaceBuilds(
 			agentScripts,
 			agentLogSources,
 			templateVersion,
+			nil,
 		)
 		if err != nil {
 			return nil, xerrors.Errorf("converting workspace build: %w", err)
@@ -826,6 +837,7 @@ func (api *API) convertWorkspaceBuild(
 	agentScripts []database.WorkspaceAgentScript,
 	agentLogSources []database.WorkspaceAgentLogSource,
 	templateVersion database.TemplateVersion,
+	matchedProvisioners *codersdk.MatchedProvisioners,
 ) (codersdk.WorkspaceBuild, error) {
 	resourcesByJobID := map[uuid.UUID][]database.WorkspaceResource{}
 	for _, resource := range workspaceResources {
@@ -918,6 +930,7 @@ func (api *API) convertWorkspaceBuild(
 		Resources:               apiResources,
 		Status:                  codersdk.ConvertWorkspaceStatus(apiJob.Status, transition),
 		DailyCost:               build.DailyCost,
+		MatchedProvisioners:     matchedProvisioners,
 	}, nil
 }
 

coderd/workspacebuilds_test.go

@@ -1097,6 +1097,12 @@ func TestPostWorkspaceBuild(t *testing.T) {
 			Transition:        codersdk.WorkspaceTransitionStart,
 		})
 		require.NoError(t, err)
+		if assert.NotNil(t, build.MatchedProvisioners) {
+			require.Equal(t, 1, build.MatchedProvisioners.Count)
+			require.Equal(t, 1, build.MatchedProvisioners.Available)
+			require.NotZero(t, build.MatchedProvisioners.MostRecentlySeen.Time)
+		}
+
 		coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, build.ID)
 
 		require.Eventually(t, func() bool {
@@ -1124,6 +1130,12 @@ func TestPostWorkspaceBuild(t *testing.T) {
 			Transition:        codersdk.WorkspaceTransitionStart,
 		})
 		require.NoError(t, err)
+		if assert.NotNil(t, build.MatchedProvisioners) {
+			require.Equal(t, 1, build.MatchedProvisioners.Count)
+			require.Equal(t, 1, build.MatchedProvisioners.Available)
+			require.NotZero(t, build.MatchedProvisioners.MostRecentlySeen.Time)
+		}
+
 		require.Equal(t, workspace.LatestBuild.BuildNumber+1, build.BuildNumber)
 	})
 
@@ -1150,6 +1162,12 @@ func TestPostWorkspaceBuild(t *testing.T) {
 			ProvisionerState:  wantState,
 		})
 		require.NoError(t, err)
+		if assert.NotNil(t, build.MatchedProvisioners) {
+			require.Equal(t, 1, build.MatchedProvisioners.Count)
+			require.Equal(t, 1, build.MatchedProvisioners.Available)
+			require.NotZero(t, build.MatchedProvisioners.MostRecentlySeen.Time)
+		}
+
 		gotState, err := client.WorkspaceBuildState(ctx, build.ID)
 		require.NoError(t, err)
 		require.Equal(t, wantState, gotState)
@@ -1173,6 +1191,12 @@ func TestPostWorkspaceBuild(t *testing.T) {
 		})
 		require.NoError(t, err)
 		require.Equal(t, workspace.LatestBuild.BuildNumber+1, build.BuildNumber)
+		if assert.NotNil(t, build.MatchedProvisioners) {
+			require.Equal(t, 1, build.MatchedProvisioners.Count)
+			require.Equal(t, 1, build.MatchedProvisioners.Available)
+			require.NotZero(t, build.MatchedProvisioners.MostRecentlySeen.Time)
+		}
+
 		coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, build.ID)
 
 		res, err := client.Workspaces(ctx, codersdk.WorkspaceFilter{
@@ -1181,6 +1205,102 @@ func TestPostWorkspaceBuild(t *testing.T) {
 		require.NoError(t, err)
 		require.Len(t, res.Workspaces, 0)
 	})
+
+	t.Run("NoProvisionersAvailable", func(t *testing.T) {
+		t.Parallel()
+		if !dbtestutil.WillUsePostgres() {
+			t.Skip("this test requires postgres")
+		}
+		// Given: a coderd instance with a provisioner daemon
+		store, ps, db := dbtestutil.NewDBWithSQLDB(t)
+		client, closeDaemon := coderdtest.NewWithProvisionerCloser(t, &coderdtest.Options{
+			Database:                 store,
+			Pubsub:                   ps,
+			IncludeProvisionerDaemon: true,
+		})
+		defer closeDaemon.Close()
+		// Given: a user, template, and workspace
+		user := coderdtest.CreateFirstUser(t, client)
+		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
+		coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
+		workspace := coderdtest.CreateWorkspace(t, client, template.ID)
+		coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
+
+		// Stop the provisioner daemon.
+		require.NoError(t, closeDaemon.Close())
+		ctx := testutil.Context(t, testutil.WaitLong)
+		// Given: no provisioner daemons exist.
+		_, err := db.ExecContext(ctx, `DELETE FROM provisioner_daemons;`)
+		require.NoError(t, err)
+
+		// When: a new workspace build is created
+		build, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{
+			TemplateVersionID: template.ActiveVersionID,
+			Transition:        codersdk.WorkspaceTransitionStart,
+		})
+		// Then: the request should succeed.
+		require.NoError(t, err)
+		// Then: the provisioner job should remain pending.
+		require.Equal(t, codersdk.ProvisionerJobPending, build.Job.Status)
+		// Then: the response should indicate no provisioners are available.
+		if assert.NotNil(t, build.MatchedProvisioners) {
+			assert.Zero(t, build.MatchedProvisioners.Count)
+			assert.Zero(t, build.MatchedProvisioners.Available)
+			assert.Zero(t, build.MatchedProvisioners.MostRecentlySeen.Time)
+			assert.False(t, build.MatchedProvisioners.MostRecentlySeen.Valid)
+		}
+	})
+
+	t.Run("AllProvisionersStale", func(t *testing.T) {
+		t.Parallel()
+		if !dbtestutil.WillUsePostgres() {
+			t.Skip("this test requires postgres")
+		}
+		// Given: a coderd instance with a provisioner daemon
+		store, ps, db := dbtestutil.NewDBWithSQLDB(t)
+		client, closeDaemon := coderdtest.NewWithProvisionerCloser(t, &coderdtest.Options{
+			Database:                 store,
+			Pubsub:                   ps,
+			IncludeProvisionerDaemon: true,
+		})
+		defer closeDaemon.Close()
+		// Given: a user, template, and workspace
+		user := coderdtest.CreateFirstUser(t, client)
+		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
+		coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
+		workspace := coderdtest.CreateWorkspace(t, client, template.ID)
+		coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
+
+		ctx := testutil.Context(t, testutil.WaitLong)
+		// Given: all provisioner daemons are stale
+		// First stop the provisioner
+		require.NoError(t, closeDaemon.Close())
+		newLastSeenAt := dbtime.Now().Add(-time.Hour)
+		// Update the last seen at for all provisioner daemons. We have to use the
+		// SQL db directly because store.UpdateProvisionerDaemonLastSeenAt has a
+		// built-in check to prevent updating the last seen at to a time in the past.
+		_, err := db.ExecContext(ctx, `UPDATE provisioner_daemons SET last_seen_at = $1;`, newLastSeenAt)
+		require.NoError(t, err)
+
+		// When: a new workspace build is created
+		build, err := client.CreateWorkspaceBuild(ctx, workspace.ID, codersdk.CreateWorkspaceBuildRequest{
+			TemplateVersionID: template.ActiveVersionID,
+			Transition:        codersdk.WorkspaceTransitionStart,
+		})
+		// Then: the request should succeed
+		require.NoError(t, err)
+		// Then: the provisioner job should remain pending
+		require.Equal(t, codersdk.ProvisionerJobPending, build.Job.Status)
+		// Then: the response should indicate no provisioners are available
+		if assert.NotNil(t, build.MatchedProvisioners) {
+			assert.Zero(t, build.MatchedProvisioners.Available)
+			assert.Equal(t, 1, build.MatchedProvisioners.Count)
+			assert.Equal(t, newLastSeenAt.UTC(), build.MatchedProvisioners.MostRecentlySeen.Time.UTC())
+			assert.True(t, build.MatchedProvisioners.MostRecentlySeen.Valid)
+		}
+	})
 }
 
 func TestWorkspaceBuildTimings(t *testing.T) {

coderd/workspaces.go

@@ -27,6 +27,7 @@ import (
 	"github.com/coder/coder/v2/coderd/httpapi"
 	"github.com/coder/coder/v2/coderd/httpmw"
 	"github.com/coder/coder/v2/coderd/notifications"
+	"github.com/coder/coder/v2/coderd/provisionerdserver"
 	"github.com/coder/coder/v2/coderd/rbac"
 	"github.com/coder/coder/v2/coderd/rbac/policy"
 	"github.com/coder/coder/v2/coderd/schedule/cron"
@@ -593,8 +594,7 @@ func createWorkspace(
 			}},
 		})
 		return
-	}
-	if err != nil && !errors.Is(err, sql.ErrNoRows) {
+	} else if !errors.Is(err, sql.ErrNoRows) {
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
 			Message: fmt.Sprintf("Internal error fetching workspace by name %q.", req.Name),
 			Detail:  err.Error(),
@@ -603,8 +603,10 @@ func createWorkspace(
 	}
 
 	var (
-		provisionerJob *database.ProvisionerJob
-		workspaceBuild *database.WorkspaceBuild
+		provisionerJob      *database.ProvisionerJob
+		workspaceBuild      *database.WorkspaceBuild
+		provisionerDaemons  []database.ProvisionerDaemon
+		matchedProvisioners codersdk.MatchedProvisioners
 	)
 	err = api.Database.InTx(func(db database.Store) error {
 		now := dbtime.Now()
@@ -645,7 +647,7 @@ func createWorkspace(
 			builder = builder.VersionID(req.TemplateVersionID)
 		}
 
-		workspaceBuild, provisionerJob, err = builder.Build(
+		workspaceBuild, provisionerJob, provisionerDaemons, err = builder.Build(
 			ctx,
 			db,
 			func(action policy.Action, object rbac.Objecter) bool {
@@ -655,6 +657,7 @@ func createWorkspace(
 		)
 		return err
 	}, nil)
+
 	var bldErr wsbuilder.BuildError
 	if xerrors.As(err, &bldErr) {
 		httpapi.Write(ctx, rw, bldErr.Status, codersdk.Response{
@@ -675,6 +678,10 @@ func createWorkspace(
 		// Client probably doesn't care about this error, so just log it.
 		api.Logger.Error(ctx, "failed to post provisioner job to pubsub", slog.Error(err))
 	}
+	if provisionerJob != nil {
+		matchedProvisioners = db2sdk.MatchedProvisioners(provisionerDaemons, provisionerJob.CreatedAt, provisionerdserver.StaleInterval)
+	}
+
 	auditReq.New = workspace.WorkspaceTable()
 
 	api.Telemetry.Report(&telemetry.Snapshot{
@@ -696,6 +703,7 @@ func createWorkspace(
 		[]database.WorkspaceAgentScript{},
 		[]database.WorkspaceAgentLogSource{},
 		database.TemplateVersion{},
+		&matchedProvisioners,
 	)
 	if err != nil {
 		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{

coderd/workspaces_test.go

@@ -766,6 +766,94 @@ func TestPostWorkspacesByOrganization(t *testing.T) {
 		require.NoError(t, err)
 		require.EqualValues(t, exp, *ws.TTLMillis)
 	})
+
+	t.Run("NoProvisionersAvailable", func(t *testing.T) {
+		t.Parallel()
+		if !dbtestutil.WillUsePostgres() {
+			t.Skip("this test requires postgres")
+		}
+		// Given: a coderd instance with a provisioner daemon
+		store, ps, db := dbtestutil.NewDBWithSQLDB(t)
+		client, closeDaemon := coderdtest.NewWithProvisionerCloser(t, &coderdtest.Options{
+			Database:                 store,
+			Pubsub:                   ps,
+			IncludeProvisionerDaemon: true,
+		})
+		defer closeDaemon.Close()
+
+		// Given: a user, template, and workspace
+		user := coderdtest.CreateFirstUser(t, client)
+		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
+		coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
+
+		// Given: all the provisioner daemons disappear
+		ctx := testutil.Context(t, testutil.WaitLong)
+		_, err := db.ExecContext(ctx, `DELETE FROM provisioner_daemons;`)
+		require.NoError(t, err)
+
+		// When: a new workspace is created
+		ws, err := client.CreateUserWorkspace(ctx, codersdk.Me, codersdk.CreateWorkspaceRequest{
+			TemplateID: template.ID,
+			Name:       "testing",
+		})
+		// Then: the request succeeds
+		require.NoError(t, err)
+		// Then: the workspace build is pending
+		require.Equal(t, codersdk.ProvisionerJobPending, ws.LatestBuild.Job.Status)
+		// Then: the workspace build has no matched provisioners
+		if assert.NotNil(t, ws.LatestBuild.MatchedProvisioners) {
+			assert.Zero(t, ws.LatestBuild.MatchedProvisioners.Count)
+			assert.Zero(t, ws.LatestBuild.MatchedProvisioners.Available)
+			assert.Zero(t, ws.LatestBuild.MatchedProvisioners.MostRecentlySeen.Time)
+			assert.False(t, ws.LatestBuild.MatchedProvisioners.MostRecentlySeen.Valid)
+		}
+	})
+
+	t.Run("AllProvisionersStale", func(t *testing.T) {
+		t.Parallel()
+		if !dbtestutil.WillUsePostgres() {
+			t.Skip("this test requires postgres")
+		}
+
+		// Given: a coderd instance with a provisioner daemon
+		store, ps, db := dbtestutil.NewDBWithSQLDB(t)
+		client, closeDaemon := coderdtest.NewWithProvisionerCloser(t, &coderdtest.Options{
+			Database:                 store,
+			Pubsub:                   ps,
+			IncludeProvisionerDaemon: true,
+		})
+		defer closeDaemon.Close()
+
+		// Given: a user, template, and workspace
+		user := coderdtest.CreateFirstUser(t, client)
+		version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
+		coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+		template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
+
+		// Given: all the provisioner daemons have not been seen for a while
+		ctx := testutil.Context(t, testutil.WaitLong)
+		newLastSeenAt := dbtime.Now().Add(-time.Hour)
+		_, err := db.ExecContext(ctx, `UPDATE provisioner_daemons SET last_seen_at = $1;`, newLastSeenAt)
+		require.NoError(t, err)
+
+		// When: a new workspace is created
+		ws, err := client.CreateUserWorkspace(ctx, codersdk.Me, codersdk.CreateWorkspaceRequest{
+			TemplateID: template.ID,
+			Name:       "testing",
+		})
+		// Then: the request succeeds
+		require.NoError(t, err)
+		// Then: the workspace build is pending
+		require.Equal(t, codersdk.ProvisionerJobPending, ws.LatestBuild.Job.Status)
+		// Then: we can see that there are some provisioners that are stale
+		if assert.NotNil(t, ws.LatestBuild.MatchedProvisioners) {
+			assert.Equal(t, 1, ws.LatestBuild.MatchedProvisioners.Count)
+			assert.Zero(t, ws.LatestBuild.MatchedProvisioners.Available)
+			assert.Equal(t, newLastSeenAt.UTC(), ws.LatestBuild.MatchedProvisioners.MostRecentlySeen.Time.UTC())
+			assert.True(t, ws.LatestBuild.MatchedProvisioners.MostRecentlySeen.Valid)
+		}
+	})
 }
 
 func TestWorkspaceByOwnerAndName(t *testing.T) {

coderd/wsbuilder/wsbuilder.go

@@ -24,6 +24,7 @@ import (
 	"github.com/coder/coder/v2/coderd/audit"
 	"github.com/coder/coder/v2/coderd/database"
 	"github.com/coder/coder/v2/coderd/database/db2sdk"
+	"github.com/coder/coder/v2/coderd/database/dbauthz"
 	"github.com/coder/coder/v2/coderd/database/dbtime"
 	"github.com/coder/coder/v2/coderd/httpapi"
 	"github.com/coder/coder/v2/coderd/provisionerdserver"
@@ -213,12 +214,12 @@ func (b *Builder) Build(
 	authFunc func(action policy.Action, object rbac.Objecter) bool,
 	auditBaggage audit.WorkspaceBuildBaggage,
 ) (
-	*database.WorkspaceBuild, *database.ProvisionerJob, error,
+	*database.WorkspaceBuild, *database.ProvisionerJob, []database.ProvisionerDaemon, error,
 ) {
 	var err error
 	b.ctx, err = audit.BaggageToContext(ctx, auditBaggage)
 	if err != nil {
-		return nil, nil, xerrors.Errorf("create audit baggage: %w", err)
+		return nil, nil, nil, xerrors.Errorf("create audit baggage: %w", err)
 	}
 
 	// Run the build in a transaction with RepeatableRead isolation, and retries.
@@ -227,16 +228,17 @@ func (b *Builder) Build(
 	// later reads are consistent with earlier ones.
 	var workspaceBuild *database.WorkspaceBuild
 	var provisionerJob *database.ProvisionerJob
+	var provisionerDaemons []database.ProvisionerDaemon
 	err = database.ReadModifyUpdate(store, func(tx database.Store) error {
 		var err error
 		b.store = tx
-		workspaceBuild, provisionerJob, err = b.buildTx(authFunc)
+		workspaceBuild, provisionerJob, provisionerDaemons, err = b.buildTx(authFunc)
 		return err
 	})
 	if err != nil {
-		return nil, nil, xerrors.Errorf("build tx: %w", err)
+		return nil, nil, nil, xerrors.Errorf("build tx: %w", err)
 	}
-	return workspaceBuild, provisionerJob, nil
+	return workspaceBuild, provisionerJob, provisionerDaemons, nil
 }
 
 // buildTx contains the business logic of computing a new build.  Attributes of the new database objects are computed
@@ -246,35 +248,35 @@ func (b *Builder) Build(
 //
 // In order to utilize this cache, the functions that compute build attributes use a pointer receiver type.
 func (b *Builder) buildTx(authFunc func(action policy.Action, object rbac.Objecter) bool) (
-	*database.WorkspaceBuild, *database.ProvisionerJob, error,
+	*database.WorkspaceBuild, *database.ProvisionerJob, []database.ProvisionerDaemon, error,
 ) {
 	if authFunc != nil {
 		err := b.authorize(authFunc)
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 	}
 	err := b.checkTemplateVersionMatchesTemplate()
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	err = b.checkTemplateJobStatus()
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 	err = b.checkRunningBuild()
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
 	template, err := b.getTemplate()
 	if err != nil {
-		return nil, nil, BuildError{http.StatusInternalServerError, "failed to fetch template", err}
+		return nil, nil, nil, BuildError{http.StatusInternalServerError, "failed to fetch template", err}
 	}
 
 	templateVersionJob, err := b.getTemplateVersionJob()
 	if err != nil {
-		return nil, nil, BuildError{
+		return nil, nil, nil, BuildError{
 			http.StatusInternalServerError, "failed to fetch template version job", err,
 		}
 	}
@@ -294,7 +296,7 @@ func (b *Builder) buildTx(authFunc func(action policy.Action, object rbac.Object
 		LogLevel:         b.logLevel,
 	})
 	if err != nil {
-		return nil, nil, BuildError{
+		return nil, nil, nil, BuildError{
 			http.StatusInternalServerError,
 			"marshal provision job",
 			err,
@@ -302,12 +304,12 @@ func (b *Builder) buildTx(authFunc func(action policy.Action, object rbac.Object
 	}
 	traceMetadataRaw, err := json.Marshal(tracing.MetadataFromContext(b.ctx))
 	if err != nil {
-		return nil, nil, BuildError{http.StatusInternalServerError, "marshal metadata", err}
+		return nil, nil, nil, BuildError{http.StatusInternalServerError, "marshal metadata", err}
 	}
 
 	tags, err := b.getProvisionerTags()
 	if err != nil {
-		return nil, nil, err // already wrapped BuildError
+		return nil, nil, nil, err // already wrapped BuildError
 	}
 
 	now := dbtime.Now()
@@ -329,20 +331,35 @@ func (b *Builder) buildTx(authFunc func(action policy.Action, object rbac.Object
 		},
 	})
 	if err != nil {
-		return nil, nil, BuildError{http.StatusInternalServerError, "insert provisioner job", err}
+		return nil, nil, nil, BuildError{http.StatusInternalServerError, "insert provisioner job", err}
+	}
+
+	// nolint:gocritic // The user performing this request may not have permission
+	// to read all provisioner daemons. We need to retrieve the eligible
+	// provisioner daemons for this job to show in the UI if there is no
+	// matching provisioner daemon.
+	provisionerDaemons, err := b.store.GetProvisionerDaemonsByOrganization(dbauthz.AsSystemReadProvisionerDaemons(b.ctx), database.GetProvisionerDaemonsByOrganizationParams{
+		OrganizationID: template.OrganizationID,
+		WantTags:       provisionerJob.Tags,
+	})
+	if err != nil {
+		// NOTE: we do **not** want to fail a workspace build if we fail to
+		// retrieve provisioner daemons. This is just to show in the UI if there
+		// is no matching provisioner daemon for the job.
+		provisionerDaemons = []database.ProvisionerDaemon{}
 	}
 
 	templateVersionID, err := b.getTemplateVersionID()
 	if err != nil {
-		return nil, nil, BuildError{http.StatusInternalServerError, "compute template version ID", err}
+		return nil, nil, nil, BuildError{http.StatusInternalServerError, "compute template version ID", err}
 	}
 	buildNum, err := b.getBuildNumber()
 	if err != nil {
-		return nil, nil, BuildError{http.StatusInternalServerError, "compute build number", err}
+		return nil, nil, nil, BuildError{http.StatusInternalServerError, "compute build number", err}
 	}
 	state, err := b.getState()
 	if err != nil {
-		return nil, nil, BuildError{http.StatusInternalServerError, "compute build state", err}
+		return nil, nil, nil, BuildError{http.StatusInternalServerError, "compute build state", err}
 	}
 
 	var workspaceBuild database.WorkspaceBuild
@@ -393,10 +410,10 @@ func (b *Builder) buildTx(authFunc func(action policy.Action, object rbac.Object
 		return nil
 	}, nil)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
-	return &workspaceBuild, &provisionerJob, nil
+	return &workspaceBuild, &provisionerJob, provisionerDaemons, nil
 }
 
 func (b *Builder) getTemplate() (*database.Template, error) {

coderd/wsbuilder/wsbuilder_test.go

@@ -61,6 +61,7 @@ func TestBuilder_NoOptions(t *testing.T) {
 		withRichParameters(nil),
 		withParameterSchemas(inactiveJobID, nil),
 		withWorkspaceTags(inactiveVersionID, nil),
+		withProvisionerDaemons([]database.ProvisionerDaemon{}),
 
 		// Outputs
 		expectProvisionerJob(func(job database.InsertProvisionerJobParams) {
@@ -94,7 +95,8 @@ func TestBuilder_NoOptions(t *testing.T) {
 
 	ws := database.Workspace{ID: workspaceID, TemplateID: templateID, OwnerID: userID}
 	uut := wsbuilder.New(ws, database.WorkspaceTransitionStart)
-	_, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
+	// nolint: dogsled
+	_, _, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
 	req.NoError(err)
 }
 
@@ -114,6 +116,7 @@ func TestBuilder_Initiator(t *testing.T) {
 		withRichParameters(nil),
 		withParameterSchemas(inactiveJobID, nil),
 		withWorkspaceTags(inactiveVersionID, nil),
+		withProvisionerDaemons([]database.ProvisionerDaemon{}),
 
 		// Outputs
 		expectProvisionerJob(func(job database.InsertProvisionerJobParams) {
@@ -130,7 +133,8 @@ func TestBuilder_Initiator(t *testing.T) {
 
 	ws := database.Workspace{ID: workspaceID, TemplateID: templateID, OwnerID: userID}
 	uut := wsbuilder.New(ws, database.WorkspaceTransitionStart).Initiator(otherUserID)
-	_, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
+	// nolint: dogsled
+	_, _, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
 	req.NoError(err)
 }
 
@@ -157,6 +161,7 @@ func TestBuilder_Baggage(t *testing.T) {
 		withRichParameters(nil),
 		withParameterSchemas(inactiveJobID, nil),
 		withWorkspaceTags(inactiveVersionID, nil),
+		withProvisionerDaemons([]database.ProvisionerDaemon{}),
 
 		// Outputs
 		expectProvisionerJob(func(job database.InsertProvisionerJobParams) {
@@ -172,7 +177,8 @@ func TestBuilder_Baggage(t *testing.T) {
 
 	ws := database.Workspace{ID: workspaceID, TemplateID: templateID, OwnerID: userID}
 	uut := wsbuilder.New(ws, database.WorkspaceTransitionStart).Initiator(otherUserID)
-	_, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{IP: "127.0.0.1"})
+	// nolint: dogsled
+	_, _, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{IP: "127.0.0.1"})
 	req.NoError(err)
 }
 
@@ -192,6 +198,7 @@ func TestBuilder_Reason(t *testing.T) {
 		withRichParameters(nil),
 		withParameterSchemas(inactiveJobID, nil),
 		withWorkspaceTags(inactiveVersionID, nil),
+		withProvisionerDaemons([]database.ProvisionerDaemon{}),
 
 		// Outputs
 		expectProvisionerJob(func(_ database.InsertProvisionerJobParams) {
@@ -207,7 +214,8 @@ func TestBuilder_Reason(t *testing.T) {
 
 	ws := database.Workspace{ID: workspaceID, TemplateID: templateID, OwnerID: userID}
 	uut := wsbuilder.New(ws, database.WorkspaceTransitionStart).Reason(database.BuildReasonAutostart)
-	_, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
+	// nolint: dogsled
+	_, _, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
 	req.NoError(err)
 }
 
@@ -226,6 +234,7 @@ func TestBuilder_ActiveVersion(t *testing.T) {
 		withLastBuildNotFound,
 		withParameterSchemas(activeJobID, nil),
 		withWorkspaceTags(activeVersionID, nil),
+		withProvisionerDaemons([]database.ProvisionerDaemon{}),
 		// previous rich parameters are not queried because there is no previous build.
 
 		// Outputs
@@ -247,7 +256,8 @@ func TestBuilder_ActiveVersion(t *testing.T) {
 
 	ws := database.Workspace{ID: workspaceID, TemplateID: templateID, OwnerID: userID}
 	uut := wsbuilder.New(ws, database.WorkspaceTransitionStart).ActiveVersion()
-	_, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
+	// nolint: dogsled
+	_, _, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
 	req.NoError(err)
 }
 
@@ -314,6 +324,7 @@ func TestWorkspaceBuildWithTags(t *testing.T) {
 		withRichParameters(nil),
 		withParameterSchemas(inactiveJobID, nil),
 		withWorkspaceTags(inactiveVersionID, workspaceTags),
+		withProvisionerDaemons([]database.ProvisionerDaemon{}),
 
 		// Outputs
 		expectProvisionerJob(func(job database.InsertProvisionerJobParams) {
@@ -343,7 +354,8 @@ func TestWorkspaceBuildWithTags(t *testing.T) {
 
 	ws := database.Workspace{ID: workspaceID, TemplateID: templateID, OwnerID: userID}
 	uut := wsbuilder.New(ws, database.WorkspaceTransitionStart).RichParameterValues(buildParameters)
-	_, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
+	// nolint: dogsled
+	_, _, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
 	req.NoError(err)
 }
 
@@ -404,6 +416,7 @@ func TestWorkspaceBuildWithRichParameters(t *testing.T) {
 			withRichParameters(initialBuildParameters),
 			withParameterSchemas(inactiveJobID, nil),
 			withWorkspaceTags(inactiveVersionID, nil),
+			withProvisionerDaemons([]database.ProvisionerDaemon{}),
 
 			// Outputs
 			expectProvisionerJob(func(job database.InsertProvisionerJobParams) {}),
@@ -422,7 +435,8 @@ func TestWorkspaceBuildWithRichParameters(t *testing.T) {
 
 		ws := database.Workspace{ID: workspaceID, TemplateID: templateID, OwnerID: userID}
 		uut := wsbuilder.New(ws, database.WorkspaceTransitionStart).RichParameterValues(nextBuildParameters)
-		_, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
+		// nolint: dogsled
+		_, _, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
 		req.NoError(err)
 	})
 	t.Run("UsePreviousParameterValues", func(t *testing.T) {
@@ -448,6 +462,7 @@ func TestWorkspaceBuildWithRichParameters(t *testing.T) {
 			withRichParameters(initialBuildParameters),
 			withParameterSchemas(inactiveJobID, nil),
 			withWorkspaceTags(inactiveVersionID, nil),
+			withProvisionerDaemons([]database.ProvisionerDaemon{}),
 
 			// Outputs
 			expectProvisionerJob(func(job database.InsertProvisionerJobParams) {}),
@@ -466,7 +481,8 @@ func TestWorkspaceBuildWithRichParameters(t *testing.T) {
 
 		ws := database.Workspace{ID: workspaceID, TemplateID: templateID, OwnerID: userID}
 		uut := wsbuilder.New(ws, database.WorkspaceTransitionStart).RichParameterValues(nextBuildParameters)
-		_, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
+		// nolint: dogsled
+		_, _, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
 		req.NoError(err)
 	})
 
@@ -502,7 +518,7 @@ func TestWorkspaceBuildWithRichParameters(t *testing.T) {
 
 		ws := database.Workspace{ID: workspaceID, TemplateID: templateID, OwnerID: userID}
 		uut := wsbuilder.New(ws, database.WorkspaceTransitionStart)
-		_, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
+		_, _, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
 		bldErr := wsbuilder.BuildError{}
 		req.ErrorAs(err, &bldErr)
 		asrt.Equal(http.StatusBadRequest, bldErr.Status)
@@ -536,7 +552,8 @@ func TestWorkspaceBuildWithRichParameters(t *testing.T) {
 
 		ws := database.Workspace{ID: workspaceID, TemplateID: templateID, OwnerID: userID}
 		uut := wsbuilder.New(ws, database.WorkspaceTransitionStart).RichParameterValues(nextBuildParameters)
-		_, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
+		// nolint: dogsled
+		_, _, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
 		bldErr := wsbuilder.BuildError{}
 		req.ErrorAs(err, &bldErr)
 		asrt.Equal(http.StatusBadRequest, bldErr.Status)
@@ -579,6 +596,7 @@ func TestWorkspaceBuildWithRichParameters(t *testing.T) {
 			withRichParameters(initialBuildParameters),
 			withParameterSchemas(activeJobID, nil),
 			withWorkspaceTags(activeVersionID, nil),
+			withProvisionerDaemons([]database.ProvisionerDaemon{}),
 
 			// Outputs
 			expectProvisionerJob(func(job database.InsertProvisionerJobParams) {}),
@@ -599,7 +617,7 @@ func TestWorkspaceBuildWithRichParameters(t *testing.T) {
 		uut := wsbuilder.New(ws, database.WorkspaceTransitionStart).
 			RichParameterValues(nextBuildParameters).
 			VersionID(activeVersionID)
-		_, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
+		_, _, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
 		req.NoError(err)
 	})
 
@@ -640,6 +658,7 @@ func TestWorkspaceBuildWithRichParameters(t *testing.T) {
 			withRichParameters(initialBuildParameters),
 			withParameterSchemas(activeJobID, nil),
 			withWorkspaceTags(activeVersionID, nil),
+			withProvisionerDaemons([]database.ProvisionerDaemon{}),
 
 			// Outputs
 			expectProvisionerJob(func(job database.InsertProvisionerJobParams) {}),
@@ -660,7 +679,7 @@ func TestWorkspaceBuildWithRichParameters(t *testing.T) {
 		uut := wsbuilder.New(ws, database.WorkspaceTransitionStart).
 			RichParameterValues(nextBuildParameters).
 			VersionID(activeVersionID)
-		_, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
+		_, _, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
 		req.NoError(err)
 	})
 
@@ -699,6 +718,7 @@ func TestWorkspaceBuildWithRichParameters(t *testing.T) {
 			withRichParameters(initialBuildParameters),
 			withParameterSchemas(activeJobID, nil),
 			withWorkspaceTags(activeVersionID, nil),
+			withProvisionerDaemons([]database.ProvisionerDaemon{}),
 
 			// Outputs
 			expectProvisionerJob(func(job database.InsertProvisionerJobParams) {}),
@@ -719,7 +739,8 @@ func TestWorkspaceBuildWithRichParameters(t *testing.T) {
 		uut := wsbuilder.New(ws, database.WorkspaceTransitionStart).
 			RichParameterValues(nextBuildParameters).
 			VersionID(activeVersionID)
-		_, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
+		// nolint: dogsled
+		_, _, _, err := uut.Build(ctx, mDB, nil, audit.WorkspaceBuildBaggage{})
 		req.NoError(err)
 	})
 }
@@ -987,3 +1008,9 @@ func expectBuildParameters(
 			)
 	}
 }
+
+func withProvisionerDaemons(provisionerDaemons []database.ProvisionerDaemon) func(mTx *dbmock.MockStore) {
+	return func(mTx *dbmock.MockStore) {
+		mTx.EXPECT().GetProvisionerDaemonsByOrganization(gomock.Any(), gomock.Any()).Return(provisionerDaemons, nil)
+	}
+}