fix: reduce agentfake manager startup time (#25669)

Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Mux <noreply@coder.com>

coderd/database/dbauthz/dbauthz.go

@@ -3457,6 +3457,20 @@ func (q *querier) GetEnabledMCPServerConfigs(ctx context.Context) ([]database.MC
 	return q.db.GetEnabledMCPServerConfigs(ctx)
 }
 
+// GetExternalAgentTokensByTemplateID is used for scaletesting purposes; the
+// scaletest agentfake path calls this query directly via a connection to the
+// database. There is no production code path that uses this method, and it is
+// deliberately not exposed over HTTP. The query filters for running
+// workspaces only (latest build has transition=start and job_status=succeeded).
+func (q *querier) GetExternalAgentTokensByTemplateID(ctx context.Context, arg database.GetExternalAgentTokensByTemplateIDParams) ([]database.GetExternalAgentTokensByTemplateIDRow, error) {
+	// ResourceSystem is used because the query spans multiple workspaces
+	// with no single RBAC object to check.
+	if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceSystem); err != nil {
+		return nil, err
+	}
+	return q.db.GetExternalAgentTokensByTemplateID(ctx, arg)
+}
+
 func (q *querier) GetExternalAuthLink(ctx context.Context, arg database.GetExternalAuthLinkParams) (database.ExternalAuthLink, error) {
 	return fetchWithAction(q.log, q.auth, policy.ActionReadPersonal, q.db.GetExternalAuthLink)(ctx, arg)
 }

coderd/database/dbauthz/dbauthz_test.go

@@ -3137,6 +3137,12 @@ func (s *MethodTestSuite) TestUser() {
 		dbm.EXPECT().UpdateGitSSHKey(gomock.Any(), arg).Return(key, nil).AnyTimes()
 		check.Args(arg).Asserts(key, policy.ActionUpdatePersonal).Returns(key)
 	}))
+	s.Run("GetExternalAgentTokensByTemplateID", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
+		arg := database.GetExternalAgentTokensByTemplateIDParams{TemplateID: uuid.New(), OwnerID: uuid.Nil}
+		row := testutil.Fake(s.T(), faker, database.GetExternalAgentTokensByTemplateIDRow{})
+		dbm.EXPECT().GetExternalAgentTokensByTemplateID(gomock.Any(), arg).Return([]database.GetExternalAgentTokensByTemplateIDRow{row}, nil).AnyTimes()
+		check.Args(arg).Asserts(rbac.ResourceSystem, policy.ActionRead).Returns(slice.New(row))
+	}))
 	s.Run("GetExternalAuthLink", s.Mocked(func(dbm *dbmock.MockStore, faker *gofakeit.Faker, check *expects) {
 		link := testutil.Fake(s.T(), faker, database.ExternalAuthLink{})
 		arg := database.GetExternalAuthLinkParams{ProviderID: link.ProviderID, UserID: link.UserID}

coderd/database/dbmetrics/querymetrics.go

@@ -1841,6 +1841,14 @@ func (m queryMetricsStore) GetEnabledMCPServerConfigs(ctx context.Context) ([]da
 	return r0, r1
 }
 
+func (m queryMetricsStore) GetExternalAgentTokensByTemplateID(ctx context.Context, arg database.GetExternalAgentTokensByTemplateIDParams) ([]database.GetExternalAgentTokensByTemplateIDRow, error) {
+	start := time.Now()
+	r0, r1 := m.s.GetExternalAgentTokensByTemplateID(ctx, arg)
+	m.queryLatencies.WithLabelValues("GetExternalAgentTokensByTemplateID").Observe(time.Since(start).Seconds())
+	m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "GetExternalAgentTokensByTemplateID").Inc()
+	return r0, r1
+}
+
 func (m queryMetricsStore) GetExternalAuthLink(ctx context.Context, arg database.GetExternalAuthLinkParams) (database.ExternalAuthLink, error) {
 	start := time.Now()
 	r0, r1 := m.s.GetExternalAuthLink(ctx, arg)

coderd/database/dbmock/dbmock.go

@@ -3420,6 +3420,21 @@ func (mr *MockStoreMockRecorder) GetEnabledMCPServerConfigs(ctx any) *gomock.Cal
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetEnabledMCPServerConfigs", reflect.TypeOf((*MockStore)(nil).GetEnabledMCPServerConfigs), ctx)
 }
 
+// GetExternalAgentTokensByTemplateID mocks base method.
+func (m *MockStore) GetExternalAgentTokensByTemplateID(ctx context.Context, arg database.GetExternalAgentTokensByTemplateIDParams) ([]database.GetExternalAgentTokensByTemplateIDRow, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetExternalAgentTokensByTemplateID", ctx, arg)
+	ret0, _ := ret[0].([]database.GetExternalAgentTokensByTemplateIDRow)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetExternalAgentTokensByTemplateID indicates an expected call of GetExternalAgentTokensByTemplateID.
+func (mr *MockStoreMockRecorder) GetExternalAgentTokensByTemplateID(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetExternalAgentTokensByTemplateID", reflect.TypeOf((*MockStore)(nil).GetExternalAgentTokensByTemplateID), ctx, arg)
+}
+
 // GetExternalAuthLink mocks base method.
 func (m *MockStore) GetExternalAuthLink(ctx context.Context, arg database.GetExternalAuthLinkParams) (database.ExternalAuthLink, error) {
 	m.ctrl.T.Helper()

coderd/database/querier.go

@@ -459,6 +459,15 @@ type sqlcQuerier interface {
 	GetEnabledChatModelConfigByID(ctx context.Context, id uuid.UUID) (ChatModelConfig, error)
 	GetEnabledChatModelConfigs(ctx context.Context) ([]ChatModelConfig, error)
 	GetEnabledMCPServerConfigs(ctx context.Context) ([]MCPServerConfig, error)
+	// GetExternalAgentTokensByTemplateID returns the auth tokens for all
+	// non-deleted external agents on the latest build of every running workspace
+	// of the given template. "Running" means the latest build has
+	// transition=start and job_status=succeeded (matches the workspace-status
+	// definition used by coderd/database/queries/workspaces.sql).
+	// An owner_id of '00000000-0000-0000-0000-000000000000' (uuid.Nil) means
+	// "all owners"; any other value restricts results to workspaces owned by
+	// that user.
+	GetExternalAgentTokensByTemplateID(ctx context.Context, arg GetExternalAgentTokensByTemplateIDParams) ([]GetExternalAgentTokensByTemplateIDRow, error)
 	GetExternalAuthLink(ctx context.Context, arg GetExternalAuthLinkParams) (ExternalAuthLink, error)
 	GetExternalAuthLinksByUserID(ctx context.Context, userID uuid.UUID) ([]ExternalAuthLink, error)
 	GetFailedWorkspaceBuildsByTemplateID(ctx context.Context, arg GetFailedWorkspaceBuildsByTemplateIDParams) ([]GetFailedWorkspaceBuildsByTemplateIDRow, error)

coderd/database/queries.sql.go

@@ -30362,6 +30362,102 @@ func (q *sqlQuerier) GetAuthenticatedWorkspaceAgentAndBuildByAuthToken(ctx conte
 	return i, err
 }
 
+const getExternalAgentTokensByTemplateID = `-- name: GetExternalAgentTokensByTemplateID :many
+SELECT
+	workspaces.id               AS workspace_id,
+	workspaces.name             AS workspace_name,
+	workspace_agents.id         AS agent_id,
+	workspace_agents.name       AS agent_name,
+	workspace_agents.auth_token AS agent_token
+FROM
+	workspaces
+JOIN (
+	-- latest build per workspace
+	SELECT DISTINCT ON (workspace_id)
+		id, workspace_id, job_id, transition, has_external_agent
+	FROM
+		workspace_builds
+	ORDER BY
+		workspace_id, build_number DESC
+) AS latest_builds
+ON
+	latest_builds.workspace_id = workspaces.id
+JOIN
+	provisioner_jobs
+ON
+	provisioner_jobs.id = latest_builds.job_id
+JOIN
+	workspace_resources
+ON
+	workspace_resources.job_id = latest_builds.job_id
+JOIN
+	workspace_agents
+ON
+	workspace_agents.resource_id = workspace_resources.id
+WHERE
+	workspaces.template_id = $1
+	AND (
+		$2 :: uuid = '00000000-0000-0000-0000-000000000000' :: uuid
+		OR workspaces.owner_id = $2
+	)
+	AND workspaces.deleted = FALSE
+	AND latest_builds.has_external_agent = TRUE
+	AND latest_builds.transition = 'start' :: workspace_transition
+	AND provisioner_jobs.job_status = 'succeeded' :: provisioner_job_status
+	AND workspace_agents.deleted = FALSE
+	AND workspace_agents.auth_instance_id IS NULL
+`
+
+type GetExternalAgentTokensByTemplateIDParams struct {
+	TemplateID uuid.UUID `db:"template_id" json:"template_id"`
+	OwnerID    uuid.UUID `db:"owner_id" json:"owner_id"`
+}
+
+type GetExternalAgentTokensByTemplateIDRow struct {
+	WorkspaceID   uuid.UUID `db:"workspace_id" json:"workspace_id"`
+	WorkspaceName string    `db:"workspace_name" json:"workspace_name"`
+	AgentID       uuid.UUID `db:"agent_id" json:"agent_id"`
+	AgentName     string    `db:"agent_name" json:"agent_name"`
+	AgentToken    uuid.UUID `db:"agent_token" json:"agent_token"`
+}
+
+// GetExternalAgentTokensByTemplateID returns the auth tokens for all
+// non-deleted external agents on the latest build of every running workspace
+// of the given template. "Running" means the latest build has
+// transition=start and job_status=succeeded (matches the workspace-status
+// definition used by coderd/database/queries/workspaces.sql).
+// An owner_id of '00000000-0000-0000-0000-000000000000' (uuid.Nil) means
+// "all owners"; any other value restricts results to workspaces owned by
+// that user.
+func (q *sqlQuerier) GetExternalAgentTokensByTemplateID(ctx context.Context, arg GetExternalAgentTokensByTemplateIDParams) ([]GetExternalAgentTokensByTemplateIDRow, error) {
+	rows, err := q.db.QueryContext(ctx, getExternalAgentTokensByTemplateID, arg.TemplateID, arg.OwnerID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []GetExternalAgentTokensByTemplateIDRow
+	for rows.Next() {
+		var i GetExternalAgentTokensByTemplateIDRow
+		if err := rows.Scan(
+			&i.WorkspaceID,
+			&i.WorkspaceName,
+			&i.AgentID,
+			&i.AgentName,
+			&i.AgentToken,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const getWorkspaceAgentAndWorkspaceByID = `-- name: GetWorkspaceAgentAndWorkspaceByID :one
 SELECT
 	workspace_agents.id, workspace_agents.created_at, workspace_agents.updated_at, workspace_agents.name, workspace_agents.first_connected_at, workspace_agents.last_connected_at, workspace_agents.disconnected_at, workspace_agents.resource_id, workspace_agents.auth_token, workspace_agents.auth_instance_id, workspace_agents.architecture, workspace_agents.environment_variables, workspace_agents.operating_system, workspace_agents.instance_metadata, workspace_agents.resource_metadata, workspace_agents.directory, workspace_agents.version, workspace_agents.last_connected_replica_id, workspace_agents.connection_timeout_seconds, workspace_agents.troubleshooting_url, workspace_agents.motd_file, workspace_agents.lifecycle_state, workspace_agents.expanded_directory, workspace_agents.logs_length, workspace_agents.logs_overflowed, workspace_agents.started_at, workspace_agents.ready_at, workspace_agents.subsystems, workspace_agents.display_apps, workspace_agents.api_version, workspace_agents.display_order, workspace_agents.parent_id, workspace_agents.api_key_scope, workspace_agents.deleted,

coderd/database/queries/workspaceagents.sql

@@ -352,6 +352,59 @@ WHERE
 	-- Filter out deleted sub agents.
 	AND workspace_agents.deleted = FALSE;
 
+-- name: GetExternalAgentTokensByTemplateID :many
+-- GetExternalAgentTokensByTemplateID returns the auth tokens for all
+-- non-deleted external agents on the latest build of every running workspace
+-- of the given template. "Running" means the latest build has
+-- transition=start and job_status=succeeded (matches the workspace-status
+-- definition used by coderd/database/queries/workspaces.sql).
+-- An owner_id of '00000000-0000-0000-0000-000000000000' (uuid.Nil) means
+-- "all owners"; any other value restricts results to workspaces owned by
+-- that user.
+SELECT
+	workspaces.id               AS workspace_id,
+	workspaces.name             AS workspace_name,
+	workspace_agents.id         AS agent_id,
+	workspace_agents.name       AS agent_name,
+	workspace_agents.auth_token AS agent_token
+FROM
+	workspaces
+JOIN (
+	-- latest build per workspace
+	SELECT DISTINCT ON (workspace_id)
+		id, workspace_id, job_id, transition, has_external_agent
+	FROM
+		workspace_builds
+	ORDER BY
+		workspace_id, build_number DESC
+) AS latest_builds
+ON
+	latest_builds.workspace_id = workspaces.id
+JOIN
+	provisioner_jobs
+ON
+	provisioner_jobs.id = latest_builds.job_id
+JOIN
+	workspace_resources
+ON
+	workspace_resources.job_id = latest_builds.job_id
+JOIN
+	workspace_agents
+ON
+	workspace_agents.resource_id = workspace_resources.id
+WHERE
+	workspaces.template_id = @template_id
+	AND (
+		@owner_id :: uuid = '00000000-0000-0000-0000-000000000000' :: uuid
+		OR workspaces.owner_id = @owner_id
+	)
+	AND workspaces.deleted = FALSE
+	AND latest_builds.has_external_agent = TRUE
+	AND latest_builds.transition = 'start' :: workspace_transition
+	AND provisioner_jobs.job_status = 'succeeded' :: provisioner_job_status
+	AND workspace_agents.deleted = FALSE
+	AND workspace_agents.auth_instance_id IS NULL;
+
 -- GetAuthenticatedWorkspaceAgentAndBuildByAuthToken returns an authenticated
 -- workspace agent and its associated build. During normal operation, this is
 -- the latest build. During shutdown, this may be the previous START build while

enterprise/cli/exp_scaletest_agentfake.go

@@ -5,9 +5,16 @@ package cli
 import (
 	"os/signal"
 
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"golang.org/x/xerrors"
 
+	"cdr.dev/slog/v3"
+	"cdr.dev/slog/v3/sloggers/sloghuman"
 	agplcli "github.com/coder/coder/v2/cli"
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/awsiamrds"
+	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/enterprise/scaletest/agentfake"
 	"github.com/coder/serpent"
 )
@@ -26,8 +33,13 @@ func (r *RootCmd) AGPLExperimental() []*serpent.Command {
 
 func (r *RootCmd) scaletestAgentFake() *serpent.Command {
 	var (
-		template string
-		owner    string
+		template                string
+		owner                   string
+		prometheusAddress       string
+		expectedAgents          int64
+		expectedAgentsTolerance int64
+		postgresURL             string
+		postgresAuth            string
 	)
 
 	cmd := &serpent.Command{
@@ -44,10 +56,15 @@ func (r *RootCmd) scaletestAgentFake() *serpent.Command {
 			"fetches each workspace agent's external-agent credentials, and supervises one in-process fake " +
 			"agent per token until the command is interrupted.\n\n" +
 			"Requires a session token whose user is template-admin (or higher) on a deployment licensed " +
-			"for the workspace external-agent feature; both the workspace builds and the credentials " +
-			"endpoint are gated server-side. Pair with `coder exp scaletest create-workspaces " +
-			"--no-wait-for-agents` to seed the workspaces this command will pick up. Workspaces created " +
-			"after this command starts are NOT picked up; rerun the command after seeding more.",
+			"for the workspace external-agent feature, and a Postgres connection URL (with credentials " +
+			"encoded into the URL) that points at the same database instance coderd is using. Intended " +
+			"to run inside the same network as coderd, not from operator machines outside the cluster. " +
+			"The workspace listing and external-agent feature are gated server-side. Pair with " +
+			"`coder exp scaletest create-workspaces --no-wait-for-agents` to seed the workspaces this " +
+			"command will pick up. Workspaces created after this command starts are NOT picked up; " +
+			"rerun the command after seeding more.\n\n" +
+			"Exposes Prometheus metrics (Go runtime and process collectors) at /metrics on " +
+			"--prometheus-address (default 0.0.0.0:21112).",
 		Handler: func(inv *serpent.Invocation) error {
 			ctx := inv.Context()
 			client, err := r.InitClient(inv)
@@ -66,11 +83,45 @@ func (r *RootCmd) scaletestAgentFake() *serpent.Command {
 			if template == "" {
 				return xerrors.New("--template is required")
 			}
+			if postgresURL == "" {
+				return xerrors.New("--postgres-url (CODER_PG_CONNECTION_URL) is required")
+			}
+			if expectedAgents > 0 && expectedAgentsTolerance < 0 {
+				return xerrors.New("--expected-agents-tolerance must be non-negative")
+			}
 
-			logger := inv.Logger
-			mgr := agentfake.NewManager(client.URL, client, logger, agentfake.ManagerOptions{
-				Template: template,
-				Owner:    owner,
+			logger := inv.Logger.AppendSinks(sloghuman.Sink(inv.Stderr))
+			if ok, _ := inv.ParsedFlags().GetBool("verbose"); ok {
+				logger = logger.Leveled(slog.LevelDebug)
+			}
+
+			sqlDriver := "postgres"
+			if codersdk.PostgresAuth(postgresAuth) == codersdk.PostgresAuthAWSIAMRDS {
+				var err error
+				sqlDriver, err = awsiamrds.Register(ctx, sqlDriver)
+				if err != nil {
+					return xerrors.Errorf("register aws rds iam auth: %w", err)
+				}
+			}
+			sqlDB, err := agplcli.ConnectToPostgres(ctx, logger, sqlDriver, postgresURL, nil)
+			if err != nil {
+				return xerrors.Errorf("dial postgres: %w", err)
+			}
+			defer sqlDB.Close()
+			db := database.New(sqlDB)
+
+			prometheusSrvClose := agplcli.ServeHandler(ctx, logger,
+				promhttp.Handler(), prometheusAddress, "prometheus")
+			defer prometheusSrvClose()
+
+			metrics := agentfake.NewMetrics(prometheus.DefaultRegisterer)
+
+			mgr := agentfake.NewManager(logger, client.URL, client, db, agentfake.ManagerOptions{
+				Template:                template,
+				Owner:                   owner,
+				Metrics:                 metrics,
+				ExpectedAgents:          expectedAgents,
+				ExpectedAgentsTolerance: expectedAgentsTolerance,
 			})
 			defer mgr.Close()
 
@@ -94,6 +145,41 @@ func (r *RootCmd) scaletestAgentFake() *serpent.Command {
 			Description: "Optional workspace-owner filter (username). When empty, all owners' workspaces of the template are included.",
 			Value:       serpent.StringOf(&owner),
 		},
+		{
+			Flag:        "prometheus-address",
+			Env:         "CODER_SCALETEST_AGENTFAKE_PROMETHEUS_ADDRESS",
+			Default:     "0.0.0.0:21112",
+			Description: "Address on which to expose Prometheus metrics (Go runtime + process collectors) at /metrics.",
+			Value:       serpent.StringOf(&prometheusAddress),
+		},
+		{
+			Flag:        "expected-agents",
+			Env:         "CODER_SCALETEST_AGENTFAKE_EXPECTED_AGENTS",
+			Default:     "0",
+			Description: "Expected number of agents to enumerate. When non-zero, the command polls until the workspace count is within expected ± expected-agents-tolerance before enumerating.",
+			Value:       serpent.Int64Of(&expectedAgents),
+		},
+		{
+			Flag:        "expected-agents-tolerance",
+			Env:         "CODER_SCALETEST_AGENTFAKE_EXPECTED_AGENTS_TOLERANCE",
+			Default:     "0",
+			Description: "Acceptable variance around --expected-agents. Ignored when --expected-agents is 0.",
+			Value:       serpent.Int64Of(&expectedAgentsTolerance),
+		},
+		{
+			Flag:        "postgres-url",
+			Env:         "CODER_PG_CONNECTION_URL",
+			Description: "URL of the Postgres database that the target coderd is using. Required; used to bulk-fetch external-agent tokens for the enumerated workspaces in a single query. The same connection string the coder server pods consume (e.g. the coder-db-url secret in scaletest deployments).",
+			Value:       serpent.StringOf(&postgresURL),
+		},
+		serpent.Option{
+			Name:        "Postgres Connection Auth",
+			Description: "Type of auth to use when connecting to postgres.",
+			Flag:        "postgres-connection-auth",
+			Env:         "CODER_PG_CONNECTION_AUTH",
+			Default:     "password",
+			Value:       serpent.EnumOf(&postgresAuth, codersdk.PostgresAuthDrivers...),
+		},
 	}
 
 	return cmd

enterprise/scaletest/agentfake/agent.go

@@ -5,6 +5,7 @@ import (
 	"encoding/base64"
 	"net/url"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/google/uuid"
@@ -55,6 +56,13 @@ type Agent struct {
 	logger   slog.Logger
 	clock    quartz.Clock
 	dialer   rpcDialer // nil → built from coderURL+token in Run
+	metrics  *Metrics  // nil → no metrics
+
+	// firstConnected guards firstConnect so reconnects don't re-report.
+	firstConnect   chan<- time.Duration
+	firstConnected atomic.Bool
+
+	start time.Time
 
 	cancel context.CancelFunc
 }
@@ -82,7 +90,25 @@ func WithDialer(d rpcDialer) Option {
 	}
 }
 
-func NewAgent(coderURL *url.URL, token string, logger slog.Logger, opts ...Option) *Agent {
+// WithMetrics injects Prometheus collectors. A nil *Metrics (the
+// default when this option is not used) is a valid no-op; every
+// collector helper method nil-guards on the receiver.
+func WithMetrics(m *Metrics) Option {
+	return func(a *Agent) {
+		a.metrics = m
+	}
+}
+
+// WithFirstConnect sets a shared channel used by the Manager to aggregate
+// time-to-first-connect across all agents without one stalled agent blocking
+// the others.
+func WithFirstConnect(ch chan<- time.Duration) Option {
+	return func(a *Agent) {
+		a.firstConnect = ch
+	}
+}
+
+func NewAgent(logger slog.Logger, coderURL *url.URL, token string, opts ...Option) *Agent {
 	a := &Agent{
 		coderURL: coderURL,
 		token:    token,
@@ -109,6 +135,7 @@ func (a *Agent) Run(ctx context.Context) error {
 	if client == nil {
 		client = agentsdk.New(a.coderURL, agentsdk.WithFixedToken(a.token))
 	}
+	a.start = a.clock.Now()
 	for {
 		if err := runCtx.Err(); err != nil {
 			return nil
@@ -130,14 +157,33 @@ func (a *Agent) Run(ctx context.Context) error {
 
 // connectAndServe opens one dRPC websocket, announces lifecycle = READY, then blocks until ctx is canceled or the
 // connection is closed by either side. Returns the underlying error, if any.
+//
+// A child ctx (connCtx) is derived from ctx and canceled when this function
+// returns. Background goroutines started for the lifetime of this single dRPC
+// connection (notably runMetadata) bind to connCtx rather than ctx so that
+// they exit promptly on remote-close + reconnect, instead of leaking and
+// continuing to issue RPCs against an already-closed rpc handle until the
+// outer ctx (the whole Agent's lifetime) eventually cancels.
 func (a *Agent) connectAndServe(ctx context.Context, client rpcDialer) error {
 	rpc, _, err := client.ConnectRPC29WithRole(ctx, "agent")
 	if err != nil {
 		return xerrors.Errorf("connect dRPC: %w", err)
 	}
+	connCtx, cancelConn := context.WithCancel(ctx)
+	defer cancelConn()
 	conn := rpc.DRPCConn()
+	a.metrics.incConnected()
+	// Non-blocking so a slow collector can never stall this agent's
+	// reconnect loop.
+	if a.firstConnect != nil && a.firstConnected.CompareAndSwap(false, true) {
+		select {
+		case a.firstConnect <- a.clock.Since(a.start):
+		default:
+		}
+	}
 	defer func() {
 		_ = conn.Close()
+		a.metrics.decConnected()
 	}()
 
 	// Real agents transition to READY once their startup script finishes. Fakes have no startup script, so they're
@@ -176,7 +222,7 @@ func (a *Agent) connectAndServe(ctx context.Context, client rpcDialer) error {
 				slog.Error(idErr))
 			workspaceID = uuid.Nil
 		}
-		go a.runMetadata(ctx, rpc, workspaceID, descs)
+		go a.runMetadata(connCtx, rpc, workspaceID, descs)
 	}
 
 	select {

enterprise/scaletest/agentfake/agent_test.go

@@ -38,7 +38,7 @@ func TestAgent_ConnectsAndReachesReady(t *testing.T) {
 	dialer := agenttest.NewClient(t, logger, agentID, manifest, statsCh, coord)
 	t.Cleanup(dialer.Close)
 
-	a := agentfake.NewAgent(nil, "", logger, agentfake.WithDialer(dialer))
+	a := agentfake.NewAgent(logger, nil, "", agentfake.WithDialer(dialer))
 	t.Cleanup(a.Close)
 
 	runCtx, cancel := context.WithCancel(ctx)
@@ -106,7 +106,7 @@ func TestAgent_SendsMetadata(t *testing.T) {
 	dialer := agenttest.NewClient(t, logger, agentID, manifest, statsCh, coord)
 	t.Cleanup(dialer.Close)
 
-	a := agentfake.NewAgent(nil, "", logger,
+	a := agentfake.NewAgent(logger, nil, "",
 		agentfake.WithDialer(dialer),
 		agentfake.WithClock(mClock),
 	)

enterprise/scaletest/agentfake/manager.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"net/http"
 	"net/url"
+	"sort"
 	"strconv"
 	"sync"
 	"time"
@@ -15,24 +16,31 @@ import (
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog/v3"
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/dbauthz"
 	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/quartz"
 )
 
-// ExternalAgentClient is the subset of *codersdk.Client the Manager
-// uses to enumerate external-agent workspaces under a template and
-// fetch each agent's auth token. *codersdk.Client satisfies this
-// interface, so production callers pass their client directly; tests
-// substitute a fake without standing up a real coderd.
+// ExternalAgentClient is the subset of *codersdk.Client the Manager uses to
+// resolve the template/owner the operator named on the command line and to
+// poll the workspace count gate. The actual external-agent auth tokens are
+// fetched in-process via a direct database query (see
+// GetExternalAgentTokensByTemplateID), not via this client. *codersdk.Client
+// satisfies this interface, so production callers pass their client
+// directly; tests substitute a fake without standing up a real coderd.
 type ExternalAgentClient interface {
+	User(ctx context.Context, userIdent string) (codersdk.User, error)
+	Template(ctx context.Context, id uuid.UUID) (codersdk.Template, error)
+	TemplatesByOrganization(ctx context.Context, orgID uuid.UUID) ([]codersdk.Template, error)
 	Workspaces(ctx context.Context, filter codersdk.WorkspaceFilter) (codersdk.WorkspacesResponse, error)
-	WorkspaceExternalAgentCredentials(ctx context.Context, workspaceID uuid.UUID, agentName string) (codersdk.ExternalAgentCredentials, error)
 }
 
 const (
-	enumeratePageSize        = 100
-	maxEnumerateRetries      = 5
-	initialEnumerateBackoff  = 1 * time.Second
-	maxEnumerateRetryBackoff = 5 * time.Second
+	maxEnumerateRetries        = 5
+	initialEnumerateBackoff    = 1 * time.Second
+	maxEnumerateRetryBackoff   = 5 * time.Second
+	workspaceCountPollInterval = 5 * time.Second
 )
 
 // TokenInfo is a single workspace-agent auth token retrieved for a coder external agent, along with the identifying
@@ -53,6 +61,16 @@ type ManagerOptions struct {
 	Template string
 	// Owner restricts enumeration to workspaces owned by the given user. Optional; if empty, all owners are included.
 	Owner string
+	// Metrics collectors. Optional; nil disables metric reporting.
+	Metrics *Metrics
+	// ExpectedAgents, when non-zero, causes Run to poll until the workspace
+	// count is within [ExpectedAgents-Tolerance, ExpectedAgents+Tolerance]
+	// before enumerating.
+	ExpectedAgents          int64
+	ExpectedAgentsTolerance int64
+	// Clock is used for the workspace-count polling interval.
+	// Defaults to the real clock; override in tests with quartz.NewMock.
+	Clock quartz.Clock
 }
 
 // Manager supervises a set of fake Agents in one process. It enumerates the agents it owns from coderd at Run time
@@ -61,21 +79,35 @@ type ManagerOptions struct {
 type Manager struct {
 	coderURL *url.URL
 	client   ExternalAgentClient
+	db       database.Store
 	logger   slog.Logger
 	opts     ManagerOptions
 
+	// templateID + ownerID are resolved once during Run from opts.Template /
+	// opts.Owner (names). ownerID stays uuid.Nil when opts.Owner is empty, which
+	// the GetExternalAgentTokensByTemplateID query treats as "match any owner".
+	templateID uuid.UUID
+	ownerID    uuid.UUID
+
 	mu     sync.Mutex
 	agents []*Agent
 }
 
-// NewManager returns an Agent Manager. The provided client must already be authenticated with sufficient privilege
-// to list workspaces by template and to call the enterprise-only WorkspaceExternalAgentCredentials endpoint
-// (template-admin or higher; FeatureWorkspaceExternalAgent must be enabled). coderURL is the URL the spawned
-// fake agents will dial.
-func NewManager(coderURL *url.URL, client ExternalAgentClient, logger slog.Logger, opts ManagerOptions) *Manager {
+// NewManager returns an Agent Manager. The provided client must already be
+// authenticated with sufficient privilege to list workspaces, look up the
+// configured template, and (when --owner is set) look up the named user
+// (template-admin or higher). db must be a database.Store connected to the
+// same Postgres database as the target coderd; it is used to bulk-fetch
+// external-agent tokens for the enumerated workspaces. coderURL is the URL
+// the spawned fake agents will dial.
+func NewManager(logger slog.Logger, coderURL *url.URL, client ExternalAgentClient, db database.Store, opts ManagerOptions) *Manager {
+	if opts.Clock == nil {
+		opts.Clock = quartz.NewReal()
+	}
 	return &Manager{
 		coderURL: coderURL,
 		client:   client,
+		db:       db,
 		logger:   logger,
 		opts:     opts,
 	}
@@ -91,15 +123,33 @@ func (m *Manager) Run(ctx context.Context) error {
 		return xerrors.New("invalid manager options: Template is required")
 	}
 
+	if m.opts.ExpectedAgents > 0 {
+		if err := m.waitForWorkspaceCount(ctx); err != nil {
+			return xerrors.Errorf("waiting for workspaces: %w", err)
+		}
+	}
+
+	if err := m.ResolveTemplateAndOwner(ctx); err != nil {
+		return xerrors.Errorf("resolve template/owner: %w", err)
+	}
+
 	tokens, err := m.enumerateWithRetry(ctx)
 	if err != nil {
 		return xerrors.Errorf("enumerate external agents: %w", err)
 	}
 
-	agents := make([]*Agent, 0, len(tokens))
+	numAgents := len(tokens)
+
+	// Buffered so a stalled collector can never block any agent's send.
+	firstConnectCh := make(chan time.Duration, numAgents)
+
+	agents := make([]*Agent, 0, numAgents)
 	for i, ti := range tokens {
-		agents = append(agents, NewAgent(m.coderURL, ti.Token,
-			m.logger.Named("agent-"+strconv.Itoa(i))))
+		agents = append(agents, NewAgent(
+			m.logger.Named("agent-"+strconv.Itoa(i)),
+			m.coderURL, ti.Token,
+			WithMetrics(m.opts.Metrics),
+			WithFirstConnect(firstConnectCh)))
 	}
 	m.mu.Lock()
 	m.agents = agents
@@ -111,6 +161,30 @@ func (m *Manager) Run(ctx context.Context) error {
 			return a.Run(egCtx)
 		})
 	}
+
+	// Bound to Run's lifetime rather than egCtx so the collector can't
+	// outlive Run when every agent returns nil (errgroup never cancels
+	// egCtx on clean shutdown).
+	collectorCtx, cancelCollector := context.WithCancel(ctx)
+	defer cancelCollector()
+	go func() {
+		durations := collectFirstConnect(collectorCtx, firstConnectCh, numAgents)
+		if len(durations) == 0 {
+			return
+		}
+		// Mean is order-independent and is computed before the sort so the
+		// dependency between the two percentile calls and sortedness is
+		// localized here.
+		mean := meanDuration(durations)
+		sort.Slice(durations, func(i, j int) bool { return durations[i] < durations[j] })
+		m.logger.Info(collectorCtx, "all agents connected",
+			slog.F("count", len(durations)),
+			slog.F("mean", mean),
+			slog.F("pct_ninety_five", percentileDuration(durations, 95)),
+			slog.F("pct_ninety_nine", percentileDuration(durations, 99)),
+		)
+	}()
+
 	err = eg.Wait()
 	if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
 		return err
@@ -118,6 +192,25 @@ func (m *Manager) Run(ctx context.Context) error {
 	return nil
 }
 
+// collectFirstConnect drains ch until expected values arrive or ctx is
+// canceled. The single shared channel ensures one stalled agent cannot
+// hold up reports from the others.
+func collectFirstConnect(ctx context.Context, ch <-chan time.Duration, expected int) []time.Duration {
+	if expected <= 0 {
+		return nil
+	}
+	durations := make([]time.Duration, 0, expected)
+	for len(durations) < expected {
+		select {
+		case d := <-ch:
+			durations = append(durations, d)
+		case <-ctx.Done():
+			return durations
+		}
+	}
+	return durations
+}
+
 // Close stops every Agent constructed during Run. Safe to call any
 // number of times.
 func (m *Manager) Close() {
@@ -135,7 +228,6 @@ func (m *Manager) enumerateWithRetry(ctx context.Context) ([]TokenInfo, error) {
 	bkoff := backoff.WithContext(backoff.WithMaxRetries(b, maxEnumerateRetries), ctx)
 
 	var tokens []TokenInfo
-	// for attempt := 0; attempt <= maxEnumerateRetries; attempt++ {
 	err := backoff.Retry(func() error {
 		var retryErr error
 		tokens, retryErr = m.EnumerateExternalAgents(ctx)
@@ -154,59 +246,172 @@ func (m *Manager) enumerateWithRetry(ctx context.Context) ([]TokenInfo, error) {
 	return tokens, nil
 }
 
-// EnumerateExternalAgents asks coderd for the list of workspaces matching the configured template, walks each
-// workspace's latest build for agents on builds with HasExternalAgent=true, and returns the auth tokens for every
-// external agent. Per-agent credential failures are logged and skipped; a non-nil error is returned only if the
-// workspace listing itself fails.
+// EnumerateExternalAgents bulk-fetches the auth tokens for every external agent on a running workspace of the
+// configured template (optionally filtered by owner) via a single direct Postgres query. resolveTemplateAndOwner
+// must have been called once before any invocation; Run handles that, but tests that call this method directly
+// must do the same.
 func (m *Manager) EnumerateExternalAgents(ctx context.Context) ([]TokenInfo, error) {
-	var workspaces []codersdk.Workspace
-	filter := codersdk.WorkspaceFilter{
-		Template: m.opts.Template,
-		Owner:    m.opts.Owner,
-		Limit:    enumeratePageSize,
-	}
-	for {
-		page, err := m.client.Workspaces(ctx, filter)
-		if err != nil {
-			return nil, xerrors.Errorf("list workspaces (offset=%d): %w", filter.Offset, err)
-		}
-		workspaces = append(workspaces, page.Workspaces...)
-		if len(page.Workspaces) < filter.Limit {
-			break
-		}
-		filter.Offset += len(page.Workspaces)
+	start := time.Now()
+	m.logger.Info(ctx, "enumerating external-agent workspaces",
+		slog.F("template", m.opts.Template),
+		slog.F("template_id", m.templateID),
+		slog.F("owner", m.opts.Owner))
+
+	// AsSystemRestricted is required because GetExternalAgentTokensByTemplateID
+	// is gated by dbauthz on ResourceSystem read. This code path runs in the
+	// agentfake scaletest manager pod, which holds a direct Postgres connection
+	// and acts as a trusted system caller; the security boundary here is Postgres
+	// authn (the coder-db-url secret), not a coder session token.
+	// nolint:gocritic
+	rows, err := m.db.GetExternalAgentTokensByTemplateID(dbauthz.AsSystemRestricted(ctx), database.GetExternalAgentTokensByTemplateIDParams{
+		TemplateID: m.templateID,
+		OwnerID:    m.ownerID,
+	})
+	if err != nil {
+		return nil, xerrors.Errorf("fetch external-agent tokens: %w", err)
 	}
 
-	tokens := make([]TokenInfo, 0, len(workspaces))
-	for _, ws := range workspaces {
-		// The credentials endpoint requires WorkspaceBuild.HasExternalAgent=true (see
-		// enterprise/coderd/workspaceagents.go:48). Skip workspaces whose latest build
-		// doesn't carry the flag rather than 404 our way through every workspace in coderd.
-		if ws.LatestBuild.HasExternalAgent == nil || !*ws.LatestBuild.HasExternalAgent {
-			continue
+	tokens := make([]TokenInfo, 0, len(rows))
+	for _, row := range rows {
+		tokens = append(tokens, TokenInfo{
+			WorkspaceID:   row.WorkspaceID,
+			WorkspaceName: row.WorkspaceName,
+			AgentID:       row.AgentID,
+			AgentName:     row.AgentName,
+			Token:         row.AgentToken.String(),
+		})
+	}
+	m.logger.Info(ctx, "enumerated external-agent workspaces",
+		slog.F("template", m.opts.Template),
+		slog.F("template_id", m.templateID),
+		slog.F("owner", m.opts.Owner),
+		slog.F("tokens", len(tokens)),
+		slog.F("duration", time.Since(start)))
+	return tokens, nil
+}
+
+// ResolveTemplateAndOwner looks up the configured template name (and, when set,
+// owner username) once and caches the resulting UUIDs on the Manager so that
+// EnumerateExternalAgents can issue a single by-ID DB query per cycle.
+// Run calls this automatically; tests that exercise EnumerateExternalAgents
+// directly must call it themselves first.
+//
+// Template resolution walks every organization the calling user belongs to,
+// matching scaletest convention (see cli.parseTemplate). Owner resolution is
+// skipped when opts.Owner is empty; the cached uuid.Nil is interpreted by the
+// underlying query as "match workspaces of any owner".
+func (m *Manager) ResolveTemplateAndOwner(ctx context.Context) error {
+	me, err := m.client.User(ctx, codersdk.Me)
+	if err != nil {
+		return xerrors.Errorf("get current user: %w", err)
+	}
+	tpl, err := parseTemplate(ctx, m.client, me.OrganizationIDs, m.opts.Template)
+	if err != nil {
+		return xerrors.Errorf("resolve template %q: %w", m.opts.Template, err)
+	}
+	m.templateID = tpl.ID
+
+	if m.opts.Owner != "" {
+		owner, err := m.client.User(ctx, m.opts.Owner)
+		if err != nil {
+			return xerrors.Errorf("resolve owner %q: %w", m.opts.Owner, err)
 		}
-		for _, res := range ws.LatestBuild.Resources {
-			for _, agent := range res.Agents {
-				creds, err := m.client.WorkspaceExternalAgentCredentials(ctx, ws.ID, agent.Name)
-				if err != nil {
-					m.logger.Warn(ctx, "fetch external-agent credentials",
-						slog.F("workspace_id", ws.ID),
-						slog.F("workspace_name", ws.Name),
-						slog.F("agent_name", agent.Name),
-						slog.Error(err))
-					continue
+		m.ownerID = owner.ID
+	}
+	return nil
+}
+
+// parseTemplate is duplicated from cli/exp_scaletest.go (AGPL) to avoid
+// exporting an internal helper as part of that package's public API for the
+// sole benefit of this enterprise consumer. Keep behavior in sync with the
+// original: accept either a UUID or a template name, search all of the user's
+// organizations for a name match.
+func parseTemplate(ctx context.Context, client ExternalAgentClient, organizationIDs []uuid.UUID, template string) (tpl codersdk.Template, err error) {
+	if id, err := uuid.Parse(template); err == nil && id != uuid.Nil {
+		tpl, err = client.Template(ctx, id)
+		if err != nil {
+			return tpl, xerrors.Errorf("get template by ID %q: %w", template, err)
+		}
+	} else {
+		// List templates in all orgs until we find a match.
+	orgLoop:
+		for _, orgID := range organizationIDs {
+			tpls, err := client.TemplatesByOrganization(ctx, orgID)
+			if err != nil {
+				return tpl, xerrors.Errorf("list templates in org %q: %w", orgID, err)
+			}
+			for _, t := range tpls {
+				if t.Name == template {
+					tpl = t
+					break orgLoop
 				}
-				tokens = append(tokens, TokenInfo{
-					WorkspaceID:   ws.ID,
-					WorkspaceName: ws.Name,
-					AgentID:       agent.ID,
-					AgentName:     agent.Name,
-					Token:         creds.AgentToken,
-				})
 			}
 		}
 	}
-	return tokens, nil
+	if tpl.ID == uuid.Nil {
+		return tpl, xerrors.Errorf("could not find template %q in any organization", template)
+	}
+	return tpl, nil
+}
+
+// waitForWorkspaceCount polls until the workspace count for the configured
+// template is within [ExpectedAgents-Tolerance, ExpectedAgents+Tolerance].
+// It uses limit=1 on each poll; the workspaces SQL query computes the total
+// count in a CTE before applying LIMIT, so Count reflects the full result set
+// regardless of page size.
+func (m *Manager) waitForWorkspaceCount(ctx context.Context) error {
+	lo := m.opts.ExpectedAgents - m.opts.ExpectedAgentsTolerance
+	hi := m.opts.ExpectedAgents + m.opts.ExpectedAgentsTolerance
+
+	// checkWorkspaceCount returns true if the current workspace count for the
+	// template is within the expected tolerance range, or an error if the
+	// workspaces endpoint fails.
+	checkWorkspaceCount := func() (bool, error) {
+		page, err := m.client.Workspaces(ctx, codersdk.WorkspaceFilter{
+			Template: m.opts.Template,
+			Owner:    m.opts.Owner,
+			Limit:    1,
+		})
+		if err != nil {
+			return false, xerrors.Errorf("check workspace count: %w", err)
+		}
+		count := int64(page.Count)
+		if count >= lo && count <= hi {
+			m.logger.Info(ctx, "workspace count ready",
+				slog.F("count", count),
+				slog.F("expected", m.opts.ExpectedAgents),
+				slog.F("tolerance", m.opts.ExpectedAgentsTolerance),
+			)
+			return true, nil
+		}
+		m.logger.Info(ctx, "waiting for workspaces",
+			slog.F("count", count),
+			slog.F("want_lo", lo),
+			slog.F("want_hi", hi),
+		)
+		return false, nil
+	}
+
+	errDone := xerrors.New("done")
+	var tickErr error
+	waiter := m.opts.Clock.TickerFunc(ctx, workspaceCountPollInterval, func() error {
+		done, err := checkWorkspaceCount()
+		if err != nil {
+			tickErr = err
+			return err
+		}
+		if done {
+			return errDone
+		}
+		return nil
+	})
+	if err := waiter.Wait(); err != nil && !errors.Is(err, errDone) {
+		if tickErr != nil {
+			return tickErr
+		}
+		return xerrors.Errorf("waiting for workspace count: %w", err)
+	}
+	return nil
 }
 
 // IsFatalEnumerationError reports whether err from a coderd API call indicates an unrecoverable misconfiguration that
@@ -231,3 +436,32 @@ func IsFatalEnumerationError(err error) bool {
 	}
 	return false
 }
+
+// meanDuration returns the mean of d, or zero if d is empty.
+func meanDuration(d []time.Duration) time.Duration {
+	if len(d) == 0 {
+		return 0
+	}
+	var total time.Duration
+	for _, v := range d {
+		total += v
+	}
+	return total / time.Duration(len(d))
+}
+
+// percentileDuration returns the p-th percentile (0-100) using nearest-rank.
+// Expects d to be sorted ascending; callers sort once before invoking this
+// for multiple percentiles.
+func percentileDuration(d []time.Duration, p float64) time.Duration {
+	if len(d) == 0 {
+		return 0
+	}
+	idx := int(p/100*float64(len(d))+0.5) - 1
+	if idx < 0 {
+		idx = 0
+	}
+	if idx >= len(d) {
+		idx = len(d) - 1
+	}
+	return d[idx]
+}

enterprise/scaletest/agentfake/manager_test.go

@@ -2,6 +2,7 @@ package agentfake_test
 
 import (
 	"context"
+	"database/sql"
 	"net/http"
 	"net/url"
 	"sort"
@@ -14,111 +15,183 @@ import (
 
 	"cdr.dev/slog/v3"
 	"cdr.dev/slog/v3/sloggers/slogtest"
+	"github.com/coder/coder/v2/coderd/database"
+	"github.com/coder/coder/v2/coderd/database/dbfake"
+	"github.com/coder/coder/v2/coderd/database/dbgen"
+	"github.com/coder/coder/v2/coderd/database/dbtestutil"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/enterprise/scaletest/agentfake"
+	sdkproto "github.com/coder/coder/v2/provisionersdk/proto"
 	"github.com/coder/coder/v2/testutil"
 )
 
-// fakeExternalAgentClient is an in-package fake for the
-// ExternalAgentClient interface used by
-// Manager.EnumerateExternalAgents. Tests populate workspaces /
-// credentials / workspacesErr before calling the Manager.
+// fakeExternalAgentClient is an in-package fake for the ExternalAgentClient
+// interface used by Manager to resolve names (template, owner) and to poll
+// the workspace-count gate. The actual external-agent auth tokens are read
+// from the real database.Store the tests seed via dbfake / dbgen.
+//
+// Tests populate me, owner, template, workspaces (the latter being a
+// codersdk-shaped view of whichever rows the test seeded into the DB).
 type fakeExternalAgentClient struct {
-	// workspaces, in the order Workspaces() should return them. Each
-	// call returns up to filter.Limit entries starting at filter.Offset
-	// to model pagination, matching real coderd behavior.
-	workspaces []codersdk.Workspace
-	// credentials, keyed by "{workspaceID}/{agentName}". A nil entry
-	// causes WorkspaceExternalAgentCredentials to error with notFoundErr.
-	credentials map[string]codersdk.ExternalAgentCredentials
+	me       codersdk.User
+	owner    codersdk.User
+	template codersdk.Template
 
-	// workspacesErr, if non-nil, is returned from every Workspaces call.
-	workspacesErr error
+	// workspaces, in the order Workspaces() should return them. Each call
+	// returns up to filter.Limit entries starting at filter.Offset to model
+	// pagination, matching real coderd behavior. Tests only need to populate
+	// this when exercising the workspace-count gate; the new EnumerateExternalAgents
+	// path doesn't list workspaces over HTTP at all.
+	workspaces []codersdk.Workspace
+
+	// meErr / templateErr are used by tests that want to verify resolution
+	// errors are classified as fatal by the enumerate retry loop.
+	meErr       error
+	templateErr error
+}
+
+func (f *fakeExternalAgentClient) User(_ context.Context, userIdent string) (codersdk.User, error) {
+	if userIdent == codersdk.Me {
+		if f.meErr != nil {
+			return codersdk.User{}, f.meErr
+		}
+		return f.me, nil
+	}
+	if userIdent == f.owner.Username {
+		return f.owner, nil
+	}
+	return codersdk.User{}, xerrors.Errorf("no user %q", userIdent)
+}
+
+func (f *fakeExternalAgentClient) Template(_ context.Context, id uuid.UUID) (codersdk.Template, error) {
+	if f.templateErr != nil {
+		return codersdk.Template{}, f.templateErr
+	}
+	if id == f.template.ID {
+		return f.template, nil
+	}
+	return codersdk.Template{}, xerrors.Errorf("no template with id %s", id)
+}
+
+func (f *fakeExternalAgentClient) TemplatesByOrganization(_ context.Context, orgID uuid.UUID) ([]codersdk.Template, error) {
+	if f.templateErr != nil {
+		return nil, f.templateErr
+	}
+	if f.template.ID == uuid.Nil || f.template.OrganizationID != orgID {
+		return nil, nil
+	}
+	return []codersdk.Template{f.template}, nil
 }
 
 func (f *fakeExternalAgentClient) Workspaces(_ context.Context, filter codersdk.WorkspaceFilter) (codersdk.WorkspacesResponse, error) {
-	if f.workspacesErr != nil {
-		return codersdk.WorkspacesResponse{}, f.workspacesErr
-	}
 	start := filter.Offset
 	if start > len(f.workspaces) {
 		start = len(f.workspaces)
 	}
 	end := start + filter.Limit
-	if end > len(f.workspaces) {
+	if filter.Limit == 0 || end > len(f.workspaces) {
 		end = len(f.workspaces)
 	}
-	page := f.workspaces[start:end]
 	return codersdk.WorkspacesResponse{
-		Workspaces: page,
+		Workspaces: f.workspaces[start:end],
 		Count:      len(f.workspaces),
 	}, nil
 }
 
-func (f *fakeExternalAgentClient) WorkspaceExternalAgentCredentials(_ context.Context, wsID uuid.UUID, agentName string) (codersdk.ExternalAgentCredentials, error) {
-	key := wsID.String() + "/" + agentName
-	creds, ok := f.credentials[key]
-	if !ok {
-		return codersdk.ExternalAgentCredentials{}, xerrors.Errorf("no credentials for %s", key)
-	}
-	return creds, nil
-}
-
-// externalAgentWorkspace returns a codersdk.Workspace whose latest
-// build has HasExternalAgent=true and one agent with the given name.
-func externalAgentWorkspace(t *testing.T, name, agentName string) (codersdk.Workspace, uuid.UUID) {
+// seedUserOrgAndTemplate sets up the minimum DB rows needed for a workspace's
+// FK constraints to hold, and returns the IDs the caller will reuse when
+// seeding workspaces and populating the fake client.
+func seedUserOrgAndTemplate(t *testing.T, db database.Store) (org database.Organization, user database.User, tpl database.Template) {
 	t.Helper()
-	wsID := uuid.New()
-	agentID := uuid.New()
-	hasExternal := true
-	return codersdk.Workspace{
-		ID:   wsID,
-		Name: name,
-		LatestBuild: codersdk.WorkspaceBuild{
-			HasExternalAgent: &hasExternal,
-			Resources: []codersdk.WorkspaceResource{{
-				Name: "external",
-				Type: "coder_external_agent",
-				Agents: []codersdk.WorkspaceAgent{{
-					ID:   agentID,
-					Name: agentName,
-				}},
-			}},
-		},
-	}, agentID
+	org = dbgen.Organization(t, db, database.Organization{})
+	user = dbgen.User(t, db, database.User{})
+	_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{
+		UserID:         user.ID,
+		OrganizationID: org.ID,
+	})
+	tv := dbgen.TemplateVersion(t, db, database.TemplateVersion{
+		OrganizationID: org.ID,
+		CreatedBy:      user.ID,
+	})
+	tpl = dbgen.Template(t, db, database.Template{
+		OrganizationID:  org.ID,
+		ActiveVersionID: tv.ID,
+		CreatedBy:       user.ID,
+	})
+	return org, user, tpl
 }
 
-// Asserts the TokenInfo shape (workspace IDs, agent names, tokens)
-// returned by the enumeration loop given a fake client.
+// buildExternalAgentWorkspace creates one workspace with a coder_external_agent
+// resource, an agent, and HasExternalAgent=true on the latest build. The
+// latest build's provisioner job is Succeeded by default (the dbfake default),
+// which is what the "running" filter in GetExternalAgentTokensByTemplateID
+// requires.
+func buildExternalAgentWorkspace(
+	t *testing.T,
+	db database.Store,
+	orgID, ownerID, templateID uuid.UUID,
+) dbfake.WorkspaceResponse {
+	t.Helper()
+	return dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
+		OrganizationID: orgID,
+		OwnerID:        ownerID,
+		TemplateID:     templateID,
+	}).
+		Seed(database.WorkspaceBuild{
+			HasExternalAgent: sql.NullBool{Bool: true, Valid: true},
+		}).
+		Resource(&sdkproto.Resource{
+			Name: "external",
+			Type: "coder_external_agent",
+		}).
+		WithAgent().
+		Do()
+}
+
+// newFakeClient builds a fakeExternalAgentClient consistent with the rows the
+// caller seeded into the DB. me is the user that the manager will call
+// User(codersdk.Me) on; its OrganizationIDs is what parseTemplate walks.
+func newFakeClient(me database.User, org database.Organization, tpl database.Template) *fakeExternalAgentClient {
+	return &fakeExternalAgentClient{
+		me: codersdk.User{
+			ReducedUser:     codersdk.ReducedUser{MinimalUser: codersdk.MinimalUser{ID: me.ID, Username: me.Username}},
+			OrganizationIDs: []uuid.UUID{org.ID},
+		},
+		template: codersdk.Template{
+			ID:             tpl.ID,
+			OrganizationID: org.ID,
+			Name:           tpl.Name,
+		},
+	}
+}
+
+// Asserts the TokenInfo shape (workspace IDs, agent names, tokens) returned by
+// the enumeration loop reads from the DB the test seeded.
 func Test_Manager_EnumerateExternalAgents_returnsAllTokens(t *testing.T) {
 	t.Parallel()
 	ctx := testutil.Context(t, testutil.WaitShort)
 
+	db, _ := dbtestutil.NewDB(t)
+	org, user, tpl := seedUserOrgAndTemplate(t, db)
+
 	const numWorkspaces = 3
-	workspaces := make([]codersdk.Workspace, 0, numWorkspaces)
-	credentials := map[string]codersdk.ExternalAgentCredentials{}
 	want := make([]agentfake.TokenInfo, 0, numWorkspaces)
 	for i := 0; i < numWorkspaces; i++ {
-		agentName := "external"
-		ws, agentID := externalAgentWorkspace(t, "ws-"+uuid.NewString(), agentName)
-		workspaces = append(workspaces, ws)
-		token := uuid.NewString()
-		credentials[ws.ID.String()+"/"+agentName] = codersdk.ExternalAgentCredentials{
-			AgentToken: token,
-		}
+		r := buildExternalAgentWorkspace(t, db, org.ID, user.ID, tpl.ID)
 		want = append(want, agentfake.TokenInfo{
-			WorkspaceID:   ws.ID,
-			WorkspaceName: ws.Name,
-			AgentID:       agentID,
-			AgentName:     agentName,
-			Token:         token,
+			WorkspaceID:   r.Workspace.ID,
+			WorkspaceName: r.Workspace.Name,
+			AgentID:       r.Agents[0].ID,
+			AgentName:     r.Agents[0].Name,
+			Token:         r.AgentToken,
 		})
 	}
 
-	client := &fakeExternalAgentClient{workspaces: workspaces, credentials: credentials}
+	client := newFakeClient(user, org, tpl)
 	coderURL, _ := url.Parse("http://fake")
 	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
-	m := agentfake.NewManager(coderURL, client, logger, agentfake.ManagerOptions{Template: "tmpl"})
+	m := agentfake.NewManager(logger, coderURL, client, db, agentfake.ManagerOptions{Template: tpl.Name})
+	require.NoError(t, m.ResolveTemplateAndOwner(ctx))
 
 	got, err := m.EnumerateExternalAgents(ctx)
 	require.NoError(t, err)
@@ -126,36 +199,119 @@ func Test_Manager_EnumerateExternalAgents_returnsAllTokens(t *testing.T) {
 	sortTokenInfosByWorkspaceID(want)
 	sortTokenInfosByWorkspaceID(got)
 
-	require.Equal(t, len(want), len(got), "expected one TokenInfo per external-agent workspace")
+	require.Equal(t, len(want), len(got),
+		"expected one TokenInfo per external-agent workspace under the template")
 	for i := range want {
 		assert.Equal(t, want[i].WorkspaceID, got[i].WorkspaceID, "WorkspaceID for entry %d", i)
+		assert.Equal(t, want[i].WorkspaceName, got[i].WorkspaceName, "WorkspaceName for entry %d", i)
 		assert.Equal(t, want[i].AgentName, got[i].AgentName, "AgentName for entry %d", i)
 		assert.Equal(t, want[i].Token, got[i].Token, "Token for entry %d", i)
 		assert.NotEmpty(t, got[i].Token, "Token must be non-empty for entry %d", i)
 	}
 }
 
-// Asserts that an authentication failure during enumeration produces a
-// fatal error, so the retry loop in enumerateWithRetry surfaces it
-// immediately rather than hammering endpoints with credentials that
-// will never work.
-func Test_Manager_EnumerateExternalAgents_invalidTokenIsFatal(t *testing.T) {
+// Asserts that an authentication failure surfaced during template/owner
+// resolution is fatal, so Run does not retry indefinitely against credentials
+// that will never work.
+func Test_Manager_ResolveTemplateAndOwner_invalidTokenIsFatal(t *testing.T) {
 	t.Parallel()
 	ctx := testutil.Context(t, testutil.WaitShort)
 
+	db, _ := dbtestutil.NewDB(t)
 	client := &fakeExternalAgentClient{
-		workspacesErr: codersdk.NewError(http.StatusUnauthorized, codersdk.Response{Message: "unauthorized"}),
+		meErr: codersdk.NewError(http.StatusUnauthorized, codersdk.Response{Message: "unauthorized"}),
 	}
 	coderURL, _ := url.Parse("http://fake")
 	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
-	m := agentfake.NewManager(coderURL, client, logger, agentfake.ManagerOptions{Template: "tmpl"})
+	m := agentfake.NewManager(logger, coderURL, client, db, agentfake.ManagerOptions{Template: "tmpl"})
 
-	_, err := m.EnumerateExternalAgents(ctx)
-	require.Error(t, err, "expected enumeration to fail with an invalid session token")
+	err := m.ResolveTemplateAndOwner(ctx)
+	require.Error(t, err, "expected resolution to fail with an invalid session token")
 	require.True(t, agentfake.IsFatalEnumerationError(err),
 		"expected error to be classified as fatal; got: %v", err)
 }
 
+// Asserts that --owner restricts results to workspaces owned by that user even
+// when other owners have external-agent workspaces under the same template.
+func Test_Manager_EnumerateExternalAgents_filtersByOwner(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	db, _ := dbtestutil.NewDB(t)
+	org, firstUser, tpl := seedUserOrgAndTemplate(t, db)
+	secondUser := dbgen.User(t, db, database.User{})
+	_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{
+		UserID:         secondUser.ID,
+		OrganizationID: org.ID,
+	})
+
+	_ = buildExternalAgentWorkspace(t, db, org.ID, firstUser.ID, tpl.ID)
+	r2 := buildExternalAgentWorkspace(t, db, org.ID, secondUser.ID, tpl.ID)
+
+	client := newFakeClient(firstUser, org, tpl)
+	client.owner = codersdk.User{
+		ReducedUser: codersdk.ReducedUser{MinimalUser: codersdk.MinimalUser{
+			ID: secondUser.ID, Username: secondUser.Username,
+		}},
+		OrganizationIDs: []uuid.UUID{org.ID},
+	}
+	coderURL, _ := url.Parse("http://fake")
+	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
+	m := agentfake.NewManager(logger, coderURL, client, db, agentfake.ManagerOptions{
+		Template: tpl.Name,
+		Owner:    secondUser.Username,
+	})
+	require.NoError(t, m.ResolveTemplateAndOwner(ctx))
+
+	got, err := m.EnumerateExternalAgents(ctx)
+	require.NoError(t, err)
+	require.Len(t, got, 1, "expected only the second user's workspace to be returned")
+	require.Equal(t, r2.Workspace.ID, got[0].WorkspaceID)
+	require.Equal(t, r2.AgentToken, got[0].Token)
+}
+
+// Asserts that workspaces whose latest build is not in the "running" state
+// (job_status != succeeded or transition != start) are excluded from
+// enumeration results.
+func Test_Manager_EnumerateExternalAgents_excludesNonRunning(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	db, _ := dbtestutil.NewDB(t)
+	org, user, tpl := seedUserOrgAndTemplate(t, db)
+
+	// Running workspace: should be included.
+	running := buildExternalAgentWorkspace(t, db, org.ID, user.ID, tpl.ID)
+
+	// Failed-build workspace under the same template: should be excluded.
+	_ = dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
+		OrganizationID: org.ID,
+		OwnerID:        user.ID,
+		TemplateID:     tpl.ID,
+	}).
+		Seed(database.WorkspaceBuild{
+			HasExternalAgent: sql.NullBool{Bool: true, Valid: true},
+		}).
+		Resource(&sdkproto.Resource{
+			Name: "external",
+			Type: "coder_external_agent",
+		}).
+		WithAgent().
+		Failed().
+		Do()
+
+	client := newFakeClient(user, org, tpl)
+	coderURL, _ := url.Parse("http://fake")
+	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).Leveled(slog.LevelDebug)
+	m := agentfake.NewManager(logger, coderURL, client, db, agentfake.ManagerOptions{Template: tpl.Name})
+	require.NoError(t, m.ResolveTemplateAndOwner(ctx))
+
+	got, err := m.EnumerateExternalAgents(ctx)
+	require.NoError(t, err)
+	require.Len(t, got, 1, "only the running workspace should be returned")
+	require.Equal(t, running.Workspace.ID, got[0].WorkspaceID)
+}
+
 func sortTokenInfosByWorkspaceID(s []agentfake.TokenInfo) {
 	sort.Slice(s, func(i, j int) bool {
 		return s[i].WorkspaceID.String() < s[j].WorkspaceID.String()

enterprise/scaletest/agentfake/metrics.go

@@ -0,0 +1,39 @@
+package agentfake
+
+import "github.com/prometheus/client_golang/prometheus"
+
+// Metrics holds the Prometheus collectors for the agentfake manager.
+// A nil *Metrics is a valid no-op.
+type Metrics struct {
+	// ConnectedAgents is the number of fake agents with an established dRPC connection.
+	ConnectedAgents prometheus.Gauge
+}
+
+// NewMetrics registers agentfake collectors on reg and returns the handle.
+func NewMetrics(reg prometheus.Registerer) *Metrics {
+	m := &Metrics{
+		ConnectedAgents: prometheus.NewGauge(prometheus.GaugeOpts{
+			Namespace: "coder",
+			Subsystem: "scaletest_agentfake",
+			Name:      "connected_agents",
+			Help:      "Number of fake agents with an established dRPC connection to coderd.",
+		}),
+	}
+	reg.MustRegister(m.ConnectedAgents)
+	m.ConnectedAgents.Set(0) // ensure the metric appears before any agent connects
+	return m
+}
+
+func (m *Metrics) incConnected() {
+	if m == nil {
+		return
+	}
+	m.ConnectedAgents.Inc()
+}
+
+func (m *Metrics) decConnected() {
+	if m == nil {
+		return
+	}
+	m.ConnectedAgents.Dec()
+}

scripts/metricsdocgen/scanner/scanner.go

@@ -42,6 +42,7 @@ var scanDirs = []string{
 var skipPaths = []string{
 	"coderd/aibridged/metrics.go",
 	"enterprise/aibridgeproxyd/metrics.go",
+	"enterprise/scaletest/agentfake/metrics.go",
 }
 
 // MetricType represents the type of Prometheus metric.