feat: add telemetry for task lifecycle events (#21922)

Relates to https://github.com/coder/internal/issues/1259

Adds new database queries and telemetry collection functions to gather
task lifecycle events (pause/resume cycles, idle time) for analytics.
    
Task events track pause/resume activity, idle duration before pausing,
paused duration, and time from resume to first app status, filtered to
recent activity based on the telemetry snapshot interval.

🤖 Created with Mux (Opus 4.6).

coderd/database/dbauthz/dbauthz.go

@@ -3166,6 +3166,13 @@ func (q *querier) GetTelemetryItems(ctx context.Context) ([]database.TelemetryIt
 	return q.db.GetTelemetryItems(ctx)
 }
 
+func (q *querier) GetTelemetryTaskEvents(ctx context.Context, arg database.GetTelemetryTaskEventsParams) ([]database.GetTelemetryTaskEventsRow, error) {
+	if err := q.authorizeContext(ctx, policy.ActionRead, rbac.ResourceTask.All()); err != nil {
+		return nil, err
+	}
+	return q.db.GetTelemetryTaskEvents(ctx, arg)
+}
+
 func (q *querier) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) {
 	if err := q.authorizeTemplateInsights(ctx, arg.TemplateIDs); err != nil {
 		return nil, err

coderd/database/dbauthz/dbauthz_test.go

@@ -1326,6 +1326,11 @@ func (s *MethodTestSuite) TestTemplate() {
 		dbm.EXPECT().GetTemplateInsightsByTemplate(gomock.Any(), arg).Return([]database.GetTemplateInsightsByTemplateRow{}, nil).AnyTimes()
 		check.Args(arg).Asserts(rbac.ResourceTemplate, policy.ActionViewInsights)
 	}))
+	s.Run("GetTelemetryTaskEvents", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
+		arg := database.GetTelemetryTaskEventsParams{}
+		dbm.EXPECT().GetTelemetryTaskEvents(gomock.Any(), arg).Return([]database.GetTelemetryTaskEventsRow{}, nil).AnyTimes()
+		check.Args(arg).Asserts(rbac.ResourceTask.All(), policy.ActionRead)
+	}))
 	s.Run("GetTemplateAppInsights", s.Mocked(func(dbm *dbmock.MockStore, _ *gofakeit.Faker, check *expects) {
 		arg := database.GetTemplateAppInsightsParams{}
 		dbm.EXPECT().GetTemplateAppInsights(gomock.Any(), arg).Return([]database.GetTemplateAppInsightsRow{}, nil).AnyTimes()

coderd/database/dbmetrics/querymetrics.go

@@ -1790,6 +1790,14 @@ func (m queryMetricsStore) GetTelemetryItems(ctx context.Context) ([]database.Te
 	return r0, r1
 }
 
+func (m queryMetricsStore) GetTelemetryTaskEvents(ctx context.Context, createdAfter database.GetTelemetryTaskEventsParams) ([]database.GetTelemetryTaskEventsRow, error) {
+	start := time.Now()
+	r0, r1 := m.s.GetTelemetryTaskEvents(ctx, createdAfter)
+	m.queryLatencies.WithLabelValues("GetTelemetryTaskEvents").Observe(time.Since(start).Seconds())
+	m.queryCounts.WithLabelValues(httpmw.ExtractHTTPRoute(ctx), httpmw.ExtractHTTPMethod(ctx), "GetTelemetryTaskEvents").Inc()
+	return r0, r1
+}
+
 func (m queryMetricsStore) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) {
 	start := time.Now()
 	r0, r1 := m.s.GetTemplateAppInsights(ctx, arg)

coderd/database/dbmock/dbmock.go

@@ -3314,6 +3314,21 @@ func (mr *MockStoreMockRecorder) GetTelemetryItems(ctx any) *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTelemetryItems", reflect.TypeOf((*MockStore)(nil).GetTelemetryItems), ctx)
 }
 
+// GetTelemetryTaskEvents mocks base method.
+func (m *MockStore) GetTelemetryTaskEvents(ctx context.Context, arg database.GetTelemetryTaskEventsParams) ([]database.GetTelemetryTaskEventsRow, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetTelemetryTaskEvents", ctx, arg)
+	ret0, _ := ret[0].([]database.GetTelemetryTaskEventsRow)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetTelemetryTaskEvents indicates an expected call of GetTelemetryTaskEvents.
+func (mr *MockStoreMockRecorder) GetTelemetryTaskEvents(ctx, arg any) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTelemetryTaskEvents", reflect.TypeOf((*MockStore)(nil).GetTelemetryTaskEvents), ctx, arg)
+}
+
 // GetTemplateAppInsights mocks base method.
 func (m *MockStore) GetTemplateAppInsights(ctx context.Context, arg database.GetTemplateAppInsightsParams) ([]database.GetTemplateAppInsightsRow, error) {
 	m.ctrl.T.Helper()

coderd/database/querier.go

@@ -361,6 +361,23 @@ type sqlcQuerier interface {
 	GetTaskSnapshot(ctx context.Context, taskID uuid.UUID) (TaskSnapshot, error)
 	GetTelemetryItem(ctx context.Context, key string) (TelemetryItem, error)
 	GetTelemetryItems(ctx context.Context) ([]TelemetryItem, error)
+	// Returns all data needed to build task lifecycle events for telemetry
+	// in a single round-trip. For each task whose workspace is in the
+	// given set, fetches:
+	//   - the latest workspace app binding (task_workspace_apps)
+	//   - the most recent stop and start builds (workspace_builds)
+	//   - the last "working" app status (workspace_app_statuses)
+	//   - the first app status after resume, for active workspaces
+	//
+	// Assumptions:
+	// - 1:1 relationship between tasks and workspaces. All builds on the
+	//   workspace are considered task-related.
+	// - Idle duration approximation: If the agent reports "working", does
+	//   work, then reports "done", we miss that working time.
+	// - lws and active_dur join across all historical app IDs for the task,
+	//   because each resume cycle provisions a new app ID. This ensures
+	//   pre-pause statuses contribute to idle duration and active duration.
+	GetTelemetryTaskEvents(ctx context.Context, arg GetTelemetryTaskEventsParams) ([]GetTelemetryTaskEventsRow, error)
 	// GetTemplateAppInsights returns the aggregate usage of each app in a given
 	// timeframe. The result can be filtered on template_ids, meaning only user data
 	// from workspaces based on those templates will be included.

coderd/database/queries.sql.go

@@ -13334,6 +13334,203 @@ func (q *sqlQuerier) GetTaskSnapshot(ctx context.Context, taskID uuid.UUID) (Tas
 	return i, err
 }
 
+const getTelemetryTaskEvents = `-- name: GetTelemetryTaskEvents :many
+WITH task_app_ids AS (
+	SELECT task_id, workspace_app_id
+	FROM task_workspace_apps
+),
+task_status_timeline AS (
+	-- All app statuses across every historical app for each task,
+	-- plus synthetic "boundary" rows at each stop/start build transition.
+	-- This allows us to correctly take gaps due to pause/resume into account.
+	SELECT tai.task_id, was.created_at, was.state::text AS state
+	FROM workspace_app_statuses was
+	JOIN task_app_ids tai ON tai.workspace_app_id = was.app_id
+	UNION ALL
+	SELECT t.id AS task_id, wb.created_at, '_boundary' AS state
+	FROM tasks t
+	JOIN workspace_builds wb ON wb.workspace_id = t.workspace_id
+	WHERE t.deleted_at IS NULL
+		AND t.workspace_id IS NOT NULL
+		AND wb.build_number > 1
+),
+task_event_data AS (
+	SELECT
+		t.id AS task_id,
+		t.workspace_id,
+		twa.workspace_app_id,
+		-- Latest stop build.
+		stop_build.created_at AS stop_build_created_at,
+		stop_build.reason AS stop_build_reason,
+		-- Latest start build (task_resume only).
+		start_build.created_at AS start_build_created_at,
+		start_build.reason AS start_build_reason,
+		start_build.build_number AS start_build_number,
+		-- Last "working" app status (for idle duration).
+		lws.created_at AS last_working_status_at,
+		-- First app status after resume (for resume-to-status duration).
+		-- Only populated for workspaces in an active phase (started more
+		-- recently than stopped).
+		fsar.created_at AS first_status_after_resume_at,
+		-- Cumulative time spent in "working" state.
+		active_dur.total_working_ms AS active_duration_ms
+	FROM tasks t
+	LEFT JOIN LATERAL (
+		SELECT task_app.workspace_app_id
+		FROM task_workspace_apps task_app
+		WHERE task_app.task_id = t.id
+		ORDER BY task_app.workspace_build_number DESC
+		LIMIT 1
+	) twa ON TRUE
+	LEFT JOIN LATERAL (
+		SELECT wb.created_at, wb.reason, wb.build_number
+		FROM workspace_builds wb
+		WHERE wb.workspace_id = t.workspace_id
+			AND wb.transition = 'stop'
+		ORDER BY wb.build_number DESC
+		LIMIT 1
+	) stop_build ON TRUE
+	LEFT JOIN LATERAL (
+		SELECT wb.created_at, wb.reason, wb.build_number
+		FROM workspace_builds wb
+		WHERE wb.workspace_id = t.workspace_id
+			AND wb.transition = 'start'
+		ORDER BY wb.build_number DESC
+		LIMIT 1
+	) start_build ON TRUE
+	LEFT JOIN LATERAL (
+		SELECT tst.created_at
+		FROM task_status_timeline tst
+		WHERE tst.task_id = t.id
+			AND tst.state = 'working'
+		-- Only consider status before the latest pause so that
+		-- post-resume statuses don't mask pre-pause idle time.
+		AND (stop_build.created_at IS NULL
+			OR tst.created_at <= stop_build.created_at)
+		ORDER BY tst.created_at DESC
+		LIMIT 1
+	) lws ON TRUE
+	LEFT JOIN LATERAL (
+		SELECT was.created_at
+		FROM workspace_app_statuses was
+		WHERE was.app_id = twa.workspace_app_id
+			AND was.created_at > start_build.created_at
+		ORDER BY was.created_at ASC
+		LIMIT 1
+	) fsar ON twa.workspace_app_id IS NOT NULL
+		AND start_build.created_at IS NOT NULL
+		AND (stop_build.created_at IS NULL
+			OR start_build.created_at > stop_build.created_at)
+	-- Active duration: cumulative time spent in "working" state across all
+	-- historical app IDs for this task. Uses LEAD() to convert point-in-time
+	-- statuses into intervals, then sums intervals where state='working'. For
+	-- the last status, falls back to stop_build time (if paused) or @now (if
+	-- still running).
+	LEFT JOIN LATERAL (
+		SELECT COALESCE(
+			SUM(EXTRACT(EPOCH FROM (interval_end - interval_start)) * 1000)::bigint,
+			0
+		)::bigint AS total_working_ms
+		FROM (
+			SELECT
+				tst.created_at AS interval_start,
+				COALESCE(
+					LEAD(tst.created_at) OVER (ORDER BY tst.created_at ASC, CASE WHEN tst.state = '_boundary' THEN 1 ELSE 0 END ASC),
+					CASE WHEN stop_build.created_at IS NOT NULL
+						AND (start_build.created_at IS NULL
+							OR stop_build.created_at > start_build.created_at)
+					THEN stop_build.created_at
+					ELSE $1::timestamptz
+					END
+				) AS interval_end,
+				tst.state
+			FROM task_status_timeline tst
+			WHERE tst.task_id = t.id
+		) intervals
+		WHERE intervals.state = 'working'
+	) active_dur ON TRUE
+	WHERE t.deleted_at IS NULL
+		AND t.workspace_id IS NOT NULL
+		AND EXISTS (
+			SELECT 1 FROM workspace_builds wb
+			WHERE wb.workspace_id = t.workspace_id
+			  AND wb.created_at > $2
+		)
+)
+SELECT task_id, workspace_id, workspace_app_id, stop_build_created_at, stop_build_reason, start_build_created_at, start_build_reason, start_build_number, last_working_status_at, first_status_after_resume_at, active_duration_ms FROM task_event_data
+ORDER BY task_id
+`
+
+type GetTelemetryTaskEventsParams struct {
+	Now          time.Time `db:"now" json:"now"`
+	CreatedAfter time.Time `db:"created_after" json:"created_after"`
+}
+
+type GetTelemetryTaskEventsRow struct {
+	TaskID                   uuid.UUID       `db:"task_id" json:"task_id"`
+	WorkspaceID              uuid.NullUUID   `db:"workspace_id" json:"workspace_id"`
+	WorkspaceAppID           uuid.NullUUID   `db:"workspace_app_id" json:"workspace_app_id"`
+	StopBuildCreatedAt       sql.NullTime    `db:"stop_build_created_at" json:"stop_build_created_at"`
+	StopBuildReason          NullBuildReason `db:"stop_build_reason" json:"stop_build_reason"`
+	StartBuildCreatedAt      sql.NullTime    `db:"start_build_created_at" json:"start_build_created_at"`
+	StartBuildReason         NullBuildReason `db:"start_build_reason" json:"start_build_reason"`
+	StartBuildNumber         sql.NullInt32   `db:"start_build_number" json:"start_build_number"`
+	LastWorkingStatusAt      sql.NullTime    `db:"last_working_status_at" json:"last_working_status_at"`
+	FirstStatusAfterResumeAt sql.NullTime    `db:"first_status_after_resume_at" json:"first_status_after_resume_at"`
+	ActiveDurationMs         int64           `db:"active_duration_ms" json:"active_duration_ms"`
+}
+
+// Returns all data needed to build task lifecycle events for telemetry
+// in a single round-trip. For each task whose workspace is in the
+// given set, fetches:
+//   - the latest workspace app binding (task_workspace_apps)
+//   - the most recent stop and start builds (workspace_builds)
+//   - the last "working" app status (workspace_app_statuses)
+//   - the first app status after resume, for active workspaces
+//
+// Assumptions:
+//   - 1:1 relationship between tasks and workspaces. All builds on the
+//     workspace are considered task-related.
+//   - Idle duration approximation: If the agent reports "working", does
+//     work, then reports "done", we miss that working time.
+//   - lws and active_dur join across all historical app IDs for the task,
+//     because each resume cycle provisions a new app ID. This ensures
+//     pre-pause statuses contribute to idle duration and active duration.
+func (q *sqlQuerier) GetTelemetryTaskEvents(ctx context.Context, arg GetTelemetryTaskEventsParams) ([]GetTelemetryTaskEventsRow, error) {
+	rows, err := q.db.QueryContext(ctx, getTelemetryTaskEvents, arg.Now, arg.CreatedAfter)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []GetTelemetryTaskEventsRow
+	for rows.Next() {
+		var i GetTelemetryTaskEventsRow
+		if err := rows.Scan(
+			&i.TaskID,
+			&i.WorkspaceID,
+			&i.WorkspaceAppID,
+			&i.StopBuildCreatedAt,
+			&i.StopBuildReason,
+			&i.StartBuildCreatedAt,
+			&i.StartBuildReason,
+			&i.StartBuildNumber,
+			&i.LastWorkingStatusAt,
+			&i.FirstStatusAfterResumeAt,
+			&i.ActiveDurationMs,
+		); err != nil {
+			return nil, err
+		}
+		items = append(items, i)
+	}
+	if err := rows.Close(); err != nil {
+		return nil, err
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const insertTask = `-- name: InsertTask :one
 INSERT INTO tasks
 	(id, organization_id, owner_id, name, display_name, workspace_id, template_version_id, template_parameters, prompt, created_at)

coderd/database/queries/tasks.sql

@@ -100,3 +100,146 @@ FROM
 	task_snapshots
 WHERE
 	task_id = $1;
+
+-- name: GetTelemetryTaskEvents :many
+-- Returns all data needed to build task lifecycle events for telemetry
+-- in a single round-trip. For each task whose workspace is in the
+-- given set, fetches:
+--   - the latest workspace app binding (task_workspace_apps)
+--   - the most recent stop and start builds (workspace_builds)
+--   - the last "working" app status (workspace_app_statuses)
+--   - the first app status after resume, for active workspaces
+--
+-- Assumptions:
+-- - 1:1 relationship between tasks and workspaces. All builds on the
+--   workspace are considered task-related.
+-- - Idle duration approximation: If the agent reports "working", does
+--   work, then reports "done", we miss that working time.
+-- - lws and active_dur join across all historical app IDs for the task,
+--   because each resume cycle provisions a new app ID. This ensures
+--   pre-pause statuses contribute to idle duration and active duration.
+WITH task_app_ids AS (
+	SELECT task_id, workspace_app_id
+	FROM task_workspace_apps
+),
+task_status_timeline AS (
+	-- All app statuses across every historical app for each task,
+	-- plus synthetic "boundary" rows at each stop/start build transition.
+	-- This allows us to correctly take gaps due to pause/resume into account.
+	SELECT tai.task_id, was.created_at, was.state::text AS state
+	FROM workspace_app_statuses was
+	JOIN task_app_ids tai ON tai.workspace_app_id = was.app_id
+	UNION ALL
+	SELECT t.id AS task_id, wb.created_at, '_boundary' AS state
+	FROM tasks t
+	JOIN workspace_builds wb ON wb.workspace_id = t.workspace_id
+	WHERE t.deleted_at IS NULL
+		AND t.workspace_id IS NOT NULL
+		AND wb.build_number > 1
+),
+task_event_data AS (
+	SELECT
+		t.id AS task_id,
+		t.workspace_id,
+		twa.workspace_app_id,
+		-- Latest stop build.
+		stop_build.created_at AS stop_build_created_at,
+		stop_build.reason AS stop_build_reason,
+		-- Latest start build (task_resume only).
+		start_build.created_at AS start_build_created_at,
+		start_build.reason AS start_build_reason,
+		start_build.build_number AS start_build_number,
+		-- Last "working" app status (for idle duration).
+		lws.created_at AS last_working_status_at,
+		-- First app status after resume (for resume-to-status duration).
+		-- Only populated for workspaces in an active phase (started more
+		-- recently than stopped).
+		fsar.created_at AS first_status_after_resume_at,
+		-- Cumulative time spent in "working" state.
+		active_dur.total_working_ms AS active_duration_ms
+	FROM tasks t
+	LEFT JOIN LATERAL (
+		SELECT task_app.workspace_app_id
+		FROM task_workspace_apps task_app
+		WHERE task_app.task_id = t.id
+		ORDER BY task_app.workspace_build_number DESC
+		LIMIT 1
+	) twa ON TRUE
+	LEFT JOIN LATERAL (
+		SELECT wb.created_at, wb.reason, wb.build_number
+		FROM workspace_builds wb
+		WHERE wb.workspace_id = t.workspace_id
+			AND wb.transition = 'stop'
+		ORDER BY wb.build_number DESC
+		LIMIT 1
+	) stop_build ON TRUE
+	LEFT JOIN LATERAL (
+		SELECT wb.created_at, wb.reason, wb.build_number
+		FROM workspace_builds wb
+		WHERE wb.workspace_id = t.workspace_id
+			AND wb.transition = 'start'
+		ORDER BY wb.build_number DESC
+		LIMIT 1
+	) start_build ON TRUE
+	LEFT JOIN LATERAL (
+		SELECT tst.created_at
+		FROM task_status_timeline tst
+		WHERE tst.task_id = t.id
+			AND tst.state = 'working'
+		-- Only consider status before the latest pause so that
+		-- post-resume statuses don't mask pre-pause idle time.
+		AND (stop_build.created_at IS NULL
+			OR tst.created_at <= stop_build.created_at)
+		ORDER BY tst.created_at DESC
+		LIMIT 1
+	) lws ON TRUE
+	LEFT JOIN LATERAL (
+		SELECT was.created_at
+		FROM workspace_app_statuses was
+		WHERE was.app_id = twa.workspace_app_id
+			AND was.created_at > start_build.created_at
+		ORDER BY was.created_at ASC
+		LIMIT 1
+	) fsar ON twa.workspace_app_id IS NOT NULL
+		AND start_build.created_at IS NOT NULL
+		AND (stop_build.created_at IS NULL
+			OR start_build.created_at > stop_build.created_at)
+	-- Active duration: cumulative time spent in "working" state across all
+	-- historical app IDs for this task. Uses LEAD() to convert point-in-time
+	-- statuses into intervals, then sums intervals where state='working'. For
+	-- the last status, falls back to stop_build time (if paused) or @now (if
+	-- still running).
+	LEFT JOIN LATERAL (
+		SELECT COALESCE(
+			SUM(EXTRACT(EPOCH FROM (interval_end - interval_start)) * 1000)::bigint,
+			0
+		)::bigint AS total_working_ms
+		FROM (
+			SELECT
+				tst.created_at AS interval_start,
+				COALESCE(
+					LEAD(tst.created_at) OVER (ORDER BY tst.created_at ASC, CASE WHEN tst.state = '_boundary' THEN 1 ELSE 0 END ASC),
+					CASE WHEN stop_build.created_at IS NOT NULL
+						AND (start_build.created_at IS NULL
+							OR stop_build.created_at > start_build.created_at)
+					THEN stop_build.created_at
+					ELSE @now::timestamptz
+					END
+				) AS interval_end,
+				tst.state
+			FROM task_status_timeline tst
+			WHERE tst.task_id = t.id
+		) intervals
+		WHERE intervals.state = 'working'
+	) active_dur ON TRUE
+	WHERE t.deleted_at IS NULL
+		AND t.workspace_id IS NOT NULL
+		AND EXISTS (
+			SELECT 1 FROM workspace_builds wb
+			WHERE wb.workspace_id = t.workspace_id
+			  AND wb.created_at > @created_after
+		)
+)
+SELECT * FROM task_event_data
+ORDER BY task_id;
+

coderd/database/queries/workspaceapps.sql

@@ -87,3 +87,4 @@ SELECT DISTINCT ON (workspace_id)
 FROM workspace_app_statuses
 WHERE workspace_id = ANY(@ids :: uuid[])
 ORDER BY workspace_id, created_at DESC;
+

coderd/database/sqlc.yaml

@@ -124,6 +124,24 @@ sql:
           - column: "tasks_with_status.workspace_app_health"
             go_type:
               type: "NullWorkspaceAppHealth"
+          # Workaround for sqlc not interpreting the left join correctly
+          # in the combined telemetry query.
+          - column: "task_event_data.start_build_number"
+            go_type: "database/sql.NullInt32"
+          - column: "task_event_data.stop_build_created_at"
+            go_type: "database/sql.NullTime"
+          - column: "task_event_data.stop_build_reason"
+            go_type:
+              type: "NullBuildReason"
+          - column: "task_event_data.start_build_created_at"
+            go_type: "database/sql.NullTime"
+          - column: "task_event_data.start_build_reason"
+            go_type:
+              type: "NullBuildReason"
+          - column: "task_event_data.last_working_status_at"
+            go_type: "database/sql.NullTime"
+          - column: "task_event_data.first_status_after_resume_at"
+            go_type: "database/sql.NullTime"
         rename:
           group_member: GroupMemberTable
           group_members_expanded: GroupMember