feat: expose template insights as Prometheus metrics (#10325)

coderd/prometheusmetrics/insights/metricscollector.go

@@ -0,0 +1,174 @@
+package insights
+
+import (
+	"context"
+	"sync/atomic"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/xerrors"
+
+	"cdr.dev/slog"
+
+	"github.com/coder/coder/v2/coderd/database"
+)
+
+var templatesActiveUsersDesc = prometheus.NewDesc("coderd_insights_templates_active_users", "The number of active users of the template.", []string{"template_name"}, nil)
+
+type MetricsCollector struct {
+	database     database.Store
+	logger       slog.Logger
+	timeWindow   time.Duration
+	tickInterval time.Duration
+
+	data atomic.Pointer[insightsData]
+}
+
+type insightsData struct {
+	templates []database.GetTemplateInsightsByTemplateRow
+
+	templateNames map[uuid.UUID]string
+}
+
+var _ prometheus.Collector = new(MetricsCollector)
+
+func NewMetricsCollector(db database.Store, logger slog.Logger, timeWindow time.Duration, tickInterval time.Duration) (*MetricsCollector, error) {
+	if timeWindow == 0 {
+		timeWindow = 5 * time.Minute
+	}
+	if timeWindow < 5*time.Minute {
+		return nil, xerrors.Errorf("time window must be at least 5 mins")
+	}
+	if tickInterval == 0 {
+		tickInterval = timeWindow
+	}
+
+	return &MetricsCollector{
+		database:     db,
+		logger:       logger.Named("insights_metrics_collector"),
+		timeWindow:   timeWindow,
+		tickInterval: tickInterval,
+	}, nil
+}
+
+func (mc *MetricsCollector) Run(ctx context.Context) (func(), error) {
+	ctx, closeFunc := context.WithCancel(ctx)
+	done := make(chan struct{})
+
+	// Use time.Nanosecond to force an initial tick. It will be reset to the
+	// correct duration after executing once.
+	ticker := time.NewTicker(time.Nanosecond)
+	doTick := func() {
+		defer ticker.Reset(mc.tickInterval)
+
+		now := time.Now()
+		startTime := now.Add(-mc.timeWindow)
+		endTime := now
+
+		// Phase 1: Fetch insights from database
+		// FIXME errorGroup will be used to fetch insights for apps and parameters
+		eg, egCtx := errgroup.WithContext(ctx)
+		eg.SetLimit(1)
+
+		var templateInsights []database.GetTemplateInsightsByTemplateRow
+
+		eg.Go(func() error {
+			var err error
+			templateInsights, err = mc.database.GetTemplateInsightsByTemplate(egCtx, database.GetTemplateInsightsByTemplateParams{
+				StartTime: startTime,
+				EndTime:   endTime,
+			})
+			if err != nil {
+				mc.logger.Error(ctx, "unable to fetch template insights from database", slog.Error(err))
+			}
+			return err
+		})
+		err := eg.Wait()
+		if err != nil {
+			return
+		}
+
+		// Phase 2: Collect template IDs, and fetch relevant details
+		templateIDs := uniqueTemplateIDs(templateInsights)
+
+		templateNames := make(map[uuid.UUID]string, len(templateIDs))
+		if len(templateIDs) > 0 {
+			templates, err := mc.database.GetTemplatesWithFilter(ctx, database.GetTemplatesWithFilterParams{
+				IDs: templateIDs,
+			})
+			if err != nil {
+				mc.logger.Error(ctx, "unable to fetch template details from database", slog.Error(err))
+				return
+			}
+			templateNames = onlyTemplateNames(templates)
+		}
+
+		// Refresh the collector state
+		mc.data.Store(&insightsData{
+			templates:     templateInsights,
+			templateNames: templateNames,
+		})
+	}
+
+	go func() {
+		defer close(done)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+				ticker.Stop()
+				doTick()
+			}
+		}
+	}()
+	return func() {
+		closeFunc()
+		<-done
+	}, nil
+}
+
+func (*MetricsCollector) Describe(descCh chan<- *prometheus.Desc) {
+	descCh <- templatesActiveUsersDesc
+}
+
+func (mc *MetricsCollector) Collect(metricsCh chan<- prometheus.Metric) {
+	// Phase 3: Collect metrics
+
+	data := mc.data.Load()
+	if data == nil {
+		return // insights data not loaded yet
+	}
+
+	for _, templateRow := range data.templates {
+		metricsCh <- prometheus.MustNewConstMetric(templatesActiveUsersDesc, prometheus.GaugeValue, float64(templateRow.ActiveUsers), data.templateNames[templateRow.TemplateID])
+	}
+}
+
+// Helper functions below.
+
+func uniqueTemplateIDs(templateInsights []database.GetTemplateInsightsByTemplateRow) []uuid.UUID {
+	tids := map[uuid.UUID]bool{}
+	for _, t := range templateInsights {
+		tids[t.TemplateID] = true
+	}
+
+	uniqueUUIDs := make([]uuid.UUID, len(tids))
+	var i int
+	for t := range tids {
+		uniqueUUIDs[i] = t
+		i++
+	}
+	return uniqueUUIDs
+}
+
+func onlyTemplateNames(templates []database.Template) map[uuid.UUID]string {
+	m := map[uuid.UUID]string{}
+	for _, t := range templates {
+		m[t.ID] = t.Name
+	}
+	return m
+}

coderd/prometheusmetrics/insights/metricscollector_test.go

@@ -0,0 +1,132 @@
+package insights_test
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"cdr.dev/slog/sloggers/slogtest"
+	"github.com/coder/coder/v2/agent/agenttest"
+	"github.com/coder/coder/v2/coderd/coderdtest"
+	"github.com/coder/coder/v2/coderd/database/dbtestutil"
+	"github.com/coder/coder/v2/coderd/prometheusmetrics/insights"
+	"github.com/coder/coder/v2/codersdk"
+	"github.com/coder/coder/v2/provisioner/echo"
+	"github.com/coder/coder/v2/testutil"
+)
+
+func TestCollect_TemplateInsights(t *testing.T) {
+	t.Parallel()
+
+	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
+	db, ps := dbtestutil.NewDB(t)
+
+	options := &coderdtest.Options{
+		IncludeProvisionerDaemon:  true,
+		AgentStatsRefreshInterval: time.Millisecond * 100,
+		Database:                  db,
+		Pubsub:                    ps,
+	}
+	client := coderdtest.New(t, options)
+
+	// Given
+	// Initialize metrics collector
+	mc, err := insights.NewMetricsCollector(db, logger, 0, time.Second)
+	require.NoError(t, err)
+
+	registry := prometheus.NewRegistry()
+	registry.Register(mc)
+
+	// Create two users, one that will appear in the report and another that
+	// won't (due to not having/using a workspace).
+	user := coderdtest.CreateFirstUser(t, client)
+	_, _ = coderdtest.CreateAnotherUser(t, client, user.OrganizationID)
+	authToken := uuid.NewString()
+	version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
+		Parse:          echo.ParseComplete,
+		ProvisionPlan:  echo.PlanComplete,
+		ProvisionApply: echo.ProvisionApplyWithAgent(authToken),
+	})
+	template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
+	require.Empty(t, template.BuildTimeStats[codersdk.WorkspaceTransitionStart])
+
+	coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
+	workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
+	coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
+
+	// Start an agent so that we can generate stats.
+	_ = agenttest.New(t, client.URL, authToken)
+	resources := coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
+
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
+	defer cancel()
+
+	// Run metrics collector
+	closeFunc, err := mc.Run(ctx)
+	require.NoError(t, err)
+	defer closeFunc()
+
+	// Connect to the agent to generate usage/latency stats.
+	conn, err := client.DialWorkspaceAgent(ctx, resources[0].Agents[0].ID, &codersdk.DialWorkspaceAgentOptions{
+		Logger: logger.Named("client"),
+	})
+	require.NoError(t, err)
+	defer conn.Close()
+
+	sshConn, err := conn.SSHClient(ctx)
+	require.NoError(t, err)
+	defer sshConn.Close()
+
+	sess, err := sshConn.NewSession()
+	require.NoError(t, err)
+	defer sess.Close()
+
+	r, w := io.Pipe()
+	defer r.Close()
+	defer w.Close()
+	sess.Stdin = r
+	sess.Stdout = io.Discard
+	err = sess.Start("cat")
+	require.NoError(t, err)
+
+	goldenFile, err := os.ReadFile("testdata/insights-metrics.json")
+	require.NoError(t, err)
+	golden := map[string]int{}
+	err = json.Unmarshal(goldenFile, &golden)
+	require.NoError(t, err)
+
+	collected := map[string]int{}
+	assert.Eventuallyf(t, func() bool {
+		// When
+		metrics, err := registry.Gather()
+		require.NoError(t, err)
+
+		// Then
+		for _, metric := range metrics {
+			switch metric.GetName() {
+			case "coderd_insights_templates_active_users":
+				for _, m := range metric.Metric {
+					collected[metric.GetName()] = int(m.Gauge.GetValue())
+				}
+			default:
+				require.FailNowf(t, "unexpected metric collected", "metric: %s", metric.GetName())
+			}
+		}
+
+		return assert.ObjectsAreEqualValues(golden, collected)
+	}, testutil.WaitMedium, testutil.IntervalFast, "template insights are missing")
+
+	// We got our latency metrics, close the connection.
+	_ = sess.Close()
+	_ = sshConn.Close()
+
+	require.EqualValues(t, golden, collected)
+}

coderd/prometheusmetrics/insights/testdata/insights-metrics.json

@@ -0,0 +1,3 @@
+{
+  "coderd_insights_templates_active_users": 1
+}