chore: add aibridge data to telemetry (#20449)

- Adds a new table to keep track of which payloads have already been
reported since we only report for the last clock hour
- Adds a query to gather and aggregate all the data by
provider/model/client

Relates to https://github.com/coder/coder-telemetry-server/issues/27

coderd/database/dbpurge/dbpurge.go

@@ -24,6 +24,12 @@ const (
 	// but we won't touch the `connection_logs` table.
 	maxAuditLogConnectionEventAge    = 90 * 24 * time.Hour // 90 days
 	auditLogConnectionEventBatchSize = 1000
+	// Telemetry heartbeats are used to deduplicate events across replicas. We
+	// don't need to persist heartbeat rows for longer than 24 hours, as they
+	// are only used for deduplication across replicas. The time needs to be
+	// long enough to cover the maximum interval of a heartbeat event (currently
+	// 1 hour) plus some buffer.
+	maxTelemetryHeartbeatAge = 24 * time.Hour
 )
 
 // New creates a new periodically purging database instance.
@@ -71,6 +77,10 @@ func New(ctx context.Context, logger slog.Logger, db database.Store, clk quartz.
 			if err := tx.ExpirePrebuildsAPIKeys(ctx, dbtime.Time(start)); err != nil {
 				return xerrors.Errorf("failed to expire prebuilds user api keys: %w", err)
 			}
+			deleteOldTelemetryLocksBefore := start.Add(-maxTelemetryHeartbeatAge)
+			if err := tx.DeleteOldTelemetryLocks(ctx, deleteOldTelemetryLocksBefore); err != nil {
+				return xerrors.Errorf("failed to delete old telemetry locks: %w", err)
+			}
 
 			deleteOldAuditLogConnectionEventsBefore := start.Add(-maxAuditLogConnectionEventAge)
 			if err := tx.DeleteOldAuditLogConnectionEvents(ctx, database.DeleteOldAuditLogConnectionEventsParams{

coderd/database/dbpurge/dbpurge_test.go

@@ -704,3 +704,56 @@ func TestExpireOldAPIKeys(t *testing.T) {
 	// Out of an abundance of caution, we do not expire explicitly named prebuilds API keys.
 	assertKeyActive(namedPrebuildsAPIKey.ID)
 }
+
+func TestDeleteOldTelemetryHeartbeats(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.Context(t, testutil.WaitLong)
+
+	db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t, dbtestutil.WithDumpOnFailure())
+	logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
+	clk := quartz.NewMock(t)
+	now := clk.Now().UTC()
+
+	// Insert telemetry heartbeats.
+	err := db.InsertTelemetryLock(ctx, database.InsertTelemetryLockParams{
+		EventType:      "aibridge_interceptions_summary",
+		PeriodEndingAt: now.Add(-25 * time.Hour), // should be purged
+	})
+	require.NoError(t, err)
+	err = db.InsertTelemetryLock(ctx, database.InsertTelemetryLockParams{
+		EventType:      "aibridge_interceptions_summary",
+		PeriodEndingAt: now.Add(-23 * time.Hour), // should be kept
+	})
+	require.NoError(t, err)
+	err = db.InsertTelemetryLock(ctx, database.InsertTelemetryLockParams{
+		EventType:      "aibridge_interceptions_summary",
+		PeriodEndingAt: now, // should be kept
+	})
+	require.NoError(t, err)
+
+	done := awaitDoTick(ctx, t, clk)
+	closer := dbpurge.New(ctx, logger, db, clk)
+	defer closer.Close()
+	<-done // doTick() has now run.
+
+	require.Eventuallyf(t, func() bool {
+		// We use an SQL queries directly here because we don't expose queries
+		// for deleting heartbeats in the application code.
+		var totalCount int
+		err := sqlDB.QueryRowContext(ctx, `
+			SELECT COUNT(*) FROM telemetry_locks;
+		`).Scan(&totalCount)
+		assert.NoError(t, err)
+
+		var oldCount int
+		err = sqlDB.QueryRowContext(ctx, `
+			SELECT COUNT(*) FROM telemetry_locks WHERE period_ending_at < $1;
+		`, now.Add(-24*time.Hour)).Scan(&oldCount)
+		assert.NoError(t, err)
+
+		// Expect 2 heartbeats remaining and none older than 24 hours.
+		t.Logf("eventually: total count: %d, old count: %d", totalCount, oldCount)
+		return totalCount == 2 && oldCount == 0
+	}, testutil.WaitShort, testutil.IntervalFast, "it should delete old telemetry heartbeats")
+}