fix: handle boundary usage across snapshots and flush races (#21805)

Previously there were two issues that could cause incorrect boundary
usage telemetry data.

1. Bad handling across snapshot intervals: After telemetry snapshot deleted
the DB row, the next flush would INSERT the stale cumulative data (which
included already-reported usage). This would then be overwritten by
subsequent UPDATE flushes, causing the delta between the last snapshot
and the reset to be lost (under-reporting usage). Additionally, if there
was no new usage after the reset, the tracker would carry over all usage
from the previous period into the next period (over-reporting usage).

2. Missed usage from a race condition: Track() calls between the first
mutex unlock and second mutex lock in FlushToDB() were lost. The data
wasn't included in the current flush (already snapshotted) and was wiped
by the subsequent reset. This is likely low impact to overall usage
numbers in the real world.

Fix by tracking unique workspace/user deltas separately from cumulative
values and always tracking delta allowed/denied requests. Deltas are used
for INSERT (fresh start after reset), cumulative for UPDATE (accurate unique
counts within a period). All counters reset atomically before the DB operation
so Track() calls during the operation are preserved for the next flush.

coderd/database/querier.go

@@ -762,9 +762,10 @@ type sqlcQuerier interface {
 	UpsertAnnouncementBanners(ctx context.Context, value string) error
 	UpsertAppSecurityKey(ctx context.Context, value string) error
 	UpsertApplicationName(ctx context.Context, value string) error
-	// Upserts boundary usage statistics for a replica. All values are replaced with
-	// the current in-memory state. Returns true if this was an insert (new period),
-	// false if update.
+	// Upserts boundary usage statistics for a replica. On INSERT (new period), uses
+	// delta values for unique counts (only data since last flush). On UPDATE, uses
+	// cumulative values for unique counts (accurate period totals). Request counts
+	// are always deltas, accumulated in DB. Returns true if insert, false if update.
 	UpsertBoundaryUsageStats(ctx context.Context, arg UpsertBoundaryUsageStatsParams) (bool, error)
 	UpsertConnectionLog(ctx context.Context, arg UpsertConnectionLogParams) (ConnectionLog, error)
 	UpsertCoordinatorResumeTokenSigningKey(ctx context.Context, value string) error

coderd/database/queries.sql.go

@@ -2051,32 +2051,37 @@ INSERT INTO boundary_usage_stats (
     NOW(),
     NOW()
 ) ON CONFLICT (replica_id) DO UPDATE SET
-    unique_workspaces_count = EXCLUDED.unique_workspaces_count,
-    unique_users_count = EXCLUDED.unique_users_count,
-    allowed_requests = EXCLUDED.allowed_requests,
-    denied_requests = EXCLUDED.denied_requests,
+    unique_workspaces_count = $6,
+    unique_users_count = $7,
+    allowed_requests = boundary_usage_stats.allowed_requests + EXCLUDED.allowed_requests,
+    denied_requests = boundary_usage_stats.denied_requests + EXCLUDED.denied_requests,
     updated_at = NOW()
 RETURNING (xmax = 0) AS new_period
 `
 
 type UpsertBoundaryUsageStatsParams struct {
 	ReplicaID             uuid.UUID `db:"replica_id" json:"replica_id"`
-	UniqueWorkspacesCount int64     `db:"unique_workspaces_count" json:"unique_workspaces_count"`
-	UniqueUsersCount      int64     `db:"unique_users_count" json:"unique_users_count"`
+	UniqueWorkspacesDelta int64     `db:"unique_workspaces_delta" json:"unique_workspaces_delta"`
+	UniqueUsersDelta      int64     `db:"unique_users_delta" json:"unique_users_delta"`
 	AllowedRequests       int64     `db:"allowed_requests" json:"allowed_requests"`
 	DeniedRequests        int64     `db:"denied_requests" json:"denied_requests"`
+	UniqueWorkspacesCount int64     `db:"unique_workspaces_count" json:"unique_workspaces_count"`
+	UniqueUsersCount      int64     `db:"unique_users_count" json:"unique_users_count"`
 }
 
-// Upserts boundary usage statistics for a replica. All values are replaced with
-// the current in-memory state. Returns true if this was an insert (new period),
-// false if update.
+// Upserts boundary usage statistics for a replica. On INSERT (new period), uses
+// delta values for unique counts (only data since last flush). On UPDATE, uses
+// cumulative values for unique counts (accurate period totals). Request counts
+// are always deltas, accumulated in DB. Returns true if insert, false if update.
 func (q *sqlQuerier) UpsertBoundaryUsageStats(ctx context.Context, arg UpsertBoundaryUsageStatsParams) (bool, error) {
 	row := q.db.QueryRowContext(ctx, upsertBoundaryUsageStats,
 		arg.ReplicaID,
-		arg.UniqueWorkspacesCount,
-		arg.UniqueUsersCount,
+		arg.UniqueWorkspacesDelta,
+		arg.UniqueUsersDelta,
 		arg.AllowedRequests,
 		arg.DeniedRequests,
+		arg.UniqueWorkspacesCount,
+		arg.UniqueUsersCount,
 	)
 	var new_period bool
 	err := row.Scan(&new_period)

coderd/database/queries/boundaryusagestats.sql

@@ -1,7 +1,8 @@
 -- name: UpsertBoundaryUsageStats :one
--- Upserts boundary usage statistics for a replica. All values are replaced with
--- the current in-memory state. Returns true if this was an insert (new period),
--- false if update.
+-- Upserts boundary usage statistics for a replica. On INSERT (new period), uses
+-- delta values for unique counts (only data since last flush). On UPDATE, uses
+-- cumulative values for unique counts (accurate period totals). Request counts
+-- are always deltas, accumulated in DB. Returns true if insert, false if update.
 INSERT INTO boundary_usage_stats (
     replica_id,
     unique_workspaces_count,
@@ -12,17 +13,17 @@ INSERT INTO boundary_usage_stats (
     updated_at
 ) VALUES (
     @replica_id,
-    @unique_workspaces_count,
-    @unique_users_count,
+    @unique_workspaces_delta,
+    @unique_users_delta,
     @allowed_requests,
     @denied_requests,
     NOW(),
     NOW()
 ) ON CONFLICT (replica_id) DO UPDATE SET
-    unique_workspaces_count = EXCLUDED.unique_workspaces_count,
-    unique_users_count = EXCLUDED.unique_users_count,
-    allowed_requests = EXCLUDED.allowed_requests,
-    denied_requests = EXCLUDED.denied_requests,
+    unique_workspaces_count = @unique_workspaces_count,
+    unique_users_count = @unique_users_count,
+    allowed_requests = boundary_usage_stats.allowed_requests + EXCLUDED.allowed_requests,
+    denied_requests = boundary_usage_stats.denied_requests + EXCLUDED.denied_requests,
     updated_at = NOW()
 RETURNING (xmax = 0) AS new_period;