fix(stats): include current exact snapshots

This commit is contained in:
saltbo
2026-07-22 01:39:34 -04:00
parent 42e2a9b707
commit e8a8232f46
4 changed files with 96 additions and 19 deletions
+22 -11
View File
@@ -32,7 +32,8 @@ export interface HourlyMetricRow {
export class AdminStatsHourlyReader {
private readonly queryFrom: Date
private readonly queryTo: Date
private readonly counterQueryTo: Date
private readonly snapshotQueryTo: Date
private readonly metricRows = new Map<string, Promise<HourlyMetricRow[]>>()
private readonly markerRowsPromises = new Map<AdminStatsRollupScope, Promise<CompatibleMarkerRow[]>>()
@@ -46,7 +47,9 @@ export class AdminStatsHourlyReader {
throw new Error('stats_range_must_align_to_utc_hours')
}
this.queryFrom = range.from
this.queryTo = new Date(Math.min(rangeToExclusive, floorHour(now).getTime()))
const currentHourStart = floorHour(now).getTime()
this.counterQueryTo = new Date(Math.min(rangeToExclusive, currentHourStart))
this.snapshotQueryTo = new Date(Math.min(rangeToExclusive, currentHourStart + HOUR_MS))
}
async rows(
@@ -76,7 +79,7 @@ export class AdminStatsHourlyReader {
async topSpaceUsage(
options: { limit?: number; personalOnly?: boolean } = {},
): Promise<Array<{ orgId: string; usedBytes: number; quotaBytes: number }>> {
if (this.queryFrom >= this.queryTo) return []
if (this.queryFrom >= this.snapshotQueryTo) return []
const limit = options.limit ?? DASHBOARD_RANKING_LIMIT
const personalFilter = options.personalOnly
? sql`AND (org.slug LIKE 'personal-%' OR (json_valid(org.metadata) = 1 AND json_extract(org.metadata, '$.type') = 'personal'))`
@@ -90,7 +93,7 @@ export class AdminStatsHourlyReader {
AND dimension_key = ''
AND dimension_value = ''
AND bucket_start >= ${this.queryFrom.getTime()}
AND bucket_start < ${this.queryTo.getTime()}
AND bucket_start < ${this.snapshotQueryTo.getTime()}
AND CASE WHEN json_valid(metadata) = 1 THEN json_extract(metadata, '$.version') END = ${ROLLUP_VERSION}
AND CASE WHEN json_valid(metadata) = 1 THEN json_extract(metadata, '$.scope') END IN ('snapshots', 'full')
AND CASE WHEN json_valid(metadata) = 1 THEN
@@ -129,7 +132,8 @@ export class AdminStatsHourlyReader {
}
async coverage(requiredScope: AdminStatsRollupScope = 'full'): Promise<AdminStatsCoverage> {
const expectedBuckets = Math.max(0, Math.floor((this.queryTo.getTime() - this.queryFrom.getTime()) / HOUR_MS))
const queryTo = this.queryTo(requiredScope)
const expectedBuckets = Math.max(0, Math.floor((queryTo.getTime() - this.queryFrom.getTime()) / HOUR_MS))
const markerRows = await this.markerRows(requiredScope)
const completedBuckets = markerRows.length
const lowerBoundBuckets = markerRows.filter(
@@ -151,9 +155,10 @@ export class AdminStatsHourlyReader {
async completeDayKeys(requiredScope: AdminStatsRollupScope): Promise<Set<string>> {
const markerBuckets = await this.markerBuckets(requiredScope)
const queryTo = this.queryTo(requiredScope)
const expectedByDay = new Map<string, number>()
const completedByDay = new Map<string, number>()
for (let at = this.queryFrom.getTime(); at < this.queryTo.getTime(); at += HOUR_MS) {
for (let at = this.queryFrom.getTime(); at < queryTo.getTime(); at += HOUR_MS) {
const day = this.dayKey(new Date(at))
expectedByDay.set(day, (expectedByDay.get(day) ?? 0) + 1)
if (markerBuckets.has(at)) completedByDay.set(day, (completedByDay.get(day) ?? 0) + 1)
@@ -166,7 +171,7 @@ export class AdminStatsHourlyReader {
}
endExclusive(): Date {
return this.queryTo
return this.counterQueryTo
}
dayKey(date: Date): string {
@@ -177,8 +182,9 @@ export class AdminStatsHourlyReader {
metric: AdminStatsMetric,
dimensionKeys: readonly (AdminStatsDimension | '')[],
): Promise<HourlyMetricRow[]> {
if (this.queryFrom >= this.queryTo) return []
const requiredScope = metricDefinition(metric).kind === 'gauge' ? 'snapshots' : 'counters'
const queryTo = this.queryTo(requiredScope)
if (this.queryFrom >= queryTo) return []
const markerBuckets = await this.markerBuckets(requiredScope)
if (markerBuckets.size === 0) return []
const rows = await this.db
@@ -198,7 +204,7 @@ export class AdminStatsHourlyReader {
eq(statsRollupsHourly.metricKey, metric),
inArray(statsRollupsHourly.dimensionKey, dimensionKeys),
gte(statsRollupsHourly.bucketStart, this.queryFrom),
lt(statsRollupsHourly.bucketStart, this.queryTo),
lt(statsRollupsHourly.bucketStart, queryTo),
sql`CASE WHEN json_valid(${statsRollupsHourly.metadata}) = 1 THEN json_extract(${statsRollupsHourly.metadata}, '$.version') END = ${ROLLUP_VERSION}`,
requiredScope === 'snapshots'
? sql`CASE WHEN json_valid(${statsRollupsHourly.metadata}) = 1 THEN json_extract(${statsRollupsHourly.metadata}, '$.scope') END IN ('snapshots', 'full')`
@@ -236,7 +242,8 @@ export class AdminStatsHourlyReader {
}
private async loadMarkerRows(requiredScope: AdminStatsRollupScope): Promise<CompatibleMarkerRow[]> {
if (this.queryFrom >= this.queryTo) return []
const queryTo = this.queryTo(requiredScope)
if (this.queryFrom >= queryTo) return []
const rows = await this.db
.select({ bucketStart: statsRollupsHourly.bucketStart, metadata: statsRollupsHourly.metadata })
.from(statsRollupsHourly)
@@ -246,7 +253,7 @@ export class AdminStatsHourlyReader {
eq(statsRollupsHourly.orgId, ''),
eq(statsRollupsHourly.dimensionKey, ''),
gte(statsRollupsHourly.bucketStart, this.queryFrom),
lt(statsRollupsHourly.bucketStart, this.queryTo),
lt(statsRollupsHourly.bucketStart, queryTo),
),
)
return rows.flatMap((row) => {
@@ -258,6 +265,10 @@ export class AdminStatsHourlyReader {
: []
})
}
private queryTo(requiredScope: AdminStatsRollupScope): Date {
return requiredScope === 'snapshots' ? this.snapshotQueryTo : this.counterQueryTo
}
}
type CompatibleMarkerRow = { bucketStart: Date; metadata: AdminStatsRollupMetadata }
@@ -559,7 +559,7 @@ describe('admin hourly stats rollup', () => {
expect(await completeReader.completeDayKeys('counters')).toEqual(new Set(['2026-07-10']))
})
it('never exposes the current open hour, even if rollup rows already exist', async () => {
it('reads exact current-hour snapshots without exposing current-hour counters', async () => {
const { db } = await createTestApp()
const bucketStart = Date.parse('2026-07-10T10:00:00.000Z')
await db.run(sql`
@@ -570,6 +570,8 @@ describe('admin hourly stats rollup', () => {
('current-hour-marker', ${bucketStart}, '', 'stats.rollup_run', '', '', 1, 0, 0,
'{"version":3,"scope":"full","quality":"exact"}', ${bucketStart}),
('current-hour-rollup', ${bucketStart}, '', 'transfer.upload', '', '', 1, 42, 0,
'{"version":3,"scope":"full","quality":"exact"}', ${bucketStart}),
('current-hour-snapshot', ${bucketStart}, '', 'storage.used', '', '', 0, 99, 0,
'{"version":3,"scope":"full","quality":"exact"}', ${bucketStart})
`)
const reader = new AdminStatsHourlyReader(
@@ -583,6 +585,12 @@ describe('admin hourly stats rollup', () => {
)
expect(await reader.rows(M.transferUpload)).toEqual([])
expect(await reader.rows(M.storageUsed)).toEqual([expect.objectContaining({ bytes: 99 })])
expect(await reader.coverage()).toMatchObject({ status: 'empty', expectedBuckets: 0, completedBuckets: 0 })
expect(await reader.coverage('snapshots')).toMatchObject({
status: 'complete',
expectedBuckets: 1,
completedBuckets: 1,
})
})
})
+7 -7
View File
@@ -62,7 +62,7 @@ async function getOverviewStatistics(
range: AdminStatsDateRange,
): Promise<AdminOverviewStatistics> {
const effective = effectiveRange(range, now)
const reader = new AdminStatsHourlyReader(db, effective, now)
const reader = new AdminStatsHourlyReader(db, range, now)
const [
inventory,
active,
@@ -251,7 +251,7 @@ async function getDashboardOverviewStats(
): Promise<AdminDashboardOverviewStats> {
const effective = effectiveRange(range, now)
const previous = previousRange(effective)
const reader = new AdminStatsHourlyReader(db, effective, now)
const reader = new AdminStatsHourlyReader(db, range, now)
const previousReader = new AdminStatsHourlyReader(db, previous, now)
const [
users,
@@ -387,7 +387,7 @@ async function getDashboardOperationsStats(
range: AdminStatsDateRange,
): Promise<AdminDashboardOperationsStats> {
const effective = effectiveRange(range, now)
const reader = new AdminStatsHourlyReader(db, effective, now)
const reader = new AdminStatsHourlyReader(db, range, now)
const [
activeBackgroundJobs,
activeRemoteDownloads,
@@ -463,7 +463,7 @@ async function getDashboardGrowthStats(
): Promise<AdminDashboardGrowthStats> {
const effective = effectiveRange(range, now)
const previous = previousRange(effective)
const reader = new AdminStatsHourlyReader(db, effective, now)
const reader = new AdminStatsHourlyReader(db, range, now)
const previousReader = new AdminStatsHourlyReader(db, previous, now)
const [
users,
@@ -544,7 +544,7 @@ async function getDashboardStorageStats(
): Promise<AdminDashboardStorageStats> {
const effective = effectiveRange(range, now)
const previous = previousRange(effective)
const reader = new AdminStatsHourlyReader(db, effective, now)
const reader = new AdminStatsHourlyReader(db, range, now)
const previousReader = new AdminStatsHourlyReader(db, previous, now)
const [
quotas,
@@ -661,7 +661,7 @@ async function getDashboardTrafficStats(
): Promise<AdminDashboardTrafficStats> {
const effective = effectiveRange(range, now)
const previous = previousRange(effective)
const reader = new AdminStatsHourlyReader(db, effective, now)
const reader = new AdminStatsHourlyReader(db, range, now)
const previousReader = new AdminStatsHourlyReader(db, previous, now)
const [
traffic,
@@ -826,7 +826,7 @@ async function getDashboardSharingStats(
): Promise<AdminDashboardSharingStats> {
const effective = effectiveRange(range, now)
const previous = previousRange(effective)
const reader = new AdminStatsHourlyReader(db, effective, now)
const reader = new AdminStatsHourlyReader(db, range, now)
const previousReader = new AdminStatsHourlyReader(db, previous, now)
const [
sharing,
@@ -324,6 +324,64 @@ describe('site stats routes', () => {
expect(sharing.typeBreakdown).toEqual([])
})
it('publishes exact current-hour snapshots without treating open counters as complete', async () => {
const { db } = await createTestApp()
const bucketStart = Date.UTC(2026, 6, 22, 10)
const now = new Date(bucketStart + 30 * 60_000)
const metadata = JSON.stringify({
version: 3,
scope: 'snapshots',
quality: 'exact',
snapshotObservedAt: now.toISOString(),
})
await db.run(sql`
INSERT INTO stats_rollups_hourly (
id, bucket_start, org_id, metric_key, dimension_key, dimension_value,
count, bytes, unique_count, metadata, updated_at
) VALUES
('current-snapshot-marker', ${bucketStart}, '', 'stats.rollup_run', '', '', 1, 0, 0,
${metadata}, ${now.getTime()}),
('current-user-inventory', ${bucketStart}, '', 'user.inventory', '', '', 42, 0, 0,
${metadata}, ${now.getTime()}),
('current-active-base', ${bucketStart}, '', 'user.active_snapshot', '', '', 7, 0, 7,
${metadata}, ${now.getTime()}),
('current-active-dau', ${bucketStart}, '', 'user.active_snapshot', 'window', 'dau', 3, 0, 3,
${metadata}, ${now.getTime()}),
('current-active-wau', ${bucketStart}, '', 'user.active_snapshot', 'window', 'wau', 5, 0, 5,
${metadata}, ${now.getTime()}),
('current-active-mau', ${bucketStart}, '', 'user.active_snapshot', 'window', 'mau', 7, 0, 7,
${metadata}, ${now.getTime()}),
('current-storage-used', ${bucketStart}, '', 'storage.used', '', '', 0, 4096, 0,
${metadata}, ${now.getTime()}),
('current-storage-quota', ${bucketStart}, '', 'storage.quota', '', '', 0, 8192, 0,
${metadata}, ${now.getTime()}),
('current-storage-quota-invalid', ${bucketStart}, '', 'storage.quota', 'status', 'invalid', 0, 0, 0,
${metadata}, ${now.getTime()})
`)
const range = {
from: new Date(Date.UTC(2026, 6, 22)),
to: new Date(Date.UTC(2026, 6, 23) - 1),
timeZone: 'UTC' as const,
}
const repo = createAdminStatsRepo(db)
const [overview, growth, storage] = await Promise.all([
repo.getOverviewStatistics(now, range),
repo.getDashboardGrowthStats(now, range),
repo.getDashboardStorageStats(now, range),
])
expect(overview.users.trend).toEqual([{ date: '2026-07-22', totalUsers: 42, activeUsers: 7, newUsers: null }])
expect(growth.summary.totalUsers).toBe(42)
expect(growth.summary.activeUsers.value).toBe(7)
expect(growth.summary.newUsers.value).toBeNull()
expect(growth.userScaleTrend).toEqual([{ date: '2026-07-22', newUsers: null, totalUsers: 42 }])
expect(growth.activeUserTrend).toEqual([{ date: '2026-07-22', dau: 3, wau: 5, mau: 7 }])
expect(growth.coverage).toMatchObject({ status: 'empty', completedBuckets: 0, expectedBuckets: 10 })
expect(growth.snapshotCoverage).toMatchObject({ status: 'partial', completedBuckets: 1, expectedBuckets: 11 })
expect(storage.summary).toMatchObject({ storageUsedBytes: 4096, quotaBytes: 8192 })
})
it('hydrates completed-hour dashboard dimensions from rollups', async () => {
const { app, db } = await createTestApp()
const headers = await adminHeaders(app)