mirror of
https://github.com/saltbo/zpan.git
synced 2026-09-19 01:51:11 +08:00
fix(stats): include current exact snapshots
This commit is contained in:
@@ -32,7 +32,8 @@ export interface HourlyMetricRow {
|
||||
|
||||
export class AdminStatsHourlyReader {
|
||||
private readonly queryFrom: Date
|
||||
private readonly queryTo: Date
|
||||
private readonly counterQueryTo: Date
|
||||
private readonly snapshotQueryTo: Date
|
||||
private readonly metricRows = new Map<string, Promise<HourlyMetricRow[]>>()
|
||||
private readonly markerRowsPromises = new Map<AdminStatsRollupScope, Promise<CompatibleMarkerRow[]>>()
|
||||
|
||||
@@ -46,7 +47,9 @@ export class AdminStatsHourlyReader {
|
||||
throw new Error('stats_range_must_align_to_utc_hours')
|
||||
}
|
||||
this.queryFrom = range.from
|
||||
this.queryTo = new Date(Math.min(rangeToExclusive, floorHour(now).getTime()))
|
||||
const currentHourStart = floorHour(now).getTime()
|
||||
this.counterQueryTo = new Date(Math.min(rangeToExclusive, currentHourStart))
|
||||
this.snapshotQueryTo = new Date(Math.min(rangeToExclusive, currentHourStart + HOUR_MS))
|
||||
}
|
||||
|
||||
async rows(
|
||||
@@ -76,7 +79,7 @@ export class AdminStatsHourlyReader {
|
||||
async topSpaceUsage(
|
||||
options: { limit?: number; personalOnly?: boolean } = {},
|
||||
): Promise<Array<{ orgId: string; usedBytes: number; quotaBytes: number }>> {
|
||||
if (this.queryFrom >= this.queryTo) return []
|
||||
if (this.queryFrom >= this.snapshotQueryTo) return []
|
||||
const limit = options.limit ?? DASHBOARD_RANKING_LIMIT
|
||||
const personalFilter = options.personalOnly
|
||||
? sql`AND (org.slug LIKE 'personal-%' OR (json_valid(org.metadata) = 1 AND json_extract(org.metadata, '$.type') = 'personal'))`
|
||||
@@ -90,7 +93,7 @@ export class AdminStatsHourlyReader {
|
||||
AND dimension_key = ''
|
||||
AND dimension_value = ''
|
||||
AND bucket_start >= ${this.queryFrom.getTime()}
|
||||
AND bucket_start < ${this.queryTo.getTime()}
|
||||
AND bucket_start < ${this.snapshotQueryTo.getTime()}
|
||||
AND CASE WHEN json_valid(metadata) = 1 THEN json_extract(metadata, '$.version') END = ${ROLLUP_VERSION}
|
||||
AND CASE WHEN json_valid(metadata) = 1 THEN json_extract(metadata, '$.scope') END IN ('snapshots', 'full')
|
||||
AND CASE WHEN json_valid(metadata) = 1 THEN
|
||||
@@ -129,7 +132,8 @@ export class AdminStatsHourlyReader {
|
||||
}
|
||||
|
||||
async coverage(requiredScope: AdminStatsRollupScope = 'full'): Promise<AdminStatsCoverage> {
|
||||
const expectedBuckets = Math.max(0, Math.floor((this.queryTo.getTime() - this.queryFrom.getTime()) / HOUR_MS))
|
||||
const queryTo = this.queryTo(requiredScope)
|
||||
const expectedBuckets = Math.max(0, Math.floor((queryTo.getTime() - this.queryFrom.getTime()) / HOUR_MS))
|
||||
const markerRows = await this.markerRows(requiredScope)
|
||||
const completedBuckets = markerRows.length
|
||||
const lowerBoundBuckets = markerRows.filter(
|
||||
@@ -151,9 +155,10 @@ export class AdminStatsHourlyReader {
|
||||
|
||||
async completeDayKeys(requiredScope: AdminStatsRollupScope): Promise<Set<string>> {
|
||||
const markerBuckets = await this.markerBuckets(requiredScope)
|
||||
const queryTo = this.queryTo(requiredScope)
|
||||
const expectedByDay = new Map<string, number>()
|
||||
const completedByDay = new Map<string, number>()
|
||||
for (let at = this.queryFrom.getTime(); at < this.queryTo.getTime(); at += HOUR_MS) {
|
||||
for (let at = this.queryFrom.getTime(); at < queryTo.getTime(); at += HOUR_MS) {
|
||||
const day = this.dayKey(new Date(at))
|
||||
expectedByDay.set(day, (expectedByDay.get(day) ?? 0) + 1)
|
||||
if (markerBuckets.has(at)) completedByDay.set(day, (completedByDay.get(day) ?? 0) + 1)
|
||||
@@ -166,7 +171,7 @@ export class AdminStatsHourlyReader {
|
||||
}
|
||||
|
||||
endExclusive(): Date {
|
||||
return this.queryTo
|
||||
return this.counterQueryTo
|
||||
}
|
||||
|
||||
dayKey(date: Date): string {
|
||||
@@ -177,8 +182,9 @@ export class AdminStatsHourlyReader {
|
||||
metric: AdminStatsMetric,
|
||||
dimensionKeys: readonly (AdminStatsDimension | '')[],
|
||||
): Promise<HourlyMetricRow[]> {
|
||||
if (this.queryFrom >= this.queryTo) return []
|
||||
const requiredScope = metricDefinition(metric).kind === 'gauge' ? 'snapshots' : 'counters'
|
||||
const queryTo = this.queryTo(requiredScope)
|
||||
if (this.queryFrom >= queryTo) return []
|
||||
const markerBuckets = await this.markerBuckets(requiredScope)
|
||||
if (markerBuckets.size === 0) return []
|
||||
const rows = await this.db
|
||||
@@ -198,7 +204,7 @@ export class AdminStatsHourlyReader {
|
||||
eq(statsRollupsHourly.metricKey, metric),
|
||||
inArray(statsRollupsHourly.dimensionKey, dimensionKeys),
|
||||
gte(statsRollupsHourly.bucketStart, this.queryFrom),
|
||||
lt(statsRollupsHourly.bucketStart, this.queryTo),
|
||||
lt(statsRollupsHourly.bucketStart, queryTo),
|
||||
sql`CASE WHEN json_valid(${statsRollupsHourly.metadata}) = 1 THEN json_extract(${statsRollupsHourly.metadata}, '$.version') END = ${ROLLUP_VERSION}`,
|
||||
requiredScope === 'snapshots'
|
||||
? sql`CASE WHEN json_valid(${statsRollupsHourly.metadata}) = 1 THEN json_extract(${statsRollupsHourly.metadata}, '$.scope') END IN ('snapshots', 'full')`
|
||||
@@ -236,7 +242,8 @@ export class AdminStatsHourlyReader {
|
||||
}
|
||||
|
||||
private async loadMarkerRows(requiredScope: AdminStatsRollupScope): Promise<CompatibleMarkerRow[]> {
|
||||
if (this.queryFrom >= this.queryTo) return []
|
||||
const queryTo = this.queryTo(requiredScope)
|
||||
if (this.queryFrom >= queryTo) return []
|
||||
const rows = await this.db
|
||||
.select({ bucketStart: statsRollupsHourly.bucketStart, metadata: statsRollupsHourly.metadata })
|
||||
.from(statsRollupsHourly)
|
||||
@@ -246,7 +253,7 @@ export class AdminStatsHourlyReader {
|
||||
eq(statsRollupsHourly.orgId, ''),
|
||||
eq(statsRollupsHourly.dimensionKey, ''),
|
||||
gte(statsRollupsHourly.bucketStart, this.queryFrom),
|
||||
lt(statsRollupsHourly.bucketStart, this.queryTo),
|
||||
lt(statsRollupsHourly.bucketStart, queryTo),
|
||||
),
|
||||
)
|
||||
return rows.flatMap((row) => {
|
||||
@@ -258,6 +265,10 @@ export class AdminStatsHourlyReader {
|
||||
: []
|
||||
})
|
||||
}
|
||||
|
||||
private queryTo(requiredScope: AdminStatsRollupScope): Date {
|
||||
return requiredScope === 'snapshots' ? this.snapshotQueryTo : this.counterQueryTo
|
||||
}
|
||||
}
|
||||
|
||||
type CompatibleMarkerRow = { bucketStart: Date; metadata: AdminStatsRollupMetadata }
|
||||
|
||||
@@ -559,7 +559,7 @@ describe('admin hourly stats rollup', () => {
|
||||
expect(await completeReader.completeDayKeys('counters')).toEqual(new Set(['2026-07-10']))
|
||||
})
|
||||
|
||||
it('never exposes the current open hour, even if rollup rows already exist', async () => {
|
||||
it('reads exact current-hour snapshots without exposing current-hour counters', async () => {
|
||||
const { db } = await createTestApp()
|
||||
const bucketStart = Date.parse('2026-07-10T10:00:00.000Z')
|
||||
await db.run(sql`
|
||||
@@ -570,6 +570,8 @@ describe('admin hourly stats rollup', () => {
|
||||
('current-hour-marker', ${bucketStart}, '', 'stats.rollup_run', '', '', 1, 0, 0,
|
||||
'{"version":3,"scope":"full","quality":"exact"}', ${bucketStart}),
|
||||
('current-hour-rollup', ${bucketStart}, '', 'transfer.upload', '', '', 1, 42, 0,
|
||||
'{"version":3,"scope":"full","quality":"exact"}', ${bucketStart}),
|
||||
('current-hour-snapshot', ${bucketStart}, '', 'storage.used', '', '', 0, 99, 0,
|
||||
'{"version":3,"scope":"full","quality":"exact"}', ${bucketStart})
|
||||
`)
|
||||
const reader = new AdminStatsHourlyReader(
|
||||
@@ -583,6 +585,12 @@ describe('admin hourly stats rollup', () => {
|
||||
)
|
||||
|
||||
expect(await reader.rows(M.transferUpload)).toEqual([])
|
||||
expect(await reader.rows(M.storageUsed)).toEqual([expect.objectContaining({ bytes: 99 })])
|
||||
expect(await reader.coverage()).toMatchObject({ status: 'empty', expectedBuckets: 0, completedBuckets: 0 })
|
||||
expect(await reader.coverage('snapshots')).toMatchObject({
|
||||
status: 'complete',
|
||||
expectedBuckets: 1,
|
||||
completedBuckets: 1,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -62,7 +62,7 @@ async function getOverviewStatistics(
|
||||
range: AdminStatsDateRange,
|
||||
): Promise<AdminOverviewStatistics> {
|
||||
const effective = effectiveRange(range, now)
|
||||
const reader = new AdminStatsHourlyReader(db, effective, now)
|
||||
const reader = new AdminStatsHourlyReader(db, range, now)
|
||||
const [
|
||||
inventory,
|
||||
active,
|
||||
@@ -251,7 +251,7 @@ async function getDashboardOverviewStats(
|
||||
): Promise<AdminDashboardOverviewStats> {
|
||||
const effective = effectiveRange(range, now)
|
||||
const previous = previousRange(effective)
|
||||
const reader = new AdminStatsHourlyReader(db, effective, now)
|
||||
const reader = new AdminStatsHourlyReader(db, range, now)
|
||||
const previousReader = new AdminStatsHourlyReader(db, previous, now)
|
||||
const [
|
||||
users,
|
||||
@@ -387,7 +387,7 @@ async function getDashboardOperationsStats(
|
||||
range: AdminStatsDateRange,
|
||||
): Promise<AdminDashboardOperationsStats> {
|
||||
const effective = effectiveRange(range, now)
|
||||
const reader = new AdminStatsHourlyReader(db, effective, now)
|
||||
const reader = new AdminStatsHourlyReader(db, range, now)
|
||||
const [
|
||||
activeBackgroundJobs,
|
||||
activeRemoteDownloads,
|
||||
@@ -463,7 +463,7 @@ async function getDashboardGrowthStats(
|
||||
): Promise<AdminDashboardGrowthStats> {
|
||||
const effective = effectiveRange(range, now)
|
||||
const previous = previousRange(effective)
|
||||
const reader = new AdminStatsHourlyReader(db, effective, now)
|
||||
const reader = new AdminStatsHourlyReader(db, range, now)
|
||||
const previousReader = new AdminStatsHourlyReader(db, previous, now)
|
||||
const [
|
||||
users,
|
||||
@@ -544,7 +544,7 @@ async function getDashboardStorageStats(
|
||||
): Promise<AdminDashboardStorageStats> {
|
||||
const effective = effectiveRange(range, now)
|
||||
const previous = previousRange(effective)
|
||||
const reader = new AdminStatsHourlyReader(db, effective, now)
|
||||
const reader = new AdminStatsHourlyReader(db, range, now)
|
||||
const previousReader = new AdminStatsHourlyReader(db, previous, now)
|
||||
const [
|
||||
quotas,
|
||||
@@ -661,7 +661,7 @@ async function getDashboardTrafficStats(
|
||||
): Promise<AdminDashboardTrafficStats> {
|
||||
const effective = effectiveRange(range, now)
|
||||
const previous = previousRange(effective)
|
||||
const reader = new AdminStatsHourlyReader(db, effective, now)
|
||||
const reader = new AdminStatsHourlyReader(db, range, now)
|
||||
const previousReader = new AdminStatsHourlyReader(db, previous, now)
|
||||
const [
|
||||
traffic,
|
||||
@@ -826,7 +826,7 @@ async function getDashboardSharingStats(
|
||||
): Promise<AdminDashboardSharingStats> {
|
||||
const effective = effectiveRange(range, now)
|
||||
const previous = previousRange(effective)
|
||||
const reader = new AdminStatsHourlyReader(db, effective, now)
|
||||
const reader = new AdminStatsHourlyReader(db, range, now)
|
||||
const previousReader = new AdminStatsHourlyReader(db, previous, now)
|
||||
const [
|
||||
sharing,
|
||||
|
||||
@@ -324,6 +324,64 @@ describe('site stats routes', () => {
|
||||
expect(sharing.typeBreakdown).toEqual([])
|
||||
})
|
||||
|
||||
it('publishes exact current-hour snapshots without treating open counters as complete', async () => {
|
||||
const { db } = await createTestApp()
|
||||
const bucketStart = Date.UTC(2026, 6, 22, 10)
|
||||
const now = new Date(bucketStart + 30 * 60_000)
|
||||
const metadata = JSON.stringify({
|
||||
version: 3,
|
||||
scope: 'snapshots',
|
||||
quality: 'exact',
|
||||
snapshotObservedAt: now.toISOString(),
|
||||
})
|
||||
await db.run(sql`
|
||||
INSERT INTO stats_rollups_hourly (
|
||||
id, bucket_start, org_id, metric_key, dimension_key, dimension_value,
|
||||
count, bytes, unique_count, metadata, updated_at
|
||||
) VALUES
|
||||
('current-snapshot-marker', ${bucketStart}, '', 'stats.rollup_run', '', '', 1, 0, 0,
|
||||
${metadata}, ${now.getTime()}),
|
||||
('current-user-inventory', ${bucketStart}, '', 'user.inventory', '', '', 42, 0, 0,
|
||||
${metadata}, ${now.getTime()}),
|
||||
('current-active-base', ${bucketStart}, '', 'user.active_snapshot', '', '', 7, 0, 7,
|
||||
${metadata}, ${now.getTime()}),
|
||||
('current-active-dau', ${bucketStart}, '', 'user.active_snapshot', 'window', 'dau', 3, 0, 3,
|
||||
${metadata}, ${now.getTime()}),
|
||||
('current-active-wau', ${bucketStart}, '', 'user.active_snapshot', 'window', 'wau', 5, 0, 5,
|
||||
${metadata}, ${now.getTime()}),
|
||||
('current-active-mau', ${bucketStart}, '', 'user.active_snapshot', 'window', 'mau', 7, 0, 7,
|
||||
${metadata}, ${now.getTime()}),
|
||||
('current-storage-used', ${bucketStart}, '', 'storage.used', '', '', 0, 4096, 0,
|
||||
${metadata}, ${now.getTime()}),
|
||||
('current-storage-quota', ${bucketStart}, '', 'storage.quota', '', '', 0, 8192, 0,
|
||||
${metadata}, ${now.getTime()}),
|
||||
('current-storage-quota-invalid', ${bucketStart}, '', 'storage.quota', 'status', 'invalid', 0, 0, 0,
|
||||
${metadata}, ${now.getTime()})
|
||||
`)
|
||||
const range = {
|
||||
from: new Date(Date.UTC(2026, 6, 22)),
|
||||
to: new Date(Date.UTC(2026, 6, 23) - 1),
|
||||
timeZone: 'UTC' as const,
|
||||
}
|
||||
const repo = createAdminStatsRepo(db)
|
||||
|
||||
const [overview, growth, storage] = await Promise.all([
|
||||
repo.getOverviewStatistics(now, range),
|
||||
repo.getDashboardGrowthStats(now, range),
|
||||
repo.getDashboardStorageStats(now, range),
|
||||
])
|
||||
|
||||
expect(overview.users.trend).toEqual([{ date: '2026-07-22', totalUsers: 42, activeUsers: 7, newUsers: null }])
|
||||
expect(growth.summary.totalUsers).toBe(42)
|
||||
expect(growth.summary.activeUsers.value).toBe(7)
|
||||
expect(growth.summary.newUsers.value).toBeNull()
|
||||
expect(growth.userScaleTrend).toEqual([{ date: '2026-07-22', newUsers: null, totalUsers: 42 }])
|
||||
expect(growth.activeUserTrend).toEqual([{ date: '2026-07-22', dau: 3, wau: 5, mau: 7 }])
|
||||
expect(growth.coverage).toMatchObject({ status: 'empty', completedBuckets: 0, expectedBuckets: 10 })
|
||||
expect(growth.snapshotCoverage).toMatchObject({ status: 'partial', completedBuckets: 1, expectedBuckets: 11 })
|
||||
expect(storage.summary).toMatchObject({ storageUsedBytes: 4096, quotaBytes: 8192 })
|
||||
})
|
||||
|
||||
it('hydrates completed-hour dashboard dimensions from rollups', async () => {
|
||||
const { app, db } = await createTestApp()
|
||||
const headers = await adminHeaders(app)
|
||||
|
||||
Reference in New Issue
Block a user