diff --git a/docs/design/admin-analytics.md b/docs/design/admin-analytics.md index a7778b7d..f189c3d3 100644 --- a/docs/design/admin-analytics.md +++ b/docs/design/admin-analytics.md @@ -58,7 +58,7 @@ Request-time work is bounded and result-only. Charts and totals read only the ag - Upload bytes mean successfully confirmed upload bytes. - Download bytes and counts mean issued downloads, not client-completed transfers. -- Cloud traffic report status describes metering synchronization and belongs to Operations, not download success. +- Cloud traffic report status is a closed-hour snapshot of the current metering queue and belongs to Operations, not download success. Mutable report rows are never modeled as additive counters. - User, storage, share lifecycle, active job/task, downloader, report, and webhook state comes from closed-hour snapshots, not live request-time reads. - Storage inventory includes normal files and image-hosting objects. - Files older than 90 days are an age cohort, not proven cold data; the UI labels this explicitly. diff --git a/scripts/backfill-admin-stats.ts b/scripts/backfill-admin-stats.ts index f20823bf..b4d48fbc 100644 --- a/scripts/backfill-admin-stats.ts +++ b/scripts/backfill-admin-stats.ts @@ -52,8 +52,6 @@ interface ValidationSummary { rollupSharePasswordPasses: number rawFailedDownloads: number rollupFailedDownloads: number - rawTrafficReports: number - rollupTrafficReports: number rawFinishedDownloadTasks: number rollupFinishedDownloadTasks: number rawFinishedBackgroundJobs: number @@ -325,14 +323,6 @@ function buildHourlyBackfillSql(now: Date): string { activitySource('share.download_issued', "ae.action = 'share_download'", { share_id: "COALESCE(CASE WHEN json_valid(ae.metadata) = 1 THEN json_extract(ae.metadata, '$.shareId') END, ae.target_id)", kind: "CASE WHEN json_valid(ae.metadata) = 1 THEN json_extract(ae.metadata, '$.kind') END", source: "COALESCE(CASE WHEN json_valid(ae.metadata) = 1 THEN json_extract(ae.metadata, '$.source') END, 'landing_share')", actor_type: "COALESCE(ae.actor_type, CASE WHEN ae.user_id IS NULL THEN 'anonymous' ELSE 'user' END)" }, true), activitySource('share.saved', "ae.action = 'save_from_share'", { share_id: "COALESCE(CASE WHEN json_valid(ae.metadata) = 1 THEN json_extract(ae.metadata, '$.shareId') END, ae.target_id)", actor_type: "COALESCE(ae.actor_type, CASE WHEN ae.user_id IS NULL THEN 'anonymous' ELSE 'user' END)" }, true), activitySource('share.password_passed', "ae.action = 'share_password_passed'", { share_id: 'ae.target_id' }), - { - metric: 'traffic.report_sync', - source: 'cloud_traffic_reports ctr', - timestampMs: 'ctr.updated_at', - org: 'ctr.org_id', - bytes: 'SUM(ctr.bytes)', - dimensions: { source: 'ctr.source', status: 'ctr.status' }, - }, { metric: 'remote_download.task_finished', source: 'download_tasks dt', @@ -526,8 +516,6 @@ function statsHistoryStartSql(): string { 'share_download', 'object_download', 'image_hosting_download', 'webdav_download', 'download_failed', 'share_view', 'save_from_share', 'share_password_passed' )), ${missing}), - COALESCE((SELECT CAST(MIN(ctr.updated_at) / 3600000 AS INTEGER) * 3600000 - FROM cloud_traffic_reports ctr WHERE ctr.updated_at >= ${MIN_VALID_TIMESTAMP_MS}), ${missing}), COALESCE((SELECT CAST(MIN(dt.finished_at) / 3600000 AS INTEGER) * 3600000 FROM download_tasks dt WHERE dt.finished_at IS NOT NULL AND dt.finished_at >= ${MIN_VALID_TIMESTAMP_MS}), ${missing}), COALESCE((SELECT CAST(MIN(bj.finished_at) / 3600000 AS INTEGER) * 3600000 @@ -628,10 +616,6 @@ export function buildValidationSql(now = new Date()): string { WHERE action = 'download_failed' AND created_at >= ${MIN_VALID_TIMESTAMP_SECONDS} AND created_at * 1000 < ${currentHour} ), - 'rawTrafficReports', ( - SELECT COUNT(*) FROM cloud_traffic_reports - WHERE updated_at >= ${MIN_VALID_TIMESTAMP_MS} AND updated_at < ${currentHour} - ), 'rawFinishedDownloadTasks', ( SELECT COUNT(*) FROM download_tasks WHERE finished_at >= ${MIN_VALID_TIMESTAMP_MS} AND finished_at < ${currentHour} @@ -756,10 +740,6 @@ SELECT json_object( SELECT COALESCE(SUM(count), 0) FROM counter_rows WHERE metric_key = 'transfer.download_failed' AND dimension_key = '' ), - 'rollupTrafficReports', ( - SELECT COALESCE(SUM(count), 0) FROM counter_rows - WHERE metric_key = 'traffic.report_sync' AND dimension_key = '' - ), 'rollupFinishedDownloadTasks', ( SELECT COALESCE(SUM(count), 0) FROM counter_rows WHERE metric_key = 'remote_download.task_finished' AND dimension_key = '' @@ -998,7 +978,6 @@ function assertBackfillValidation(summary: ValidationSummary): void { ['share saves', summary.rawShareSaves, summary.rollupShareSaves], ['share password passes', summary.rawSharePasswordPasses, summary.rollupSharePasswordPasses], ['failed downloads', summary.rawFailedDownloads, summary.rollupFailedDownloads], - ['traffic reports', summary.rawTrafficReports, summary.rollupTrafficReports], ['finished download tasks', summary.rawFinishedDownloadTasks, summary.rollupFinishedDownloadTasks], ['finished background jobs', summary.rawFinishedBackgroundJobs, summary.rollupFinishedBackgroundJobs], ['missing byte events', summary.rawMissingByteEvents, summary.rollupMissingByteEvents], diff --git a/server/adapters/repos/admin-stats-rollup.integration.test.ts b/server/adapters/repos/admin-stats-rollup.integration.test.ts index e8011614..66cd9eb3 100644 --- a/server/adapters/repos/admin-stats-rollup.integration.test.ts +++ b/server/adapters/repos/admin-stats-rollup.integration.test.ts @@ -190,7 +190,7 @@ describe('admin hourly stats rollup', () => { expect(row(M.userSignup, '', '', '')).toMatchObject({ count: 2 }) expect(row(M.userSignup, 'provider', 'direct', '')).toMatchObject({ count: 1 }) expect(row(M.shareCreated)).toMatchObject({ count: 4 }) - expect(row(M.trafficReportSync)).toMatchObject({ count: 2, bytes: 300 }) + expect(rows.some((value) => value.metric === 'traffic.report_sync')).toBe(false) expect(row(M.remoteDownloadTaskFinished)).toMatchObject({ count: 1, bytes: 60 }) expect(row(M.backgroundJobFinished)).toMatchObject({ count: 1 }) expect(row(M.storageInventory)).toMatchObject({ count: 2, bytes: 500 }) diff --git a/server/adapters/repos/admin-stats-rollup.ts b/server/adapters/repos/admin-stats-rollup.ts index b7abea31..a231345b 100644 --- a/server/adapters/repos/admin-stats-rollup.ts +++ b/server/adapters/repos/admin-stats-rollup.ts @@ -36,7 +36,6 @@ const COUNTER_METRICS: AdminStatsMetric[] = [ M.shareView, M.statsMissingBytes, M.statsRollupRun, - M.trafficReportSync, M.transferDownloadFailed, M.transferDownloadIssued, M.transferUpload, @@ -269,18 +268,7 @@ async function addUserMetrics(db: Database, rollups: RollupAccumulator, from: Da } async function addOperationalMetrics(db: Database, rollups: RollupAccumulator, from: Date, to: Date): Promise { - const [trafficRows, taskFinishedRows, jobRows] = await Promise.all([ - db - .select({ - orgId: cloudTrafficReports.orgId, - source: cloudTrafficReports.source, - status: cloudTrafficReports.status, - count: sql`COUNT(*)`, - bytes: sql`COALESCE(SUM(${cloudTrafficReports.bytes}), 0)`, - }) - .from(cloudTrafficReports) - .where(and(gte(cloudTrafficReports.updatedAt, from), lt(cloudTrafficReports.updatedAt, to))) - .groupBy(cloudTrafficReports.orgId, cloudTrafficReports.source, cloudTrafficReports.status), + const [taskFinishedRows, jobRows] = await Promise.all([ db .select({ orgId: downloadTasks.orgId, @@ -305,12 +293,6 @@ async function addOperationalMetrics(db: Database, rollups: RollupAccumulator, f .groupBy(backgroundJobs.orgId, backgroundJobs.type, backgroundJobs.status), ]) - for (const row of trafficRows) { - rollups.add(M.trafficReportSync, row.orgId, Number(row.count), Number(row.bytes), { - source: row.source, - status: row.status, - }) - } for (const row of taskFinishedRows) { rollups.add(M.remoteDownloadTaskFinished, row.orgId, Number(row.count), Number(row.bytes), { category: row.category ?? 'uncategorized', diff --git a/server/adapters/repos/admin-stats.ts b/server/adapters/repos/admin-stats.ts index b58e6df8..bd3f7968 100644 --- a/server/adapters/repos/admin-stats.ts +++ b/server/adapters/repos/admin-stats.ts @@ -837,7 +837,7 @@ async function getActivityMetricDimensionTotalsFromRollup( } async function getCloudReportOutcomes(reader: AdminStatsHourlyReader): Promise> { - return getActivityMetricDimensionTotalsFromRollup(reader, ADMIN_STATS_METRICS.trafficReportSync, 'status', 'count') + return getLatestGaugeDimensions(reader, ADMIN_STATS_METRICS.trafficReportSnapshot, 'status') } async function getActivityMetricTotal( diff --git a/server/domain/admin-stats-metrics.ts b/server/domain/admin-stats-metrics.ts index 2d2bcc44..5f77f23a 100644 --- a/server/domain/admin-stats-metrics.ts +++ b/server/domain/admin-stats-metrics.ts @@ -52,7 +52,6 @@ export const ADMIN_STATS_METRICS = { storageInventory: 'storage.inventory', storageQuota: 'storage.quota', storageUsed: 'storage.used', - trafficReportSync: 'traffic.report_sync', trafficReportSnapshot: 'traffic.report_snapshot', transferDownloadFailed: 'transfer.download_failed', transferDownloadIssued: 'transfer.download_issued', @@ -91,7 +90,6 @@ export const ADMIN_STATS_METRIC_REGISTRY = { [M.storageInventory]: gauge(['age_bucket', 'file_type_group', 'size_bucket', 'storage_id'], 'entities', true), [M.storageQuota]: gauge(['status'], null, true), [M.storageUsed]: gauge(['storage_id'], null, true), - [M.trafficReportSync]: counter(['source', 'status'], true), [M.trafficReportSnapshot]: gauge(['status'], 'entities', true), [M.transferDownloadFailed]: counter(['reason', 'source'], true), [M.transferDownloadIssued]: counter(['actor_type', 'source', 'storage_id'], true), diff --git a/server/http/admin-stats.integration.test.ts b/server/http/admin-stats.integration.test.ts index 3347c2c7..b1241f1a 100644 --- a/server/http/admin-stats.integration.test.ts +++ b/server/http/admin-stats.integration.test.ts @@ -602,12 +602,20 @@ describe('site stats routes', () => { const headers = await adminHeaders(app) await seedProLicense(db) const { orgId, bucketStart, eventMs } = await seedStatsFixture(db) + const previousBucket = new Date(bucketStart.getTime() - 3_600_000) + const reportAt = previousBucket.getTime() + 60_000 await db.run(sql`UPDATE download_tasks SET finished_at = updated_at WHERE id IN ('task-1', 'task-2')`) await db.run(sql` INSERT INTO cloud_traffic_reports (id, org_id, period, source, source_id, event_id, bytes, status, created_at, updated_at) VALUES ('operations-cloud-report', ${orgId}, '2026-07', 'object_download', 'stats-file', - 'operations-cloud-report', 64, 'pending', ${eventMs}, ${eventMs}) + 'operations-cloud-report', 64, 'pending', ${reportAt}, ${reportAt}) + `) + await rebuildAdminStatsHour(db, previousBucket, new Date(reportAt + 3_600_000), true) + await db.run(sql` + UPDATE cloud_traffic_reports + SET status = 'reported', updated_at = ${eventMs} + WHERE id = 'operations-cloud-report' `) await rebuildAdminStatsHour(db, bucketStart, new Date(), true) @@ -636,7 +644,11 @@ describe('site stats routes', () => { expect.objectContaining({ name: 'failed', value: 1 }), ]), ) - expect(body.cloudReportStatus).toContainEqual(expect.objectContaining({ name: 'pending', value: 1 })) + expect(body.cloudReportStatus).toEqual([{ name: 'reported', value: 1, percent: 100 }]) + const [{ count: mutableCounters }] = await db.all<{ count: number }>(sql` + SELECT COUNT(*) AS count FROM stats_rollups_hourly WHERE metric_key = 'traffic.report_sync' + `) + expect(mutableCounters).toBe(0) }) it('applies dashboard ranges to sharing drill-down data', async () => { diff --git a/server/scripts/backfill-admin-stats.test.ts b/server/scripts/backfill-admin-stats.test.ts index 46850aef..a9186200 100644 --- a/server/scripts/backfill-admin-stats.test.ts +++ b/server/scripts/backfill-admin-stats.test.ts @@ -92,6 +92,8 @@ describe('admin stats backfill', () => { ('open-marker', ${currentHourMs}, '', 'stats.rollup_run', '', '', 1, 0, 0, '{"version":2,"scope":"full","quality":"exact"}', ${currentHourMs}), ('stale-task', ${eventMs - 3_600_000}, 'o1', 'remote_download.task_finished', '', '', 1, 512, 0, + '{"version":2,"scope":"counters","quality":"exact"}', ${eventMs}), + ('stale-traffic', ${eventMs - 3_600_000}, 'o1', 'traffic.report_sync', '', '', 2, 640, 0, '{"version":2,"scope":"counters","quality":"exact"}', ${eventMs}); `) @@ -142,8 +144,6 @@ describe('admin stats backfill', () => { rollupSharesCreated: 1, rawShareDownloads: 1, rollupShareDownloads: 1, - rawTrafficReports: 2, - rollupTrafficReports: 2, rawFinishedDownloadTasks: 1, rollupFinishedDownloadTasks: 1, rawMissingByteEvents: 0, @@ -173,6 +173,11 @@ describe('admin stats backfill', () => { ) .get(), ).toEqual({ value: 1 }) + expect( + db.prepare("SELECT COUNT(*) AS value FROM stats_rollups_hourly WHERE metric_key = 'traffic.report_sync'").get(), + ).toEqual({ + value: 0, + }) db.close() }) }) diff --git a/src/routes/_authenticated/admin/index.tsx b/src/routes/_authenticated/admin/index.tsx index 61e3614f..a400ad9a 100644 --- a/src/routes/_authenticated/admin/index.tsx +++ b/src/routes/_authenticated/admin/index.tsx @@ -1275,7 +1275,7 @@ function OperationsSection({ stats }: { stats: AdminDashboardOperationsStats }) - + )