diff --git a/V2_ROADMAP.md b/V2_ROADMAP.md index 31a4432e..495aa65c 100644 --- a/V2_ROADMAP.md +++ b/V2_ROADMAP.md @@ -36,7 +36,7 @@ Each version ships 1–2 major features. Ship small, ship often. | [v2.5](docs/roadmap/v2.5.md) | **Multi-Platform Deployment** | 7 first-class targets via Turso (CF, Docker, AWS Lambda, Vercel, Netlify, Azure, GCP); avatar upload | | [v2.6](docs/roadmap/v2.6.md) | **Pro / Business Launch** | Cloud binding, entitlement system, white-label, audit log, quota store machinery, site announcements, retroactive gates | | [v2.7](docs/roadmap/v2.7.md) | **WebDAV & File Processing** | WebDAV protocol access, small-file zip compression/extraction, and Community remote-download orchestration through Aria2, qBittorrent, and future adapters | -| [v2.8](docs/roadmap/v2.8.md) | **Admin Analytics & Dashboard** | Admin overview, usage/cost/reliability metrics, share analytics, and report exports | +| [v2.8](docs/roadmap/v2.8.md) | **Admin Analytics & Dashboard** | Admin overview, usage/cost/reliability metrics, share analytics, and offline result coverage | | [v2.9](docs/roadmap/v2.9.md) | **Agent CLI** | Scriptable CLI for agents and CI: upload/manage files, shares, spaces, quota, and tasks | | [v2.10](docs/roadmap/v2.10.md) | **Desktop Sync Foundation** | Sync device model, change log, sync API contract, conflict model, and protocol tests for external clients | | Future | **Native Client Projects** | macOS File Provider, Windows/Linux sync clients, Flutter/mobile clients, and other OS-specific apps in separate repositories | @@ -100,7 +100,7 @@ ZPan. - Quota store, subscription-plan catalog, gift-card / credit flows, and credit-backed traffic billing - Site announcements -- Analytics dashboard and reporting exports +- Analytics dashboard backed by completed offline results - Future webhook notifications and integration automation - Future enterprise / legacy identity such as SAML, LDAP / SCIM, and group mapping if demand proves real - Future advanced governance such as custom roles, retention policies, and admin support mode diff --git a/docs/design/admin-analytics.md b/docs/design/admin-analytics.md index f48d91c8..99f53435 100644 --- a/docs/design/admin-analytics.md +++ b/docs/design/admin-analytics.md @@ -1,39 +1,81 @@ # Admin analytics -## Storage model +## Boundary -Admin analytics has one derived table: `stats_rollups_hourly`. Every bucket starts on a UTC hour. The server has no configured reporting time zone and does not persist daily aggregates or time-zone-specific rows. +Admin analytics separates operational facts from queryable results: -Each row is identified by UTC bucket, organization, metric, dimension, and dimension value. Metric and dimension combinations must be declared in `server/domain/admin-stats-metrics.ts`. Counters are additive; gauges are point-in-time snapshots; distinct values must never be added across buckets. +- Operational tables such as `activity_events`, `shares`, `matters`, `org_quotas`, jobs, tasks, and webhook/report tables are rollup inputs. Dashboard requests never query or aggregate these tables. +- `stats_rollups_hourly` is the only dashboard result store. Every bucket starts on a UTC hour. +- The current open UTC hour is never visible. A result becomes queryable only after the background rollup commits the hour and its completion marker atomically. +- The browser only renders server-returned values. Rates, deltas, percentages, alerts, rankings, and data-quality totals are computed by the server from result rows. -The ten-minute scheduler rebuilds the current and previous UTC hours. Rebuilding an hour deletes and inserts that hour atomically, so retries are safe. A `stats.rollup_run` marker means the counter rows for an hour are complete. Snapshot rows are finalized near the next hour and remain useful as waterlines, not event totals. +The dashboard is read-only and does not support report or CSV export. + +## Fact writes + +Metrics derived from activity facts use a validated producer contract. New transfer and sharing facts fail fast when required fields such as `bytes`, `source`, `trafficEventId`, or `shareId` are missing. Historical rows that predate this contract remain diagnosable through quality metrics. + +A deduplicated share view updates `shares.views` and inserts its `share_view` fact in one database transaction. Download flows record an issued fact only after quota/metering and presigning succeed; a failed fact records the stable reason and traffic event id. If the issued-fact write fails, the flow refunds the counters it already reserved. + +## Result writes + +Metric and dimension combinations are declared in `server/domain/admin-stats-metrics.ts`: + +- Counters are additive event totals for a closed hour. +- Gauges are point-in-time snapshots written for that hour's close. +- High-volume source rows are grouped by the database before the Worker builds result rows; the scheduler does not load a whole hour of raw events into memory. + +The ten-minute scheduler always rebuilds the latest closed UTC hour with counters and snapshots. It also checks the previous 48 hours and repairs at most three missing counter buckets per run. Historical repair does not invent old snapshots from current state. + +Each row and `stats.rollup_run` marker carries: + +- the rollup schema version; +- `scope=full` for counters plus snapshots, or `scope=counters` for a historical counter repair; +- `quality=exact` or `quality=lower_bound`. + +A transaction deletes and reinserts the affected result scope, including its marker, so retries are idempotent. Readers reject old-version rows, rows without valid metadata, orphan rows without a compatible marker, and gauges backed only by a counter-repair marker. ## Query model -The client sends an IANA time zone with its selected range. Date-only inputs are interpreted as local calendar dates in that zone, including daylight-saving transitions and non-whole-hour offsets. +Dashboard ranges use UTC calendar boundaries only. Date-only input is normalized to complete UTC days; explicit timestamps must align to complete UTC hours. Ranges are limited to one year. -Queries combine: +Each response includes result coverage: -- complete UTC hours from `stats_rollups_hourly`; -- raw rows for the current partial hour, incomplete hours, and partial hours cut by the requested boundary; -- raw rows for a UTC hour that crosses a local-day boundary, such as `Asia/Kathmandu` at `:15` UTC. +- `complete`: every expected bucket has a compatible completion marker; +- `partial`: at least one compatible bucket exists, but one or more are missing; +- `empty`: no compatible result bucket exists; +- `dataThrough`: the exclusive end of the newest completed bucket. -This preserves exact local-day results without storing a daily table. A missing completion marker always fails back to the raw source instead of returning a partial aggregate. +Responses with period-over-period deltas also include `comparisonCoverage`. A delta is only +fully reconcilable when both the requested range and its comparison range are complete; the UI +shows either side's missing buckets instead of presenting a partial comparison as authoritative. -Exact DAU, WAU, and MAU remain a deliberate exception because hourly distinct counts cannot be added. Current inventory, quota pressure, active jobs, downloader status, and active shares are also queried live because they describe current state rather than historical event volume. +Event-only traffic coverage accepts `counters` and `full` markers. Views that depend on snapshots require `full` markers. + +Request-time work is bounded and result-only. Charts and totals read only the aggregate dimensions they need; high-cardinality share dimensions are not loaded when querying totals. Top-share and top-space queries aggregate the result table and return at most eight ids; only those ids are hydrated from the operational tables for display names. No dashboard endpoint scans users, files, activities, shares, jobs, tasks, reports, or downloaders. ## Metric semantics - Upload bytes mean successfully confirmed upload bytes. - Download bytes and counts mean issued downloads, not client-completed transfers. - Cloud traffic report status describes metering synchronization and belongs to Operations, not download success. -- Files older than 90 days are an age cohort, not cold data; access recency is not inferred. -- Sharing views, downloads, and saves are independent event totals, not a user-level funnel. +- User, storage, share lifecycle, active job/task, downloader, report, and webhook state comes from closed-hour snapshots, not live request-time reads. +- Storage inventory includes normal files and image-hosting objects. +- Files older than 90 days are an age cohort, not proven cold data; the UI labels this explicitly. +- Sharing views, downloads, and saves are independent event totals. Downloads/saves per 100 views are intensity ratios, not user-level conversion funnels. -Rows with transfer events but no recoverable byte count carry `quality=lower_bound`. The `stats.quality_missing_bytes` metric exposes the affected event count by direction and source. +Rows with transfer events but no recoverable byte count carry `quality=lower_bound`. `stats.quality_missing_bytes` exposes affected event counts by direction and source. ## Backfill and validation -Run `pnpm stats:backfill -- --apply ...` after the migration in each environment. The script first recovers audit metadata where an exact source still exists, then rebuilds historical hourly counters with conflict-aware updates. Re-running the same script produces no data changes. +Run `pnpm stats:backfill -- --apply ...` after the migration in each environment. The script repairs recoverable historical fact metadata and rebuilds historical counters idempotently. Backfilled markers use `scope=counters`; the script does not fabricate historical inventory or active-user snapshots, so snapshot-backed views honestly report partial or empty coverage for those hours. -Validation must reconcile raw event totals with hourly rows, verify completion markers, report missing transfer byte metadata, and compare dashboard responses for a UTC range, a DST date, and a non-whole-hour time zone. +Validation must cover: + +- fact-write contracts and atomic share-view writes; +- UTC range alignment and exclusion of the current open hour; +- rollup idempotency and database-side grouping; +- version and scope rejection, including counter-only versus full coverage; +- missing-bucket repair and lower-bound byte quality; +- dashboard endpoints never writing results or falling back to raw data; +- browser rendering without metric calculations. diff --git a/docs/roadmap/v2.8.md b/docs/roadmap/v2.8.md index 4e971669..084ddb22 100644 --- a/docs/roadmap/v2.8.md +++ b/docs/roadmap/v2.8.md @@ -9,7 +9,8 @@ desktop sync, webhooks, or native clients. ## Product Boundary -- **Business** gets the full admin analytics surface and report export. +- **Pro and Business** get the full admin analytics surface through the + `analytics` feature gate. - **Pro** keeps the existing operator features already shipped in v2.6, such as audit log, white-label, open registration, and higher free limits. - **Community** keeps normal file workflows and can continue to see local, @@ -29,21 +30,20 @@ ZPan already records enough local data to build a useful first dashboard: - `org_quotas`, `org_quota_entitlements`, and `cloud_traffic_reports` - `background_jobs` -v2.8 should first turn these into a coherent admin view. Add new analytics -tables only where a metric cannot be reconstructed cheaply or safely from -existing records. +v2.8 turns these into background-computed result rows. Dashboard requests must +not reconstruct metrics from operational records. ## Major Features ### Admin Overview -Add a Business-gated analytics page under Admin. +Add a Pro+-gated analytics page under Admin. Top-level cards: - Total storage used, quota allocated, and quota utilization - Download traffic issued by ZPan -- Active users, new signups, active spaces, and team growth +- Active users, new signups, and user status - Share views/downloads and top shared files - Remote-download task volume, success rate, failure reasons, and downloader health @@ -58,26 +58,26 @@ Use additive rollups for metrics that would otherwise become expensive. Suggested storage: -- **Hourly rollups** by metric, org, source, target type, and time bucket +- **Hourly rollups** by metric, org, declared dimension, and UTC time bucket - Daily and longer-range charts are composed from UTC hourly rows at query time Retention defaults: - Hourly rollups are the only persisted aggregate and are retained until an explicit pruning policy is introduced -- Reporting time zones are query inputs; no server reporting time zone or time-zone-specific aggregate is stored +- Dashboard ranges use UTC calendar boundaries; no time-zone-specific aggregate is stored -Do not create a broad raw event pipeline in v2.8 unless a specific chart needs -it. The audit log remains the record-level source for operational events. +The current open hour is excluded. Versioned completion markers distinguish full +snapshot results from counter-only historical repairs. Missing coverage is shown +to the operator and never falls back to request-time raw aggregation. ### Usage And Cost Views Operators need to understand where resource pressure comes from: - Storage used by space and file type -- Quota allocation by base quota vs entitlements -- Traffic usage by storage backend and period -- Credit-backed traffic usage where enabled -- Remote-download usage by downloader and task category +- Quota allocation and utilization by space +- Upload/download volume by source and period +- Remote-download and background-job outcomes - Over-quota or near-quota spaces This is a dashboard, not a billing engine. Billing still belongs to Cloud and @@ -89,7 +89,7 @@ The current share detail panel exposes simple counters. v2.8 turns them into an admin-operable view: - Top shares by views and downloads -- Share conversion: landing views -> file downloads +- Downloads and saves per 100 landing views, explicitly not a user funnel - Expired, revoked, and download-limit-hit counts - Anonymous vs signed-in access where known - Top public user-homepage items where data is available @@ -100,26 +100,19 @@ Avoid per-recipient behavioral analytics in this version. Give admins a place to diagnose operational health: -- API error rate by route family where locally available -- Background job failure list -- Remote-download task failure reasons -- Downloader heartbeat and offline history -- Cloud entitlement / quota-store webhook failures -- License refresh status +- Background-job outcomes and active backlog +- Remote-download outcomes and active backlog +- Current closed-hour downloader online/offline snapshot +- Cloud traffic-report backlog and outcomes +- Webhook failure snapshot The goal is to answer "what is broken?" before adding deeper observability. -### Report Export +### Read-only Dashboard Boundary -Business operators need portable reports: - -- CSV export for visible dashboard tables -- Audit-event export using the existing audit filters -- Download-task and downloader export for troubleshooting -- Quota and entitlement export for support - -Large exports should be streamed or job-backed; do not hold large exports in -memory on Workers. +The browser queries completed result data and does not calculate metrics from +detail rows. Request-time lookups are limited to hydrating the top eight result +ids with display names. Report and CSV export are not part of v2.8. ## Non-goals for v2.8 @@ -128,6 +121,7 @@ memory on Workers. - Folder-level ACLs, custom roles, or complex team governance - CLI, desktop sync, Flutter, or native client implementation - Hosted analytics pipeline in ZPan Cloud +- Report, CSV, audit-event, task, downloader, quota, or entitlement export - Content moderation, media transcoding, or managed large-file processing - Replacing audit log; analytics summarizes behavior, audit log remains the record-level review surface @@ -143,8 +137,8 @@ memory on Workers. > the most traffic before users ask for help. **Agency sharing files with clients:** -> I export this month's share-link traffic and top downloads as CSV, then attach -> the report to the client invoice. +> I can inspect this month's share-link traffic and top downloads without +> running database queries or waiting for a large live aggregation. ## v1 Issues Resolved diff --git a/scripts/backfill-admin-stats.ts b/scripts/backfill-admin-stats.ts index a5f594f7..fea5bbba 100644 --- a/scripts/backfill-admin-stats.ts +++ b/scripts/backfill-admin-stats.ts @@ -217,8 +217,6 @@ WHERE ctr.source IN ('direct_share', 'landing_share', 'image_hosting', 'object_d ) ); -DELETE FROM stats_rollups_hourly WHERE metric_key = 'storage.used.bytes'; - ${buildHourlyBackfillSql(now)} ` } @@ -233,6 +231,7 @@ type HourlySource = { bytes?: string uniqueCount?: string quality?: string + scope?: 'counters' | 'full' dimensions?: Record } @@ -253,20 +252,6 @@ function buildHourlyBackfillSql(now: Date): string { storage_id: "CASE WHEN json_valid(ae.metadata) = 1 THEN json_extract(ae.metadata, '$.storageId') END", }, }, - { - metric: 'storage.ingress', - source: 'activity_events ae', - timestampMs: 'ae.created_at * 1000', - org: 'ae.org_id', - where: "ae.action = 'upload_confirm'", - bytes: "SUM(CASE WHEN json_valid(ae.metadata) = 1 THEN COALESCE(json_extract(ae.metadata, '$.bytes'), 0) ELSE 0 END)", - quality: "CASE WHEN SUM(CASE WHEN ae.metadata IS NULL OR json_valid(ae.metadata) = 0 OR json_type(ae.metadata, '$.bytes') IS NULL THEN 1 ELSE 0 END) > 0 THEN 'lower_bound' ELSE 'exact' END", - dimensions: { - source: "COALESCE(CASE WHEN json_valid(ae.metadata) = 1 THEN json_extract(ae.metadata, '$.source') END, 'web_upload')", - status: "'success'", - storage_id: "CASE WHEN json_valid(ae.metadata) = 1 THEN json_extract(ae.metadata, '$.storageId') END", - }, - }, { metric: 'transfer.download_issued', source: 'activity_events ae', @@ -312,13 +297,6 @@ function buildHourlyBackfillSql(now: Date): string { bytes: 'SUM(ctr.bytes)', dimensions: { source: 'ctr.source', status: 'ctr.status' }, }, - { - metric: 'remote_download.task_created', - source: 'download_tasks dt', - timestampMs: 'dt.created_at', - org: 'dt.org_id', - dimensions: { category: "COALESCE(dt.category, 'uncategorized')", source: 'dt.source_type' }, - }, { metric: 'remote_download.task_finished', source: 'download_tasks dt', @@ -336,23 +314,6 @@ function buildHourlyBackfillSql(now: Date): string { where: 'bj.finished_at IS NOT NULL', dimensions: { job_type: 'bj.type', outcome: 'bj.status' }, }, - { - metric: 'remote_download.usage', - source: 'remote_download_usage_reports ru', - timestampMs: 'ru.created_at', - org: 'ru.org_id', - count: 'SUM(ru.credits_per_unit)', - bytes: 'SUM(ru.unit_bytes)', - dimensions: { downloader_id: 'ru.downloader_id', status: 'ru.status' }, - }, - { - metric: 'webhook.processed', - source: 'webhook_events we', - timestampMs: 'we.processed_at', - org: "''", - where: 'we.processed_at IS NOT NULL', - dimensions: { outcome: 'we.status' }, - }, { metric: 'stats.quality_missing_bytes', source: 'activity_events ae', @@ -367,7 +328,7 @@ function buildHourlyBackfillSql(now: Date): string { ] const statements = sources.flatMap((source) => hourlyStatements(source)) - statements.push(userSignupBackfillSql(), activeUserBackfillSql(), rollupMarkerBackfillSql(now)) + statements.push(userSignupBackfillSql(), rollupMarkerBackfillSql(now)) return statements.join('\n\n') } @@ -405,7 +366,7 @@ SELECT CAST(bucket_start AS TEXT) || ':' || COALESCE(NULLIF(org_id, ''), 'global') || ':${source.metric}:${dimensionKey || 'all'}:' || hex(dimension_value), bucket_start, org_id, '${source.metric}', '${dimensionKey}', dimension_value, count_value, bytes_value, unique_value, - json_object('version', 1, 'quality', quality_value), + json_object('version', 2, 'scope', '${source.scope ?? 'counters'}', 'quality', quality_value), bucket_start + 3600000 FROM ( SELECT ${bucket} AS bucket_start, ${source.org} AS org_id, ${dimension} AS dimension_value, @@ -440,19 +401,6 @@ function userSignupBackfillSql(): string { }) } -function activeUserBackfillSql(): string { - return hourlyInsert({ - metric: 'user.active_hour', - source: `(SELECT created_at AS at, user_id FROM session - UNION ALL - SELECT ae.created_at * 1000 AS at, ae.user_id FROM activity_events ae JOIN user u ON u.id = ae.user_id) active`, - timestampMs: 'active.at', - org: "''", - count: '0', - uniqueCount: 'COUNT(DISTINCT active.user_id)', - }) -} - function rollupMarkerBackfillSql(now: Date): string { const currentHour = Math.floor(now.getTime() / 3_600_000) * 3_600_000 return hourlyInsert({ @@ -463,6 +411,7 @@ function rollupMarkerBackfillSql(now: Date): string { timestampMs: 'buckets.at', org: "''", count: '1', + scope: 'counters', }) } diff --git a/server/adapters/repos/activity.ts b/server/adapters/repos/activity.ts index ef0e8fe5..0a87bf9c 100644 --- a/server/adapters/repos/activity.ts +++ b/server/adapters/repos/activity.ts @@ -2,8 +2,26 @@ import { and, count, desc, eq, gte, lte } from 'drizzle-orm' import { nanoid } from 'nanoid' import { organization, user } from '../../db/auth-schema' import { activityEvents } from '../../db/schema' +import { assertAdminStatsEvent } from '../../domain/admin-stats-events' import type { Database } from '../../platform/interface' -import type { ActivityActorType, ActivityRepo } from '../../usecases/ports' +import type { ActivityActorType, ActivityRepo, RecordActivityInput } from '../../usecases/ports' + +export function activityEventValues(event: RecordActivityInput): typeof activityEvents.$inferInsert { + assertAdminStatsEvent(event.action, event.metadata) + return { + id: nanoid(), + orgId: event.orgId, + userId: event.userId ?? null, + actorType: event.actorType ?? (event.userId ? 'user' : 'anonymous'), + actorRef: event.actorRef ?? null, + action: event.action, + targetType: event.targetType, + targetId: event.targetId ?? null, + targetName: event.targetName, + metadata: event.metadata ? JSON.stringify(event.metadata) : null, + createdAt: new Date(), + } +} function normalizeActorType(value: string | null, userId?: string | null): ActivityActorType { if (value === 'anonymous' || value === 'system' || value === 'downloader') return value @@ -21,19 +39,7 @@ function actorDisplayName(actorType: ActivityActorType, actorRef: string | null) export function createActivityRepo(db: Database): ActivityRepo { return { async record(event) { - await db.insert(activityEvents).values({ - id: nanoid(), - orgId: event.orgId, - userId: event.userId ?? null, - actorType: event.actorType ?? (event.userId ? 'user' : 'anonymous'), - actorRef: event.actorRef ?? null, - action: event.action, - targetType: event.targetType, - targetId: event.targetId ?? null, - targetName: event.targetName, - metadata: event.metadata ? JSON.stringify(event.metadata) : null, - createdAt: new Date(), - }) + await db.insert(activityEvents).values(activityEventValues(event)) }, async list(orgId, opts) { diff --git a/server/adapters/repos/admin-stats-hourly.ts b/server/adapters/repos/admin-stats-hourly.ts index 408e6119..de40e3b0 100644 --- a/server/adapters/repos/admin-stats-hourly.ts +++ b/server/adapters/repos/admin-stats-hourly.ts @@ -1,11 +1,22 @@ -import { and, eq, gte, lt } from 'drizzle-orm' +import type { AdminStatsCoverage } from '@shared/types' +import { and, eq, gte, inArray, lt, sql } from 'drizzle-orm' import { statsRollupsHourly } from '../../db/schema' -import type { AdminStatsMetric } from '../../domain/admin-stats-metrics' +import { + ADMIN_STATS_METRICS, + type AdminStatsDimension, + type AdminStatsMetric, + type AdminStatsRollupScope, + assertMetricDimension, + metricDefinition, + parseAdminStatsRollupMetadata, + ROLLUP_VERSION, +} from '../../domain/admin-stats-metrics' import { statsDayKey } from '../../domain/admin-stats-time' import type { Database } from '../../platform/interface' import type { AdminStatsDateRange } from '../../usecases/ports' const HOUR_MS = 3_600_000 +const DASHBOARD_RANKING_LIMIT = 8 export interface HourlyMetricRow { bucketStart: Date @@ -19,23 +30,158 @@ export interface HourlyMetricRow { } export class AdminStatsHourlyReader { - private readonly toExclusive: Date private readonly queryFrom: Date private readonly queryTo: Date + private readonly metricRows = new Map>() + private readonly markerBucketsPromises = new Map>>() constructor( private readonly db: Database, - private readonly range: AdminStatsDateRange, + range: AdminStatsDateRange, now: Date, ) { - const nextWholeSecond = Math.ceil((now.getTime() + 1) / 1000) * 1000 - this.toExclusive = new Date(Math.min(range.to.getTime() + 1, nextWholeSecond)) - this.queryFrom = ceilHour(range.from) - this.queryTo = ceilHour(this.toExclusive) + const rangeToExclusive = range.to.getTime() + 1 + if (range.from.getTime() % HOUR_MS !== 0 || rangeToExclusive % HOUR_MS !== 0) { + throw new Error('stats_range_must_align_to_utc_hours') + } + this.queryFrom = range.from + this.queryTo = new Date(Math.min(rangeToExclusive, floorHour(now).getTime())) } - async rows(metric: AdminStatsMetric): Promise { + async rows( + metric: AdminStatsMetric, + dimensionKeys: readonly (AdminStatsDimension | '')[] = [''], + ): Promise { + const normalizedKeys = [...new Set(dimensionKeys)].sort() + if (normalizedKeys.length === 0) return [] + for (const dimensionKey of normalizedKeys) assertMetricDimension(metric, dimensionKey) + const cacheKey = `${metric}\u0000${normalizedKeys.join(',')}` + const cached = this.metricRows.get(cacheKey) + if (cached) return cached + const rows = this.loadRows(metric, normalizedKeys) + this.metricRows.set(cacheKey, rows) + return rows + } + + async latestRows( + metric: AdminStatsMetric, + dimensionKeys: readonly (AdminStatsDimension | '')[] = [''], + ): Promise { + const rows = await this.rows(metric, dimensionKeys) + const latest = rows.reduce((value, row) => Math.max(value, row.bucketStart.getTime()), Number.NEGATIVE_INFINITY) + return rows.filter((row) => row.bucketStart.getTime() === latest) + } + + async topShareActivity(): Promise> { if (this.queryFrom >= this.queryTo) return [] + const rows = await this.db.all<{ shareId: string; views: number; downloads: number }>(sql` + SELECT + result.dimension_value AS shareId, + SUM(CASE WHEN result.metric_key = ${ADMIN_STATS_METRICS.shareView} THEN result.count ELSE 0 END) AS views, + SUM(CASE WHEN result.metric_key = ${ADMIN_STATS_METRICS.shareDownloadIssued} THEN result.count ELSE 0 END) AS downloads + FROM stats_rollups_hourly result + INNER JOIN stats_rollups_hourly marker + ON marker.bucket_start = result.bucket_start + AND marker.org_id = '' + AND marker.metric_key = ${ADMIN_STATS_METRICS.statsRollupRun} + AND marker.dimension_key = '' + AND marker.dimension_value = '' + WHERE result.metric_key IN (${ADMIN_STATS_METRICS.shareView}, ${ADMIN_STATS_METRICS.shareDownloadIssued}) + AND result.dimension_key = 'share_id' + AND result.dimension_value <> '' + AND result.bucket_start >= ${this.queryFrom.getTime()} + AND result.bucket_start < ${this.queryTo.getTime()} + AND CASE WHEN json_valid(result.metadata) = 1 THEN json_extract(result.metadata, '$.version') END = ${ROLLUP_VERSION} + AND CASE WHEN json_valid(result.metadata) = 1 THEN json_extract(result.metadata, '$.scope') END IN ('counters', 'full') + AND CASE WHEN json_valid(result.metadata) = 1 THEN json_extract(result.metadata, '$.quality') END IN ('exact', 'lower_bound') + AND CASE WHEN json_valid(marker.metadata) = 1 THEN json_extract(marker.metadata, '$.version') END = ${ROLLUP_VERSION} + AND CASE WHEN json_valid(marker.metadata) = 1 THEN json_extract(marker.metadata, '$.scope') END IN ('counters', 'full') + AND CASE WHEN json_valid(marker.metadata) = 1 THEN json_extract(marker.metadata, '$.quality') END IN ('exact', 'lower_bound') + GROUP BY result.dimension_value + ORDER BY views DESC, downloads DESC, result.dimension_value + LIMIT ${DASHBOARD_RANKING_LIMIT} + `) + return rows.map((row: { shareId: string; views: number; downloads: number }) => ({ + shareId: row.shareId, + views: Number(row.views), + downloads: Number(row.downloads), + })) + } + + async topSpaceUsage(): Promise> { + if (this.queryFrom >= this.queryTo) return [] + const rows = await this.db.all<{ orgId: string; usedBytes: number; quotaBytes: number }>(sql` + WITH latest AS ( + SELECT MAX(bucket_start) AS bucketStart + FROM stats_rollups_hourly + WHERE org_id = '' + AND metric_key = ${ADMIN_STATS_METRICS.statsRollupRun} + AND dimension_key = '' + AND dimension_value = '' + AND bucket_start >= ${this.queryFrom.getTime()} + AND bucket_start < ${this.queryTo.getTime()} + AND CASE WHEN json_valid(metadata) = 1 THEN json_extract(metadata, '$.version') END = ${ROLLUP_VERSION} + AND CASE WHEN json_valid(metadata) = 1 THEN json_extract(metadata, '$.scope') END = 'full' + AND CASE WHEN json_valid(metadata) = 1 THEN json_extract(metadata, '$.quality') END IN ('exact', 'lower_bound') + ) + SELECT used.org_id AS orgId, used.bytes AS usedBytes, COALESCE(quota.bytes, 0) AS quotaBytes + FROM stats_rollups_hourly used + LEFT JOIN stats_rollups_hourly quota + ON quota.bucket_start = used.bucket_start + AND quota.org_id = used.org_id + AND quota.metric_key = ${ADMIN_STATS_METRICS.storageQuota} + AND quota.dimension_key = '' + AND quota.dimension_value = '' + AND CASE WHEN json_valid(quota.metadata) = 1 THEN json_extract(quota.metadata, '$.version') END = ${ROLLUP_VERSION} + AND CASE WHEN json_valid(quota.metadata) = 1 THEN json_extract(quota.metadata, '$.scope') END = 'full' + AND CASE WHEN json_valid(quota.metadata) = 1 THEN json_extract(quota.metadata, '$.quality') END IN ('exact', 'lower_bound') + WHERE used.bucket_start = (SELECT bucketStart FROM latest) + AND used.org_id <> '' + AND used.metric_key = ${ADMIN_STATS_METRICS.storageUsed} + AND used.dimension_key = '' + AND used.dimension_value = '' + AND CASE WHEN json_valid(used.metadata) = 1 THEN json_extract(used.metadata, '$.version') END = ${ROLLUP_VERSION} + AND CASE WHEN json_valid(used.metadata) = 1 THEN json_extract(used.metadata, '$.scope') END = 'full' + AND CASE WHEN json_valid(used.metadata) = 1 THEN json_extract(used.metadata, '$.quality') END IN ('exact', 'lower_bound') + ORDER BY used.bytes DESC, used.org_id + LIMIT ${DASHBOARD_RANKING_LIMIT} + `) + return rows.map((row: { orgId: string; usedBytes: number; quotaBytes: number }) => ({ + orgId: row.orgId, + usedBytes: Number(row.usedBytes), + quotaBytes: Number(row.quotaBytes), + })) + } + + async coverage(requiredScope: AdminStatsRollupScope = 'full'): Promise { + const expectedBuckets = Math.max(0, Math.floor((this.queryTo.getTime() - this.queryFrom.getTime()) / HOUR_MS)) + const markerBuckets = await this.markerBuckets(requiredScope) + const completedBuckets = markerBuckets.size + const latest = Math.max(...markerBuckets, Number.NEGATIVE_INFINITY) + return { + status: completedBuckets === 0 ? 'empty' : completedBuckets === expectedBuckets ? 'complete' : 'partial', + expectedBuckets, + completedBuckets, + dataThrough: Number.isFinite(latest) ? new Date(latest + HOUR_MS).toISOString() : null, + } + } + + endExclusive(): Date { + return this.queryTo + } + + dayKey(date: Date): string { + return statsDayKey(date) + } + + private async loadRows( + metric: AdminStatsMetric, + dimensionKeys: readonly (AdminStatsDimension | '')[], + ): Promise { + if (this.queryFrom >= this.queryTo) return [] + const requiredScope = metricDefinition(metric).kind === 'gauge' ? 'full' : 'counters' + const markerBuckets = await this.markerBuckets(requiredScope) + if (markerBuckets.size === 0) return [] const rows = await this.db .select({ bucketStart: statsRollupsHourly.bucketStart, @@ -51,48 +197,70 @@ export class AdminStatsHourlyReader { .where( and( eq(statsRollupsHourly.metricKey, metric), + inArray(statsRollupsHourly.dimensionKey, dimensionKeys), + gte(statsRollupsHourly.bucketStart, this.queryFrom), + lt(statsRollupsHourly.bucketStart, this.queryTo), + sql`CASE WHEN json_valid(${statsRollupsHourly.metadata}) = 1 THEN json_extract(${statsRollupsHourly.metadata}, '$.version') END = ${ROLLUP_VERSION}`, + requiredScope === 'full' + ? sql`CASE WHEN json_valid(${statsRollupsHourly.metadata}) = 1 THEN json_extract(${statsRollupsHourly.metadata}, '$.scope') END = 'full'` + : sql`CASE WHEN json_valid(${statsRollupsHourly.metadata}) = 1 THEN json_extract(${statsRollupsHourly.metadata}, '$.scope') END IN ('counters', 'full')`, + sql`CASE WHEN json_valid(${statsRollupsHourly.metadata}) = 1 THEN json_extract(${statsRollupsHourly.metadata}, '$.quality') END IN ('exact', 'lower_bound')`, + ), + ) + return rows + .filter((row) => { + const metadata = parseAdminStatsRollupMetadata(row.metadata) + return markerBuckets.has(row.bucketStart.getTime()) && supportsScope(metadata?.scope, requiredScope) + }) + .map((row) => ({ + bucketStart: row.bucketStart, + orgId: row.orgId, + dimensionKey: row.dimensionKey, + dimensionValue: row.dimensionValue, + count: row.count, + bytes: row.bytes, + uniqueCount: row.uniqueCount, + lowerBound: parseAdminStatsRollupMetadata(row.metadata)?.quality === 'lower_bound', + })) + } + + private markerBuckets(requiredScope: AdminStatsRollupScope): Promise> { + const cached = this.markerBucketsPromises.get(requiredScope) + if (cached) return cached + const buckets = this.loadMarkerBuckets(requiredScope) + this.markerBucketsPromises.set(requiredScope, buckets) + return buckets + } + + private async loadMarkerBuckets(requiredScope: AdminStatsRollupScope): Promise> { + if (this.queryFrom >= this.queryTo) return new Set() + const rows = await this.db + .select({ bucketStart: statsRollupsHourly.bucketStart, metadata: statsRollupsHourly.metadata }) + .from(statsRollupsHourly) + .where( + and( + eq(statsRollupsHourly.metricKey, ADMIN_STATS_METRICS.statsRollupRun), + eq(statsRollupsHourly.orgId, ''), + eq(statsRollupsHourly.dimensionKey, ''), gte(statsRollupsHourly.bucketStart, this.queryFrom), lt(statsRollupsHourly.bucketStart, this.queryTo), ), ) - return rows.map((row) => ({ - bucketStart: row.bucketStart, - orgId: row.orgId, - dimensionKey: row.dimensionKey, - dimensionValue: row.dimensionValue, - count: row.count, - bytes: row.bytes, - uniqueCount: row.uniqueCount, - lowerBound: parseQuality(row.metadata) === 'lower_bound', - })) - } - - endExclusive(): Date { - return this.toExclusive - } - - dayKey(date: Date): string { - return statsDayKey(date, this.range.timeZone) + return new Set( + rows + .filter((row) => { + const metadata = parseAdminStatsRollupMetadata(row.metadata) + return supportsScope(metadata?.scope, requiredScope) + }) + .map((row) => row.bucketStart.getTime()), + ) } } -function parseQuality(metadata: string | null): string { - if (!metadata) return 'unknown' - try { - const value: unknown = JSON.parse(metadata) - if (!value || typeof value !== 'object' || Array.isArray(value)) return 'unknown' - const quality = (value as Record).quality - return typeof quality === 'string' ? quality : 'unknown' - } catch { - return 'unknown' - } +function supportsScope(scope: AdminStatsRollupScope | undefined, requiredScope: AdminStatsRollupScope): boolean { + return scope === 'full' || (requiredScope === 'counters' && scope === 'counters') } function floorHour(date: Date): Date { return new Date(Math.floor(date.getTime() / HOUR_MS) * HOUR_MS) } - -function ceilHour(date: Date): Date { - const floor = floorHour(date) - return floor.getTime() === date.getTime() ? floor : new Date(floor.getTime() + HOUR_MS) -} diff --git a/server/adapters/repos/admin-stats-rollup.integration.test.ts b/server/adapters/repos/admin-stats-rollup.integration.test.ts index f183eacf..e8011614 100644 --- a/server/adapters/repos/admin-stats-rollup.integration.test.ts +++ b/server/adapters/repos/admin-stats-rollup.integration.test.ts @@ -184,25 +184,18 @@ describe('admin hourly stats rollup', () => { }) expect(first.lowerBoundRows).toBeGreaterThan(0) expect(row(M.transferUpload)).toMatchObject({ count: 4, bytes: 100 }) - expect(row(M.storageIngress)).toMatchObject({ count: 2, bytes: 100 }) expect(row(M.transferDownloadIssued)).toMatchObject({ count: 4, bytes: 90 }) - expect(row(M.trafficQuotaBlocked)).toMatchObject({ count: 1, bytes: 5 }) expect(row(M.shareDownloadIssued)).toMatchObject({ count: 1, bytes: 20 }) expect(row(M.statsMissingBytes)).toMatchObject({ count: 2 }) - expect(row(M.teamMembershipChange)).toMatchObject({ count: 2 }) expect(row(M.userSignup, '', '', '')).toMatchObject({ count: 2 }) expect(row(M.userSignup, 'provider', 'direct', '')).toMatchObject({ count: 1 }) - expect(row(M.userActiveHour, '', '', '')).toMatchObject({ uniqueCount: 1 }) - expect(row(M.spaceCreated, '', '', '')).toMatchObject({ count: 2 }) expect(row(M.shareCreated)).toMatchObject({ count: 4 }) expect(row(M.trafficReportSync)).toMatchObject({ count: 2, bytes: 300 }) - expect(row(M.trafficCreditConsumed)).toMatchObject({ count: 4, bytes: 100 }) - expect(row(M.remoteDownloadTaskCreated)).toMatchObject({ count: 2 }) expect(row(M.remoteDownloadTaskFinished)).toMatchObject({ count: 1, bytes: 60 }) expect(row(M.backgroundJobFinished)).toMatchObject({ count: 1 }) - expect(row(M.remoteDownloadUsage)).toMatchObject({ count: 3, bytes: 70 }) - expect(row(M.webhookProcessed, '', '', '')).toMatchObject({ count: 1 }) expect(row(M.storageInventory)).toMatchObject({ count: 2, bytes: 500 }) + expect(row(M.storageUsed)).toMatchObject({ bytes: 500 }) + expect(row(M.storageQuota)).toMatchObject({ bytes: 1000 }) expect(row(M.shareInventory, 'lifecycle', 'usable')).toMatchObject({ count: 1 }) expect(row(M.shareInventory, 'lifecycle', 'revoked')).toMatchObject({ count: 1 }) expect(row(M.shareInventory, 'lifecycle', 'expired')).toMatchObject({ count: 1 }) @@ -210,8 +203,12 @@ describe('admin hourly stats rollup', () => { expect(row(M.backgroundJobSnapshot)).toMatchObject({ count: 1 }) expect(row(M.remoteDownloadTaskSnapshot)).toMatchObject({ count: 1 }) expect(row(M.downloaderSnapshot, '', '', '')).toMatchObject({ count: 2 }) + expect(row(M.userInventory, '', '', '')).toMatchObject({ count: 2 }) + expect(row(M.userActiveSnapshot, 'window', 'mau', '')).toMatchObject({ count: 1 }) + expect(row(M.trafficReportSnapshot, '', '', '')).toMatchObject({ count: 2, bytes: 300 }) + expect(row(M.webhookSnapshot, 'status', 'processed', '')).toMatchObject({ count: 1 }) expect(row(M.statsRollupRun, '', '', '')).toMatchObject({ count: 1 }) - expect(JSON.parse(row(M.transferUpload)?.metadata ?? '{}')).toMatchObject({ quality: 'lower_bound' }) + expect(JSON.parse(row(M.transferUpload)?.metadata ?? '{}')).toMatchObject({ version: 2, quality: 'lower_bound' }) const second = await rebuildAdminStatsHour(db, bucketStart, generatedAt, true) const [{ count: storedRows }] = await db.all<{ count: number }>(sql` @@ -238,7 +235,7 @@ describe('admin hourly stats rollup', () => { ).rejects.toThrow('stats_rollup_query_failed:quota') }) - it('treats absent and malformed rollup quality metadata as exact data', async () => { + it('reads only current-version result rows with a compatible completion scope', async () => { const { db } = await createTestApp() const bucketStart = Date.parse('2026-07-10T10:00:00.000Z') await db.run(sql` @@ -246,11 +243,16 @@ describe('admin hourly stats rollup', () => { (id, bucket_start, org_id, metric_key, dimension_key, dimension_value, count, bytes, unique_count, metadata, updated_at) VALUES - ('quality-marker', ${bucketStart}, '', 'stats.rollup_run', '', '', 1, 0, 0, '{}', ${bucketStart}), + ('quality-marker', ${bucketStart}, '', 'stats.rollup_run', '', '', 1, 0, 0, + '{"version":2,"scope":"full","quality":"exact"}', ${bucketStart}), ('quality-null', ${bucketStart}, 'org-null', 'transfer.upload', '', '', 1, 1, 0, NULL, ${bucketStart}), ('quality-array', ${bucketStart}, 'org-array', 'transfer.upload', '', '', 1, 2, 0, '[]', ${bucketStart}), - ('quality-invalid', ${bucketStart}, 'org-invalid', 'transfer.upload', '', '', 1, 3, 0, '{', ${bucketStart}), - ('quality-object', ${bucketStart}, 'org-object', 'transfer.upload', '', '', 1, 4, 0, '{}', ${bucketStart}) + ('quality-v1', ${bucketStart}, 'org-v1', 'transfer.upload', '', '', 1, 3, 0, + '{"version":1,"scope":"full","quality":"exact"}', ${bucketStart}), + ('quality-exact', ${bucketStart}, 'org-exact', 'transfer.upload', '', '', 1, 4, 0, + '{"version":2,"scope":"full","quality":"exact"}', ${bucketStart}), + ('quality-lower', ${bucketStart}, 'org-lower', 'transfer.upload', '', '', 1, 5, 0, + '{"version":2,"scope":"full","quality":"lower_bound"}', ${bucketStart}) `) const reader = new AdminStatsHourlyReader( db, @@ -263,35 +265,70 @@ describe('admin hourly stats rollup', () => { ) expect(reader.endExclusive()).toEqual(new Date(bucketStart + 3_600_000)) - expect(await reader.rows(M.transferUpload)).toHaveLength(4) - expect((await reader.rows(M.transferUpload)).every((row) => row.lowerBound === false)).toBe(true) + expect(await reader.rows(M.transferUpload)).toEqual([ + expect.objectContaining({ orgId: 'org-exact', bytes: 4, lowerBound: false }), + expect.objectContaining({ orgId: 'org-lower', bytes: 5, lowerBound: true }), + ]) expect(metricDefinition(M.transferUpload)).toMatchObject({ kind: 'counter', bytesUnit: 'bytes' }) expect(() => assertMetricDimension(M.transferUpload, 'not-a-dimension')).toThrow( 'Unsupported stats dimension: transfer.upload/not-a-dimension', ) }) - it('reads a current-hour rollup without querying raw source tables', async () => { + it('keeps counter-only repairs separate from full snapshot results', async () => { + const { db } = await createTestApp() + const bucketStart = Date.parse('2026-07-10T09:00:00.000Z') + const metadata = '{"version":2,"scope":"counters","quality":"exact"}' + await db.run(sql` + INSERT INTO stats_rollups_hourly + (id, bucket_start, org_id, metric_key, dimension_key, dimension_value, + count, bytes, unique_count, metadata, updated_at) + VALUES + ('counter-marker', ${bucketStart}, '', 'stats.rollup_run', '', '', 1, 0, 0, ${metadata}, ${bucketStart}), + ('counter-result', ${bucketStart}, '', 'transfer.upload', '', '', 1, 42, 0, ${metadata}, ${bucketStart}), + ('orphan-gauge', ${bucketStart}, '', 'storage.used', '', '', 0, 99, 0, + '{"version":2,"scope":"full","quality":"exact"}', ${bucketStart}) + `) + const reader = new AdminStatsHourlyReader( + db, + { + from: new Date(bucketStart), + to: new Date(bucketStart + 3_600_000 - 1), + timeZone: 'UTC', + }, + new Date(bucketStart + 7_200_000), + ) + + expect(await reader.rows(M.transferUpload)).toEqual([expect.objectContaining({ bytes: 42 })]) + expect(await reader.rows(M.storageUsed)).toEqual([]) + expect(await reader.coverage('counters')).toMatchObject({ status: 'complete', completedBuckets: 1 }) + expect(await reader.coverage()).toMatchObject({ status: 'empty', completedBuckets: 0 }) + }) + + it('never exposes the current open hour, even if rollup rows already exist', async () => { const { db } = await createTestApp() const bucketStart = Date.parse('2026-07-10T10:00:00.000Z') await db.run(sql` INSERT INTO stats_rollups_hourly (id, bucket_start, org_id, metric_key, dimension_key, dimension_value, count, bytes, unique_count, metadata, updated_at) - VALUES ('current-hour-rollup', ${bucketStart}, '', 'transfer.upload', '', '', 1, 42, 0, '{}', ${bucketStart}) + VALUES + ('current-hour-marker', ${bucketStart}, '', 'stats.rollup_run', '', '', 1, 0, 0, + '{"version":2,"scope":"full","quality":"exact"}', ${bucketStart}), + ('current-hour-rollup', ${bucketStart}, '', 'transfer.upload', '', '', 1, 42, 0, + '{"version":2,"scope":"full","quality":"exact"}', ${bucketStart}) `) const reader = new AdminStatsHourlyReader( db, { from: new Date(bucketStart), - to: new Date(bucketStart + 30 * 60_000), + to: new Date(bucketStart + 3_600_000 - 1), timeZone: 'UTC', }, new Date(bucketStart + 30 * 60_000), ) - expect(await reader.rows(M.transferUpload)).toEqual([ - expect.objectContaining({ bucketStart: new Date(bucketStart), count: 1, bytes: 42 }), - ]) + expect(await reader.rows(M.transferUpload)).toEqual([]) + expect(await reader.coverage()).toMatchObject({ status: 'empty', expectedBuckets: 0, completedBuckets: 0 }) }) }) diff --git a/server/adapters/repos/admin-stats-rollup.ts b/server/adapters/repos/admin-stats-rollup.ts index 21bd792b..b7abea31 100644 --- a/server/adapters/repos/admin-stats-rollup.ts +++ b/server/adapters/repos/admin-stats-rollup.ts @@ -1,8 +1,5 @@ -import { isPersonalOrgLike } from '@shared/org-slugs' import { and, eq, gte, inArray, lt, sql } from 'drizzle-orm' -import { account, organization, session, user } from '../../db/auth-schema' import { - activityEvents, backgroundJobs, cloudTrafficReports, downloaders, @@ -10,10 +7,8 @@ import { imageHostings, matters, orgQuotas, - remoteDownloadUsageReports, shares, statsRollupsHourly, - storages, webhookEvents, } from '../../db/schema' import { type AtomicQuery, executeWriteTransaction } from '../../db/transaction' @@ -33,32 +28,19 @@ export const ADMIN_STATS_ROLLUP_WRITE_BATCH_SIZE = Math.floor(D1_MAX_BOUND_PARAM const DOWNLOAD_ACTIONS = ['share_download', 'object_download', 'image_hosting_download', 'webdav_download'] const COUNTER_METRICS: AdminStatsMetric[] = [ M.backgroundJobFinished, - M.licenseRefresh, - M.remoteDownloadTaskCreated, M.remoteDownloadTaskFinished, - M.remoteDownloadUsage, M.shareCreated, M.shareDownloadIssued, M.sharePasswordPassed, M.shareSaved, M.shareView, - M.spaceCreated, M.statsMissingBytes, M.statsRollupRun, - M.storageIngress, - M.storageRemoved, - M.storageRestored, - M.teamMembershipChange, - M.trafficCreditConsumed, - M.trafficQuotaBlocked, M.trafficReportSync, M.transferDownloadFailed, M.transferDownloadIssued, M.transferUpload, - M.userActiveHour, - M.userSessionStarted, M.userSignup, - M.webhookProcessed, ] type RollupValue = { @@ -114,6 +96,7 @@ export async function rebuildAdminStatsHour( uniqueCount: row.uniqueCount, metadata: JSON.stringify({ version: ROLLUP_VERSION, + scope: includeSnapshots ? 'full' : 'counters', quality: row.lowerBound ? 'lower_bound' : 'exact', generatedAt: generatedAt.toISOString(), }), @@ -132,25 +115,51 @@ export async function rebuildAdminStatsHour( } async function addEventMetrics(db: Database, rollups: RollupAccumulator, from: Date, to: Date): Promise { - const rows = await db - .select({ - orgId: activityEvents.orgId, - userId: activityEvents.userId, - actorType: activityEvents.actorType, - action: activityEvents.action, - targetId: activityEvents.targetId, - metadata: activityEvents.metadata, - }) - .from(activityEvents) - .where(and(gte(activityEvents.createdAt, from), lt(activityEvents.createdAt, to))) + const rows = await db.all(sql` + WITH facts AS ( + SELECT + org_id AS orgId, + action, + COALESCE(actor_type, CASE WHEN user_id IS NULL THEN 'anonymous' ELSE 'user' END) AS actorType, + CASE WHEN json_valid(metadata) = 1 THEN json_extract(metadata, '$.source') END AS source, + CASE WHEN json_valid(metadata) = 1 THEN json_extract(metadata, '$.storageId') END AS storageId, + CASE WHEN json_valid(metadata) = 1 THEN json_extract(metadata, '$.reason') END AS reason, + CASE WHEN action IN ('share_view', 'share_download', 'save_from_share', 'share_password_passed') + THEN COALESCE(CASE WHEN json_valid(metadata) = 1 THEN json_extract(metadata, '$.shareId') END, target_id) + END AS shareId, + CASE WHEN json_valid(metadata) = 1 THEN json_extract(metadata, '$.kind') END AS kind, + CASE WHEN json_valid(metadata) = 1 THEN + CASE WHEN json_type(metadata, '$.bytes') IN ('integer', 'real') THEN json_extract(metadata, '$.bytes') ELSE 0 END + ELSE 0 END AS bytes, + CASE WHEN action IN ('upload_confirm', 'share_download', 'object_download', 'image_hosting_download', 'webdav_download') + THEN CASE + WHEN json_valid(metadata) = 0 THEN 1 + WHEN json_type(metadata, '$.bytes') IN ('integer', 'real') THEN 0 + ELSE 1 + END + ELSE 0 + END AS missingBytes + FROM activity_events + WHERE created_at >= ${Math.floor(from.getTime() / 1000)} + AND created_at < ${Math.floor(to.getTime() / 1000)} + AND action IN ( + 'upload_confirm', 'upload_cancel', 'upload_failed', + 'share_download', 'object_download', 'image_hosting_download', 'webdav_download', 'download_failed', + 'share_view', 'save_from_share', 'share_password_passed' + ) + ) + SELECT + orgId, action, actorType, source, storageId, reason, shareId, kind, + COUNT(*) AS count, COALESCE(SUM(bytes), 0) AS bytes, SUM(missingBytes) AS missingBytes + FROM facts + GROUP BY orgId, action, actorType, source, storageId, reason, shareId, kind + `) for (const row of rows) { - const metadata = parseMetadata(row.metadata) - const bytes = metadataNumber(metadata, 'bytes') - const source = metadataString(metadata, 'source') - const storageId = metadataString(metadata, 'storageId') - const actorType = row.actorType ?? (row.userId ? 'user' : 'anonymous') - const lowerBound = needsBytes(row.action) && !hasFiniteNumber(metadata.bytes) + const count = Number(row.count) + const bytes = Number(row.bytes) + const missingBytes = Number(row.missingBytes) + const lowerBound = missingBytes > 0 if (row.action === 'upload_confirm' || row.action === 'upload_cancel' || row.action === 'upload_failed') { const status = @@ -158,226 +167,170 @@ async function addEventMetrics(db: Database, rollups: RollupAccumulator, from: D const reason = row.action === 'upload_confirm' ? null - : (metadataString(metadata, 'reason') ?? - (row.action === 'upload_cancel' ? 'upload_canceled' : 'upload_failed')) + : (row.reason ?? (row.action === 'upload_cancel' ? 'upload_canceled' : 'upload_failed')) const confirmedBytes = row.action === 'upload_confirm' ? bytes : 0 rollups.add( M.transferUpload, row.orgId, - 1, + count, confirmedBytes, - { source: source ?? 'upload', status, reason, storage_id: storageId }, + { source: row.source ?? 'upload', status, reason, storage_id: row.storageId }, lowerBound, ) - if (row.action === 'upload_confirm') { - rollups.add( - M.storageIngress, - row.orgId, - 1, - bytes, - { source: source ?? 'web_upload', status, storage_id: storageId }, - lowerBound, - ) - } } if (DOWNLOAD_ACTIONS.includes(row.action)) { rollups.add( M.transferDownloadIssued, row.orgId, - 1, + count, bytes, { - source: source ?? downloadSourceForAction(row.action), - storage_id: storageId, - actor_type: actorType, + source: row.source ?? downloadSourceForAction(row.action), + storage_id: row.storageId, + actor_type: row.actorType, }, lowerBound, ) } if (row.action === 'download_failed') { - const reason = metadataString(metadata, 'reason') ?? 'unknown' - rollups.add(M.transferDownloadFailed, row.orgId, 1, bytes, { source: source ?? 'unknown', reason }) - if (reason === 'quota_exceeded') rollups.add(M.trafficQuotaBlocked, row.orgId, 1, bytes, { source }) + rollups.add(M.transferDownloadFailed, row.orgId, count, bytes, { + source: row.source ?? 'unknown', + reason: row.reason ?? 'unknown', + }) } if (row.action === 'share_view') { - rollups.add(M.shareView, row.orgId, 1, 0, { share_id: row.targetId, actor_type: actorType }) + rollups.add(M.shareView, row.orgId, count, 0, { share_id: row.shareId, actor_type: row.actorType }) } if (row.action === 'share_download') { - rollups.add(M.shareDownloadIssued, row.orgId, 1, bytes, { - share_id: metadataString(metadata, 'shareId') ?? row.targetId, - kind: metadataString(metadata, 'kind'), - source: source ?? downloadSourceForAction(row.action), - actor_type: actorType, + rollups.add(M.shareDownloadIssued, row.orgId, count, bytes, { + share_id: row.shareId, + kind: row.kind, + source: row.source ?? downloadSourceForAction(row.action), + actor_type: row.actorType, }) } if (row.action === 'save_from_share') { - rollups.add(M.shareSaved, row.orgId, 1, bytes, { - share_id: metadataString(metadata, 'shareId') ?? row.targetId, - actor_type: actorType, - }) + rollups.add(M.shareSaved, row.orgId, count, bytes, { share_id: row.shareId, actor_type: row.actorType }) } if (row.action === 'share_password_passed') { - rollups.add(M.sharePasswordPassed, row.orgId, 1, 0, { share_id: row.targetId }) - } - if (row.action === 'restore') rollups.add(M.storageRestored, row.orgId, 1, bytes, { source: 'trash' }, lowerBound) - if (row.action === 'object_purge') { - rollups.add(M.storageRemoved, row.orgId, 1, bytes, { reason: 'purged' }, lowerBound) - } - if (row.action === 'team_member_join') rollups.add(M.teamMembershipChange, row.orgId, 1, 0, { outcome: 'join' }) - if (row.action === 'team_member_remove') rollups.add(M.teamMembershipChange, row.orgId, 1, 0, { outcome: 'remove' }) - if (row.action === 'license_refresh') { - rollups.add(M.licenseRefresh, '', 1, 0, { outcome: metadataString(metadata, 'status') ?? 'success' }) + rollups.add(M.sharePasswordPassed, row.orgId, count, 0, { share_id: row.shareId }) } if (lowerBound) { - rollups.add(M.statsMissingBytes, row.orgId, 1, 0, { + rollups.add(M.statsMissingBytes, row.orgId, missingBytes, 0, { direction: row.action === 'upload_confirm' ? 'upload' : 'download', - source: source ?? row.action, + source: row.source ?? row.action, }) } } } +type EventMetricGroup = { + orgId: string + action: string + actorType: string + source: string | null + storageId: string | null + reason: string | null + shareId: string | null + kind: string | null + count: number + bytes: number + missingBytes: number +} + async function addUserMetrics(db: Database, rollups: RollupAccumulator, from: Date, to: Date): Promise { - const [signupRows, sessionRows, activeActivityRows, spaceRows, shareRows] = await Promise.all([ + const [signupRows, shareRows] = await Promise.all([ + db.all<{ provider: string; count: number }>(sql` + SELECT provider, COUNT(*) AS count + FROM ( + SELECT COALESCE(( + SELECT a.provider_id FROM account a + WHERE a.user_id = u.id + ORDER BY a.created_at, a.id + LIMIT 1 + ), 'direct') AS provider + FROM "user" u + WHERE u.created_at >= ${from.getTime()} AND u.created_at < ${to.getTime()} + ) signups + GROUP BY provider + `), db - .select({ userId: user.id, provider: account.providerId, accountCreatedAt: account.createdAt }) - .from(user) - .leftJoin(account, eq(account.userId, user.id)) - .where(and(gte(user.createdAt, from), lt(user.createdAt, to))), - db - .select({ userId: session.userId }) - .from(session) - .where(and(gte(session.createdAt, from), lt(session.createdAt, to))), - db - .select({ userId: activityEvents.userId }) - .from(activityEvents) - .innerJoin(user, eq(activityEvents.userId, user.id)) - .where(and(gte(activityEvents.createdAt, from), lt(activityEvents.createdAt, to))), - db - .select({ slug: organization.slug, metadata: organization.metadata }) - .from(organization) - .where(and(gte(organization.createdAt, from), lt(organization.createdAt, to))), - db - .select({ orgId: shares.orgId, kind: shares.kind }) + .select({ orgId: shares.orgId, kind: shares.kind, count: sql`COUNT(*)` }) .from(shares) - .where(and(gte(shares.createdAt, from), lt(shares.createdAt, to))), + .where(and(gte(shares.createdAt, from), lt(shares.createdAt, to))) + .groupBy(shares.orgId, shares.kind), ]) - const providers = new Map() - for (const row of signupRows) { - if (!row.provider || !row.accountCreatedAt) continue - const current = providers.get(row.userId) - if (!current || row.accountCreatedAt < current.at) - providers.set(row.userId, { provider: row.provider, at: row.accountCreatedAt }) - } - for (const userId of new Set(signupRows.map((row) => row.userId))) { - rollups.add(M.userSignup, '', 1, 0, { provider: providers.get(userId)?.provider ?? 'direct' }) - } - rollups.add(M.userSessionStarted, '', sessionRows.length) - rollups.addDistinct( - M.userActiveHour, - '', - [...sessionRows, ...activeActivityRows].map((row) => row.userId).filter(Boolean) as string[], - ) - for (const row of spaceRows) { - const orgType = isPersonalOrgLike({ slug: row.slug, metadata: row.metadata }) ? 'personal' : 'team' - rollups.add(M.spaceCreated, '', 1, 0, { org_type: orgType }) - } - for (const row of shareRows) rollups.add(M.shareCreated, row.orgId, 1, 0, { kind: row.kind }) + for (const row of signupRows) rollups.add(M.userSignup, '', Number(row.count), 0, { provider: row.provider }) + for (const row of shareRows) rollups.add(M.shareCreated, row.orgId, Number(row.count), 0, { kind: row.kind }) } async function addOperationalMetrics(db: Database, rollups: RollupAccumulator, from: Date, to: Date): Promise { - const [trafficRows, taskCreatedRows, taskFinishedRows, jobRows, usageRows, webhookRows] = await Promise.all([ + const [trafficRows, taskFinishedRows, jobRows] = await Promise.all([ db .select({ orgId: cloudTrafficReports.orgId, source: cloudTrafficReports.source, status: cloudTrafficReports.status, - bytes: cloudTrafficReports.bytes, - storageId: cloudTrafficReports.storageId, - unitBytes: cloudTrafficReports.unitBytes, - creditsPerUnit: cloudTrafficReports.creditsPerUnit, + count: sql`COUNT(*)`, + bytes: sql`COALESCE(SUM(${cloudTrafficReports.bytes}), 0)`, }) .from(cloudTrafficReports) - .where(and(gte(cloudTrafficReports.updatedAt, from), lt(cloudTrafficReports.updatedAt, to))), - db - .select({ orgId: downloadTasks.orgId, category: downloadTasks.category, sourceType: downloadTasks.sourceType }) - .from(downloadTasks) - .where(and(gte(downloadTasks.createdAt, from), lt(downloadTasks.createdAt, to))), + .where(and(gte(cloudTrafficReports.updatedAt, from), lt(cloudTrafficReports.updatedAt, to))) + .groupBy(cloudTrafficReports.orgId, cloudTrafficReports.source, cloudTrafficReports.status), db .select({ orgId: downloadTasks.orgId, category: downloadTasks.category, downloaderId: downloadTasks.assignedDownloaderId, status: downloadTasks.status, - bytes: downloadTasks.billingChargedBytes, + count: sql`COUNT(*)`, + bytes: sql`COALESCE(SUM(${downloadTasks.billingChargedBytes}), 0)`, }) .from(downloadTasks) - .where(and(gte(downloadTasks.finishedAt, from), lt(downloadTasks.finishedAt, to))), - db - .select({ orgId: backgroundJobs.orgId, type: backgroundJobs.type, status: backgroundJobs.status }) - .from(backgroundJobs) - .where(and(gte(backgroundJobs.finishedAt, from), lt(backgroundJobs.finishedAt, to))), + .where(and(gte(downloadTasks.finishedAt, from), lt(downloadTasks.finishedAt, to))) + .groupBy(downloadTasks.orgId, downloadTasks.category, downloadTasks.assignedDownloaderId, downloadTasks.status), db .select({ - orgId: remoteDownloadUsageReports.orgId, - downloaderId: remoteDownloadUsageReports.downloaderId, - status: remoteDownloadUsageReports.status, - bytes: remoteDownloadUsageReports.unitBytes, - credits: remoteDownloadUsageReports.creditsPerUnit, + orgId: backgroundJobs.orgId, + type: backgroundJobs.type, + status: backgroundJobs.status, + count: sql`COUNT(*)`, }) - .from(remoteDownloadUsageReports) - .where(and(gte(remoteDownloadUsageReports.createdAt, from), lt(remoteDownloadUsageReports.createdAt, to))), - db - .select({ status: webhookEvents.status }) - .from(webhookEvents) - .where(and(gte(webhookEvents.processedAt, from), lt(webhookEvents.processedAt, to))), + .from(backgroundJobs) + .where(and(gte(backgroundJobs.finishedAt, from), lt(backgroundJobs.finishedAt, to))) + .groupBy(backgroundJobs.orgId, backgroundJobs.type, backgroundJobs.status), ]) for (const row of trafficRows) { - rollups.add(M.trafficReportSync, row.orgId, 1, row.bytes, { source: row.source, status: row.status }) - if (row.status !== 'blocked' && row.unitBytes && row.creditsPerUnit) { - rollups.add( - M.trafficCreditConsumed, - row.orgId, - Math.ceil(row.bytes / row.unitBytes) * row.creditsPerUnit, - row.bytes, - { source: row.source, storage_id: row.storageId }, - ) - } - } - for (const row of taskCreatedRows) { - rollups.add(M.remoteDownloadTaskCreated, row.orgId, 1, 0, { - category: row.category ?? 'uncategorized', - source: row.sourceType, + rollups.add(M.trafficReportSync, row.orgId, Number(row.count), Number(row.bytes), { + source: row.source, + status: row.status, }) } for (const row of taskFinishedRows) { - rollups.add(M.remoteDownloadTaskFinished, row.orgId, 1, row.bytes, { + rollups.add(M.remoteDownloadTaskFinished, row.orgId, Number(row.count), Number(row.bytes), { category: row.category ?? 'uncategorized', downloader_id: row.downloaderId, outcome: row.status, }) } for (const row of jobRows) { - rollups.add(M.backgroundJobFinished, row.orgId, 1, 0, { job_type: row.type, outcome: row.status }) - } - for (const row of usageRows) { - rollups.add(M.remoteDownloadUsage, row.orgId, row.credits, row.bytes, { - downloader_id: row.downloaderId, - status: row.status, + rollups.add(M.backgroundJobFinished, row.orgId, Number(row.count), 0, { + job_type: row.type, + outcome: row.status, }) } - for (const row of webhookRows) rollups.add(M.webhookProcessed, '', 1, 0, { outcome: row.status }) } async function addSnapshotMetrics(db: Database, rollups: RollupAccumulator, now: Date): Promise { const [ + userRows, + activeUserRows, quotaRows, - storageRows, inventoryRows, inventoryByType, inventoryBySize, @@ -386,7 +339,11 @@ async function addSnapshotMetrics(db: Database, rollups: RollupAccumulator, now: jobRows, taskRows, downloaderRows, + trafficReportRows, + webhookRows, ] = await Promise.all([ + queryStage('user-inventory', userInventorySnapshot(db, now)), + queryStage('user-activity', activeUserSnapshot(db, now)), queryStage( 'quota', db @@ -394,47 +351,26 @@ async function addSnapshotMetrics(db: Database, rollups: RollupAccumulator, now: orgId: orgQuotas.orgId, used: orgQuotas.used, quota: orgQuotas.quota, - trafficUsed: orgQuotas.trafficUsed, - trafficQuota: orgQuotas.trafficQuota, }) .from(orgQuotas), ), - queryStage( - 'storage', - db.select({ id: storages.id, used: storages.used }).from(storages).where(eq(storages.status, 'active')), - ), queryStage('inventory-base', inventoryGroups(db, now, 'base')), queryStage('inventory-type', inventoryGroups(db, now, 'file_type_group')), queryStage('inventory-size', inventoryGroups(db, now, 'size_bucket')), queryStage('inventory-age', inventoryGroups(db, now, 'age_bucket')), - queryStage( - 'shares', - db - .select({ - orgId: shares.orgId, - kind: shares.kind, - status: shares.status, - expiresAt: shares.expiresAt, - downloadLimit: shares.downloadLimit, - downloads: shares.downloads, - }) - .from(shares), - ), + queryStage('shares', shareSnapshotGroups(db, now)), queryStage( 'jobs', db - .select({ orgId: backgroundJobs.orgId, type: backgroundJobs.type, status: backgroundJobs.status }) + .select({ orgId: backgroundJobs.orgId, count: sql`COUNT(*)` }) .from(backgroundJobs) - .where(inArray(backgroundJobs.status, ['queued', 'running'])), + .where(inArray(backgroundJobs.status, ['queued', 'running'])) + .groupBy(backgroundJobs.orgId), ), queryStage( 'tasks', db - .select({ - orgId: downloadTasks.orgId, - downloaderId: downloadTasks.assignedDownloaderId, - status: downloadTasks.status, - }) + .select({ orgId: downloadTasks.orgId, count: sql`COUNT(*)` }) .from(downloadTasks) .where( inArray(downloadTasks.status, [ @@ -446,43 +382,161 @@ async function addSnapshotMetrics(db: Database, rollups: RollupAccumulator, now: 'paused', 'interrupted', ]), - ), + ) + .groupBy(downloadTasks.orgId), + ), + queryStage( + 'downloaders', + db + .select({ status: downloaders.status, count: sql`COUNT(*)` }) + .from(downloaders) + .groupBy(downloaders.status), + ), + queryStage( + 'traffic-report-snapshot', + db + .select({ + status: cloudTrafficReports.status, + count: sql`COUNT(*)`, + bytes: sql`COALESCE(SUM(${cloudTrafficReports.bytes}), 0)`, + }) + .from(cloudTrafficReports) + .groupBy(cloudTrafficReports.status), + ), + queryStage( + 'webhook-snapshot', + db + .select({ status: webhookEvents.status, count: sql`COUNT(*)` }) + .from(webhookEvents) + .groupBy(webhookEvents.status), ), - queryStage('downloaders', db.select({ id: downloaders.id, status: downloaders.status }).from(downloaders)), ]) + rollups.setGauge(M.userInventory, '', userRows.total, 0) + rollups.setGauge(M.userInventory, '', userRows.normal, 0, 'status', 'normal') + rollups.setGauge(M.userInventory, '', userRows.verified, 0, 'status', 'verified') + rollups.setGauge(M.userInventory, '', userRows.unverified, 0, 'status', 'unverified') + rollups.setGauge(M.userInventory, '', userRows.banned, 0, 'status', 'banned') + rollups.setGauge(M.userInventory, '', userRows.silent, 0, 'status', 'silent') + for (const row of activeUserRows) rollups.setGauge(M.userActiveSnapshot, '', row.count, 0, 'window', row.window) for (const row of quotaRows) { rollups.setGauge(M.storageUsed, row.orgId, 0, row.used) rollups.setGauge(M.storageQuota, row.orgId, 0, row.quota) - rollups.setGauge(M.trafficQuotaUsed, row.orgId, 0, row.trafficUsed) - rollups.setGauge(M.trafficQuota, row.orgId, 0, row.trafficQuota) + rollups.incrementGauge(M.storageUsed, '', 0, row.used) + rollups.incrementGauge(M.storageQuota, '', 1, row.quota) + const status = + row.quota > 0 && row.used >= row.quota + ? 'over' + : row.quota > 0 && row.used >= row.quota * 0.8 + ? 'near' + : 'healthy' + rollups.incrementGauge(M.storageQuota, '', 1, 0, 'status', status) } - for (const row of storageRows) rollups.setGauge(M.storageUsed, '', 0, row.used, 'storage_id', row.id) for (const row of [...inventoryRows, ...inventoryByType, ...inventoryBySize, ...inventoryByAge]) { rollups.incrementGauge(M.storageInventory, row.orgId, row.files, row.bytes, row.dimensionKey, row.dimensionValue) + rollups.incrementGauge(M.storageInventory, '', row.files, row.bytes, row.dimensionKey, row.dimensionValue) } for (const row of shareRows) { - const lifecycle = shareLifecycle(row, now) - rollups.incrementGauge(M.shareInventory, row.orgId, 1, 0) - rollups.incrementGauge(M.shareInventory, row.orgId, 1, 0, 'kind', row.kind) - rollups.incrementGauge(M.shareInventory, row.orgId, 1, 0, 'lifecycle', lifecycle) + rollups.incrementGauge(M.shareInventory, row.orgId, Number(row.count), 0, 'lifecycle', row.lifecycle) + rollups.incrementGauge(M.shareInventory, '', Number(row.count), 0, 'lifecycle', row.lifecycle) } for (const row of jobRows) { - rollups.incrementGauge(M.backgroundJobSnapshot, row.orgId, 1, 0) - rollups.incrementGauge(M.backgroundJobSnapshot, row.orgId, 1, 0, 'job_type', row.type) - rollups.incrementGauge(M.backgroundJobSnapshot, row.orgId, 1, 0, 'status', row.status) + rollups.incrementGauge(M.backgroundJobSnapshot, row.orgId, Number(row.count), 0) + rollups.incrementGauge(M.backgroundJobSnapshot, '', Number(row.count), 0) } for (const row of taskRows) { - rollups.incrementGauge(M.remoteDownloadTaskSnapshot, row.orgId, 1, 0) - if (row.downloaderId) - rollups.incrementGauge(M.remoteDownloadTaskSnapshot, row.orgId, 1, 0, 'downloader_id', row.downloaderId) - rollups.incrementGauge(M.remoteDownloadTaskSnapshot, row.orgId, 1, 0, 'status', row.status) + rollups.incrementGauge(M.remoteDownloadTaskSnapshot, row.orgId, Number(row.count), 0) + rollups.incrementGauge(M.remoteDownloadTaskSnapshot, '', Number(row.count), 0) } for (const row of downloaderRows) { - rollups.incrementGauge(M.downloaderSnapshot, '', 1, 0) - rollups.incrementGauge(M.downloaderSnapshot, '', 1, 0, 'downloader_id', row.id) - rollups.incrementGauge(M.downloaderSnapshot, '', 1, 0, 'status', row.status) + rollups.incrementGauge(M.downloaderSnapshot, '', Number(row.count), 0) + rollups.incrementGauge(M.downloaderSnapshot, '', Number(row.count), 0, 'status', row.status) } + for (const row of trafficReportRows) { + rollups.incrementGauge(M.trafficReportSnapshot, '', Number(row.count), Number(row.bytes)) + rollups.incrementGauge(M.trafficReportSnapshot, '', Number(row.count), Number(row.bytes), 'status', row.status) + } + for (const row of webhookRows) { + rollups.incrementGauge(M.webhookSnapshot, '', Number(row.count), 0) + rollups.incrementGauge(M.webhookSnapshot, '', Number(row.count), 0, 'status', row.status) + } +} + +async function shareSnapshotGroups( + db: Database, + now: Date, +): Promise> { + const nowSeconds = Math.floor(now.getTime() / 1000) + const lifecycle = sql`CASE + WHEN ${shares.status} <> 'active' THEN 'revoked' + WHEN ${shares.expiresAt} IS NOT NULL AND ${shares.expiresAt} <= ${nowSeconds} THEN 'expired' + WHEN ${shares.downloadLimit} IS NOT NULL AND ${shares.downloads} >= ${shares.downloadLimit} THEN 'download_limit_reached' + ELSE 'usable' + END` + return await db + .select({ orgId: shares.orgId, lifecycle, count: sql`COUNT(*)` }) + .from(shares) + .groupBy(shares.orgId, lifecycle) +} + +type UserInventorySnapshot = { + total: number + normal: number + unverified: number + banned: number + silent: number + verified: number +} + +async function userInventorySnapshot(db: Database, now: Date): Promise { + const activeCutoffMs = now.getTime() - 30 * 86_400_000 + const activeCutoffSec = Math.floor(activeCutoffMs / 1000) + const rows = await db.all(sql` + SELECT + COUNT(*) AS total, + SUM(CASE WHEN email_verified = 1 THEN 1 ELSE 0 END) AS verified, + SUM(CASE WHEN email_verified = 0 AND COALESCE(banned, 0) = 0 THEN 1 ELSE 0 END) AS unverified, + SUM(CASE WHEN COALESCE(banned, 0) = 1 THEN 1 ELSE 0 END) AS banned, + SUM(CASE WHEN email_verified = 1 AND COALESCE(banned, 0) = 0 + AND NOT EXISTS (SELECT 1 FROM activity_events ae WHERE ae.user_id = user.id AND ae.created_at >= ${activeCutoffSec}) + AND NOT EXISTS (SELECT 1 FROM session s WHERE s.user_id = user.id AND s.created_at >= ${activeCutoffMs}) + THEN 1 ELSE 0 END) AS silent + FROM user + `) + const row = rows[0] ?? { total: 0, verified: 0, unverified: 0, banned: 0, silent: 0, normal: 0 } + const total = Number(row.total) + const unverified = Number(row.unverified) + const banned = Number(row.banned) + const silent = Number(row.silent) + const verified = Number(row.verified) + return { total, verified, unverified, banned, silent, normal: Math.max(0, total - unverified - banned - silent) } +} + +async function activeUserSnapshot( + db: Database, + now: Date, +): Promise> { + const windows = [ + { window: 'dau' as const, days: 1 }, + { window: 'wau' as const, days: 7 }, + { window: 'mau' as const, days: 30 }, + ] + return await Promise.all( + windows.map(async ({ window, days }) => { + const cutoffMs = now.getTime() - days * 86_400_000 + const cutoffSec = Math.floor(cutoffMs / 1000) + const rows = await db.all<{ count: number }>(sql` + SELECT COUNT(DISTINCT user_id) AS count + FROM ( + SELECT s.user_id AS user_id FROM session s JOIN user u ON u.id = s.user_id WHERE s.created_at >= ${cutoffMs} + UNION ALL + SELECT ae.user_id AS user_id FROM activity_events ae JOIN user u ON u.id = ae.user_id + WHERE ae.user_id IS NOT NULL AND ae.created_at >= ${cutoffSec} + ) active_users + `) + return { window, count: Number(rows[0]?.count ?? 0) } + }), + ) } type InventoryGroup = { @@ -572,10 +626,6 @@ class RollupAccumulator { } } - addDistinct(metric: AdminStatsMetric, orgId: string, values: string[]): void { - this.increment(metric, orgId, '', '', 0, 0, new Set(values).size, false) - } - setGauge( metric: AdminStatsMetric, orgId: string, @@ -654,34 +704,6 @@ function rollupId(bucketStart: Date, row: RollupValue): string { return `${bucketStart.getTime()}:${row.orgId || 'global'}:${row.metric}:${row.dimensionKey || 'all'}:${row.dimensionValue || 'all'}` } -function parseMetadata(value: string | null): Record { - if (!value) return {} - try { - const parsed: unknown = JSON.parse(value) - return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? (parsed as Record) : {} - } catch { - return {} - } -} - -function metadataString(metadata: Record, keyValue: string): string | null { - const value = metadata[keyValue] - return typeof value === 'string' && value.length > 0 ? value : null -} - -function metadataNumber(metadata: Record, keyValue: string): number { - const value = metadata[keyValue] - return hasFiniteNumber(value) ? value : 0 -} - -function hasFiniteNumber(value: unknown): value is number { - return typeof value === 'number' && Number.isFinite(value) -} - -function needsBytes(action: string): boolean { - return action === 'upload_confirm' || DOWNLOAD_ACTIONS.includes(action) -} - function downloadSourceForAction(action: string): string { if (action === 'object_download') return 'object_download' if (action === 'image_hosting_download') return 'image_hosting' @@ -689,16 +711,6 @@ function downloadSourceForAction(action: string): string { return 'landing_share' } -function shareLifecycle( - share: { status: string; expiresAt: Date | null; downloadLimit: number | null; downloads: number }, - now: Date, -): string { - if (share.status !== 'active') return 'revoked' - if (share.expiresAt && share.expiresAt <= now) return 'expired' - if (share.downloadLimit !== null && share.downloads >= share.downloadLimit) return 'download_limit_reached' - return 'usable' -} - async function queryStage(name: string, query: PromiseLike): Promise { try { return await query diff --git a/server/adapters/repos/admin-stats.ts b/server/adapters/repos/admin-stats.ts index e6b2e2d1..b58e6df8 100644 --- a/server/adapters/repos/admin-stats.ts +++ b/server/adapters/repos/admin-stats.ts @@ -10,24 +10,16 @@ import type { AdminTopShare, AdminTransferDataQuality, } from '@shared/types' -import type { SQL } from 'drizzle-orm' -import { and, count, desc, eq, gt, gte, inArray, isNull, lte, or, sql } from 'drizzle-orm' -import type { SQLiteTable } from 'drizzle-orm/sqlite-core' -import { organization, session, user } from '../../db/auth-schema' +import { and, eq, gte, inArray, lte } from 'drizzle-orm' +import { organization, user } from '../../db/auth-schema' +import { matters, shares, statsRollupsHourly } from '../../db/schema' import { - activityEvents, - backgroundJobs, - cloudTrafficReports, - downloaders, - downloadTasks, - matters, - orgQuotas, - shares, - statsRollupsHourly, - webhookEvents, -} from '../../db/schema' -import { ADMIN_STATS_METRICS } from '../../domain/admin-stats-metrics' -import { addCalendarDays, statsDayKey as dayKey, localDateStart } from '../../domain/admin-stats-time' + ADMIN_STATS_METRICS, + type AdminStatsDimension, + type AdminStatsMetric, + parseAdminStatsRollupMetadata, +} from '../../domain/admin-stats-metrics' +import { addCalendarDays, statsDayKey as dayKey, utcDateStart } from '../../domain/admin-stats-time' import type { Database } from '../../platform/interface' import type { AdminStatsDateRange, AdminStatsRepo } from '../../usecases/ports' import { AdminStatsHourlyReader } from './admin-stats-hourly' @@ -35,10 +27,6 @@ import { rebuildAdminStatsHour } from './admin-stats-rollup' const DOWNLOAD_ACTIVITY_ACTIONS = ['share_download', 'object_download', 'image_hosting_download', 'webdav_download'] const DOWNLOAD_FAILURE_ACTION = 'download_failed' -const STORAGE_USED_ROLLUP_METRIC = ADMIN_STATS_METRICS.storageUsed -const GLOBAL_ROLLUP_DIMENSION_KEY = '' -const GLOBAL_ROLLUP_DIMENSION_VALUE = '' - export function createAdminStatsRepo(db: Database): AdminStatsRepo { return { refreshHourlyRollups: (now) => refreshHourlyRollups(db, now), @@ -52,50 +40,34 @@ export function createAdminStatsRepo(db: Database): AdminStatsRepo { } async function refreshHourlyRollups(db: Database, now: Date) { - const currentHour = startOfHour(now) - const previousHour = new Date(currentHour.getTime() - 3_600_000) - const finalizePreviousSnapshot = now.getUTCMinutes() < 10 - const previous = await rebuildAdminStatsHour(db, previousHour, now, finalizePreviousSnapshot) - const current = await rebuildAdminStatsHour(db, currentHour, now, true) - return [previous, current] -} - -async function countRows(db: Database, table: typeof user): Promise { - const rows = await db.select({ value: count() }).from(table) - return toNumber(rows[0]?.value) -} - -async function countRowsWhere(db: Database, table: SQLiteTable, where: SQL | undefined): Promise { - const rows = await db.select({ value: count() }).from(table).where(where) - return toNumber(rows[0]?.value) -} - -async function getStorageByType(db: Database): Promise> { - const rows = await db - .select({ - type: matters.type, - files: count(), - bytes: sql`COALESCE(SUM(${matters.size}), 0)`, - }) - .from(matters) - .where(and(eq(matters.status, 'active'), eq(matters.dirtype, 0))) - .groupBy(matters.type) - .orderBy(desc(sql`COALESCE(SUM(${matters.size}), 0)`)) - - const result = rows.map((row) => ({ - type: row.type || 'unknown', - files: toNumber(row.files), - bytes: toNumber(row.bytes), - })) - if (result.length <= 8) return result - const other = result - .slice(8) - .reduce((total, row) => ({ type: 'other', files: total.files + row.files, bytes: total.bytes + row.bytes }), { - type: 'other', - files: 0, - bytes: 0, - }) - return [...result.slice(0, 8), other] + const latestClosedHour = new Date(startOfHour(now).getTime() - 3_600_000) + const repairFrom = new Date(latestClosedHour.getTime() - 47 * 3_600_000) + const markers = await db + .select({ bucketStart: statsRollupsHourly.bucketStart, metadata: statsRollupsHourly.metadata }) + .from(statsRollupsHourly) + .where( + and( + eq(statsRollupsHourly.metricKey, ADMIN_STATS_METRICS.statsRollupRun), + eq(statsRollupsHourly.orgId, ''), + eq(statsRollupsHourly.dimensionKey, ''), + gte(statsRollupsHourly.bucketStart, repairFrom), + lte(statsRollupsHourly.bucketStart, latestClosedHour), + ), + ) + const completed = new Set( + markers + .filter((row) => parseAdminStatsRollupMetadata(row.metadata) !== null) + .map((row) => row.bucketStart.getTime()), + ) + const repairTargets: Date[] = [] + for (let at = repairFrom.getTime(); at < latestClosedHour.getTime(); at += 3_600_000) { + if (!completed.has(at)) repairTargets.push(new Date(at)) + if (repairTargets.length === 3) break + } + const latest = await rebuildAdminStatsHour(db, latestClosedHour, now, true) + const repaired = [] + for (const bucketStart of repairTargets) repaired.push(await rebuildAdminStatsHour(db, bucketStart, now, false)) + return [latest, ...repaired] } async function getDashboardOverviewStats( @@ -107,7 +79,7 @@ async function getDashboardOverviewStats( const reader = new AdminStatsHourlyReader(db, range, now) const previousReader = new AdminStatsHourlyReader(db, previous, now) const [ - totalUsers, + users, newUsers, previousNewUsers, activeUsers, @@ -118,23 +90,27 @@ async function getDashboardOverviewStats( sharing, previousSharing, dataQuality, + coverage, + comparisonCoverage, ] = await Promise.all([ - countRows(db, user), + getUserInventory(reader), getSignupTotal(reader), getSignupTotal(previousReader), - countActiveUsers(db, range), - countActiveUsers(db, previous), - getQuotaTotals(db), + getActiveUserSnapshot(reader), + getActiveUserSnapshot(previousReader), + getQuotaTotals(reader), getTrafficTotals(reader), getTrafficTotals(previousReader), - getSharingEventTotals(db, now, reader), - getSharingEventTotals(db, now, previousReader), - getTransferDataQuality(db, range, now), + getSharingEventTotals(reader), + getSharingComparisonTotals(previousReader), + getTransferDataQuality(reader, previousReader), + reader.coverage(), + previousReader.coverage(), ]) const [trendNewUsers, activeByDay, storageUsedByDay, uploadByDay, downloadByDay] = await Promise.all([ getSignupsByDay(reader), - getActiveUsersByDay(db, range), - getStorageUsedByDay(db, range), + getActiveUsersByDay(reader), + getStorageUsedByDay(reader), getActivityMetricByDay(reader, metricSpec(['upload_confirm']), 'bytes'), getActivityMetricByDay(reader, metricSpec(DOWNLOAD_ACTIVITY_ACTIONS), 'bytes'), ]) @@ -142,7 +118,7 @@ async function getDashboardOverviewStats( return { date, newUsers: trendNewUsers.get(date) ?? 0, - activeUsers: activeByDay.get(date)?.size ?? 0, + activeUsers: activeByDay.get(date) ?? 0, storageUsedBytes: storageUsedByDay.get(date) ?? null, uploadBytes: uploadByDay.get(date) ?? 0, downloadBytes: downloadByDay.get(date) ?? 0, @@ -150,14 +126,16 @@ async function getDashboardOverviewStats( }) return { - ...statsFrame(now, range), + ...statsFrame(now, range, coverage, comparisonCoverage), dataQuality, totals: { - users: totalUsers, + users: users.total, newUsers: delta(newUsers, previousNewUsers), - activeUsers: delta(activeUsers, previousActiveUsers), + activeUsers: delta(activeUsers.mau, previousActiveUsers.mau), + activeUserRate: nullablePercent(activeUsers.mau, users.total), storageUsedBytes: quotas.usedBytes, storageQuotaBytes: quotas.quotaBytes, + storageUtilization: nullablePercent(quotas.usedBytes, quotas.quotaBytes), trafficBytes: delta( traffic.uploadBytes + traffic.downloadBytes, previousTraffic.uploadBytes + previousTraffic.downloadBytes, @@ -178,11 +156,10 @@ async function getDashboardOperationsStats( range: AdminStatsDateRange, ): Promise { const reader = new AdminStatsHourlyReader(db, range, now) - const activeTaskStatuses = ['queued', 'assigned', 'downloading', 'uploading', 'suspended', 'paused', 'interrupted'] const [ activeBackgroundJobs, activeRemoteDownloads, - downloaderRows, + downloaderStatus, cloudReportBacklog, webhookFailures, backgroundJobOutcomes, @@ -190,26 +167,27 @@ async function getDashboardOperationsStats( cloudReportStatus, backgroundJobsByDay, remoteDownloadsByDay, + coverage, ] = await Promise.all([ - countRowsWhere(db, backgroundJobs, inArray(backgroundJobs.status, ['queued', 'running'])), - countRowsWhere(db, downloadTasks, inArray(downloadTasks.status, activeTaskStatuses)), - db.select({ status: downloaders.status }).from(downloaders), - countRowsWhere(db, cloudTrafficReports, inArray(cloudTrafficReports.status, ['pending', 'failed'])), - countRowsWhere(db, webhookEvents, eq(webhookEvents.status, 'failed')), + getLatestGaugeTotal(reader, ADMIN_STATS_METRICS.backgroundJobSnapshot), + getLatestGaugeTotal(reader, ADMIN_STATS_METRICS.remoteDownloadTaskSnapshot), + getLatestGaugeDimensions(reader, ADMIN_STATS_METRICS.downloaderSnapshot, 'status'), + getLatestGaugeDimensionSum(reader, ADMIN_STATS_METRICS.trafficReportSnapshot, 'status', ['pending', 'failed']), + getLatestGaugeDimensionSum(reader, ADMIN_STATS_METRICS.webhookSnapshot, 'status', ['failed']), getOperationalOutcomes(reader, 'background_job'), getOperationalOutcomes(reader, 'remote_download'), getCloudReportOutcomes(reader), getOperationalOutcomesByDay(reader, 'background_job'), getOperationalOutcomesByDay(reader, 'remote_download'), + reader.coverage(), ]) - const downloaderStatus = countBy(downloaderRows, (row) => row.status) const completedJobs = backgroundJobOutcomes.get('completed') ?? 0 const failedJobs = backgroundJobOutcomes.get('failed') ?? 0 const completedRemoteDownloads = remoteDownloadOutcomes.get('completed') ?? 0 const failedRemoteDownloads = remoteDownloadOutcomes.get('failed') ?? 0 return { - ...statsFrame(now, range), + ...statsFrame(now, range, coverage), summary: { activeBackgroundJobs, activeRemoteDownloads, @@ -222,6 +200,7 @@ async function getDashboardOperationsStats( ), cloudReportBacklog, webhookFailures, + alertCount: cloudReportBacklog + webhookFailures, }, trend: createDateBuckets(range).map((date) => ({ date, @@ -245,50 +224,55 @@ async function getDashboardGrowthStats( const previous = previousRange(range) const reader = new AdminStatsHourlyReader(db, range, now) const previousReader = new AdminStatsHourlyReader(db, previous, now) - const [usersRows, newUsers, previousNewUsers, activeUsers, previousActiveUsers, activeByDay, registrationSources] = - await Promise.all([ - db - .select({ id: user.id, emailVerified: user.emailVerified, banned: user.banned, createdAt: user.createdAt }) - .from(user), - getSignupTotal(reader), - getSignupTotal(previousReader), - countActiveUsers(db, range), - countActiveUsers(db, previous), - getRollingActiveUserTrend(db, range), - getRegistrationSources(reader), - ]) - const activeLast30 = await activeUserIds(db, { from: daysAgo(now, 30), to: now, timeZone: range.timeZone }) - const totalUsers = usersRows.length - const verifiedUsers = usersRows.filter((row) => row.emailVerified).length - const bannedUsers = usersRows.filter((row) => row.banned).length - const unverifiedUsers = usersRows.filter((row) => !row.emailVerified && !row.banned).length - const silentUsers = usersRows.filter((row) => row.emailVerified && !row.banned && !activeLast30.has(row.id)).length - const normalUsers = Math.max(0, totalUsers - unverifiedUsers - bannedUsers - silentUsers) + const [ + users, + newUsers, + previousNewUsers, + activeUsers, + previousActiveUsers, + activeByDay, + registrationSources, + totalsByDay, + coverage, + comparisonCoverage, + ] = await Promise.all([ + getUserInventory(reader), + getSignupTotal(reader), + getSignupTotal(previousReader), + getActiveUserSnapshot(reader), + getActiveUserSnapshot(previousReader), + getRollingActiveUserTrend(reader, range), + getRegistrationSources(reader), + getUserTotalsByDay(reader), + reader.coverage(), + previousReader.coverage(), + ]) const newUsersByDay = await getSignupsByDay(reader) - let totalAtStart = usersRows.filter((row) => row.createdAt < range.from).length - const userScaleTrend = createDateBuckets(range).map((date) => { - const dailyNew = newUsersByDay.get(date) ?? 0 - totalAtStart += dailyNew - return { date, newUsers: dailyNew, totalUsers: totalAtStart } - }) + const userScaleTrend = createDateBuckets(range).map((date) => ({ + date, + newUsers: newUsersByDay.get(date) ?? 0, + totalUsers: totalsByDay.get(date) ?? 0, + })) return { - ...statsFrame(now, range), + ...statsFrame(now, range, coverage, comparisonCoverage), summary: { - totalUsers, + totalUsers: users.total, newUsers: delta(newUsers, previousNewUsers), - activeUsers: delta(activeUsers, previousActiveUsers), - verifiedUsers, - bannedUsers, - silentUsers, + activeUsers: delta(activeUsers.mau, previousActiveUsers.mau), + verifiedUsers: users.verified, + bannedUsers: users.banned, + silentUsers: users.silent, + activeUserRate: nullablePercent(activeUsers.mau, users.total), + silentUserRate: nullablePercent(users.silent, users.total), }, userScaleTrend, activeUserTrend: activeByDay, userStatus: percentRows([ - { name: 'normal', value: normalUsers }, - { name: 'unverified', value: unverifiedUsers }, - { name: 'banned', value: bannedUsers }, - { name: 'silent', value: silentUsers }, + { name: 'normal', value: users.normal }, + { name: 'unverified', value: users.unverified }, + { name: 'banned', value: users.banned }, + { name: 'silent', value: users.silent }, ]), registrationSources, } @@ -304,7 +288,7 @@ async function getDashboardStorageStats( const previousReader = new AdminStatsHourlyReader(db, previous, now) const [ quotas, - files, + inventory, storageUsedByDay, typeBreakdown, newFiles, @@ -315,24 +299,30 @@ async function getDashboardStorageStats( uploadFilesByDay, dataQuality, spaceUsage, + quotaPressure, + sizeBreakdown, + ageBreakdown, + coverage, + comparisonCoverage, ] = await Promise.all([ - getQuotaTotals(db), - db - .select({ id: matters.id, type: matters.type, size: matters.size, createdAt: matters.createdAt }) - .from(matters) - .where(and(eq(matters.status, 'active'), eq(matters.dirtype, 0))), - getStorageUsedByDay(db, range), - getStorageByType(db), + getQuotaTotals(reader), + getStorageInventory(reader), + getStorageUsedByDay(reader), + getLatestInventoryBreakdown(reader, 'file_type_group'), getActivityMetricTotal(reader, metricSpec(['upload_confirm']), 'count'), getActivityMetricTotal(previousReader, metricSpec(['upload_confirm']), 'count'), getActivityMetricTotal(reader, metricSpec(['upload_confirm']), 'bytes'), getActivityMetricTotal(previousReader, metricSpec(['upload_confirm']), 'bytes'), getActivityMetricByDay(reader, metricSpec(['upload_confirm']), 'bytes'), getActivityMetricByDay(reader, metricSpec(['upload_confirm']), 'count'), - getTransferDataQuality(db, range, now), - getUsageBySpaceRows(db), + getTransferDataQuality(reader, previousReader), + getUsageBySpaceRows(db, reader), + getLatestGaugeDimensions(reader, ADMIN_STATS_METRICS.storageQuota, 'status'), + getLatestInventoryBreakdown(reader, 'size_bucket'), + getLatestInventoryBreakdown(reader, 'age_bucket'), + reader.coverage(), + previousReader.coverage('counters'), ]) - const fileItems = files.map((row) => ({ bytes: row.size ?? 0, createdAt: row.createdAt })) const storageTrend = createDateBuckets(range).map((date) => { return { date, @@ -341,36 +331,33 @@ async function getDashboardStorageStats( newFiles: uploadFilesByDay.get(date) ?? 0, } }) - const coldCutoff = daysAgo(now, 90) - const coldFileBytes = fileItems.filter((row) => row.createdAt < coldCutoff).reduce((sum, row) => sum + row.bytes, 0) + const coldFileBytes = ['90-180d', '>180d'].reduce( + (total, bucket) => total + (ageBreakdown.find((row) => row.name === bucket)?.bytes ?? 0), + 0, + ) return { - ...statsFrame(now, range), + ...statsFrame(now, range, coverage, comparisonCoverage), dataQuality, summary: { storageUsedBytes: quotas.usedBytes, quotaBytes: quotas.quotaBytes, - fileCount: files.length, + fileCount: inventory.files, newFiles: delta(newFiles, previousNewFiles), newBytes: delta(uploadBytes, previousUploadBytes), coldFileBytes, - nearQuotaSpaces: spaceUsage.filter((space) => space.utilization >= 80 && space.utilization < 100).length, - overQuotaSpaces: spaceUsage.filter((space) => space.utilization >= 100).length, + storageUtilization: nullablePercent(quotas.usedBytes, quotas.quotaBytes), + coldFilePercent: nullablePercent(coldFileBytes, quotas.usedBytes), + nearQuotaSpaces: quotaPressure.get('near') ?? 0, + overQuotaSpaces: quotaPressure.get('over') ?? 0, }, storageTrend, - typeBreakdown: percentByBytes( - typeBreakdown.map((row) => ({ name: row.type, bytes: row.bytes, files: row.files })), - ).map(({ name, ...row }) => ({ + typeBreakdown: typeBreakdown.map(({ name, ...row }) => ({ type: name, ...row, })), - sizeBreakdown: fileBucketBreakdown(fileItems, [ - { name: '<10MB', max: 10 * 1024 * 1024 }, - { name: '10-100MB', max: 100 * 1024 * 1024 }, - { name: '100MB-1GB', max: 1024 * 1024 * 1024 }, - { name: '>1GB', max: Number.POSITIVE_INFINITY }, - ]), - ageBreakdown: ageBucketBreakdown(fileItems, now), + sizeBreakdown, + ageBreakdown, topSpaces: spaceUsage.slice(0, 8), } } @@ -399,6 +386,8 @@ async function getDashboardTrafficStats( downloadFailureReasons, uploadFailureReasons, dataQuality, + coverage, + comparisonCoverage, ] = await Promise.all([ getTrafficTotals(reader), getTrafficTotals(previousReader), @@ -414,7 +403,9 @@ async function getDashboardTrafficStats( getActivityMetricByDay(reader, metricSpec([DOWNLOAD_FAILURE_ACTION]), 'count'), getActivityMetricDimensionTotals(reader, metricSpec([DOWNLOAD_FAILURE_ACTION]), 'reason', 'count'), getActivityMetricDimensionTotals(reader, metricSpec(['upload_cancel', 'upload_failed']), 'reason', 'count'), - getTransferDataQuality(db, range, now), + getTransferDataQuality(reader, previousReader), + reader.coverage('counters'), + previousReader.coverage('counters'), ]) const sourceRows = new Map() if (traffic.uploadBytes > 0 || traffic.uploadRequests > 0) { @@ -463,7 +454,7 @@ async function getDashboardTrafficStats( const issuedDownloads = Math.max(0, traffic.downloadRequests - blockedDownloads) return { - ...statsFrame(now, range), + ...statsFrame(now, range, coverage, comparisonCoverage), dataQuality, summary: { totalBytes: delta( @@ -473,7 +464,7 @@ async function getDashboardTrafficStats( requestCount: delta(totalRequests, previousTraffic.uploadRequests + previousTraffic.downloadRequests), issuedDownloads, blockedDownloads, - issueRate: nullablePercent(issuedDownloads, issuedDownloads + blockedDownloads), + downloadIssueSuccessRate: nullablePercent(issuedDownloads, issuedDownloads + blockedDownloads), peakDailyBytes: Math.max(0, ...trafficTrend.map((row) => row.uploadBytes + row.downloadBytes)), }, trafficTrend, @@ -497,31 +488,25 @@ async function getDashboardSharingStats( const [ sharing, previousSharing, - shareRows, createdInRange, createdPrevious, typeCounts, saveCount, previousSaveCount, downloadSources, + coverage, + comparisonCoverage, ] = await Promise.all([ - getSharingEventTotals(db, now, reader), - getSharingEventTotals(db, now, previousReader), - db - .select({ - kind: shares.kind, - status: shares.status, - expiresAt: shares.expiresAt, - downloadLimit: shares.downloadLimit, - downloads: shares.downloads, - }) - .from(shares), + getSharingEventTotals(reader), + getSharingComparisonTotals(previousReader), getShareCreatedTotal(reader), getShareCreatedTotal(previousReader), getShareCreatedKinds(reader), getActivityMetricTotal(reader, metricSpec(['save_from_share']), 'count'), getActivityMetricTotal(previousReader, metricSpec(['save_from_share']), 'count'), getActivityMetricDimensionTotals(reader, metricSpec(['share_download']), 'source', 'count'), + reader.coverage(), + previousReader.coverage('counters'), ]) const landingDownloads = downloadSources.get('landing_share') ?? 0 const directDownloads = downloadSources.get('direct_share') ?? 0 @@ -536,21 +521,21 @@ async function getDashboardSharingStats( downloads: downloadsByDay.get(date) ?? 0, saves: savesByDay.get(date) ?? 0, })) - const activeShares = shareRows.filter((row) => isShareActive(row, now)).length const topShares = await getTopSharesWithPercent(db, reader, { totalViews: sharing.views, totalDownloads: sharing.downloads, }) return { - ...statsFrame(now, range), + ...statsFrame(now, range, coverage, comparisonCoverage), summary: { - activeShares, + activeShares: sharing.activeShares, createdShares: delta(createdInRange, createdPrevious), views: delta(sharing.views, previousSharing.views), downloads: delta(sharing.downloads, previousSharing.downloads), saves: delta(saveCount, previousSaveCount), - downloadConversionRate: nullablePercent(landingDownloads, sharing.views), + downloadsPer100Views: nullablePercent(landingDownloads, sharing.views), + savesPer100Views: nullablePercent(saveCount, sharing.views), passwordPasses: sharing.passwordPasses, }, trend, @@ -570,7 +555,7 @@ async function getShareCreatedTotal(reader: AdminStatsHourlyReader): Promise> { - const rows = await reader.rows(ADMIN_STATS_METRICS.shareCreated) + const rows = await reader.rows(ADMIN_STATS_METRICS.shareCreated, ['kind']) const result = new Map() for (const row of rows) { if (row.dimensionKey === 'kind') incrementMap(result, row.dimensionValue, row.count) @@ -578,21 +563,33 @@ async function getShareCreatedKinds(reader: AdminStatsHourlyReader): Promise>, + comparisonCoverage?: Awaited>, +) { + return { + generatedAt: now.toISOString(), + from: range.from.toISOString(), + to: range.to.toISOString(), + timeZone: 'UTC' as const, + coverage, + comparisonCoverage, + } } function previousRange(range: AdminStatsDateRange): AdminStatsDateRange { - const firstDate = dayKey(range.from, range.timeZone) - const lastDate = dayKey(range.to, range.timeZone) - const localStart = localDateStart(firstDate, range.timeZone) - const localEnd = new Date(localDateStart(addCalendarDays(lastDate, 1), range.timeZone).getTime() - 1) + const firstDate = dayKey(range.from) + const lastDate = dayKey(range.to) + const localStart = utcDateStart(firstDate) + const localEnd = new Date(utcDateStart(addCalendarDays(lastDate, 1)).getTime() - 1) if (localStart.getTime() === range.from.getTime() && localEnd.getTime() === range.to.getTime()) { const days = Math.floor((dateOrdinal(lastDate) - dateOrdinal(firstDate)) / 86_400_000) + 1 const previousStartDate = addCalendarDays(firstDate, -days) return { - from: localDateStart(previousStartDate, range.timeZone), - to: new Date(localDateStart(firstDate, range.timeZone).getTime() - 1), + from: utcDateStart(previousStartDate), + to: new Date(utcDateStart(firstDate).getTime() - 1), timeZone: range.timeZone, } } @@ -602,110 +599,145 @@ function previousRange(range: AdminStatsDateRange): AdminStatsDateRange { } function delta(value: number, previousValue: number): AdminStatsDelta { - return { value, previousValue, changePercent: nullablePercent(value - previousValue, previousValue) } + return { + value, + previousValue, + change: value - previousValue, + changePercent: nullablePercent(value - previousValue, previousValue), + } } function createDateBuckets(range: AdminStatsDateRange): string[] { const dates = new Set() for (let timestamp = range.from.getTime(); timestamp <= range.to.getTime(); timestamp += 6 * 60 * 60 * 1000) { - dates.add(dayKey(new Date(timestamp), range.timeZone)) + dates.add(dayKey(new Date(timestamp))) } - dates.add(dayKey(range.to, range.timeZone)) + dates.add(dayKey(range.to)) return [...dates] } -async function getQuotaTotals(db: Database): Promise<{ usedBytes: number; quotaBytes: number }> { - const rows = await db - .select({ - usedBytes: sql`COALESCE(SUM(${orgQuotas.used}), 0)`, - quotaBytes: sql`COALESCE(SUM(${orgQuotas.quota}), 0)`, - }) - .from(orgQuotas) - return { usedBytes: toNumber(rows[0]?.usedBytes), quotaBytes: toNumber(rows[0]?.quotaBytes) } +async function getLatestGaugeTotal(reader: AdminStatsHourlyReader, metric: AdminStatsMetric): Promise { + const rows = await reader.latestRows(metric) + return rows.find((row) => row.orgId === '' && row.dimensionKey === '')?.count ?? 0 } -async function countActiveUsers(db: Database, range: AdminStatsDateRange): Promise { - return (await activeUserIds(db, range)).size -} - -async function activeUserIds(db: Database, range: AdminStatsDateRange): Promise> { - const [activityRows, sessionRows] = await Promise.all([ - db - .select({ userId: activityEvents.userId }) - .from(activityEvents) - .innerJoin(user, eq(activityEvents.userId, user.id)) - .where(and(gte(activityEvents.createdAt, range.from), lte(activityEvents.createdAt, range.to))) - .groupBy(activityEvents.userId), - db - .select({ userId: session.userId }) - .from(session) - .innerJoin(user, eq(session.userId, user.id)) - .where(and(gte(session.createdAt, range.from), lte(session.createdAt, range.to))) - .groupBy(session.userId), - ]) - const ids = new Set() - for (const row of activityRows) if (row.userId) ids.add(row.userId) - for (const row of sessionRows) ids.add(row.userId) - return ids -} - -async function getActiveUsersByDay(db: Database, range: AdminStatsDateRange): Promise>> { - const [activityRows, sessionRows] = await Promise.all([ - db - .select({ userId: activityEvents.userId, createdAt: activityEvents.createdAt }) - .from(activityEvents) - .innerJoin(user, eq(activityEvents.userId, user.id)) - .where(and(gte(activityEvents.createdAt, range.from), lte(activityEvents.createdAt, range.to))), - db - .select({ userId: session.userId, createdAt: session.createdAt }) - .from(session) - .innerJoin(user, eq(session.userId, user.id)) - .where(and(gte(session.createdAt, range.from), lte(session.createdAt, range.to))), - ]) - const byDay = new Map>() - for (const row of activityRows) { - if (row.userId) addSetMap(byDay, dayKey(row.createdAt, range.timeZone), row.userId) +async function getLatestGaugeDimensions( + reader: AdminStatsHourlyReader, + metric: AdminStatsMetric, + dimensionKey: AdminStatsDimension, + field: 'count' | 'bytes' = 'count', +): Promise> { + const result = new Map() + for (const row of await reader.latestRows(metric, [dimensionKey])) { + if (row.orgId === '' && row.dimensionKey === dimensionKey) incrementMap(result, row.dimensionValue, row[field]) } - for (const row of sessionRows) addSetMap(byDay, dayKey(row.createdAt, range.timeZone), row.userId) - return byDay + return result +} + +async function getLatestGaugeDimensionSum( + reader: AdminStatsHourlyReader, + metric: AdminStatsMetric, + dimensionKey: AdminStatsDimension, + values: string[], +): Promise { + const dimensions = await getLatestGaugeDimensions(reader, metric, dimensionKey) + return values.reduce((sum, value) => sum + (dimensions.get(value) ?? 0), 0) +} + +async function getQuotaTotals(reader: AdminStatsHourlyReader): Promise<{ usedBytes: number; quotaBytes: number }> { + const [usedRows, quotaRows] = await Promise.all([ + reader.latestRows(ADMIN_STATS_METRICS.storageUsed), + reader.latestRows(ADMIN_STATS_METRICS.storageQuota), + ]) + return { + usedBytes: usedRows.find((row) => row.orgId === '' && row.dimensionKey === '')?.bytes ?? 0, + quotaBytes: quotaRows.find((row) => row.orgId === '' && row.dimensionKey === '')?.bytes ?? 0, + } +} + +type UserInventory = { + total: number + normal: number + unverified: number + banned: number + silent: number + verified: number +} + +async function getUserInventory(reader: AdminStatsHourlyReader): Promise { + const rows = await reader.latestRows(ADMIN_STATS_METRICS.userInventory, ['', 'status']) + const dimensions = new Map( + rows.filter((row) => row.dimensionKey === 'status').map((row) => [row.dimensionValue, row.count]), + ) + return { + total: rows.find((row) => row.dimensionKey === '')?.count ?? 0, + normal: dimensions.get('normal') ?? 0, + unverified: dimensions.get('unverified') ?? 0, + banned: dimensions.get('banned') ?? 0, + silent: dimensions.get('silent') ?? 0, + verified: dimensions.get('verified') ?? 0, + } +} + +type ActiveUserSnapshot = { dau: number; wau: number; mau: number } + +async function getActiveUserSnapshot(reader: AdminStatsHourlyReader): Promise { + const dimensions = await getLatestGaugeDimensions(reader, ADMIN_STATS_METRICS.userActiveSnapshot, 'window') + return { dau: dimensions.get('dau') ?? 0, wau: dimensions.get('wau') ?? 0, mau: dimensions.get('mau') ?? 0 } +} + +async function getActiveUsersByDay(reader: AdminStatsHourlyReader): Promise> { + return getLatestGaugeValueByDay(reader, ADMIN_STATS_METRICS.userActiveSnapshot, 'count', 'window', 'dau') +} + +async function getUserTotalsByDay(reader: AdminStatsHourlyReader): Promise> { + return getLatestGaugeValueByDay(reader, ADMIN_STATS_METRICS.userInventory, 'count', '', '') } async function getRollingActiveUserTrend( - db: Database, + reader: AdminStatsHourlyReader, range: AdminStatsDateRange, ): Promise> { - // Query one extra UTC day so a 30-day calendar window is complete across DST boundaries. - const extendedRange = { from: daysAgo(range.from, 30), to: range.to } - const [activityRows, sessionRows] = await Promise.all([ - db - .select({ userId: activityEvents.userId, createdAt: activityEvents.createdAt }) - .from(activityEvents) - .innerJoin(user, eq(activityEvents.userId, user.id)) - .where(and(gte(activityEvents.createdAt, extendedRange.from), lte(activityEvents.createdAt, extendedRange.to))), - db - .select({ userId: session.userId, createdAt: session.createdAt }) - .from(session) - .innerJoin(user, eq(session.userId, user.id)) - .where(and(gte(session.createdAt, extendedRange.from), lte(session.createdAt, extendedRange.to))), + const [dau, wau, mau] = await Promise.all([ + getLatestGaugeValueByDay(reader, ADMIN_STATS_METRICS.userActiveSnapshot, 'count', 'window', 'dau'), + getLatestGaugeValueByDay(reader, ADMIN_STATS_METRICS.userActiveSnapshot, 'count', 'window', 'wau'), + getLatestGaugeValueByDay(reader, ADMIN_STATS_METRICS.userActiveSnapshot, 'count', 'window', 'mau'), ]) - const records: Array<{ userId: string; at: Date }> = [] - for (const row of activityRows) if (row.userId) records.push({ userId: row.userId, at: row.createdAt }) - for (const row of sessionRows) records.push({ userId: row.userId, at: row.createdAt }) - return createDateBuckets(range).map((dateKeyValue) => { - return { - date: dateKeyValue, - dau: distinctUsersInCalendarWindow(records, dateKeyValue, 1, range.timeZone), - wau: distinctUsersInCalendarWindow(records, dateKeyValue, 7, range.timeZone), - mau: distinctUsersInCalendarWindow(records, dateKeyValue, 30, range.timeZone), - } - }) + return createDateBuckets(range).map((date) => ({ + date, + dau: dau.get(date) ?? 0, + wau: wau.get(date) ?? 0, + mau: mau.get(date) ?? 0, + })) +} + +async function getLatestGaugeValueByDay( + reader: AdminStatsHourlyReader, + metric: AdminStatsMetric, + field: 'count' | 'bytes', + dimensionKey: AdminStatsDimension | '', + dimensionValue: string, +): Promise> { + const byHour = new Map() + for (const row of await reader.rows(metric, [dimensionKey])) { + if (row.orgId !== '' || row.dimensionKey !== dimensionKey || row.dimensionValue !== dimensionValue) continue + const at = row.bucketStart.getTime() + byHour.set(at, (byHour.get(at) ?? 0) + row[field]) + } + const latestByDay = new Map() + for (const [at, value] of byHour) { + const date = reader.dayKey(new Date(at)) + const current = latestByDay.get(date) + if (!current || at > current.at) latestByDay.set(date, { at, value }) + } + return new Map([...latestByDay].map(([date, value]) => [date, value.value])) } async function getRegistrationSources( reader: AdminStatsHourlyReader, ): Promise> { const counts = new Map() - const rows = await reader.rows(ADMIN_STATS_METRICS.userSignup) + const rows = await reader.rows(ADMIN_STATS_METRICS.userSignup, ['provider']) for (const row of rows) { if (row.dimensionKey === 'provider') incrementMap(counts, row.dimensionValue, row.count) } @@ -729,7 +761,7 @@ async function getSignupsByDay(reader: AdminStatsHourlyReader): Promise>() for (const row of rows) { if (row.dimensionKey !== 'outcome') continue @@ -793,10 +825,10 @@ async function getOperationalOutcomesByDay( async function getActivityMetricDimensionTotalsFromRollup( reader: AdminStatsHourlyReader, metric: (typeof ADMIN_STATS_METRICS)[keyof typeof ADMIN_STATS_METRICS], - dimensionKey: string, + dimensionKey: AdminStatsDimension, field: 'count' | 'bytes', ): Promise> { - const rows = await reader.rows(metric) + const rows = await reader.rows(metric, [dimensionKey]) const result = new Map() for (const row of rows) { if (row.dimensionKey === dimensionKey) incrementMap(result, row.dimensionValue, row[field]) @@ -813,7 +845,7 @@ async function getActivityMetricTotal( spec: ActivityMetricSpec, field: 'count' | 'bytes', ): Promise { - const rows = await reader.rows(spec.metric) + const rows = await reader.rows(spec.metric, [spec.dimensionKey ?? '']) return filterMetricRows(rows, spec).reduce((sum, row) => sum + row[field], 0) } @@ -822,7 +854,7 @@ async function getActivityMetricByDay( spec: ActivityMetricSpec, field: 'count' | 'bytes', ): Promise> { - const rows = await reader.rows(spec.metric) + const rows = await reader.rows(spec.metric, [spec.dimensionKey ?? '']) const result = new Map() for (const row of filterMetricRows(rows, spec)) { incrementMap(result, reader.dayKey(row.bucketStart), row[field]) @@ -833,10 +865,10 @@ async function getActivityMetricByDay( async function getActivityMetricDimensionTotals( reader: AdminStatsHourlyReader, spec: ActivityMetricSpec, - dimensionKey: string, + dimensionKey: AdminStatsDimension, field: 'count' | 'bytes', ): Promise> { - const rows = await reader.rows(spec.metric) + const rows = await reader.rows(spec.metric, [dimensionKey]) const result = new Map() for (const row of rows) { if (row.dimensionKey === dimensionKey) incrementMap(result, row.dimensionValue, row[field]) @@ -853,25 +885,25 @@ function filterMetricRows(rows: Awaited { - const previous = previousRange(range) const [current, previousCounts] = await Promise.all([ - getMissingTransferBytes(new AdminStatsHourlyReader(db, range, now)), - getMissingTransferBytes(new AdminStatsHourlyReader(db, previous, now)), + getMissingTransferBytes(reader), + getMissingTransferBytes(previousReader), ]) return { missingUploadBytesEvents: current.upload, previousMissingUploadBytesEvents: previousCounts.upload, missingDownloadBytesEvents: current.download, previousMissingDownloadBytesEvents: previousCounts.download, + missingBytesEvents: current.upload + current.download, + previousMissingBytesEvents: previousCounts.upload + previousCounts.download, } } async function getMissingTransferBytes(reader: AdminStatsHourlyReader): Promise<{ upload: number; download: number }> { - const rows = await reader.rows(ADMIN_STATS_METRICS.statsMissingBytes) + const rows = await reader.rows(ADMIN_STATS_METRICS.statsMissingBytes, ['direction']) const result = { upload: 0, download: 0 } for (const row of rows) { if (row.dimensionKey !== 'direction') continue @@ -881,41 +913,33 @@ async function getMissingTransferBytes(reader: AdminStatsHourlyReader): Promise< return result } -async function getStorageUsedByDay(db: Database, range: AdminStatsDateRange): Promise> { - const buckets = createDateBuckets(range) - if (buckets.length === 0) return new Map() +async function getStorageUsedByDay(reader: AdminStatsHourlyReader): Promise> { + return getLatestGaugeValueByDay(reader, ADMIN_STATS_METRICS.storageUsed, 'bytes', '', '') +} - const firstBucketStart = dateKeyStart(buckets[0]) - const lastBucketStart = dateKeyStart(buckets[buckets.length - 1]) - const rows = await db - .select({ - bucketStart: statsRollupsHourly.bucketStart, - orgId: statsRollupsHourly.orgId, - bytes: statsRollupsHourly.bytes, +async function getStorageInventory(reader: AdminStatsHourlyReader): Promise<{ files: number; bytes: number }> { + const rows = await reader.latestRows(ADMIN_STATS_METRICS.storageInventory) + return rows + .filter((row) => row.orgId === '' && row.dimensionKey === '') + .reduce((total, row) => ({ files: total.files + row.count, bytes: total.bytes + row.bytes }), { + files: 0, + bytes: 0, }) - .from(statsRollupsHourly) - .where( - and( - eq(statsRollupsHourly.metricKey, STORAGE_USED_ROLLUP_METRIC), - eq(statsRollupsHourly.dimensionKey, GLOBAL_ROLLUP_DIMENSION_KEY), - eq(statsRollupsHourly.dimensionValue, GLOBAL_ROLLUP_DIMENSION_VALUE), - gte(statsRollupsHourly.bucketStart, daysAgo(firstBucketStart, 1)), - lte(statsRollupsHourly.bucketStart, daysAgo(lastBucketStart, -1)), - ), - ) +} - const byBucket = new Map() - for (const row of rows) { - const at = row.bucketStart.getTime() - byBucket.set(at, (byBucket.get(at) ?? 0) + toNumber(row.bytes)) +async function getLatestInventoryBreakdown( + reader: AdminStatsHourlyReader, + dimensionKey: 'file_type_group' | 'size_bucket' | 'age_bucket', +): Promise> { + const values = new Map() + for (const row of await reader.latestRows(ADMIN_STATS_METRICS.storageInventory, [dimensionKey])) { + if (row.orgId !== '' || row.dimensionKey !== dimensionKey) continue + const value = values.get(row.dimensionValue) ?? { name: row.dimensionValue, bytes: 0, files: 0 } + value.bytes += row.bytes + value.files += row.count + values.set(row.dimensionValue, value) } - const byDay = new Map() - for (const [at, bytes] of byBucket) { - const date = dayKey(new Date(at), range.timeZone) - const current = byDay.get(date) - if (!current || at > current.at) byDay.set(date, { at, bytes }) - } - return new Map([...byDay].map(([date, value]) => [date, value.bytes])) + return percentByBytes([...values.values()].sort((a, b) => b.bytes - a.bytes)) } async function getDownloadRequestsByDay(reader: AdminStatsHourlyReader): Promise> { @@ -947,20 +971,10 @@ async function getTrafficTotals( } async function getSharingEventTotals( - db: Database, - now: Date, reader: AdminStatsHourlyReader, ): Promise<{ activeShares: number; views: number; passwordPasses: number; downloads: number }> { const [activeShares, views, passwordPasses, downloads] = await Promise.all([ - countRowsWhere( - db, - shares, - and( - eq(shares.status, 'active'), - or(isNull(shares.expiresAt), gt(shares.expiresAt, now)), - or(isNull(shares.downloadLimit), sql`${shares.downloads} < ${shares.downloadLimit}`), - ), - ), + getLatestGaugeDimensionSum(reader, ADMIN_STATS_METRICS.shareInventory, 'lifecycle', ['usable']), getActivityMetricTotal(reader, metricSpec(['share_view']), 'count'), getActivityMetricTotal(reader, metricSpec(['share_password_passed']), 'count'), getActivityMetricTotal(reader, metricSpec(['share_download']), 'count'), @@ -968,6 +982,16 @@ async function getSharingEventTotals( return { activeShares, views, passwordPasses, downloads } } +async function getSharingComparisonTotals( + reader: AdminStatsHourlyReader, +): Promise<{ views: number; downloads: number }> { + const [views, downloads] = await Promise.all([ + getActivityMetricTotal(reader, metricSpec(['share_view']), 'count'), + getActivityMetricTotal(reader, metricSpec(['share_download']), 'count'), + ]) + return { views, downloads } +} + async function getTopSharesWithPercent( db: Database, reader: AdminStatsHourlyReader, @@ -988,21 +1012,8 @@ async function getTopSharesWithPercent( } async function getTopSharesByActivity(db: Database, reader: AdminStatsHourlyReader): Promise { - const [viewCounts, downloadCounts] = await Promise.all([ - getActivityMetricDimensionTotals(reader, metricSpec(['share_view']), 'share_id', 'count'), - getActivityMetricDimensionTotals(reader, metricSpec(['share_download']), 'share_id', 'count'), - ]) - const counts = new Map() - for (const [shareId, views] of viewCounts) counts.set(shareId, { views, downloads: 0 }) - for (const [shareId, downloads] of downloadCounts) { - const item = counts.get(shareId) ?? { views: 0, downloads: 0 } - item.downloads = downloads - counts.set(shareId, item) - } - const topIds = [...counts.entries()] - .sort((a, b) => b[1].views - a[1].views || b[1].downloads - a[1].downloads) - .slice(0, 8) - .map(([id]) => id) + const activity = await reader.topShareActivity() + const topIds = activity.map((row) => row.shareId) if (topIds.length === 0) return [] const shareRows = await db @@ -1019,10 +1030,11 @@ async function getTopSharesByActivity(db: Database, reader: AdminStatsHourlyRead .leftJoin(user, eq(user.id, shares.creatorId)) .where(inArray(shares.id, topIds)) const shareById = new Map(shareRows.map((row) => [row.id, row])) + const activityById = new Map(activity.map((row) => [row.shareId, row])) return topIds.flatMap((id) => { const row = shareById.get(id) - const countValue = counts.get(id) + const countValue = activityById.get(id) if (!row || !countValue) return [] return { id: row.id, @@ -1037,27 +1049,34 @@ async function getTopSharesByActivity(db: Database, reader: AdminStatsHourlyRead }) } -async function getUsageBySpaceRows(db: Database): Promise { - const rows = await db - .select({ - orgId: orgQuotas.orgId, - usedBytes: orgQuotas.used, - quotaBytes: orgQuotas.quota, - orgName: organization.name, - slug: organization.slug, - metadata: organization.metadata, - }) - .from(orgQuotas) - .leftJoin(organization, eq(organization.id, orgQuotas.orgId)) - .orderBy(desc(orgQuotas.used)) - return rows.map((row) => ({ - orgId: row.orgId, - orgName: row.orgName ?? row.orgId, - orgType: isPersonalOrgLike({ slug: row.slug ?? '', metadata: row.metadata ?? null }) ? 'personal' : 'team', - usedBytes: row.usedBytes, - quotaBytes: row.quotaBytes, - utilization: percent(row.usedBytes, row.quotaBytes), - })) +async function getUsageBySpaceRows( + db: Database, + reader: AdminStatsHourlyReader, +): Promise { + const topUsage = await reader.topSpaceUsage() + const topIds = topUsage.map((row) => row.orgId) + const orgRows = + topIds.length === 0 + ? [] + : await db + .select({ + id: organization.id, + name: organization.name, + slug: organization.slug, + metadata: organization.metadata, + }) + .from(organization) + .where(inArray(organization.id, topIds)) + const orgById = new Map(orgRows.map((row) => [row.id, row])) + return topUsage.map((row) => { + const org = orgById.get(row.orgId) + return { + ...row, + orgName: org?.name ?? row.orgId, + orgType: org && isPersonalOrgLike({ slug: org.slug, metadata: org.metadata }) ? 'personal' : 'team', + utilization: percent(row.usedBytes, row.quotaBytes), + } + }) } function percentRows( @@ -1071,55 +1090,11 @@ function percentRows( }) } -function isShareActive( - share: { status: string; expiresAt: Date | null; downloadLimit: number | null; downloads: number }, - now: Date, -): boolean { - return ( - share.status === 'active' && - (!share.expiresAt || share.expiresAt > now) && - (share.downloadLimit === null || share.downloads < share.downloadLimit) - ) -} - function percentByBytes(rows: T[]): Array { const total = rows.reduce((sum, row) => sum + row.bytes, 0) return rows.map((row) => ({ ...row, percent: percent(row.bytes, total) })) } -function fileBucketBreakdown( - files: Array<{ bytes: number }>, - buckets: Array<{ name: string; max: number }>, -): Array<{ name: string; bytes: number; files: number; percent: number }> { - const rows = buckets.map((bucket) => ({ name: bucket.name, bytes: 0, files: 0, max: bucket.max })) - for (const file of files) { - const bucket = rows.find((row, index) => file.bytes <= row.max && (index === 0 || file.bytes > rows[index - 1].max)) - if (!bucket) continue - bucket.files += 1 - bucket.bytes += file.bytes - } - return percentByBytes(rows.map(({ max: _max, ...row }) => row)) -} - -function ageBucketBreakdown( - files: Array<{ bytes: number; createdAt: Date }>, - now: Date, -): Array<{ name: string; bytes: number; files: number; percent: number }> { - const rows = [ - { name: '<30d', bytes: 0, files: 0 }, - { name: '30-90d', bytes: 0, files: 0 }, - { name: '90-180d', bytes: 0, files: 0 }, - { name: '>180d', bytes: 0, files: 0 }, - ] - for (const file of files) { - const ageDays = Math.floor((now.getTime() - file.createdAt.getTime()) / 86_400_000) - const bucket = ageDays < 30 ? rows[0] : ageDays < 90 ? rows[1] : ageDays < 180 ? rows[2] : rows[3] - bucket.files += 1 - bucket.bytes += file.bytes - } - return percentByBytes(rows) -} - function incrementMap(map: Map, key: string, value: number): void { map.set(key, (map.get(key) ?? 0) + value) } @@ -1135,52 +1110,14 @@ function incrementNestedMap( map.set(outerKey, inner) } -function addSetMap(map: Map>, key: string, value: string): void { - const set = map.get(key) ?? new Set() - set.add(value) - map.set(key, set) -} - -function countBy(rows: T[], getKey: (row: T) => string): Map { - const map = new Map() - for (const row of rows) incrementMap(map, getKey(row), 1) - return map -} - -function distinctUsersInCalendarWindow( - records: Array<{ userId: string; at: Date }>, - endDate: string, - days: number, - timeZone: string, -): number { - const endOrdinal = dateOrdinal(endDate) - const startOrdinal = endOrdinal - (days - 1) * 86_400_000 - const ids = new Set() - for (const record of records) { - const ordinal = dateOrdinal(dayKey(record.at, timeZone)) - if (ordinal >= startOrdinal && ordinal <= endOrdinal) ids.add(record.userId) - } - return ids.size -} - function dateOrdinal(date: string): number { return Date.parse(`${date}T00:00:00.000Z`) } -function daysAgo(now: Date, days: number): Date { - const date = new Date(now) - date.setUTCDate(date.getUTCDate() - days) - return date -} - function startOfHour(date: Date): Date { return new Date(Math.floor(date.getTime() / 3_600_000) * 3_600_000) } -function dateKeyStart(date: string): Date { - return new Date(`${date}T00:00:00.000Z`) -} - function percent(part: number, total: number): number { if (total <= 0) return 0 return Math.round((part / total) * 1000) / 10 diff --git a/server/adapters/repos/share.integration.test.ts b/server/adapters/repos/share.integration.test.ts index fe5e1baa..4e63b134 100644 --- a/server/adapters/repos/share.integration.test.ts +++ b/server/adapters/repos/share.integration.test.ts @@ -3,7 +3,7 @@ import { nanoid } from 'nanoid' import { describe, expect, it } from 'vitest' import { DirType } from '../../../shared/constants' import type { CreateShareInput } from '../../../shared/schemas/share' -import { matters } from '../../db/schema' +import { activityEvents, matters } from '../../db/schema' import { isAccessibleByUser } from '../../domain/share' import { verifyPassword as verifyPasswordHash } from '../../lib/password' import type { Database } from '../../platform/interface' @@ -14,7 +14,16 @@ import { createShareRepo } from './share.js' // under test, constructed per call from the test db. const createShare = (db: Database, input: CreateShareInput) => createShareRepo(db).create(input) const resolveShareByToken = (db: Database, token: string) => createShareRepo(db).resolveByToken(token) -const incrementViews = (db: Database, shareId: string) => createShareRepo(db).incrementViews(shareId) +const recordView = (db: Database, shareId: string, orgId: string) => + createShareRepo(db).recordView(shareId, { + orgId, + actorType: 'anonymous', + action: 'share_view', + targetType: 'share', + targetId: shareId, + targetName: 'file.bin', + metadata: { shareId }, + }) const incrementDownloadsAtomic = (db: Database, shareId: string) => createShareRepo(db).incrementDownloadsAtomic(shareId) const revokeShareByToken = (db: Database, token: string, creatorId: string) => @@ -305,21 +314,48 @@ describe('isAccessibleByUser', () => { }) }) -// ─── incrementViews ─────────────────────────────────────────────────────────── +// ─── recordView ─────────────────────────────────────────────────────────────── -describe('incrementViews', () => { - it('increments view count correctly', async () => { +describe('recordView', () => { + it('atomically increments the view count and records the stats fact', async () => { const { db } = await createTestApp() const orgId = nanoid() const matter = await seedMatter(db, { orgId }) const share = await createShare(db, { matterId: matter.id, orgId, creatorId: 'u1', kind: 'landing' }) - await incrementViews(db, share.id) - await incrementViews(db, share.id) + await recordView(db, share.id, orgId) + await recordView(db, share.id, orgId) const resolved = await resolveShareByToken(db, share.token) if (resolved.status !== 'ok') throw new Error('expected found') expect(resolved.share.views).toBe(2) + const events = await db.select().from(activityEvents).where(eq(activityEvents.targetId, share.id)) + expect(events).toHaveLength(2) + expect(events.every((event) => event.action === 'share_view')).toBe(true) + }) + + it('does not increment the view count when the stats fact is invalid', async () => { + const { db } = await createTestApp() + const orgId = nanoid() + const matter = await seedMatter(db, { orgId }) + const share = await createShare(db, { matterId: matter.id, orgId, creatorId: 'u1', kind: 'landing' }) + + await expect( + createShareRepo(db).recordView(share.id, { + orgId, + actorType: 'anonymous', + action: 'share_view', + targetType: 'share', + targetId: share.id, + targetName: 'file.bin', + metadata: {}, + }), + ).rejects.toThrow('invalid_admin_stats_event:share_view:shareId') + + const resolved = await resolveShareByToken(db, share.token) + if (resolved.status !== 'ok') throw new Error('expected found') + expect(resolved.share.views).toBe(0) + await expect(db.select().from(activityEvents).where(eq(activityEvents.targetId, share.id))).resolves.toEqual([]) }) }) diff --git a/server/adapters/repos/share.ts b/server/adapters/repos/share.ts index c7009072..119b358a 100644 --- a/server/adapters/repos/share.ts +++ b/server/adapters/repos/share.ts @@ -3,7 +3,7 @@ import type { CreateShareInput } from '@shared/schemas/share' import { and, count, desc, eq, inArray, isNotNull, isNull, like, or, sql } from 'drizzle-orm' import { nanoid } from 'nanoid' import { user } from '../../db/auth-schema' -import { matters, shareRecipients, shares } from '../../db/schema' +import { activityEvents, matters, shareRecipients, shares } from '../../db/schema' import { type AtomicQuery, executeWriteTransaction } from '../../db/transaction' import { hashPassword } from '../../lib/password' import type { Database } from '../../platform/interface' @@ -15,6 +15,7 @@ import { type ShareRepo, type ShareResolution, } from '../../usecases/ports' +import { activityEventValues } from './activity' import { createQuotaRepo } from './quota' function buildPath(parent: string, name: string): string { @@ -97,11 +98,14 @@ export function createShareRepo(db: Database): ShareRepo { return { status: 'ok', share: row.share, matter: row.matter, recipients } }, - async incrementViews(shareId: string): Promise { - await db - .update(shares) - .set({ views: sql`${shares.views} + 1` }) - .where(eq(shares.id, shareId)) + async recordView(shareId, activity): Promise { + await executeWriteTransaction(db, [ + db + .update(shares) + .set({ views: sql`${shares.views} + 1` }) + .where(eq(shares.id, shareId)), + db.insert(activityEvents).values(activityEventValues(activity)), + ]) }, async hasDownloadsAvailable(shareId: string): Promise { diff --git a/server/domain/admin-stats-events.test.ts b/server/domain/admin-stats-events.test.ts new file mode 100644 index 00000000..c1b49758 --- /dev/null +++ b/server/domain/admin-stats-events.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest' +import { assertAdminStatsEvent } from './admin-stats-events' + +describe('admin stats event contract', () => { + it('accepts complete transfer and sharing facts', () => { + expect(() => + assertAdminStatsEvent('share_download', { + bytes: 42, + source: 'landing_share', + trafficEventId: 'traffic-1', + shareId: 'share-1', + }), + ).not.toThrow() + expect(() => assertAdminStatsEvent('share_view', { shareId: 'share-1' })).not.toThrow() + }) + + it('rejects new stats facts with incomplete dimensions', () => { + expect(() => assertAdminStatsEvent('upload_confirm', { source: 'upload' })).toThrow( + 'invalid_admin_stats_event:upload_confirm:bytes', + ) + expect(() => assertAdminStatsEvent('object_download', { bytes: 42, source: 'object_download' })).toThrow( + 'invalid_admin_stats_event:object_download:trafficEventId', + ) + expect(() => assertAdminStatsEvent('share_view', {})).toThrow('invalid_admin_stats_event:share_view:shareId') + }) + + it('does not constrain unrelated audit events', () => { + expect(() => assertAdminStatsEvent('branding_update', undefined)).not.toThrow() + }) +}) diff --git a/server/domain/admin-stats-events.ts b/server/domain/admin-stats-events.ts new file mode 100644 index 00000000..5cab65b3 --- /dev/null +++ b/server/domain/admin-stats-events.ts @@ -0,0 +1,48 @@ +const BYTE_ACTIONS = new Set([ + 'upload_confirm', + 'upload_failed', + 'share_download', + 'object_download', + 'image_hosting_download', + 'webdav_download', + 'download_failed', + 'save_from_share', +]) + +const UPLOAD_ACTIONS = new Set(['upload_confirm', 'upload_cancel', 'upload_failed']) +const DOWNLOAD_ACTIONS = new Set([ + 'share_download', + 'object_download', + 'image_hosting_download', + 'webdav_download', + 'download_failed', +]) +const SHARE_ACTIONS = new Set(['share_view', 'share_download', 'save_from_share', 'share_password_passed']) + +export function assertAdminStatsEvent(action: string, metadata: Record | undefined): void { + if (!BYTE_ACTIONS.has(action) && !UPLOAD_ACTIONS.has(action) && !SHARE_ACTIONS.has(action)) return + + if (BYTE_ACTIONS.has(action)) requireFiniteNonNegativeNumber(action, metadata, 'bytes') + if (UPLOAD_ACTIONS.has(action)) requireNonEmptyString(action, metadata, 'source') + if (DOWNLOAD_ACTIONS.has(action)) { + requireNonEmptyString(action, metadata, 'source') + requireNonEmptyString(action, metadata, 'trafficEventId') + } + if (SHARE_ACTIONS.has(action)) requireNonEmptyString(action, metadata, 'shareId') +} + +function requireFiniteNonNegativeNumber( + action: string, + metadata: Record | undefined, + key: string, +): void { + const value = metadata?.[key] + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) { + throw new Error(`invalid_admin_stats_event:${action}:${key}`) + } +} + +function requireNonEmptyString(action: string, metadata: Record | undefined, key: string): void { + const value = metadata?.[key] + if (typeof value !== 'string' || value.length === 0) throw new Error(`invalid_admin_stats_event:${action}:${key}`) +} diff --git a/server/domain/admin-stats-metrics.ts b/server/domain/admin-stats-metrics.ts index 39553f46..2d2bcc44 100644 --- a/server/domain/admin-stats-metrics.ts +++ b/server/domain/admin-stats-metrics.ts @@ -1,6 +1,14 @@ -export const ROLLUP_VERSION = 1 +export const ROLLUP_VERSION = 2 -export type AdminStatsMetricKind = 'counter' | 'gauge' | 'distinct' +export type AdminStatsRollupScope = 'counters' | 'full' + +export interface AdminStatsRollupMetadata { + version: number + scope: AdminStatsRollupScope + quality: 'exact' | 'lower_bound' +} + +export type AdminStatsMetricKind = 'counter' | 'gauge' export const ADMIN_STATS_DIMENSIONS = { actorType: 'actor_type', @@ -22,6 +30,7 @@ export const ADMIN_STATS_DIMENSIONS = { source: 'source', status: 'status', storage: 'storage_id', + window: 'window', } as const export type AdminStatsDimension = (typeof ADMIN_STATS_DIMENSIONS)[keyof typeof ADMIN_STATS_DIMENSIONS] @@ -30,39 +39,28 @@ export const ADMIN_STATS_METRICS = { backgroundJobFinished: 'background_job.finished', backgroundJobSnapshot: 'background_job.snapshot', downloaderSnapshot: 'downloader.snapshot', - licenseRefresh: 'license.refresh', - remoteDownloadTaskCreated: 'remote_download.task_created', remoteDownloadTaskFinished: 'remote_download.task_finished', remoteDownloadTaskSnapshot: 'remote_download.task_snapshot', - remoteDownloadUsage: 'remote_download.usage', shareCreated: 'share.created', shareDownloadIssued: 'share.download_issued', shareInventory: 'share.inventory', sharePasswordPassed: 'share.password_passed', shareSaved: 'share.saved', shareView: 'share.view', - spaceCreated: 'space.created', statsMissingBytes: 'stats.quality_missing_bytes', statsRollupRun: 'stats.rollup_run', - storageIngress: 'storage.ingress', storageInventory: 'storage.inventory', storageQuota: 'storage.quota', - storageRemoved: 'storage.removed', - storageRestored: 'storage.restored', storageUsed: 'storage.used', - teamMembershipChange: 'team.membership_change', - trafficCreditConsumed: 'traffic.credit_consumed', - trafficQuota: 'traffic.quota', - trafficQuotaBlocked: 'traffic.quota_blocked', - trafficQuotaUsed: 'traffic.quota_used', trafficReportSync: 'traffic.report_sync', + trafficReportSnapshot: 'traffic.report_snapshot', transferDownloadFailed: 'transfer.download_failed', transferDownloadIssued: 'transfer.download_issued', transferUpload: 'transfer.upload', - userActiveHour: 'user.active_hour', - userSessionStarted: 'user.session_started', userSignup: 'user.signup', - webhookProcessed: 'webhook.processed', + userActiveSnapshot: 'user.active_snapshot', + userInventory: 'user.inventory', + webhookSnapshot: 'webhook.snapshot', } as const export type AdminStatsMetric = (typeof ADMIN_STATS_METRICS)[keyof typeof ADMIN_STATS_METRICS] @@ -74,56 +72,34 @@ export interface AdminStatsMetricDefinition { bytesUnit: 'bytes' | null } -const D = ADMIN_STATS_DIMENSIONS const M = ADMIN_STATS_METRICS export const ADMIN_STATS_METRIC_REGISTRY = { [M.backgroundJobFinished]: counter(['job_type', 'outcome']), [M.backgroundJobSnapshot]: gauge(['job_type', 'status'], 'entities'), [M.downloaderSnapshot]: gauge(['downloader_id', 'status'], 'entities'), - [M.licenseRefresh]: counter(['outcome']), - [M.remoteDownloadTaskCreated]: counter(['category', 'source']), [M.remoteDownloadTaskFinished]: counter(['category', 'downloader_id', 'outcome'], true), [M.remoteDownloadTaskSnapshot]: gauge(['downloader_id', 'status'], 'entities'), - [M.remoteDownloadUsage]: { - kind: 'counter', - dimensions: [D.downloader, D.status], - countUnit: 'credits', - bytesUnit: 'bytes', - }, [M.shareCreated]: counter(['kind']), [M.shareDownloadIssued]: counter(['actor_type', 'kind', 'share_id', 'source'], true), [M.shareInventory]: gauge(['kind', 'lifecycle'], 'entities'), [M.sharePasswordPassed]: counter(['share_id']), [M.shareSaved]: counter(['actor_type', 'share_id'], true), [M.shareView]: counter(['actor_type', 'share_id']), - [M.spaceCreated]: counter(['org_type']), [M.statsMissingBytes]: counter(['direction', 'source']), [M.statsRollupRun]: counter(['outcome']), - [M.storageIngress]: counter(['source', 'status', 'storage_id'], true), [M.storageInventory]: gauge(['age_bucket', 'file_type_group', 'size_bucket', 'storage_id'], 'entities', true), - [M.storageQuota]: gauge([], null, true), - [M.storageRemoved]: counter(['reason'], true), - [M.storageRestored]: counter(['source'], true), + [M.storageQuota]: gauge(['status'], null, true), [M.storageUsed]: gauge(['storage_id'], null, true), - [M.teamMembershipChange]: counter(['outcome']), - [M.trafficCreditConsumed]: { - kind: 'counter', - dimensions: [D.source, D.storage], - countUnit: 'credits', - bytesUnit: 'bytes', - }, - [M.trafficQuota]: gauge([], null, true), - [M.trafficQuotaBlocked]: counter(['source'], true), - [M.trafficQuotaUsed]: gauge([], null, true), [M.trafficReportSync]: counter(['source', 'status'], true), + [M.trafficReportSnapshot]: gauge(['status'], 'entities', true), [M.transferDownloadFailed]: counter(['reason', 'source'], true), [M.transferDownloadIssued]: counter(['actor_type', 'source', 'storage_id'], true), [M.transferUpload]: counter(['reason', 'source', 'status', 'storage_id'], true), - [M.userActiveHour]: { kind: 'distinct', dimensions: [], countUnit: 'entities', bytesUnit: null }, - [M.userSessionStarted]: counter([]), [M.userSignup]: counter(['provider']), - [M.webhookProcessed]: counter(['outcome']), + [M.userActiveSnapshot]: gauge(['window'], 'entities'), + [M.userInventory]: gauge(['status'], 'entities'), + [M.webhookSnapshot]: gauge(['status'], 'entities'), } satisfies Record function counter(dimensions: readonly AdminStatsDimension[], bytes = false): AdminStatsMetricDefinition { @@ -149,3 +125,18 @@ export function assertMetricDimension( if (dimension === '' || metricDefinition(metric).dimensions.includes(dimension as AdminStatsDimension)) return throw new Error(`Unsupported stats dimension: ${metric}/${dimension}`) } + +export function parseAdminStatsRollupMetadata(metadata: string | null): AdminStatsRollupMetadata | null { + if (!metadata) return null + try { + const value: unknown = JSON.parse(metadata) + if (!value || typeof value !== 'object' || Array.isArray(value)) return null + const record = value as Record + if (record.version !== ROLLUP_VERSION) return null + if (record.scope !== 'counters' && record.scope !== 'full') return null + if (record.quality !== 'exact' && record.quality !== 'lower_bound') return null + return { version: record.version, scope: record.scope, quality: record.quality } + } catch { + return null + } +} diff --git a/server/domain/admin-stats-time.ts b/server/domain/admin-stats-time.ts index dbb9cc41..b630edb3 100644 --- a/server/domain/admin-stats-time.ts +++ b/server/domain/admin-stats-time.ts @@ -1,28 +1,11 @@ -const HOUR_MS = 3_600_000 - -export function statsDayKey(date: Date, timeZone = 'UTC'): string { - const parts = new Intl.DateTimeFormat('en-CA', { - timeZone, - year: 'numeric', - month: '2-digit', - day: '2-digit', - }).formatToParts(date) - const values = new Map(parts.map((part) => [part.type, part.value])) - return `${values.get('year')}-${values.get('month')}-${values.get('day')}` +export function statsDayKey(date: Date): string { + return date.toISOString().slice(0, 10) } -export function localDateStart(date: string, timeZone: string): Date { - const center = Date.parse(`${date}T00:00:00.000Z`) - if (!Number.isFinite(center)) throw new Error(`Invalid local date: ${date}`) - let low = center - 18 * HOUR_MS - let high = center + 18 * HOUR_MS - while (low + 1 < high) { - const middle = Math.floor((low + high) / 2) - if (statsDayKey(new Date(middle), timeZone) < date) low = middle - else high = middle - } - if (statsDayKey(new Date(high), timeZone) !== date) throw new Error(`Local date does not exist: ${date}/${timeZone}`) - return new Date(high) +export function utcDateStart(date: string): Date { + const value = new Date(`${date}T00:00:00.000Z`) + if (Number.isNaN(value.getTime()) || statsDayKey(value) !== date) throw new Error(`Invalid UTC date: ${date}`) + return value } export function addCalendarDays(date: string, days: number): string { diff --git a/server/http/admin-stats.integration.test.ts b/server/http/admin-stats.integration.test.ts index e5af3411..3347c2c7 100644 --- a/server/http/admin-stats.integration.test.ts +++ b/server/http/admin-stats.integration.test.ts @@ -66,67 +66,43 @@ describe('site stats routes', () => { ]) }) - it('preserves explicit ISO offsets instead of shifting the selected local-day boundary', async () => { + it('accepts exact UTC hourly boundaries', async () => { const { app } = await createTestApp() const headers = await adminHeaders(app) - const from = encodeURIComponent('2026-07-01T04:00:00.000Z') - const to = encodeURIComponent('2026-07-02T03:59:59.999Z') + const from = encodeURIComponent('2026-07-01T10:00:00.000Z') + const to = encodeURIComponent('2026-07-01T10:59:59.999Z') - const res = await app.request(`/api/site/stats/overview?from=${from}&to=${to}&timeZone=America%2FToronto`, { + const res = await app.request(`/api/site/stats/overview?from=${from}&to=${to}&timeZone=UTC`, { headers, }) const body = (await res.json()) as { from: string; to: string; trends: Array<{ date: string }> } expect(res.status).toBe(200) - expect(body.from).toBe('2026-07-01T04:00:00.000Z') - expect(body.to).toBe('2026-07-02T03:59:59.999Z') + expect(body.from).toBe('2026-07-01T10:00:00.000Z') + expect(body.to).toBe('2026-07-01T10:59:59.999Z') expect(body.trends.map((point) => point.date)).toEqual(['2026-07-01']) }) - it('interprets date-only ranges in the requested time zone', async () => { + it('rejects partial-hour ranges that cannot map to offline result buckets', async () => { + const { app } = await createTestApp() + const headers = await adminHeaders(app) + const from = encodeURIComponent('2026-07-01T10:30:00.000Z') + const to = encodeURIComponent('2026-07-01T10:59:59.999Z') + + const res = await app.request(`/api/site/stats/overview?from=${from}&to=${to}&timeZone=UTC`, { headers }) + + expect(res.status).toBe(400) + }) + + it('rejects local reporting zones because offline result buckets are UTC-only', async () => { const { app } = await createTestApp() const headers = await adminHeaders(app) const res = await app.request('/api/site/stats/overview?from=2026-07-01&to=2026-07-01&timeZone=America%2FToronto', { headers, }) - const body = (await res.json()) as { from: string; to: string; trends: Array<{ date: string }> } - expect(res.status).toBe(200) - expect(body.from).toBe('2026-07-01T04:00:00.000Z') - expect(body.to).toBe('2026-07-02T03:59:59.999Z') - expect(body.trends.map((point) => point.date)).toEqual(['2026-07-01']) - }) - - it('supports non-whole-hour time zones and daylight-saving day lengths', async () => { - const { app } = await createTestApp() - const headers = await adminHeaders(app) - - const kathmandu = await app.request( - '/api/site/stats/overview?from=2026-07-01&to=2026-07-01&timeZone=Asia%2FKathmandu', - { headers }, - ) - const spring = await app.request( - '/api/site/stats/overview?from=2026-03-08&to=2026-03-08&timeZone=America%2FToronto', - { headers }, - ) - const fall = await app.request( - '/api/site/stats/overview?from=2026-11-01&to=2026-11-01&timeZone=America%2FToronto', - { headers }, - ) - - expect(await kathmandu.json()).toMatchObject({ - from: '2026-06-30T18:15:00.000Z', - to: '2026-07-01T18:14:59.999Z', - }) - expect(await spring.json()).toMatchObject({ - from: '2026-03-08T05:00:00.000Z', - to: '2026-03-09T03:59:59.999Z', - }) - expect(await fall.json()).toMatchObject({ - from: '2026-11-01T04:00:00.000Z', - to: '2026-11-02T04:59:59.999Z', - }) + expect(res.status).toBe(400) }) it('rejects dashboard ranges longer than one year', async () => { @@ -178,8 +154,11 @@ describe('site stats routes', () => { count, bytes, unique_count, metadata, updated_at ) VALUES ( - 'storage-used-2026-01-01', ${Date.UTC(2026, 0, 1)}, 'org-1', 'storage.used', '', '', - 0, 4096, 0, NULL, ${Date.UTC(2026, 0, 2)} + 'storage-marker-2026-01-01', ${Date.UTC(2026, 0, 1)}, '', 'stats.rollup_run', '', '', + 1, 0, 0, '{"version":2,"scope":"full","quality":"exact"}', ${Date.UTC(2026, 0, 2)} + ), ( + 'storage-used-2026-01-01', ${Date.UTC(2026, 0, 1)}, '', 'storage.used', '', '', + 0, 4096, 0, '{"version":2,"scope":"full","quality":"exact"}', ${Date.UTC(2026, 0, 2)} ) `) @@ -211,14 +190,20 @@ describe('site stats routes', () => { id, bucket_start, org_id, metric_key, dimension_key, dimension_value, count, bytes, unique_count, metadata, updated_at ) VALUES + ('hourly-reader-marker', ${at}, '', 'stats.rollup_run', '', '', 1, 0, 0, + '{"version":2,"scope":"full","quality":"exact"}', ${at + 3_600_000}), ('hourly-reader-upload', ${at}, ${orgId}, 'transfer.upload', 'status', 'success', 1, 999, 0, - '{"version":1,"quality":"exact"}', ${at + 3_600_000}) + '{"version":2,"scope":"full","quality":"exact"}', ${at + 3_600_000}) `) const query = '/api/site/stats/traffic?from=2026-07-01T10%3A00%3A00.000Z&to=2026-07-01T10%3A59%3A59.999Z&timeZone=UTC' const rollupRes = await app.request(query, { headers }) - const rollupBody = (await rollupRes.json()) as { summary: { totalBytes: { value: number } } } + const rollupBody = (await rollupRes.json()) as { + coverage: { status: string; completedBuckets: number; expectedBuckets: number } + comparisonCoverage: { status: string; completedBuckets: number; expectedBuckets: number } + summary: { totalBytes: { value: number } } + } await db.run(sql`DELETE FROM stats_rollups_hourly WHERE id = 'hourly-reader-upload'`) const emptyRes = await app.request(query, { headers }) @@ -226,6 +211,8 @@ describe('site stats routes', () => { expect(rollupRes.status).toBe(200) expect(rollupBody.summary.totalBytes.value).toBe(999) + expect(rollupBody.coverage).toMatchObject({ status: 'complete', completedBuckets: 1, expectedBuckets: 1 }) + expect(rollupBody.comparisonCoverage).toMatchObject({ status: 'empty', completedBuckets: 0, expectedBuckets: 1 }) expect(emptyRes.status).toBe(200) expect(emptyBody.summary.totalBytes.value).toBe(0) }) @@ -241,18 +228,18 @@ describe('site stats routes', () => { (id, bucket_start, org_id, metric_key, dimension_key, dimension_value, count, bytes, unique_count, metadata, updated_at) VALUES - ('dimensions-marker', ${at}, '', 'stats.rollup_run', '', '', 1, 0, 0, '{"quality":"exact"}', ${at}), - ('dimensions-share-total', ${at}, ${orgId}, 'share.created', '', '', 1, 0, 0, '{"quality":"exact"}', ${at}), - ('dimensions-share-kind', ${at}, ${orgId}, 'share.created', 'kind', 'landing', 1, 0, 0, '{"quality":"exact"}', ${at}), - ('dimensions-job-total', ${at}, ${orgId}, 'background_job.finished', '', '', 1, 0, 0, '{"quality":"exact"}', ${at}), - ('dimensions-job-outcome', ${at}, ${orgId}, 'background_job.finished', 'outcome', 'failed', 1, 0, 0, '{"quality":"exact"}', ${at}), - ('dimensions-download-total', ${at}, ${orgId}, 'transfer.download_issued', '', '', 1, 10, 0, '{"quality":"exact"}', ${at}), - ('dimensions-download-source', ${at}, ${orgId}, 'transfer.download_issued', 'source', 'object_download', 1, 10, 0, '{"quality":"exact"}', ${at}), - ('dimensions-failure-total', ${at}, ${orgId}, 'transfer.download_failed', '', '', 1, 4, 0, '{"quality":"exact"}', ${at}), - ('dimensions-failure-reason', ${at}, ${orgId}, 'transfer.download_failed', 'reason', 'network', 1, 4, 0, '{"quality":"exact"}', ${at}), - ('dimensions-quality-total', ${at}, ${orgId}, 'stats.quality_missing_bytes', '', '', 5, 0, 0, '{"quality":"exact"}', ${at}), - ('dimensions-quality-upload', ${at}, ${orgId}, 'stats.quality_missing_bytes', 'direction', 'upload', 2, 0, 0, '{"quality":"exact"}', ${at}), - ('dimensions-quality-download', ${at}, ${orgId}, 'stats.quality_missing_bytes', 'direction', 'download', 3, 0, 0, '{"quality":"exact"}', ${at}) + ('dimensions-marker', ${at}, '', 'stats.rollup_run', '', '', 1, 0, 0, '{"version":2,"scope":"full","quality":"exact"}', ${at}), + ('dimensions-share-total', ${at}, ${orgId}, 'share.created', '', '', 1, 0, 0, '{"version":2,"scope":"full","quality":"exact"}', ${at}), + ('dimensions-share-kind', ${at}, ${orgId}, 'share.created', 'kind', 'landing', 1, 0, 0, '{"version":2,"scope":"full","quality":"exact"}', ${at}), + ('dimensions-job-total', ${at}, ${orgId}, 'background_job.finished', '', '', 1, 0, 0, '{"version":2,"scope":"full","quality":"exact"}', ${at}), + ('dimensions-job-outcome', ${at}, ${orgId}, 'background_job.finished', 'outcome', 'failed', 1, 0, 0, '{"version":2,"scope":"full","quality":"exact"}', ${at}), + ('dimensions-download-total', ${at}, ${orgId}, 'transfer.download_issued', '', '', 1, 10, 0, '{"version":2,"scope":"full","quality":"exact"}', ${at}), + ('dimensions-download-source', ${at}, ${orgId}, 'transfer.download_issued', 'source', 'object_download', 1, 10, 0, '{"version":2,"scope":"full","quality":"exact"}', ${at}), + ('dimensions-failure-total', ${at}, ${orgId}, 'transfer.download_failed', '', '', 1, 4, 0, '{"version":2,"scope":"full","quality":"exact"}', ${at}), + ('dimensions-failure-reason', ${at}, ${orgId}, 'transfer.download_failed', 'reason', 'network', 1, 4, 0, '{"version":2,"scope":"full","quality":"exact"}', ${at}), + ('dimensions-quality-total', ${at}, ${orgId}, 'stats.quality_missing_bytes', '', '', 5, 0, 0, '{"version":2,"scope":"full","quality":"exact"}', ${at}), + ('dimensions-quality-upload', ${at}, ${orgId}, 'stats.quality_missing_bytes', 'direction', 'upload', 2, 0, 0, '{"version":2,"scope":"full","quality":"exact"}', ${at}), + ('dimensions-quality-download', ${at}, ${orgId}, 'stats.quality_missing_bytes', 'direction', 'download', 3, 0, 0, '{"version":2,"scope":"full","quality":"exact"}', ${at}) `) const query = 'from=2026-07-01T10%3A00%3A00.000Z&to=2026-07-01T10%3A59%3A59.999Z&timeZone=UTC' @@ -302,17 +289,17 @@ describe('site stats routes', () => { const rows = await db.all<{ bucketStart: number; bytes: number; metadata: string }>(sql` SELECT bucket_start AS bucketStart, bytes, metadata FROM stats_rollups_hourly - WHERE metric_key = 'storage.used' AND dimension_key = '' + WHERE metric_key = 'storage.used' AND org_id = '' AND dimension_key = '' ORDER BY bucket_start, org_id `) - expect(rows.map((row) => Number(row.bucketStart))).toContain(Date.UTC(2026, 6, 10, 18)) + expect(rows.map((row) => Number(row.bucketStart))).toContain(Date.UTC(2026, 6, 10, 17)) expect( rows - .filter((row) => Number(row.bucketStart) === Date.UTC(2026, 6, 10, 18)) + .filter((row) => Number(row.bucketStart) === Date.UTC(2026, 6, 10, 17)) .reduce((sum, row) => sum + row.bytes, 0), ).toBe(expected) - expect(JSON.parse(rows.at(-1)?.metadata ?? '{}')).toMatchObject({ version: 1, quality: 'exact' }) + expect(JSON.parse(rows.at(-1)?.metadata ?? '{}')).toMatchObject({ version: 2, quality: 'exact' }) }) it('does not write rollups while serving storage stats', async () => { @@ -375,13 +362,15 @@ describe('site stats routes', () => { count, bytes, unique_count, metadata, updated_at) VALUES ('growth-rollup-marker', ${at}, '', 'stats.rollup_run', '', '', 1, 0, 0, - '{"version":1,"quality":"exact"}', ${at + 3_600_000}), + '{"version":2,"scope":"full","quality":"exact"}', ${at + 3_600_000}), ('growth-rollup-total', ${at}, '', 'user.signup', '', '', 2, 0, 0, - '{"version":1,"quality":"exact"}', ${at + 3_600_000}), + '{"version":2,"scope":"full","quality":"exact"}', ${at + 3_600_000}), + ('growth-inventory-total', ${at}, '', 'user.inventory', '', '', 2, 0, 0, + '{"version":2,"scope":"full","quality":"exact"}', ${at + 3_600_000}), ('growth-rollup-credential', ${at}, '', 'user.signup', 'provider', 'credential', 1, 0, 0, - '{"version":1,"quality":"exact"}', ${at + 3_600_000}), + '{"version":2,"scope":"full","quality":"exact"}', ${at + 3_600_000}), ('growth-rollup-github', ${at}, '', 'user.signup', 'provider', 'github', 1, 0, 0, - '{"version":1,"quality":"exact"}', ${at + 3_600_000}) + '{"version":2,"scope":"full","quality":"exact"}', ${at + 3_600_000}) `) const res = await app.request('/api/site/stats/growth?from=2026-01-01&to=2026-01-01', { headers }) @@ -402,11 +391,11 @@ describe('site stats routes', () => { ) }) - it('groups storage file types beyond the top eight into other', async () => { + it('does not expose raw storage changes until a snapshot rollup completes', async () => { const { app, db } = await createTestApp() const headers = await adminHeaders(app) await seedProLicense(db) - const { orgId } = await seedStatsFixture(db) + const { orgId, bucketStart } = await seedStatsFixture(db) const nowSec = Math.floor(Date.now() / 1000) for (let index = 0; index < 9; index += 1) { await db.run(sql` @@ -417,15 +406,53 @@ describe('site stats routes', () => { `) } - const res = await app.request('/api/site/stats/storage', { headers }) - const body = (await res.json()) as { typeBreakdown: Array<{ type: string; files: number; bytes: number }> } + const beforeRes = await app.request('/api/site/stats/storage', { headers }) + const before = (await beforeRes.json()) as { typeBreakdown: Array<{ type: string; files: number; bytes: number }> } - expect(res.status).toBe(200) - expect(body.typeBreakdown).toHaveLength(9) - expect(body.typeBreakdown.at(-1)).toMatchObject({ type: 'other', files: 2 }) + await rebuildAdminStatsHour(db, bucketStart, new Date(), true) + const afterRes = await app.request('/api/site/stats/storage', { headers }) + const after = (await afterRes.json()) as { typeBreakdown: Array<{ type: string; files: number; bytes: number }> } + + expect(beforeRes.status).toBe(200) + expect(before.typeBreakdown).not.toContainEqual(expect.objectContaining({ type: 'custom' })) + expect(afterRes.status).toBe(200) + expect(after.typeBreakdown).toContainEqual(expect.objectContaining({ type: 'custom', files: 9 })) }) - it('excludes orphan actors and uses immutable session creation time for historical active users', async () => { + it('counts quota pressure across every space while bounding the ranking to eight', async () => { + const { app, db } = await createTestApp() + const headers = await adminHeaders(app) + await seedProLicense(db) + const { bucketStart } = await seedStatsFixture(db) + for (let index = 0; index < 12; index += 1) { + await db.run(sql` + INSERT INTO org_quotas (id, org_id, quota, used, traffic_quota, traffic_used, traffic_period) + VALUES ( + ${`quota-pressure-${index}`}, + ${`quota-pressure-org-${index}`}, + 100, + ${index < 9 ? 90 : 110}, + 0, + 0, + '2026-07' + ) + `) + } + await rebuildAdminStatsHour(db, bucketStart, new Date(), true) + + const res = await app.request('/api/site/stats/storage', { headers }) + const body = (await res.json()) as { + summary: { nearQuotaSpaces: number; overQuotaSpaces: number } + topSpaces: Array<{ orgId: string }> + } + + expect(res.status).toBe(200) + expect(body.summary.nearQuotaSpaces).toBe(9) + expect(body.summary.overQuotaSpaces).toBe(3) + expect(body.topSpaces).toHaveLength(8) + }) + + it('reads historical active users from completed snapshots instead of raw activity', async () => { const { app, db } = await createTestApp() const headers = await adminHeaders(app) const [{ id: orgId }] = await db.all<{ id: string }>(sql`SELECT id FROM organization LIMIT 1`) @@ -437,6 +464,17 @@ describe('site stats routes', () => { ('valid-historical-activity', ${orgId}, ${userId}, 'user', 'login', 'user', 'valid', ${created}), ('orphan-historical-activity', ${orgId}, 'deleted-user', 'user', 'login', 'user', 'orphan', ${created}) `) + const at = Date.UTC(2026, 0, 2, 12) + await db.run(sql` + INSERT INTO stats_rollups_hourly + (id, bucket_start, org_id, metric_key, dimension_key, dimension_value, + count, bytes, unique_count, metadata, updated_at) + VALUES + ('active-snapshot-marker', ${at}, '', 'stats.rollup_run', '', '', 1, 0, 0, + '{"version":2,"scope":"full","quality":"exact"}', ${at + 3_600_000}), + ('active-snapshot-mau', ${at}, '', 'user.active_snapshot', 'window', 'mau', 1, 0, 0, + '{"version":2,"scope":"full","quality":"exact"}', ${at + 3_600_000}) + `) const res = await app.request('/api/site/stats/overview?from=2026-01-02&to=2026-01-02', { headers }) const body = (await res.json()) as { totals: { activeUsers: { value: number } } } @@ -493,6 +531,8 @@ describe('site stats routes', () => { expect(res.status).toBe(200) expect(body.dataQuality).toEqual({ + missingBytesEvents: 1, + previousMissingBytesEvents: 1, missingUploadBytesEvents: 1, previousMissingUploadBytesEvents: 0, missingDownloadBytesEvents: 0, @@ -655,6 +695,14 @@ describe('site stats routes', () => { ('activity-download-heavy-3', ${orgId}, NULL, 'anonymous', 'share_download', 'share', 'share-download-heavy', 'report.pdf', '{"bytes":512,"source":"landing_share","anonymous":true}', ${eventSec}) `) await rebuildAdminStatsHour(db, bucketStart, new Date(), true) + await db.run(sql` + INSERT INTO stats_rollups_hourly + (id, bucket_start, org_id, metric_key, dimension_key, dimension_value, + count, bytes, unique_count, metadata, updated_at) + VALUES + ('ranking-invalid-quality', ${bucketStart.getTime()}, ${orgId}, 'share.view', 'share_id', + 'share-download-heavy', 1000, 0, 0, '{"version":2,"scope":"full"}', ${bucketStart.getTime()}) + `) const res = await app.request('/api/site/stats/sharing', { headers }) const body = (await res.json()) as { topShares: Array<{ token: string; views: number; downloads: number }> } diff --git a/server/http/admin-stats.ts b/server/http/admin-stats.ts index 249a8d8e..e7311e6d 100644 --- a/server/http/admin-stats.ts +++ b/server/http/admin-stats.ts @@ -1,7 +1,7 @@ import { zValidator } from '@hono/zod-validator' import { Hono } from 'hono' import { z } from 'zod' -import { addCalendarDays, localDateStart } from '../domain/admin-stats-time' +import { addCalendarDays, utcDateStart } from '../domain/admin-stats-time' import { requireAdmin } from '../middleware/auth' import type { Env } from '../middleware/platform' import { requireFeature } from '../middleware/require-feature' @@ -21,24 +21,14 @@ const dashboardDateSchema = z const rangeQuerySchema = z.object({ from: dashboardDateSchema.optional(), to: dashboardDateSchema.optional(), - timeZone: z.string().max(64).refine(isValidTimeZone, 'Invalid IANA time zone').optional(), + timeZone: z.literal('UTC').optional(), }) -function parseRange(query: z.infer): { from?: Date; to?: Date; timeZone?: string } { - const timeZone = query.timeZone ?? 'UTC' +function parseRange(query: z.infer): { from?: Date; to?: Date; timeZone: 'UTC' } { return { - from: query.from ? parseDashboardDate(query.from, 'start', timeZone) : undefined, - to: query.to ? parseDashboardDate(query.to, 'end', timeZone) : undefined, - timeZone, - } -} - -function isValidTimeZone(value: string): boolean { - try { - new Intl.DateTimeFormat('en-CA', { timeZone: value }).format() - return true - } catch { - return false + from: query.from ? parseDashboardDate(query.from, 'start') : undefined, + to: query.to ? parseDashboardDate(query.to, 'end') : undefined, + timeZone: 'UTC', } } @@ -48,10 +38,10 @@ function isValidDashboardDate(value: string): boolean { return !Number.isNaN(date.getTime()) && date.toISOString().startsWith(value) } -function parseDashboardDate(value: string, boundary: 'start' | 'end', timeZone: string): Date { +function parseDashboardDate(value: string, boundary: 'start' | 'end'): Date { if (/^\d{4}-\d{2}-\d{2}$/.test(value)) { - if (boundary === 'start') return localDateStart(value, timeZone) - return new Date(localDateStart(addCalendarDays(value, 1), timeZone).getTime() - 1) + if (boundary === 'start') return utcDateStart(value) + return new Date(utcDateStart(addCalendarDays(value, 1)).getTime() - 1) } return new Date(value) } diff --git a/server/usecases/admin-stats.ts b/server/usecases/admin-stats.ts index d33c56b1..7ca3ea96 100644 --- a/server/usecases/admin-stats.ts +++ b/server/usecases/admin-stats.ts @@ -6,7 +6,7 @@ import type { AdminDashboardStorageStats, AdminDashboardTrafficStats, } from '@shared/types' -import { addCalendarDays, localDateStart, statsDayKey } from '../domain/admin-stats-time' +import { addCalendarDays, statsDayKey, utcDateStart } from '../domain/admin-stats-time' import { type AdminStatsDateRange, type AdminStatsRepo, badRequest } from './ports' export type AdminStatsDeps = { @@ -16,17 +16,25 @@ export type AdminStatsDeps = { export interface AdminStatsRangeInput { from?: Date to?: Date - timeZone?: string + timeZone?: 'UTC' } const MAX_DASHBOARD_RANGE_DAYS = 366 export function normalizeStatsRange(input: AdminStatsRangeInput, now = new Date()): AdminStatsDateRange { const timeZone = input.timeZone ?? 'UTC' - const today = statsDayKey(now, timeZone) - const to = input.to ?? new Date(localDateStart(addCalendarDays(today, 1), timeZone).getTime() - 1) - const from = input.from ?? localDateStart(addCalendarDays(statsDayKey(to, timeZone), -29), timeZone) + if (timeZone !== 'UTC') throw badRequest('admin analytics uses UTC calendar boundaries', 'UNSUPPORTED_TIME_ZONE') + const today = statsDayKey(now) + const to = input.to ?? new Date(utcDateStart(addCalendarDays(today, 1)).getTime() - 1) + const from = input.from ?? utcDateStart(addCalendarDays(statsDayKey(to), -29)) if (from > to) throw badRequest('from must be before to', 'INVALID_TIME_RANGE') - if (calendarDayCount(from, to, timeZone) > MAX_DASHBOARD_RANGE_DAYS) { + if (from.getUTCMinutes() !== 0 || from.getUTCSeconds() !== 0 || from.getUTCMilliseconds() !== 0) { + throw badRequest('from must align to a UTC hour', 'INVALID_TIME_RANGE') + } + const toExclusive = to.getTime() + 1 + if (toExclusive % 3_600_000 !== 0) { + throw badRequest('to must end on a UTC hour', 'INVALID_TIME_RANGE') + } + if (calendarDayCount(from, to) > MAX_DASHBOARD_RANGE_DAYS) { throw badRequest(`time range cannot exceed ${MAX_DASHBOARD_RANGE_DAYS} days`, 'TIME_RANGE_TOO_LARGE') } return { from, to, timeZone } @@ -80,8 +88,8 @@ export function getAdminDashboardSharingStats( return deps.adminStats.getDashboardSharingStats(now, normalizeStatsRange(input, now)) } -function calendarDayCount(from: Date, to: Date, timeZone: string): number { - const fromDay = Date.parse(`${statsDayKey(from, timeZone)}T00:00:00.000Z`) - const toDay = Date.parse(`${statsDayKey(to, timeZone)}T00:00:00.000Z`) +function calendarDayCount(from: Date, to: Date): number { + const fromDay = Date.parse(`${statsDayKey(from)}T00:00:00.000Z`) + const toDay = Date.parse(`${statsDayKey(to)}T00:00:00.000Z`) return Math.floor((toDay - fromDay) / 86_400_000) + 1 } diff --git a/server/usecases/object.integration.test.ts b/server/usecases/object.integration.test.ts index 360482ee..e4d2f945 100644 --- a/server/usecases/object.integration.test.ts +++ b/server/usecases/object.integration.test.ts @@ -371,7 +371,7 @@ describe('saveShareToDrive', () => { expect(activities[0].userId).toBe('u2') const metadata = JSON.parse(activities[0].metadata ?? '{}') - expect(metadata.sourceShareId).toBe(share.id) + expect(metadata.shareId).toBe(share.id) }) it('does NOT increment share downloads counter', async () => { @@ -791,7 +791,7 @@ describe('POST /api/shares/:token/objects', () => { expect(updatedShare?.downloads).toBe(downloadsBefore) }) - it('activity_events row created with save_from_share and sourceShareId metadata', async () => { + it('activity_events row created with save_from_share and shareId metadata', async () => { const { app, db, share, headers, personalOrgId } = await setup() await app.request(`/api/shares/${share.token}/objects`, { @@ -806,7 +806,7 @@ describe('POST /api/shares/:token/objects', () => { const saveActivity = activities.find((a) => a.action === 'save_from_share') expect(saveActivity).toBeTruthy() const metadata = JSON.parse(saveActivity?.metadata ?? '{}') - expect(metadata.sourceShareId).toBe(share.id) + expect(metadata.shareId).toBe(share.id) }) }) diff --git a/server/usecases/object.ts b/server/usecases/object.ts index 2a139b28..cd4204cb 100644 --- a/server/usecases/object.ts +++ b/server/usecases/object.ts @@ -1222,7 +1222,7 @@ export async function saveShareToDrive(deps: SaveToDriveDeps, input: SaveShareIn activity: { action: 'save_from_share', metadata: { - sourceShareId: share.id, + shareId: share.id, sourceMatterId: sourceMatter.id, creatorId: share.creatorId, bytes: sourceMatter.size ?? 0, diff --git a/server/usecases/ports/admin-stats.ts b/server/usecases/ports/admin-stats.ts index f40a75fa..476da0a9 100644 --- a/server/usecases/ports/admin-stats.ts +++ b/server/usecases/ports/admin-stats.ts @@ -22,5 +22,5 @@ export interface AdminStatsRepo { export interface AdminStatsDateRange { from: Date to: Date - timeZone: string + timeZone: 'UTC' } diff --git a/server/usecases/ports/share.ts b/server/usecases/ports/share.ts index e1d7efdf..7c888b74 100644 --- a/server/usecases/ports/share.ts +++ b/server/usecases/ports/share.ts @@ -1,4 +1,5 @@ import type { CreateShareInput } from '@shared/schemas/share' +import type { RecordActivityInput } from './activity' import type { Matter } from './matter' // ─── DTOs ────────────────────────────────────────────────────────────────── @@ -66,7 +67,7 @@ export class CreateShareError extends Error { export interface ShareRepo { create(input: CreateShareInput): Promise resolveByToken(token: string): Promise - incrementViews(shareId: string): Promise + recordView(shareId: string, activity: RecordActivityInput): Promise hasDownloadsAvailable(shareId: string): Promise incrementDownloadsAtomic(shareId: string): Promise<{ ok: boolean; downloads: number }> decrementDownloads(shareId: string): Promise diff --git a/server/usecases/share.test.ts b/server/usecases/share.test.ts index 92a3e349..15357d5f 100644 --- a/server/usecases/share.test.ts +++ b/server/usecases/share.test.ts @@ -129,7 +129,7 @@ const trashedResolution = ( function makeShareRepo(over: Partial = {}): ShareRepo { return { resolveByToken: async () => okResolution(), - incrementViews: async () => {}, + recordView: async () => {}, hasDownloadsAvailable: async () => true, incrementDownloadsAtomic: async () => ({ ok: true, downloads: 3 }), decrementDownloads: async () => {}, @@ -163,7 +163,7 @@ function makeDeps( } = {}, ) { const record = vi.fn(async () => {}) - const incrementViews = vi.fn(async () => {}) + const recordView = vi.fn(async () => {}) const decrementDownloads = vi.fn(async () => {}) const refundTraffic = vi.fn(async () => {}) const presignDownload = vi.fn(async () => PRESIGNED_URL) @@ -175,7 +175,7 @@ function makeDeps( const isEmailConfigured = vi.fn(async () => true) const deps = { - share: makeShareRepo({ incrementViews, decrementDownloads, ...over.share }), + share: makeShareRepo({ recordView, decrementDownloads, ...over.share }), matter: { list: async () => emptyMatterList, ...over.matter } as MatterRepo, storages: { get: async () => sampleStorage, ...over.storages } as StorageRepo, s3: { presignDownload, ...over.s3 } as S3Gateway, @@ -195,7 +195,7 @@ function makeDeps( return { deps, record, - incrementViews, + recordView, decrementDownloads, refundTraffic, presignDownload, @@ -251,7 +251,7 @@ describe('viewShare', () => { }) it('returns the viewer DTO and signals the view cookie for an anonymous viewer', async () => { - const { deps, incrementViews } = makeDeps() + const { deps, recordView } = makeDeps() const out = await viewShare(deps, { token: 'sk_token1', viewerId: null, @@ -261,7 +261,14 @@ describe('viewShare', () => { expect(out.ok).toBe(true) if (!out.ok) throw new Error('expected ok') expect(out.setViewCookie).toBe(true) - expect(incrementViews).toHaveBeenCalledWith('s-1') + expect(recordView).toHaveBeenCalledWith( + 's-1', + expect.objectContaining({ + action: 'share_view', + targetId: 's-1', + metadata: expect.objectContaining({ bytes: 1024 }), + }), + ) expect(out.dto).toMatchObject({ token: 'sk_token1', kind: 'landing', @@ -277,7 +284,7 @@ describe('viewShare', () => { }) it('does not increment views (no cookie) when already seen', async () => { - const { deps, incrementViews } = makeDeps() + const { deps, recordView } = makeDeps() const out = await viewShare(deps, { token: 'sk_token1', viewerId: null, @@ -286,11 +293,11 @@ describe('viewShare', () => { }) if (!out.ok) throw new Error('expected ok') expect(out.setViewCookie).toBe(false) - expect(incrementViews).not.toHaveBeenCalled() + expect(recordView).not.toHaveBeenCalled() }) it('returns the richer creator DTO and never bumps views for the creator', async () => { - const { deps, incrementViews } = makeDeps() + const { deps, recordView } = makeDeps() const out = await viewShare(deps, { token: 'sk_token1', viewerId: 'creator-1', @@ -299,7 +306,7 @@ describe('viewShare', () => { }) if (!out.ok) throw new Error('expected ok') expect(out.setViewCookie).toBe(false) - expect(incrementViews).not.toHaveBeenCalled() + expect(recordView).not.toHaveBeenCalled() expect(out.dto).toMatchObject({ id: 's-1', matterId: 'm-1', diff --git a/server/usecases/share.ts b/server/usecases/share.ts index c717f91c..cb87f734 100644 --- a/server/usecases/share.ts +++ b/server/usecases/share.ts @@ -175,29 +175,24 @@ export async function viewShare(deps: ShareDeps, params: ViewShareParams): Promi // handler sets the cookie when setViewCookie is true. const setViewCookie = !isCreator && viewCookie !== 'seen' if (setViewCookie) { - await deps.share.incrementViews(share.id) - try { - await deps.activity.record({ - orgId: share.orgId, - userId: viewerId, - actorType: viewerId ? 'user' : 'anonymous', - action: 'share_view', - targetType: 'share', - targetId: share.id, - targetName: matter.name, - metadata: { - shareId: share.id, - matterId: matter.id, - creatorId: share.creatorId, - kind: share.kind, - requiresPassword: !!share.passwordHash, - matterType: matter.type, - bytes: matter.size ?? 0, - }, - }) - } catch (error) { - console.error('[shares] recordActivity failed:', error) - } + await deps.share.recordView(share.id, { + orgId: share.orgId, + userId: viewerId, + actorType: viewerId ? 'user' : 'anonymous', + action: 'share_view', + targetType: 'share', + targetId: share.id, + targetName: matter.name, + metadata: { + shareId: share.id, + matterId: matter.id, + creatorId: share.creatorId, + kind: share.kind, + requiresPassword: !!share.passwordHash, + matterType: matter.type, + bytes: matter.size ?? 0, + }, + }) } const dto = await composeShareView(deps, { share, matter, recipients }, { viewerId, accessCookie, now }) @@ -231,25 +226,21 @@ export async function verifySharePassword( ? new Date(Math.min(share.expiresAt.getTime(), now.getTime() + oneDayMs)) : new Date(now.getTime() + oneDayMs) - try { - await deps.activity.record({ - orgId: share.orgId, - userId: viewerId, - actorType: viewerId ? 'user' : 'anonymous', - action: 'share_password_passed', - targetType: 'share', - targetId: share.id, - targetName: matter.name, - metadata: { - shareId: share.id, - matterId: matter.id, - creatorId: share.creatorId, - kind: share.kind, - }, - }) - } catch (error) { - console.error('[shares] recordActivity failed:', error) - } + await deps.activity.record({ + orgId: share.orgId, + userId: viewerId, + actorType: viewerId ? 'user' : 'anonymous', + action: 'share_password_passed', + targetType: 'share', + targetId: share.id, + targetName: matter.name, + metadata: { + shareId: share.id, + matterId: matter.id, + creatorId: share.creatorId, + kind: share.kind, + }, + }) return { ok: true, setAccessCookieExpiry } } diff --git a/server/usecases/transfer-activity.ts b/server/usecases/transfer-activity.ts index b35d55aa..5bd74ea2 100644 --- a/server/usecases/transfer-activity.ts +++ b/server/usecases/transfer-activity.ts @@ -12,7 +12,7 @@ export function recordDownloadIssued( orgId: string userId?: string | null actorType?: 'user' | 'anonymous' | 'system' | 'downloader' - action: string + action: 'share_download' | 'object_download' | 'image_hosting_download' | 'webdav_download' targetType: string targetId: string targetName: string @@ -31,12 +31,12 @@ export function recordDownloadIssued( targetId: input.targetId, targetName: input.targetName, metadata: { + ...input.metadata, direction: 'download', status: 'issued', source: input.source, bytes: input.bytes, trafficEventId: input.trafficEventId, - ...input.metadata, }, }) } @@ -66,13 +66,13 @@ export function recordDownloadFailed( targetId: input.targetId, targetName: input.targetName, metadata: { + ...input.metadata, direction: 'download', status: 'failed', source: input.source, bytes: input.bytes, trafficEventId: input.trafficEventId, reason: input.reason, - ...input.metadata, }, }) } diff --git a/shared/types/admin-stats.ts b/shared/types/admin-stats.ts index 3f4abb61..fddea681 100644 --- a/shared/types/admin-stats.ts +++ b/shared/types/admin-stats.ts @@ -22,11 +22,22 @@ export interface AdminStatsRange { generatedAt: string from: string to: string + timeZone: 'UTC' + coverage: AdminStatsCoverage + comparisonCoverage?: AdminStatsCoverage +} + +export interface AdminStatsCoverage { + status: 'complete' | 'partial' | 'empty' + expectedBuckets: number + completedBuckets: number + dataThrough: string | null } export interface AdminStatsDelta { value: number previousValue: number + change: number changePercent: number | null } @@ -35,6 +46,8 @@ export interface AdminTransferDataQuality { previousMissingUploadBytesEvents: number missingDownloadBytesEvents: number previousMissingDownloadBytesEvents: number + missingBytesEvents: number + previousMissingBytesEvents: number } export interface AdminDashboardOverviewStats extends AdminStatsRange { @@ -43,8 +56,10 @@ export interface AdminDashboardOverviewStats extends AdminStatsRange { users: number newUsers: AdminStatsDelta activeUsers: AdminStatsDelta + activeUserRate: number | null storageUsedBytes: number storageQuotaBytes: number + storageUtilization: number | null trafficBytes: AdminStatsDelta uploadBytes: AdminStatsDelta downloadBytes: AdminStatsDelta @@ -70,6 +85,8 @@ export interface AdminDashboardGrowthStats extends AdminStatsRange { verifiedUsers: number bannedUsers: number silentUsers: number + activeUserRate: number | null + silentUserRate: number | null } userScaleTrend: Array<{ date: string; newUsers: number; totalUsers: number }> activeUserTrend: Array<{ date: string; dau: number; wau: number; mau: number }> @@ -86,6 +103,8 @@ export interface AdminDashboardStorageStats extends AdminStatsRange { newFiles: AdminStatsDelta newBytes: AdminStatsDelta coldFileBytes: number + storageUtilization: number | null + coldFilePercent: number | null nearQuotaSpaces: number overQuotaSpaces: number } @@ -103,7 +122,7 @@ export interface AdminDashboardTrafficStats extends AdminStatsRange { requestCount: AdminStatsDelta issuedDownloads: number blockedDownloads: number - issueRate: number | null + downloadIssueSuccessRate: number | null peakDailyBytes: number } trafficTrend: Array<{ date: string; uploadBytes: number; downloadBytes: number; requests: number }> @@ -120,7 +139,8 @@ export interface AdminDashboardSharingStats extends AdminStatsRange { views: AdminStatsDelta downloads: AdminStatsDelta saves: AdminStatsDelta - downloadConversionRate: number | null + downloadsPer100Views: number | null + savesPer100Views: number | null passwordPasses: number } trend: Array<{ date: string; views: number; downloads: number; saves: number }> @@ -139,6 +159,7 @@ export interface AdminDashboardOperationsStats extends AdminStatsRange { remoteDownloadSuccessRate: number | null cloudReportBacklog: number webhookFailures: number + alertCount: number } trend: Array<{ date: string diff --git a/shared/types/index.ts b/shared/types/index.ts index 7e5e4a34..48ac2f93 100644 --- a/shared/types/index.ts +++ b/shared/types/index.ts @@ -216,6 +216,7 @@ export type { AdminDashboardSharingStats, AdminDashboardStorageStats, AdminDashboardTrafficStats, + AdminStatsCoverage, AdminStatsDelta, AdminStatsRange, AdminTopShare, diff --git a/src/lib/api.test.ts b/src/lib/api.test.ts index 71c973e8..51ae5626 100644 --- a/src/lib/api.test.ts +++ b/src/lib/api.test.ts @@ -3824,7 +3824,7 @@ describe('api', () => { const range = { from: '2026-07-01T00:00:00.000Z', to: '2026-07-09T00:00:00.000Z', - timeZone: 'America/Toronto', + timeZone: 'UTC' as const, } const dashboardEndpoints = [ { name: 'getAdminDashboardOverviewStats', path: '/api/site/stats/overview', fn: getAdminDashboardOverviewStats }, diff --git a/src/lib/api.ts b/src/lib/api.ts index c7e6e539..fd1b204b 100644 --- a/src/lib/api.ts +++ b/src/lib/api.ts @@ -437,7 +437,7 @@ export function retryBackgroundJob(id: string) { export interface AdminStatsRangeFilter { from?: string to?: string - timeZone?: string + timeZone?: 'UTC' } function statsRangeQuery(filter: AdminStatsRangeFilter = {}): Record { diff --git a/src/routes/_authenticated/admin/index.test.tsx b/src/routes/_authenticated/admin/index.test.tsx index decebe56..be8e89aa 100644 --- a/src/routes/_authenticated/admin/index.test.tsx +++ b/src/routes/_authenticated/admin/index.test.tsx @@ -45,24 +45,35 @@ const overviewStats: AdminDashboardOverviewStats = { generatedAt: '2026-07-09T00:00:00.000Z', from: '2026-07-01T00:00:00.000Z', to: '2026-07-09T00:00:00.000Z', + timeZone: 'UTC', + coverage: { + status: 'complete', + expectedBuckets: 192, + completedBuckets: 192, + dataThrough: '2026-07-09T00:00:00.000Z', + }, dataQuality: { missingUploadBytesEvents: 0, previousMissingUploadBytesEvents: 0, missingDownloadBytesEvents: 0, previousMissingDownloadBytesEvents: 0, + missingBytesEvents: 0, + previousMissingBytesEvents: 0, }, totals: { users: 42, - newUsers: { value: 5, previousValue: 3, changePercent: 66.7 }, - activeUsers: { value: 18, previousValue: 12, changePercent: 50 }, + newUsers: { value: 5, previousValue: 3, change: 2, changePercent: 66.7 }, + activeUsers: { value: 18, previousValue: 12, change: 6, changePercent: 50 }, + activeUserRate: 42.9, storageUsedBytes: 1024, storageQuotaBytes: 4096, - trafficBytes: { value: 512, previousValue: 256, changePercent: 100 }, - uploadBytes: { value: 128, previousValue: 64, changePercent: 100 }, - downloadBytes: { value: 384, previousValue: 192, changePercent: 100 }, + storageUtilization: 25, + trafficBytes: { value: 512, previousValue: 256, change: 256, changePercent: 100 }, + uploadBytes: { value: 128, previousValue: 64, change: 64, changePercent: 100 }, + downloadBytes: { value: 384, previousValue: 192, change: 192, changePercent: 100 }, activeShares: 4, - shareViews: { value: 120, previousValue: 60, changePercent: 100 }, - shareDownloads: { value: 30, previousValue: 15, changePercent: 100 }, + shareViews: { value: 120, previousValue: 60, change: 60, changePercent: 100 }, + shareDownloads: { value: 30, previousValue: 15, change: 15, changePercent: 100 }, }, trends: [ { @@ -80,6 +91,13 @@ const operationsStats: AdminDashboardOperationsStats = { generatedAt: '2026-07-09T00:00:00.000Z', from: '2026-07-01T00:00:00.000Z', to: '2026-07-09T00:00:00.000Z', + timeZone: 'UTC', + coverage: { + status: 'complete', + expectedBuckets: 192, + completedBuckets: 192, + dataThrough: '2026-07-09T00:00:00.000Z', + }, summary: { activeBackgroundJobs: 2, activeRemoteDownloads: 3, @@ -89,6 +107,7 @@ const operationsStats: AdminDashboardOperationsStats = { remoteDownloadSuccessRate: 95, cloudReportBacklog: 6, webhookFailures: 7, + alertCount: 13, }, trend: [], backgroundJobOutcomes: [], @@ -134,6 +153,7 @@ describe('Admin overview dashboard', () => { renderOverviewPage() expect(await screen.findByText('42')).toBeTruthy() + expect(screen.getByText('所选范围的离线结果完整。')).toBeTruthy() expect(screen.getByText('用户与增长')).toBeTruthy() expect(getAdminDashboardGrowthStats).not.toHaveBeenCalled() expect(getAdminDashboardStorageStats).not.toHaveBeenCalled() @@ -155,13 +175,40 @@ describe('Admin overview dashboard', () => { }) vi.mocked(getAdminDashboardOverviewStats).mockResolvedValue({ ...overviewStats, - dataQuality: { ...overviewStats.dataQuality, missingUploadBytesEvents: 3 }, + dataQuality: { ...overviewStats.dataQuality, missingUploadBytesEvents: 3, missingBytesEvents: 3 }, }) renderOverviewPage() expect(await screen.findByText('历史数据不完整')).toBeTruthy() - expect(screen.getByText(/当前区间有 3条/)).toBeTruthy() + expect(screen.getByText(/当前区间有 3 条/)).toBeTruthy() + }) + + it('warns when the comparison range is missing offline result buckets', async () => { + vi.mocked(useEntitlement).mockReturnValue({ + bound: false, + active: false, + edition: null, + licenseId: null, + cloudDashboardUrl: null, + hasFeature: () => false, + isLoading: false, + isError: false, + }) + vi.mocked(getAdminDashboardOverviewStats).mockResolvedValue({ + ...overviewStats, + comparisonCoverage: { + status: 'partial', + expectedBuckets: 192, + completedBuckets: 144, + dataThrough: '2026-06-30T00:00:00.000Z', + }, + }) + + renderOverviewPage() + + expect(await screen.findByText('对比区间存在缺失的小时结果，环比数据不完整。')).toBeTruthy() + expect(screen.getByText(/对比 144\/192 小时/)).toBeTruthy() }) it('loads the operations dashboard only when an entitled admin expands it', async () => { diff --git a/src/routes/_authenticated/admin/index.tsx b/src/routes/_authenticated/admin/index.tsx index ad3661ea..61e3614f 100644 --- a/src/routes/_authenticated/admin/index.tsx +++ b/src/routes/_authenticated/admin/index.tsx @@ -5,6 +5,7 @@ import type { AdminDashboardSharingStats, AdminDashboardStorageStats, AdminDashboardTrafficStats, + AdminStatsRange, AdminTransferDataQuality, } from '@shared/types' import { useQuery } from '@tanstack/react-query' @@ -160,57 +161,53 @@ const SECTION_META: Array<{ ] export function OverviewPage() { - const today = useMemo(() => new Date(), []) + const today = useMemo(() => utcCalendarDate(new Date()), []) const [openSections, setOpenSections] = useState>(() => new Set(['overview'])) - const [ranges, setRanges] = useState>(() => initialRanges(today)) + const [range, setRange] = useState(() => ({ from: startOfDay(subDays(today, 29)), to: endOfDay(today) })) const { hasFeature, isLoading: entitlementLoading } = useEntitlement() const hasAnalytics = hasFeature('analytics') const overviewQuery = useQuery({ - queryKey: ['admin', 'dashboard', 'overview', rangeKey(ranges.overview)], - queryFn: () => getAdminDashboardOverviewStats(toRangeFilter(ranges.overview)), + queryKey: ['admin', 'dashboard', 'overview', rangeKey(range)], + queryFn: () => getAdminDashboardOverviewStats(toRangeFilter(range)), staleTime: 30_000, }) const growthQuery = useDashboardSectionQuery( 'growth', - ranges.growth, + range, openSections, hasAnalytics, getAdminDashboardGrowthStats, ) const storageQuery = useDashboardSectionQuery( 'storage', - ranges.storage, + range, openSections, hasAnalytics, getAdminDashboardStorageStats, ) const trafficQuery = useDashboardSectionQuery( 'traffic', - ranges.traffic, + range, openSections, hasAnalytics, getAdminDashboardTrafficStats, ) const sharingQuery = useDashboardSectionQuery( 'sharing', - ranges.sharing, + range, openSections, hasAnalytics, getAdminDashboardSharingStats, ) const operationsQuery = useDashboardSectionQuery( 'operations', - ranges.operations, + range, openSections, hasAnalytics, getAdminDashboardOperationsStats, ) - function updateRange(section: SectionId, range: DateRange) { - setRanges((current) => ({ ...current, [section]: range })) - } - function toggleSection(section: SectionId) { if (section !== 'overview' && !hasAnalytics) return setOpenSections((current) => { @@ -224,6 +221,13 @@ export function OverviewPage() { return (
+
+
+

站点统计

+

统一使用 UTC 日期与已完成的离线小时结果。

+
+ +
{SECTION_META.map((section) => ( updateRange(section.id, range)} onToggle={() => toggleSection(section.id)} > {section.id === 'overview' ? ( @@ -282,18 +284,14 @@ function DashboardSection({ section, open, locked, - range, children, onToggle, - onRangeChange, }: { section: (typeof SECTION_META)[number] open: boolean locked?: boolean - range: DateRange children: ReactNode onToggle: () => void - onRangeChange: (range: DateRange) => void }) { const Icon = section.icon @@ -323,7 +321,6 @@ function DashboardSection({
- {open && !locked && } + )}
) if (!query.data) return - return <>{children(query.data)} + return ( +
+ + {children(query.data)} +
+ ) +} + +function StatsCoverageNotice({ stats }: { stats: AdminStatsRange }) { + const { coverage } = stats + const comparisonCoverage = stats.comparisonCoverage + const through = coverage.dataThrough + ? new Date(coverage.dataThrough).toISOString().replace('T', ' ').slice(0, 16) + : null + const comparisonIncomplete = comparisonCoverage ? comparisonCoverage.status !== 'complete' : false + const incomplete = coverage.status !== 'complete' || comparisonIncomplete + return ( +
+ + {coverage.status === 'empty' + ? '所选范围还没有可用的离线结果。' + : coverage.status === 'partial' + ? '所选范围存在缺失的小时结果，当前数据不完整。' + : comparisonCoverage?.status === 'empty' + ? '对比区间还没有可用的离线结果，环比不可对账。' + : comparisonCoverage?.status === 'partial' + ? '对比区间存在缺失的小时结果，环比数据不完整。' + : '所选范围的离线结果完整。'} + + + {through ? `数据截至 ${through} UTC · ` : ''} + 当前 {coverage.completedBuckets}/{coverage.expectedBuckets} 小时 + {comparisonCoverage + ? ` · 对比 ${comparisonCoverage.completedBuckets}/${comparisonCoverage.expectedBuckets} 小时` + : ''} + +
+ ) } function SectionSkeleton() { @@ -1536,8 +1579,8 @@ function SectionSkeleton() { } function TransferDataQualityNotice({ quality }: { quality: AdminTransferDataQuality }) { - const currentMissing = quality.missingUploadBytesEvents + quality.missingDownloadBytesEvents - const previousMissing = quality.previousMissingUploadBytesEvents + quality.previousMissingDownloadBytesEvents + const currentMissing = quality.missingBytesEvents + const previousMissing = quality.previousMissingBytesEvents if (currentMissing === 0 && previousMissing === 0) return null return ( @@ -1549,8 +1592,8 @@ function TransferDataQualityNotice({ quality }: { quality: AdminTransferDataQual 历史数据不完整 - 当前区间有 {formatNumber(currentMissing)}条、对比区间有 {formatNumber(previousMissing)} - 条传输事件缺少可恢复的字节数； 流量与新增容量仅代表已知下限，事件数量不受影响。 + 当前区间有 {formatNumber(currentMissing)} 条、对比区间有 {formatNumber(previousMissing)} + 条传输事件缺少可恢复的字节数；流量与新增容量仅代表已知下限，事件数量不受影响。
) @@ -1564,35 +1607,26 @@ function EmptyState() { ) } -function initialRanges(today: Date): Record { - const last30 = { from: startOfDay(subDays(today, 29)), to: endOfDay(today) } - return { - overview: last30, - growth: last30, - storage: { from: startOfDay(subDays(today, 89)), to: endOfDay(today) }, - traffic: { from: startOfDay(subDays(today, 6)), to: endOfDay(today) }, - sharing: last30, - operations: last30, - } -} - function dateRangePresets(): Array<{ label: string; range: DateRange }> { - const today = new Date() + const today = utcCalendarDate(new Date()) return [ { label: '最近 3 天', range: { from: startOfDay(subDays(today, 2)), to: endOfDay(today) } }, { label: '最近 7 天', range: { from: startOfDay(subDays(today, 6)), to: endOfDay(today) } }, { label: '最近 30 天', range: { from: startOfDay(subDays(today, 29)), to: endOfDay(today) } }, - { label: '最近一个月', range: { from: startOfDay(subDays(today, 30)), to: endOfDay(today) } }, { label: '本月', range: { from: startOfMonth(today), to: endOfDay(today) } }, { label: '上月', range: { from: startOfMonth(subMonths(today, 1)), to: endOfMonth(subMonths(today, 1)) } }, ] } +function utcCalendarDate(value: Date): Date { + return new Date(value.getUTCFullYear(), value.getUTCMonth(), value.getUTCDate()) +} + function toRangeFilter(range: DateRange): AdminStatsRangeFilter { return { - ...(range.from ? { from: startOfDay(range.from).toISOString() } : {}), - ...(range.to ? { to: endOfDay(range.to).toISOString() } : {}), - timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, + ...(range.from ? { from: format(range.from, 'yyyy-MM-dd') } : {}), + ...(range.to ? { to: format(range.to, 'yyyy-MM-dd') } : {}), + timeZone: 'UTC', } } @@ -1632,17 +1666,16 @@ function formatPercentLabel(value: unknown): string { return `${value}%` } -function ratio(part: number, total: number): number { - if (total <= 0) return 0 - return (part / total) * 100 +function formatPer100(value: number | null): string { + return value === null ? '—' : value.toLocaleString('zh-CN', { maximumFractionDigits: 1 }) } function formatDelta( - delta: { value: number; previousValue: number; changePercent: number | null }, + delta: { value: number; previousValue: number; change: number; changePercent: number | null }, valueFormatter: (value: number) => string = formatNumber, ): string { - const sign = delta.value >= delta.previousValue ? '+' : '' - return `${sign}${valueFormatter(delta.value - delta.previousValue)} (${formatPercent(delta.changePercent)})` + const sign = delta.change >= 0 ? '+' : '' + return `${sign}${valueFormatter(delta.change)} (${formatPercent(delta.changePercent)})` } function formatCompactNumber(value: number): string {