diff --git a/migrations/0060_traffic-issued-at.sql b/migrations/0060_traffic-issued-at.sql new file mode 100644 index 00000000..70df8cba --- /dev/null +++ b/migrations/0060_traffic-issued-at.sql @@ -0,0 +1,2 @@ +ALTER TABLE `cloud_traffic_reports` ADD `issued_at` integer;--> statement-breakpoint +CREATE INDEX `cloud_traffic_reports_issued_idx` ON `cloud_traffic_reports` (`issued_at`); \ No newline at end of file diff --git a/migrations/meta/0060_snapshot.json b/migrations/meta/0060_snapshot.json new file mode 100644 index 00000000..0f16e73c --- /dev/null +++ b/migrations/meta/0060_snapshot.json @@ -0,0 +1,4380 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "86ed1792-a472-4f59-aa17-82733c707c98", + "prevId": "b6d63ed5-2212-40d2-90f8-304d37ac1309", + "tables": { + "activity_events": { + "name": "activity_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_name": { + "name": "target_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "actor_type": { + "name": "actor_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_ref": { + "name": "actor_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "activity_events_org_created_idx": { + "name": "activity_events_org_created_idx", + "columns": [ + "org_id", + "created_at" + ], + "isUnique": false + }, + "activity_events_user_created_idx": { + "name": "activity_events_user_created_idx", + "columns": [ + "user_id", + "created_at" + ], + "isUnique": false + }, + "activity_events_action_created_idx": { + "name": "activity_events_action_created_idx", + "columns": [ + "action", + "created_at" + ], + "isUnique": false + }, + "activity_events_target_created_idx": { + "name": "activity_events_target_created_idx", + "columns": [ + "target_type", + "target_id", + "created_at" + ], + "isUnique": false + }, + "activity_events_created_idx": { + "name": "activity_events_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "announcements": { + "name": "announcements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "announcements_status_priority_idx": { + "name": "announcements_status_priority_idx", + "columns": [ + "status", + "priority" + ], + "isUnique": false + }, + "announcements_published_idx": { + "name": "announcements_published_idx", + "columns": [ + "published_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "background_jobs": { + "name": "background_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_folder": { + "name": "target_folder", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_path": { + "name": "target_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "input_bytes": { + "name": "input_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "output_bytes": { + "name": "output_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processed_bytes": { + "name": "processed_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "current_filename": { + "name": "current_filename", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "result_metadata": { + "name": "result_metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "retryable": { + "name": "retryable", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "cancelable": { + "name": "cancelable", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "retried_from_job_id": { + "name": "retried_from_job_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "background_jobs_org_created_idx": { + "name": "background_jobs_org_created_idx", + "columns": [ + "org_id", + "created_at" + ], + "isUnique": false + }, + "background_jobs_org_status_idx": { + "name": "background_jobs_org_status_idx", + "columns": [ + "org_id", + "status" + ], + "isUnique": false + }, + "background_jobs_org_type_idx": { + "name": "background_jobs_org_type_idx", + "columns": [ + "org_id", + "type" + ], + "isUnique": false + }, + "background_jobs_created_idx": { + "name": "background_jobs_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cloud_traffic_reports": { + "name": "cloud_traffic_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "period": { + "name": "period", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "unit_bytes": { + "name": "unit_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "credits_per_unit": { + "name": "credits_per_unit", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "next_retry_at": { + "name": "next_retry_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "issued_at": { + "name": "issued_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "cloud_traffic_reports_event_uniq": { + "name": "cloud_traffic_reports_event_uniq", + "columns": [ + "event_id" + ], + "isUnique": true + }, + "cloud_traffic_reports_org_period_idx": { + "name": "cloud_traffic_reports_org_period_idx", + "columns": [ + "org_id", + "period" + ], + "isUnique": false + }, + "cloud_traffic_reports_status_idx": { + "name": "cloud_traffic_reports_status_idx", + "columns": [ + "status" + ], + "isUnique": false + }, + "cloud_traffic_reports_retry_idx": { + "name": "cloud_traffic_reports_retry_idx", + "columns": [ + "status", + "next_retry_at", + "created_at" + ], + "isUnique": false + }, + "cloud_traffic_reports_issued_idx": { + "name": "cloud_traffic_reports_issued_idx", + "columns": [ + "issued_at" + ], + "isUnique": false + }, + "cloud_traffic_reports_updated_idx": { + "name": "cloud_traffic_reports_updated_idx", + "columns": [ + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "download_tasks": { + "name": "download_tasks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by_user_id": { + "name": "created_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_uri": { + "name": "source_uri", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_folder": { + "name": "target_folder", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "assigned_downloader_id": { + "name": "assigned_downloader_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "billing_authorized_bytes": { + "name": "billing_authorized_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "billing_charged_bytes": { + "name": "billing_charged_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "billing_charged_credits": { + "name": "billing_charged_credits", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "billing_status": { + "name": "billing_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'none'" + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "result_object_id": { + "name": "result_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "runtime": { + "name": "runtime", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolve_started_at": { + "name": "resolve_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolve_completed_at": { + "name": "resolve_completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "download_completed_at": { + "name": "download_completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ingest_started_at": { + "name": "ingest_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ingest_completed_at": { + "name": "ingest_completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "seeding_started_at": { + "name": "seeding_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "seeding_stopped_at": { + "name": "seeding_stopped_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "assigned_at": { + "name": "assigned_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "download_tasks_org_created_idx": { + "name": "download_tasks_org_created_idx", + "columns": [ + "org_id", + "created_at" + ], + "isUnique": false + }, + "download_tasks_org_status_idx": { + "name": "download_tasks_org_status_idx", + "columns": [ + "org_id", + "status" + ], + "isUnique": false + }, + "download_tasks_org_category_idx": { + "name": "download_tasks_org_category_idx", + "columns": [ + "org_id", + "category" + ], + "isUnique": false + }, + "download_tasks_org_tags_idx": { + "name": "download_tasks_org_tags_idx", + "columns": [ + "org_id", + "tags" + ], + "isUnique": false + }, + "download_tasks_downloader_idx": { + "name": "download_tasks_downloader_idx", + "columns": [ + "assigned_downloader_id", + "status" + ], + "isUnique": false + }, + "download_tasks_created_idx": { + "name": "download_tasks_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + }, + "download_tasks_finished_idx": { + "name": "download_tasks_finished_idx", + "columns": [ + "finished_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "downloaders": { + "name": "downloaders", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_jti": { + "name": "token_jti", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'offline'" + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "hostname": { + "name": "hostname", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "platform": { + "name": "platform", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "arch": { + "name": "arch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "engine": { + "name": "engine", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'http'" + }, + "capabilities": { + "name": "capabilities", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "max_concurrent_tasks": { + "name": "max_concurrent_tasks", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "current_tasks": { + "name": "current_tasks", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "download_bps": { + "name": "download_bps", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "upload_bps": { + "name": "upload_bps", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "free_disk_bytes": { + "name": "free_disk_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "remote_download_credit_billing_enabled": { + "name": "remote_download_credit_billing_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "remote_download_credit_unit_bytes": { + "name": "remote_download_credit_unit_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 104857600 + }, + "remote_download_credit_per_unit": { + "name": "remote_download_credit_per_unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "last_heartbeat_at": { + "name": "last_heartbeat_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "downloaders_token_jti_unique": { + "name": "downloaders_token_jti_unique", + "columns": [ + "token_jti" + ], + "isUnique": true + }, + "downloaders_status_idx": { + "name": "downloaders_status_idx", + "columns": [ + "status" + ], + "isUnique": false + }, + "downloaders_enabled_idx": { + "name": "downloaders_enabled_idx", + "columns": [ + "enabled" + ], + "isUnique": false + }, + "downloaders_created_idx": { + "name": "downloaders_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "image_hosting_configs": { + "name": "image_hosting_configs", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "custom_domain": { + "name": "custom_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cf_hostname_id": { + "name": "cf_hostname_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "domain_verified_at": { + "name": "domain_verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "referer_allowlist": { + "name": "referer_allowlist", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "image_hosting_configs_custom_domain_unique": { + "name": "image_hosting_configs_custom_domain_unique", + "columns": [ + "custom_domain" + ], + "isUnique": true + } + }, + "foreignKeys": { + "image_hosting_configs_org_id_organization_id_fk": { + "name": "image_hosting_configs_org_id_organization_id_fk", + "tableFrom": "image_hosting_configs", + "tableTo": "organization", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "image_hostings": { + "name": "image_hostings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime": { + "name": "mime", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "purged_at": { + "name": "purged_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_count": { + "name": "access_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_accessed_at": { + "name": "last_accessed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "image_hostings_token_unique": { + "name": "image_hostings_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "image_hostings_org_path_uniq": { + "name": "image_hostings_org_path_uniq", + "columns": [ + "org_id", + "path" + ], + "isUnique": true, + "where": "\"image_hostings\".\"purged_at\" IS NULL" + }, + "image_hostings_org_created_idx": { + "name": "image_hostings_org_created_idx", + "columns": [ + "org_id", + "created_at" + ], + "isUnique": false + }, + "image_hostings_token_idx": { + "name": "image_hostings_token_idx", + "columns": [ + "token" + ], + "isUnique": false + } + }, + "foreignKeys": { + "image_hostings_org_id_organization_id_fk": { + "name": "image_hostings_org_id_organization_id_fk", + "tableFrom": "image_hostings", + "tableTo": "organization", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "image_hostings_storage_id_storages_id_fk": { + "name": "image_hostings_storage_id_storages_id_fk", + "tableFrom": "image_hostings", + "tableTo": "storages", + "columnsFrom": [ + "storage_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invite_codes": { + "name": "invite_codes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "used_by": { + "name": "used_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "used_at": { + "name": "used_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invite_codes_code_unique": { + "name": "invite_codes_code_unique", + "columns": [ + "code" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "license_bindings": { + "name": "license_bindings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "cloud_binding_id": { + "name": "cloud_binding_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cloud_store_id": { + "name": "cloud_store_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cloud_account_id": { + "name": "cloud_account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cloud_account_email": { + "name": "cloud_account_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cached_certificate": { + "name": "cached_certificate", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cached_certificate_expires_at": { + "name": "cached_certificate_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bound_at": { + "name": "bound_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "disconnected_at": { + "name": "disconnected_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refresh_at": { + "name": "last_refresh_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refresh_error": { + "name": "last_refresh_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "license_bindings_active_uniq": { + "name": "license_bindings_active_uniq", + "columns": [ + "status" + ], + "isUnique": true, + "where": "status = 'active'" + }, + "license_bindings_cloud_binding_idx": { + "name": "license_bindings_cloud_binding_idx", + "columns": [ + "cloud_binding_id" + ], + "isUnique": false + }, + "license_bindings_instance_idx": { + "name": "license_bindings_instance_idx", + "columns": [ + "instance_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "matters": { + "name": "matters", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "alias": { + "name": "alias", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "dirtype": { + "name": "dirtype", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "parent": { + "name": "parent", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "object": { + "name": "object", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "trashed_at": { + "name": "trashed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "purged_at": { + "name": "purged_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "matters_alias_unique": { + "name": "matters_alias_unique", + "columns": [ + "alias" + ], + "isUnique": true + }, + "matters_status_dir_created_idx": { + "name": "matters_status_dir_created_idx", + "columns": [ + "status", + "dirtype", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notifications": { + "name": "notifications", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "ref_type": { + "name": "ref_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_id": { + "name": "ref_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "read_at": { + "name": "read_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "notifications_user_created_idx": { + "name": "notifications_user_created_idx", + "columns": [ + "user_id", + "created_at" + ], + "isUnique": false + }, + "notifications_user_read_idx": { + "name": "notifications_user_read_idx", + "columns": [ + "user_id", + "read_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "object_upload_sessions": { + "name": "object_upload_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_id": { + "name": "object_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "upload_id": { + "name": "upload_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "part_size": { + "name": "part_size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "on_conflict": { + "name": "on_conflict", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'fail'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "object_upload_sessions_object_idx": { + "name": "object_upload_sessions_object_idx", + "columns": [ + "org_id", + "object_id" + ], + "isUnique": false + }, + "object_upload_sessions_expires_idx": { + "name": "object_upload_sessions_expires_idx", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_quota_entitlements": { + "name": "org_quota_entitlements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entitlement_type": { + "name": "entitlement_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'grant'" + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "starts_at": { + "name": "starts_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "org_quota_entitlements_org_resource_idx": { + "name": "org_quota_entitlements_org_resource_idx", + "columns": [ + "org_id", + "resource_type", + "status" + ], + "isUnique": false + }, + "org_quota_entitlements_org_type_idx": { + "name": "org_quota_entitlements_org_type_idx", + "columns": [ + "org_id", + "resource_type", + "entitlement_type", + "status" + ], + "isUnique": false + }, + "org_quota_entitlements_active_plan_uniq": { + "name": "org_quota_entitlements_active_plan_uniq", + "columns": [ + "org_id", + "resource_type", + "entitlement_type" + ], + "isUnique": true, + "where": "status = 'active' AND entitlement_type = 'plan' AND source <> 'free_plan'" + }, + "org_quota_entitlements_source_resource_uniq": { + "name": "org_quota_entitlements_source_resource_uniq", + "columns": [ + "source", + "source_id", + "resource_type" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_quotas": { + "name": "org_quotas", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "quota": { + "name": "quota", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "used": { + "name": "used", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "traffic_quota": { + "name": "traffic_quota", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "traffic_used": { + "name": "traffic_used", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "traffic_period": { + "name": "traffic_period", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01'" + } + }, + "indexes": { + "org_quotas_org_uniq": { + "name": "org_quotas_org_uniq", + "columns": [ + "org_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "remote_download_usage_reports": { + "name": "remote_download_usage_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "downloader_id": { + "name": "downloader_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "task_id": { + "name": "task_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unit_index": { + "name": "unit_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unit_bytes": { + "name": "unit_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credits_per_unit": { + "name": "credits_per_unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "remote_download_usage_reports_event_id_unique": { + "name": "remote_download_usage_reports_event_id_unique", + "columns": [ + "event_id" + ], + "isUnique": true + }, + "remote_download_usage_task_unit_uniq": { + "name": "remote_download_usage_task_unit_uniq", + "columns": [ + "task_id", + "unit_index" + ], + "isUnique": true + }, + "remote_download_usage_org_idx": { + "name": "remote_download_usage_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + }, + "remote_download_usage_status_idx": { + "name": "remote_download_usage_status_idx", + "columns": [ + "status" + ], + "isUnique": false + }, + "remote_download_usage_created_idx": { + "name": "remote_download_usage_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "share_recipients": { + "name": "share_recipients", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "share_id": { + "name": "share_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "recipient_user_id": { + "name": "recipient_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "recipient_email": { + "name": "recipient_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "share_recipients_share_id_idx": { + "name": "share_recipients_share_id_idx", + "columns": [ + "share_id" + ], + "isUnique": false + }, + "share_recipients_user_id_idx": { + "name": "share_recipients_user_id_idx", + "columns": [ + "recipient_user_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "shares": { + "name": "shares", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "matter_id": { + "name": "matter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "creator_id": { + "name": "creator_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "download_limit": { + "name": "download_limit", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "views": { + "name": "views", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "downloads": { + "name": "downloads", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "shares_token_unique": { + "name": "shares_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "shares_creator_status_created_idx": { + "name": "shares_creator_status_created_idx", + "columns": [ + "creator_id", + "status", + "created_at" + ], + "isUnique": false + }, + "shares_created_idx": { + "name": "shares_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "site_invitations": { + "name": "site_invitations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_by": { + "name": "accepted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "accepted_at": { + "name": "accepted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "site_invitations_token_unique": { + "name": "site_invitations_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "site_invitations_email_idx": { + "name": "site_invitations_email_idx", + "columns": [ + "email" + ], + "isUnique": false + }, + "site_invitations_created_idx": { + "name": "site_invitations_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + }, + "site_invitations_expires_idx": { + "name": "site_invitations_expires_idx", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "stats_rollups_hourly": { + "name": "stats_rollups_hourly", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "metric_key": { + "name": "metric_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dimension_key": { + "name": "dimension_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "dimension_value": { + "name": "dimension_value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "unique_count": { + "name": "unique_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "stats_rollups_hourly_bucket_metric_dim_uniq": { + "name": "stats_rollups_hourly_bucket_metric_dim_uniq", + "columns": [ + "bucket_start", + "org_id", + "metric_key", + "dimension_key", + "dimension_value" + ], + "isUnique": true + }, + "stats_rollups_hourly_metric_bucket_idx": { + "name": "stats_rollups_hourly_metric_bucket_idx", + "columns": [ + "metric_key", + "bucket_start" + ], + "isUnique": false + }, + "stats_rollups_hourly_dimension_bucket_idx": { + "name": "stats_rollups_hourly_dimension_bucket_idx", + "columns": [ + "metric_key", + "dimension_key", + "bucket_start" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_usage_ledger": { + "name": "storage_usage_ledger", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "event_key": { + "name": "event_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "delta_bytes": { + "name": "delta_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "storage_usage_ledger_event_key_unique": { + "name": "storage_usage_ledger_event_key_unique", + "columns": [ + "event_key" + ], + "isUnique": true + }, + "storage_usage_ledger_occurred_idx": { + "name": "storage_usage_ledger_occurred_idx", + "columns": [ + "occurred_at" + ], + "isUnique": false + }, + "storage_usage_ledger_org_occurred_idx": { + "name": "storage_usage_ledger_org_occurred_idx", + "columns": [ + "org_id", + "occurred_at" + ], + "isUnique": false + }, + "storage_usage_ledger_storage_occurred_idx": { + "name": "storage_usage_ledger_storage_occurred_idx", + "columns": [ + "storage_id", + "occurred_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storages": { + "name": "storages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "bucket": { + "name": "bucket", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'auto'" + }, + "access_key": { + "name": "access_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret_key": { + "name": "secret_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "custom_host": { + "name": "custom_host", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "''" + }, + "capacity": { + "name": "capacity", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "egress_credit_billing_enabled": { + "name": "egress_credit_billing_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "egress_credit_unit_bytes": { + "name": "egress_credit_unit_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 104857600 + }, + "egress_credit_per_unit": { + "name": "egress_credit_per_unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "force_path_style": { + "name": "force_path_style", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "used": { + "name": "used", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "system_options": { + "name": "system_options", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "team_invite_links": { + "name": "team_invite_links", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "team_invite_links_token_unique": { + "name": "team_invite_links_token_unique", + "columns": [ + "token" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webdav_dead_properties": { + "name": "webdav_dead_properties", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_path": { + "name": "resource_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "webdav_dead_properties_resource_prop_uniq": { + "name": "webdav_dead_properties_resource_prop_uniq", + "columns": [ + "org_id", + "resource_path", + "namespace", + "name" + ], + "isUnique": true + }, + "webdav_dead_properties_resource_idx": { + "name": "webdav_dead_properties_resource_idx", + "columns": [ + "org_id", + "resource_path" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webdav_locks": { + "name": "webdav_locks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_path": { + "name": "resource_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "depth": { + "name": "depth", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'infinity'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "webdav_locks_token_unique": { + "name": "webdav_locks_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "webdav_locks_resource_idx": { + "name": "webdav_locks_resource_idx", + "columns": [ + "org_id", + "resource_path" + ], + "isUnique": false + }, + "webdav_locks_expires_idx": { + "name": "webdav_locks_expires_idx", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhook_events": { + "name": "webhook_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'cloud'" + }, + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'order.quota_changed'" + }, + "payload_hash": { + "name": "payload_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "raw_payload": { + "name": "raw_payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhook_events_source_event_uniq": { + "name": "webhook_events_source_event_uniq", + "columns": [ + "source", + "event_id" + ], + "isUnique": true + }, + "webhook_events_source_created_idx": { + "name": "webhook_events_source_created_idx", + "columns": [ + "source", + "created_at" + ], + "isUnique": false + }, + "webhook_events_status_idx": { + "name": "webhook_events_status_idx", + "columns": [ + "status" + ], + "isUnique": false + }, + "webhook_events_processed_idx": { + "name": "webhook_events_processed_idx", + "columns": [ + "processed_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_request": { + "name": "last_request", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apikey_config_id_idx": { + "name": "apikey_config_id_idx", + "columns": [ + "config_id" + ], + "isUnique": false + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": [ + "reference_id" + ], + "isUnique": false + }, + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": [ + "key" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "deviceCode": { + "name": "deviceCode", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "device_code": { + "name": "device_code", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_code": { + "name": "user_code", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_polled_at": { + "name": "last_polled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "polling_interval": { + "name": "polling_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "deviceCode_device_code_idx": { + "name": "deviceCode_device_code_idx", + "columns": [ + "device_code" + ], + "isUnique": false + }, + "deviceCode_user_code_idx": { + "name": "deviceCode_user_code_idx", + "columns": [ + "user_code" + ], + "isUnique": false + }, + "deviceCode_status_idx": { + "name": "deviceCode_status_idx", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invitation": { + "name": "invitation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invitation_organizationId_idx": { + "name": "invitation_organizationId_idx", + "columns": [ + "organization_id" + ], + "isUnique": false + }, + "invitation_email_idx": { + "name": "invitation_email_idx", + "columns": [ + "email" + ], + "isUnique": false + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": [ + "inviter_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "member": { + "name": "member", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "member_organizationId_idx": { + "name": "member_organizationId_idx", + "columns": [ + "organization_id" + ], + "isUnique": false + }, + "member_userId_idx": { + "name": "member_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organization": { + "name": "organization", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "session_created_idx": { + "name": "session_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "banned": { + "name": "banned", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "display_username": { + "name": "display_username", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + }, + "user_username_unique": { + "name": "user_username_unique", + "columns": [ + "username" + ], + "isUnique": true + }, + "user_created_idx": { + "name": "user_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/migrations/meta/_journal.json b/migrations/meta/_journal.json index f1257076..949fbf22 100644 --- a/migrations/meta/_journal.json +++ b/migrations/meta/_journal.json @@ -414,6 +414,13 @@ "when": 1784606180765, "tag": "0059_storage-usage-ledger", "breakpoints": true + }, + { + "idx": 60, + "version": "6", + "when": 1784614533084, + "tag": "0060_traffic-issued-at", + "breakpoints": true } ] } \ No newline at end of file diff --git a/scripts/backfill-admin-stats.ts b/scripts/backfill-admin-stats.ts index e2eddc89..e1b58ada 100644 --- a/scripts/backfill-admin-stats.ts +++ b/scripts/backfill-admin-stats.ts @@ -77,11 +77,34 @@ interface BackfillPlan { } export function buildBackfillSql(now = new Date()): string { + const trafficPeriod = now.toISOString().slice(0, 7) return ` +INSERT OR IGNORE INTO cloud_traffic_reports ( + id, org_id, period, source, source_id, event_id, bytes, storage_id, + unit_bytes, credits_per_unit, status, error, attempt_count, next_retry_at, + issued_at, created_at, updated_at +) VALUES ( + 'traffic_ledger_opening_v1', '', '${trafficPeriod}', 'object_download', + 'traffic_ledger_opening_v1', 'traffic_ledger_opening_v1', 0, NULL, + NULL, NULL, 'ledger_opening', NULL, 0, NULL, NULL, ${now.getTime()}, ${now.getTime()} +); + UPDATE activity_events SET actor_type = CASE WHEN user_id IS NULL THEN 'anonymous' ELSE 'user' END WHERE actor_type IS NULL; +UPDATE activity_events +SET actor_type = 'system', actor_ref = 'legacy-download-task-worker' +WHERE action IN ( + 'download_task_assigned', 'download_task_started', 'download_task_ingesting', + 'download_task_completed', 'download_task_failed', 'download_task_suspended', + 'download_task_queued', 'download_task_error', 'download_task_billing_suspended', + 'download_resolve_started', 'download_resolve_completed', 'download_completed', + 'download_ingest_started', 'download_ingest_completed', + 'download_seeding_started', 'download_seeding_stopped', + 'download_stale_control_resolved', 'download_stale_requeued' +); + UPDATE activity_events AS ae SET metadata = json_set( CASE WHEN metadata IS NULL OR metadata = '' OR json_valid(metadata) = 0 THEN '{}' ELSE metadata END, @@ -144,7 +167,7 @@ SET metadata = json_set( ELSE '' END ) - AND ctr.status <> 'blocked' + AND ctr.status NOT IN ('blocked', 'reversed', 'ledger_opening') AND ABS(CAST(ctr.created_at / 1000 AS INTEGER) - ae.created_at) <= 5 ORDER BY ctr.created_at, ctr.event_id LIMIT 1 @@ -161,7 +184,7 @@ SET metadata = json_set( ELSE '' END ) - AND ctr.status <> 'blocked' + AND ctr.status NOT IN ('blocked', 'reversed', 'ledger_opening') AND ABS(CAST(ctr.created_at / 1000 AS INTEGER) - ae.created_at) <= 5 ORDER BY ctr.created_at, ctr.event_id LIMIT 1 @@ -179,7 +202,7 @@ SET metadata = json_set( ELSE '' END ) - AND ctr.status <> 'blocked' + AND ctr.status NOT IN ('blocked', 'reversed', 'ledger_opening') AND ABS(CAST(ctr.created_at / 1000 AS INTEGER) - ae.created_at) <= 5 ORDER BY ctr.created_at, ctr.event_id LIMIT 1 @@ -200,10 +223,31 @@ WHERE action IN ('share_download', 'object_download', 'image_hosting_download', ELSE '' END ) - AND ctr.status <> 'blocked' + AND ctr.status NOT IN ('blocked', 'reversed', 'ledger_opening') AND ABS(CAST(ctr.created_at / 1000 AS INTEGER) - ae.created_at) <= 5 ); +UPDATE cloud_traffic_reports AS ctr +SET issued_at = ( + SELECT ae.created_at * 1000 + FROM activity_events ae + WHERE ae.action IN ('share_download', 'object_download', 'image_hosting_download', 'webdav_download') + AND ae.id NOT LIKE 'backfill_%' + AND json_valid(ae.metadata) = 1 + AND json_extract(ae.metadata, '$.trafficEventId') = ctr.event_id + ORDER BY ae.created_at + LIMIT 1 +) +WHERE ctr.issued_at IS NULL + AND ctr.status NOT IN ('blocked', 'reversed', 'ledger_opening') + AND EXISTS ( + SELECT 1 FROM activity_events ae + WHERE ae.action IN ('share_download', 'object_download', 'image_hosting_download', 'webdav_download') + AND ae.id NOT LIKE 'backfill_%' + AND json_valid(ae.metadata) = 1 + AND json_extract(ae.metadata, '$.trafficEventId') = ctr.event_id + ); + INSERT OR IGNORE INTO activity_events ( id, org_id, user_id, actor_type, actor_ref, action, target_type, target_id, target_name, metadata, created_at @@ -236,6 +280,7 @@ SELECT CAST(ctr.created_at / 1000 AS INTEGER) FROM cloud_traffic_reports ctr WHERE ctr.source IN ('direct_share', 'landing_share', 'image_hosting', 'object_download', 'webdav_download') + AND ctr.status <> 'ledger_opening' AND NOT EXISTS ( SELECT 1 FROM activity_events ae WHERE ae.id = 'backfill_' || ctr.event_id @@ -433,19 +478,19 @@ function buildHourlyBackfillSql(now: Date): string { }, { metric: 'transfer.download_issued', - source: 'activity_events ae', - timestampMs: 'ae.created_at * 1000', - org: 'ae.org_id', - where: "ae.action IN ('share_download','object_download','image_hosting_download','webdav_download')", - bytes: "SUM(CASE WHEN json_valid(ae.metadata) = 1 THEN COALESCE(json_extract(ae.metadata, '$.bytes'), 0) ELSE 0 END)", - quality: `CASE WHEN - SUM(CASE WHEN ae.metadata IS NULL OR json_valid(ae.metadata) = 0 OR json_type(ae.metadata, '$.bytes') IS NULL THEN 1 ELSE 0 END) > 0 - OR (${missingShareDownloadsQuality}) = 'lower_bound' - THEN 'lower_bound' ELSE 'exact' END`, + source: 'cloud_traffic_reports ctr', + timestampMs: 'ctr.issued_at', + org: 'ctr.org_id', + where: "ctr.issued_at IS NOT NULL AND ctr.status <> 'reversed'", + bytes: 'SUM(ctr.bytes)', + quality: `CASE WHEN MIN(ctr.issued_at) < ( + SELECT CAST((created_at + 3599999) / 3600000 AS INTEGER) * 3600000 + FROM cloud_traffic_reports + WHERE event_id = 'traffic_ledger_opening_v1' + ) THEN 'lower_bound' ELSE 'exact' END`, dimensions: { - source: "COALESCE(CASE WHEN json_valid(ae.metadata) = 1 THEN json_extract(ae.metadata, '$.source') END, CASE ae.action WHEN 'object_download' THEN 'object_download' WHEN 'image_hosting_download' THEN 'image_hosting' WHEN 'webdav_download' THEN 'webdav_download' ELSE 'landing_share' END)", - actor_type: "COALESCE(ae.actor_type, CASE WHEN ae.user_id IS NULL THEN 'anonymous' ELSE 'user' END)", - storage_id: "CASE WHEN json_valid(ae.metadata) = 1 THEN json_extract(ae.metadata, '$.storageId') END", + source: 'ctr.source', + storage_id: 'ctr.storage_id', }, }, { @@ -507,9 +552,9 @@ function buildHourlyBackfillSql(now: Date): string { source: 'activity_events ae', timestampMs: 'ae.created_at * 1000', org: 'ae.org_id', - where: "ae.action IN ('upload_confirm','share_download','object_download','image_hosting_download','webdav_download') AND (ae.metadata IS NULL OR json_valid(ae.metadata) = 0 OR json_type(ae.metadata, '$.bytes') IS NULL)", + where: "ae.action = 'upload_confirm' AND (ae.metadata IS NULL OR json_valid(ae.metadata) = 0 OR json_type(ae.metadata, '$.bytes') IS NULL)", dimensions: { - direction: "CASE WHEN ae.action = 'upload_confirm' THEN 'upload' ELSE 'download' END", + direction: "'upload'", source: "COALESCE(CASE WHEN json_valid(ae.metadata) = 1 THEN json_extract(ae.metadata, '$.source') END, ae.action)", }, }, @@ -785,6 +830,10 @@ function statsHistoryStartSql(): string { SELECT CAST(MIN(occurred_at) / 3600000 AS INTEGER) * 3600000 AS start_at FROM storage_usage_ledger WHERE reason = 'opening_balance_complete' + UNION ALL + SELECT CAST(MIN(issued_at) / 3600000 AS INTEGER) * 3600000 AS start_at + FROM cloud_traffic_reports + WHERE issued_at IS NOT NULL AND status <> 'reversed' ) history_starts WHERE start_at IS NOT NULL)` } @@ -830,15 +879,14 @@ export function buildValidationSql(now = new Date()): string { AND created_at >= ${MIN_VALID_TIMESTAMP_SECONDS} AND created_at * 1000 < ${currentHour} ), 'rawDownloadEvents', ( - SELECT COUNT(*) FROM activity_events - WHERE action IN ('share_download', 'object_download', 'image_hosting_download', 'webdav_download') - AND created_at >= ${MIN_VALID_TIMESTAMP_SECONDS} AND created_at * 1000 < ${currentHour} + SELECT COUNT(*) FROM cloud_traffic_reports + WHERE issued_at IS NOT NULL AND status <> 'reversed' + AND issued_at >= ${MIN_VALID_TIMESTAMP_MS} AND issued_at < ${currentHour} ), 'rawDownloadBytes', ( - SELECT COALESCE(SUM(CASE WHEN json_valid(metadata) = 1 THEN COALESCE(json_extract(metadata, '$.bytes'), 0) ELSE 0 END), 0) - FROM activity_events - WHERE action IN ('share_download', 'object_download', 'image_hosting_download', 'webdav_download') - AND created_at >= ${MIN_VALID_TIMESTAMP_SECONDS} AND created_at * 1000 < ${currentHour} + SELECT COALESCE(SUM(bytes), 0) FROM cloud_traffic_reports + WHERE issued_at IS NOT NULL AND status <> 'reversed' + AND issued_at >= ${MIN_VALID_TIMESTAMP_MS} AND issued_at < ${currentHour} ), 'rawShareViews', ( SELECT COUNT(*) FROM activity_events @@ -895,7 +943,7 @@ export function buildValidationSql(now = new Date()): string { ), 'rawMissingByteEvents', ( SELECT COUNT(*) FROM activity_events - WHERE action IN ('upload_confirm', 'share_download', 'object_download', 'image_hosting_download', 'webdav_download') + WHERE action = 'upload_confirm' AND (metadata IS NULL OR json_valid(metadata) = 0 OR json_type(metadata, '$.bytes') IS NULL) AND created_at >= ${MIN_VALID_TIMESTAMP_SECONDS} AND created_at * 1000 < ${currentHour} ) @@ -930,7 +978,6 @@ required_dimensions(metric_key, dimension_key, check_bytes) AS ( VALUES ('transfer.upload', 'source', 1), ('transfer.upload', 'status', 1), - ('transfer.download_issued', 'actor_type', 1), ('transfer.download_issued', 'source', 1), ('share.download_issued', 'actor_type', 1), ('share.download_issued', 'source', 1), @@ -1172,6 +1219,7 @@ SELECT json_object( 'cloudEventsToRecover', ( SELECT COUNT(*) FROM cloud_traffic_reports ctr WHERE ctr.source IN ('direct_share', 'landing_share', 'image_hosting', 'object_download', 'webdav_download') + AND ctr.status <> 'ledger_opening' AND NOT EXISTS ( SELECT 1 FROM activity_events ae WHERE ae.id = 'backfill_' || ctr.event_id diff --git a/server/adapters/repos/admin-stats-rollup.integration.test.ts b/server/adapters/repos/admin-stats-rollup.integration.test.ts index 86ae110e..d66f4e43 100644 --- a/server/adapters/repos/admin-stats-rollup.integration.test.ts +++ b/server/adapters/repos/admin-stats-rollup.integration.test.ts @@ -45,7 +45,11 @@ describe('admin hourly stats rollup', () => { await db.run(sql`UPDATE organization SET created_at = ${atMs}, updated_at = ${atMs} WHERE id = ${orgId}`) await db.run(sql` INSERT INTO user (id, name, email, email_verified, created_at, updated_at) - VALUES ('rollup-direct-user', 'Direct User', 'direct@example.com', 0, ${atMs}, ${atMs}) + VALUES ('rollup-direct-user', 'Direct User', 'direct@example.com', 1, ${atMs}, ${atMs}) + `) + await db.run(sql` + INSERT INTO session (id, expires_at, token, created_at, updated_at, user_id) + VALUES ('rollup-direct-session', ${atMs + 86_400_000}, 'rollup-direct-session', ${atMs}, ${atMs}, 'rollup-direct-user') `) await db.run(sql` INSERT INTO organization (id, name, slug, metadata, created_at, updated_at) @@ -148,10 +152,13 @@ describe('admin hourly stats rollup', () => { `) await db.run(sql` INSERT INTO cloud_traffic_reports - (id, org_id, period, source, source_id, storage_id, event_id, bytes, unit_bytes, credits_per_unit, status, created_at, updated_at) + (id, org_id, period, source, source_id, storage_id, event_id, bytes, unit_bytes, credits_per_unit, status, issued_at, created_at, updated_at) VALUES - ('rollup-traffic-ok', ${orgId}, '2026-07', 'object_download', 'rollup-file', 'rollup-storage', 'traffic-ok', 100, 50, 2, 'reported', ${atMs}, ${atMs}), - ('rollup-traffic-blocked', ${orgId}, '2026-07', 'landing_share', 'rollup-share-usable', NULL, 'traffic-blocked', 200, 50, 2, 'blocked', ${atMs}, ${atMs}) + ('rollup-traffic-share', ${orgId}, '2026-07', 'landing_share', 'rollup-share-usable', 'rollup-storage', 'traffic-share', 20, 50, 2, 'reported', ${atMs}, ${atMs}, ${atMs}), + ('rollup-traffic-object', ${orgId}, '2026-07', 'object_download', 'rollup-file', 'rollup-storage', 'traffic-object', 30, 50, 2, 'reported', ${atMs}, ${atMs}, ${atMs}), + ('rollup-traffic-image', ${orgId}, '2026-07', 'image_hosting', 'rollup-image', 'rollup-storage', 'traffic-image', 0, 50, 2, 'reported', ${atMs}, ${atMs}, ${atMs}), + ('rollup-traffic-webdav', ${orgId}, '2026-07', 'webdav_download', 'rollup-file', 'rollup-storage', 'traffic-webdav', 40, 50, 2, 'reported', ${atMs}, ${atMs}, ${atMs}), + ('rollup-traffic-blocked', ${orgId}, '2026-07', 'landing_share', 'rollup-share-usable', NULL, 'traffic-blocked', 200, 50, 2, 'blocked', NULL, ${atMs}, ${atMs}) `) await db.run(sql` INSERT INTO downloaders @@ -218,7 +225,7 @@ describe('admin hourly stats rollup', () => { expect(row(M.transferUpload)).toMatchObject({ count: 4, bytes: 100 }) expect(row(M.transferDownloadIssued)).toMatchObject({ count: 4, bytes: 90 }) expect(row(M.shareDownloadIssued)).toMatchObject({ count: 1, bytes: 20 }) - expect(row(M.statsMissingBytes)).toMatchObject({ count: 2 }) + expect(row(M.statsMissingBytes)).toMatchObject({ count: 1 }) expect(row(M.userSignup, '', '', '')).toMatchObject({ count: 2 }) expect(row(M.userSignup, 'provider', 'direct', '')).toMatchObject({ count: 1 }) expect(row(M.shareCreated)).toMatchObject({ count: 4 }) @@ -246,8 +253,9 @@ describe('admin hourly stats rollup', () => { expect(row(M.remoteDownloadTaskSnapshot)).toMatchObject({ count: 1 }) expect(row(M.downloaderSnapshot, '', '', '')).toMatchObject({ count: 2 }) expect(row(M.userInventory, '', '', '')).toMatchObject({ count: 2 }) + expect(row(M.userInventory, 'status', 'silent', '')).toMatchObject({ count: 1 }) expect(row(M.userActiveSnapshot, 'window', 'mau', '')).toMatchObject({ count: 1 }) - expect(row(M.trafficReportSnapshot, '', '', '')).toMatchObject({ count: 2, bytes: 300 }) + expect(row(M.trafficReportSnapshot, '', '', '')).toMatchObject({ count: 5, bytes: 290 }) expect(row(M.webhookSnapshot, 'status', 'processed', '')).toMatchObject({ count: 1 }) expect(row(M.statsRollupRun, '', '', '')).toMatchObject({ count: 1 }) expect(JSON.parse(row(M.transferUpload)?.metadata ?? '{}')).toMatchObject({ version: 3, quality: 'lower_bound' }) diff --git a/server/adapters/repos/admin-stats-rollup.ts b/server/adapters/repos/admin-stats-rollup.ts index 243f57a6..da443dfc 100644 --- a/server/adapters/repos/admin-stats-rollup.ts +++ b/server/adapters/repos/admin-stats-rollup.ts @@ -23,6 +23,7 @@ import { ROLLUP_VERSION, } from '../../domain/admin-stats-metrics' import type { Database } from '../../platform/interface' +import { createCloudTrafficReportRepo, trafficLedgerExactFrom } from './cloud-traffic-report' import { getEffectiveQuotasByOrg } from './quota' import { ensureStorageUsageOpeningBalances } from './storage-usage-ledger' @@ -30,7 +31,6 @@ const HOUR_MS = 3_600_000 const D1_MAX_BOUND_PARAMS = 100 const ROLLUP_BOUND_PARAMS_PER_ROW = 11 export const ADMIN_STATS_ROLLUP_WRITE_BATCH_SIZE = Math.floor(D1_MAX_BOUND_PARAMS / ROLLUP_BOUND_PARAMS_PER_ROW) -const DOWNLOAD_ACTIONS = ['share_download', 'object_download', 'image_hosting_download', 'webdav_download'] const COUNTER_METRICS: AdminStatsMetric[] = [ M.backgroundJobFinished, M.remoteDownloadTaskFinished, @@ -95,7 +95,9 @@ export async function rebuildAdminStatsHour( const capturedSnapshot = await compatibleSnapshotMarker(db, bucketStart) await ensureStorageUsageOpeningBalances(db, generatedAt) + await createCloudTrafficReportRepo(db).ensureLedgerOpening(generatedAt) await addEventMetrics(db, rollups, bucketStart, bucketEnd) + await addTransferDownloadMetrics(db, rollups, bucketStart, bucketEnd) await addUserMetrics(db, rollups, bucketStart, bucketEnd) await addOperationalMetrics(db, rollups, bucketStart, bucketEnd) await addStorageLedgerBalance(db, rollups, bucketEnd) @@ -247,7 +249,7 @@ async function addEventMetrics(db: Database, rollups: RollupAccumulator, from: D CASE WHEN json_valid(metadata) = 1 THEN CASE WHEN json_type(metadata, '$.bytes') IN ('integer', 'real') THEN json_extract(metadata, '$.bytes') ELSE 0 END ELSE 0 END AS bytes, - CASE WHEN action IN ('upload_confirm', 'share_download', 'object_download', 'image_hosting_download', 'webdav_download') + CASE WHEN action = 'upload_confirm' THEN CASE WHEN json_valid(metadata) = 0 THEN 1 WHEN json_type(metadata, '$.bytes') IN ('integer', 'real') THEN 0 @@ -295,21 +297,6 @@ async function addEventMetrics(db: Database, rollups: RollupAccumulator, from: D ) } - if (DOWNLOAD_ACTIONS.includes(row.action)) { - rollups.add( - M.transferDownloadIssued, - row.orgId, - count, - bytes, - { - source: row.source ?? downloadSourceForAction(row.action), - storage_id: row.storageId, - actor_type: row.actorType, - }, - lowerBound, - ) - } - if (row.action === 'download_failed') { rollups.add(M.transferDownloadFailed, row.orgId, count, bytes, { source: row.source ?? 'unknown', @@ -343,6 +330,46 @@ async function addEventMetrics(db: Database, rollups: RollupAccumulator, from: D } } +async function addTransferDownloadMetrics( + db: Database, + rollups: RollupAccumulator, + from: Date, + to: Date, +): Promise { + const opening = await createCloudTrafficReportRepo(db).getLedgerOpening() + if (!opening) throw new Error('traffic_ledger_opening_missing') + const lowerBound = from < trafficLedgerExactFrom(opening) + const rows = await db.all<{ + orgId: string + source: string + storageId: string | null + count: number + bytes: number + }>(sql` + SELECT + org_id AS orgId, + source, + storage_id AS storageId, + COUNT(*) AS count, + COALESCE(SUM(bytes), 0) AS bytes + FROM cloud_traffic_reports + WHERE issued_at >= ${from.getTime()} + AND issued_at < ${to.getTime()} + AND status <> 'reversed' + GROUP BY org_id, source, storage_id + `) + for (const row of rows) { + rollups.add( + M.transferDownloadIssued, + row.orgId, + Number(row.count), + Number(row.bytes), + { source: row.source, storage_id: row.storageId }, + lowerBound, + ) + } +} + type EventMetricGroup = { orgId: string action: string @@ -548,6 +575,16 @@ async function addSnapshotMetrics(db: Database, rollups: RollupAccumulator, now: bytes: sql`COALESCE(SUM(${cloudTrafficReports.bytes}), 0)`, }) .from(cloudTrafficReports) + .where( + inArray(cloudTrafficReports.status, [ + 'pending', + 'reported', + 'skipped_unbound', + 'failed', + 'dead_letter', + 'blocked', + ]), + ) .groupBy(cloudTrafficReports.status), ), queryStage( @@ -793,8 +830,7 @@ type UserInventorySnapshot = { } async function userInventorySnapshot(db: Database, now: Date): Promise { - const activeCutoffMs = now.getTime() - 30 * 86_400_000 - const activeCutoffSec = Math.floor(activeCutoffMs / 1000) + const activeCutoffSec = Math.floor((now.getTime() - 30 * 86_400_000) / 1000) const rows = await db.all(sql` SELECT COUNT(*) AS total, @@ -802,8 +838,22 @@ async function userInventorySnapshot(db: Database, now: Date): Promise= ${activeCutoffSec}) - AND NOT EXISTS (SELECT 1 FROM session s WHERE s.user_id = user.id AND s.created_at >= ${activeCutoffMs}) + AND NOT EXISTS ( + SELECT 1 FROM activity_events ae + WHERE ae.user_id = user.id + AND ae.created_at >= ${activeCutoffSec} + AND COALESCE(ae.actor_type, 'user') = 'user' + AND ae.action NOT IN ( + 'download_task_assigned', 'download_task_started', 'download_task_ingesting', + 'download_task_completed', 'download_task_failed', 'download_task_suspended', + 'download_task_queued', 'download_task_error', 'download_task_billing_suspended', + 'download_resolve_started', 'download_resolve_completed', 'download_completed', + 'download_ingest_started', 'download_ingest_completed', + 'download_seeding_started', 'download_seeding_stopped', + 'download_stale_control_resolved', 'download_stale_requeued' + ) + AND ae.action NOT LIKE 'stats_%' + ) THEN 1 ELSE 0 END) AS silent FROM user `) @@ -827,16 +877,24 @@ async function activeUserSnapshot( ] return await Promise.all( windows.map(async ({ window, days }) => { - const cutoffMs = now.getTime() - days * 86_400_000 - const cutoffSec = Math.floor(cutoffMs / 1000) + const cutoffSec = Math.floor((now.getTime() - days * 86_400_000) / 1000) const rows = await db.all<{ count: number }>(sql` - SELECT COUNT(DISTINCT user_id) AS count - FROM ( - SELECT s.user_id AS user_id FROM session s JOIN user u ON u.id = s.user_id WHERE s.created_at >= ${cutoffMs} - UNION ALL - SELECT ae.user_id AS user_id FROM activity_events ae JOIN user u ON u.id = ae.user_id - WHERE ae.user_id IS NOT NULL AND ae.created_at >= ${cutoffSec} - ) active_users + SELECT COUNT(DISTINCT ae.user_id) AS count + FROM activity_events ae + JOIN user u ON u.id = ae.user_id + WHERE ae.user_id IS NOT NULL + AND ae.created_at >= ${cutoffSec} + AND COALESCE(ae.actor_type, 'user') = 'user' + AND ae.action NOT IN ( + 'download_task_assigned', 'download_task_started', 'download_task_ingesting', + 'download_task_completed', 'download_task_failed', 'download_task_suspended', + 'download_task_queued', 'download_task_error', 'download_task_billing_suspended', + 'download_resolve_started', 'download_resolve_completed', 'download_completed', + 'download_ingest_started', 'download_ingest_completed', + 'download_seeding_started', 'download_seeding_stopped', + 'download_stale_control_resolved', 'download_stale_requeued' + ) + AND ae.action NOT LIKE 'stats_%' `) return { window, count: Number(rows[0]?.count ?? 0) } }), diff --git a/server/adapters/repos/admin-stats.ts b/server/adapters/repos/admin-stats.ts index 772d4db1..726015e3 100644 --- a/server/adapters/repos/admin-stats.ts +++ b/server/adapters/repos/admin-stats.ts @@ -27,6 +27,7 @@ import type { Database } from '../../platform/interface' import type { AdminStatsDateRange, AdminStatsRepo } from '../../usecases/ports' import { AdminStatsHourlyReader } from './admin-stats-hourly' import { captureAdminStatsSnapshot, rebuildAdminStatsHour } from './admin-stats-rollup' +import { createCloudTrafficReportRepo, trafficLedgerExactFrom } from './cloud-traffic-report' const DOWNLOAD_ACTIVITY_ACTIONS = ['share_download', 'object_download', 'image_hosting_download', 'webdav_download'] const DOWNLOAD_FAILURE_ACTION = 'download_failed' @@ -50,27 +51,41 @@ async function getOverviewStatistics( ): Promise { const effective = effectiveRange(range, now) const reader = new AdminStatsHourlyReader(db, effective, now) - const [inventory, active, newUsersByDay, totalUsersByDay, activeByDay, storageUsedByDay, topUsage] = - await Promise.all([ - getUserInventory(reader), - getActiveUserSnapshot(reader), - getSignupsByDay(reader), - getUserTotalsByDay(reader), - getRollingActiveUserTrend(reader, effective), - getStorageUsedByDay(reader), - getTopPersonalUsage(db, reader), - ]) + const [ + inventory, + active, + newUsersByDay, + totalUsersByDay, + activeByDay, + storageUsedByDay, + topUsage, + storageDataQuality, + ] = await Promise.all([ + getUserInventory(reader), + getActiveUserSnapshot(reader), + getSignupsByDay(reader), + getUserTotalsByDay(reader), + getRollingActiveUserTrend(reader, effective), + getStorageUsedByDay(reader), + getTopPersonalUsage(db, reader), + getStorageDataQuality(reader), + ]) const dates = createDateBuckets(effective) const activeByDate = new Map(activeByDay.map((row) => [row.date, row])) const dau = active?.dau ?? null const wau = active?.wau ?? null const mau = active?.mau ?? null + const recentSignups = dates.slice(-7).map((date) => (newUsersByDay.has(date) ? (newUsersByDay.get(date) ?? null) : 0)) + const exactUsage = storageDataQuality.usageDriftSpaces === null || storageDataQuality.usageDriftSpaces === 0 + const exactLedger = storageDataQuality.ledgerDriftSpaces === null || storageDataQuality.ledgerDriftSpaces === 0 return { users: { total: inventory?.total ?? null, active30Days: mau, - new7Days: dates.slice(-7).reduce((total, date) => total + (newUsersByDay.get(date) ?? 0), 0), + new7Days: recentSignups.some((value) => value === null) + ? null + : recentSignups.reduce((total, value) => total + (value ?? 0), 0), activity: { today: dau, last7Days: dau === null || wau === null ? null : Math.max(0, wau - dau), @@ -81,11 +96,14 @@ async function getOverviewStatistics( date, totalUsers: totalUsersByDay.get(date) ?? null, activeUsers: activeByDate.get(date)?.mau ?? null, - newUsers: newUsersByDay.get(date) ?? 0, + newUsers: newUsersByDay.has(date) ? (newUsersByDay.get(date) ?? null) : 0, })), - topUsage, + topUsage: exactUsage ? topUsage : [], }, - storageTrend: dates.map((date) => ({ date, usedBytes: storageUsedByDay.get(date) ?? null })), + storageTrend: dates.map((date) => ({ + date, + usedBytes: exactLedger ? (storageUsedByDay.get(date) ?? null) : null, + })), } } @@ -186,6 +204,8 @@ async function getDashboardOverviewStats( comparisonCoverage, snapshotCoverage, comparisonSnapshotCoverage, + trafficLedgerComplete, + previousTrafficLedgerComplete, ] = await Promise.all([ getUserInventory(reader), getSignupTotal(reader), @@ -204,29 +224,48 @@ async function getDashboardOverviewStats( previousReader.coverage('counters'), reader.coverage('snapshots'), previousReader.coverage('snapshots'), + trafficLedgerCoversRange(db, effective), + trafficLedgerCoversRange(db, previous), ]) - const [trendNewUsers, activeByDay, storageUsedByDay, uploadByDay, downloadByDay] = await Promise.all([ - getSignupsByDay(reader), - getActiveUsersByDay(reader), - getStorageUsedByDay(reader), - getActivityMetricByDay(reader, metricSpec(['upload_confirm']), 'bytes'), - getActivityMetricByDay(reader, metricSpec(DOWNLOAD_ACTIVITY_ACTIONS), 'bytes'), - ]) + const [trendNewUsers, activeByDay, storageUsedByDay, uploadByDay, downloadByDay, missingBytesByDay] = + await Promise.all([ + getSignupsByDay(reader), + getActiveUsersByDay(reader), + getStorageUsedByDay(reader), + getActivityMetricByDay(reader, metricSpec(['upload_confirm']), 'bytes'), + getActivityMetricByDay(reader, metricSpec(DOWNLOAD_ACTIVITY_ACTIONS), 'bytes'), + getMissingTransferBytesByDay(reader), + ]) const trends = createDateBuckets(effective).map((date) => { + const missingBytes = missingBytesByDay.get(date) return { date, - newUsers: trendNewUsers.get(date) ?? 0, + newUsers: trendNewUsers.has(date) ? (trendNewUsers.get(date) ?? null) : 0, activeUsers: activeByDay.get(date) ?? null, storageUsedBytes: storageUsedByDay.get(date) ?? null, - uploadBytes: uploadByDay.get(date) ?? 0, - downloadBytes: downloadByDay.get(date) ?? 0, + uploadBytes: missingBytes?.upload ? null : (uploadByDay.get(date) ?? 0), + downloadBytes: !trafficLedgerComplete || missingBytes?.download ? null : (downloadByDay.get(date) ?? 0), } }) const sharingComparable = comparable(coverage, comparisonCoverage) && hasExactSharingHistory(sharingDataQuality) && hasExactSharingHistory(previousSharingDataQuality) + const exactSharing = hasExactSharingHistory(sharingDataQuality) + const exactPreviousSharing = hasExactSharingHistory(previousSharingDataQuality) const validQuotaBytes = quotas && quotas.invalidQuotaSpaces === 0 ? quotas.quotaBytes : null + const currentUploadBytes = dataQuality.missingUploadBytesEvents === 0 ? traffic.uploadBytes : null + const previousUploadBytes = dataQuality.previousMissingUploadBytesEvents === 0 ? previousTraffic.uploadBytes : null + const currentDownloadBytes = + trafficLedgerComplete && dataQuality.missingDownloadBytesEvents === 0 ? traffic.downloadBytes : null + const previousDownloadBytes = + previousTrafficLedgerComplete && dataQuality.previousMissingDownloadBytesEvents === 0 + ? previousTraffic.downloadBytes + : null + const currentTrafficBytes = + currentUploadBytes === null || currentDownloadBytes === null ? null : currentUploadBytes + currentDownloadBytes + const previousTrafficBytes = + previousUploadBytes === null || previousDownloadBytes === null ? null : previousUploadBytes + previousDownloadBytes return { ...statsFrame(now, effective, coverage, comparisonCoverage, snapshotCoverage, comparisonSnapshotCoverage), @@ -243,20 +282,20 @@ async function getDashboardOverviewStats( storageUsedBytes: quotas?.usedBytes ?? null, storageQuotaBytes: validQuotaBytes, storageUtilization: nullablePercent(quotas?.usedBytes ?? null, validQuotaBytes), - trafficBytes: delta( - traffic.uploadBytes + traffic.downloadBytes, - previousTraffic.uploadBytes + previousTraffic.downloadBytes, - comparable(coverage, comparisonCoverage), - ), - uploadBytes: delta(traffic.uploadBytes, previousTraffic.uploadBytes, comparable(coverage, comparisonCoverage)), - downloadBytes: delta( - traffic.downloadBytes, - previousTraffic.downloadBytes, - comparable(coverage, comparisonCoverage), - ), + trafficBytes: delta(currentTrafficBytes, previousTrafficBytes, comparable(coverage, comparisonCoverage)), + uploadBytes: delta(currentUploadBytes, previousUploadBytes, comparable(coverage, comparisonCoverage)), + downloadBytes: delta(currentDownloadBytes, previousDownloadBytes, comparable(coverage, comparisonCoverage)), activeShares: sharing.activeShares, - shareViews: delta(sharing.views, previousSharing.views, sharingComparable), - shareDownloads: delta(sharing.downloads, previousSharing.downloads, sharingComparable), + shareViews: delta( + exactSharing ? sharing.views : null, + exactPreviousSharing ? previousSharing.views : null, + sharingComparable, + ), + shareDownloads: delta( + exactSharing ? sharing.downloads : null, + exactPreviousSharing ? previousSharing.downloads : null, + sharingComparable, + ), }, trends, } @@ -376,7 +415,7 @@ async function getDashboardGrowthStats( const newUsersByDay = await getSignupsByDay(reader) const userScaleTrend = createDateBuckets(effective).map((date) => ({ date, - newUsers: newUsersByDay.get(date) ?? 0, + newUsers: newUsersByDay.has(date) ? (newUsersByDay.get(date) ?? null) : 0, totalUsers: totalsByDay.get(date) ?? null, })) @@ -441,6 +480,7 @@ async function getDashboardStorageStats( comparisonCoverage, snapshotCoverage, comparisonSnapshotCoverage, + missingBytesByDay, ] = await Promise.all([ getQuotaTotals(reader), getStorageInventory(reader), @@ -463,12 +503,15 @@ async function getDashboardStorageStats( previousReader.coverage('counters'), reader.coverage('snapshots'), previousReader.coverage('snapshots'), + getMissingTransferBytesByDay(reader), ]) + const exactUsage = storageDataQuality.usageDriftSpaces === null || storageDataQuality.usageDriftSpaces === 0 + const exactLedger = storageDataQuality.ledgerDriftSpaces === null || storageDataQuality.ledgerDriftSpaces === 0 const storageTrend = createDateBuckets(effective).map((date) => { return { date, - usedBytes: storageUsedByDay.get(date) ?? null, - newBytes: uploadsByDay.get(date) ?? 0, + usedBytes: exactLedger ? (storageUsedByDay.get(date) ?? null) : null, + newBytes: missingBytesByDay.get(date)?.upload ? null : (uploadsByDay.get(date) ?? 0), newFiles: uploadFilesByDay.get(date) ?? 0, } }) @@ -484,19 +527,23 @@ async function getDashboardStorageStats( ...statsFrame(now, effective, coverage, comparisonCoverage, snapshotCoverage, comparisonSnapshotCoverage), dataQuality: { ...transferDataQuality, ...storageDataQuality }, summary: { - storageUsedBytes: quotas?.usedBytes ?? null, + storageUsedBytes: exactUsage ? (quotas?.usedBytes ?? null) : null, quotaBytes: validQuotaBytes, fileCount: inventory?.files ?? null, trashFileCount: trashInventory?.files ?? null, trashBytes: trashInventory?.bytes ?? null, newFiles: delta(newFiles, previousNewFiles, comparable(coverage, comparisonCoverage)), - newBytes: delta(uploadBytes, previousUploadBytes, comparable(coverage, comparisonCoverage)), + newBytes: delta( + transferDataQuality.missingUploadBytesEvents === 0 ? uploadBytes : null, + transferDataQuality.previousMissingUploadBytesEvents === 0 ? previousUploadBytes : null, + comparable(coverage, comparisonCoverage), + ), coldFileBytes, - storageUtilization: nullablePercent(quotas?.usedBytes ?? null, validQuotaBytes), + storageUtilization: nullablePercent(exactUsage ? (quotas?.usedBytes ?? null) : null, validQuotaBytes), coldFilePercent: nullablePercent(coldFileBytes, inventory?.bytes ?? null), - nearQuotaSpaces: quotaPressure?.get('near') ?? null, - overQuotaSpaces: quotaPressure?.get('over') ?? null, - invalidQuotaSpaces: quotaPressure?.get('invalid') ?? null, + nearQuotaSpaces: exactUsage ? (quotaPressure?.get('near') ?? null) : null, + overQuotaSpaces: exactUsage ? (quotaPressure?.get('over') ?? null) : null, + invalidQuotaSpaces: exactUsage ? (quotaPressure?.get('invalid') ?? null) : null, }, storageTrend, typeBreakdown: typeBreakdown.map(({ name, ...row }) => ({ @@ -505,7 +552,7 @@ async function getDashboardStorageStats( })), sizeBreakdown, ageBreakdown, - topSpaces: spaceUsage.slice(0, 8), + topSpaces: exactUsage ? spaceUsage.slice(0, 8) : [], } } @@ -536,6 +583,9 @@ async function getDashboardTrafficStats( dataQuality, coverage, comparisonCoverage, + missingBytesByDay, + trafficLedgerComplete, + previousTrafficLedgerComplete, ] = await Promise.all([ getTrafficTotals(reader), getTrafficTotals(previousReader), @@ -554,6 +604,9 @@ async function getDashboardTrafficStats( getTransferDataQuality(reader, previousReader), reader.coverage('counters'), previousReader.coverage('counters'), + getMissingTransferBytesByDay(reader), + trafficLedgerCoversRange(db, effective), + trafficLedgerCoversRange(db, previous), ]) const sourceRows = new Map() if (traffic.uploadBytes > 0 || traffic.uploadRequests > 0) { @@ -580,9 +633,12 @@ async function getDashboardTrafficStats( } const trafficTrend = createDateBuckets(effective).map((date) => ({ date, - uploadBytes: uploadByDay.get(date) ?? 0, - downloadBytes: downloadByDay.get(date) ?? 0, - requests: (uploadRequestsByDay.get(date) ?? 0) + (downloadRequestsByDay.get(date) ?? 0), + uploadBytes: missingBytesByDay.get(date)?.upload ? null : (uploadByDay.get(date) ?? 0), + downloadBytes: + !trafficLedgerComplete || missingBytesByDay.get(date)?.download ? null : (downloadByDay.get(date) ?? 0), + requests: trafficLedgerComplete + ? (uploadRequestsByDay.get(date) ?? 0) + (downloadRequestsByDay.get(date) ?? 0) + : null, })) const successTrend = createDateBuckets(effective).map((date) => { const uploadSuccesses = uploadSuccessByDay.get(date) ?? 0 @@ -594,39 +650,48 @@ async function getDashboardTrafficStats( return { date, uploadSuccessRate: uploadRequests > 0 ? percent(uploadSuccesses, uploadRequests) : null, - downloadSuccessRate: downloadRequests > 0 ? percent(downloadSuccesses, downloadRequests) : null, + downloadSuccessRate: + trafficLedgerComplete && downloadRequests > 0 ? percent(downloadSuccesses, downloadRequests) : null, } }) const totalRequests = traffic.uploadRequests + traffic.downloadRequests const blockedDownloads = [...downloadFailureReasons.values()].reduce((sum, value) => sum + value, 0) const issuedDownloads = Math.max(0, traffic.downloadRequests - blockedDownloads) + const exactCurrentBytes = trafficLedgerComplete && dataQuality.missingBytesEvents === 0 + const exactPreviousBytes = previousTrafficLedgerComplete && dataQuality.previousMissingBytesEvents === 0 return { ...statsFrame(now, effective, coverage, comparisonCoverage), dataQuality, summary: { totalBytes: delta( - traffic.uploadBytes + traffic.downloadBytes, - previousTraffic.uploadBytes + previousTraffic.downloadBytes, + exactCurrentBytes ? traffic.uploadBytes + traffic.downloadBytes : null, + exactPreviousBytes ? previousTraffic.uploadBytes + previousTraffic.downloadBytes : null, comparable(coverage, comparisonCoverage), ), requestCount: delta( - totalRequests, - previousTraffic.uploadRequests + previousTraffic.downloadRequests, + trafficLedgerComplete ? totalRequests : null, + previousTrafficLedgerComplete ? previousTraffic.uploadRequests + previousTraffic.downloadRequests : null, comparable(coverage, comparisonCoverage), ), - issuedDownloads, - blockedDownloads, - downloadIssueSuccessRate: nullablePercent(issuedDownloads, issuedDownloads + blockedDownloads), - peakDailyBytes: Math.max(0, ...trafficTrend.map((row) => row.uploadBytes + row.downloadBytes)), + issuedDownloads: trafficLedgerComplete ? issuedDownloads : null, + blockedDownloads: trafficLedgerComplete ? blockedDownloads : null, + downloadIssueSuccessRate: trafficLedgerComplete + ? nullablePercent(issuedDownloads, issuedDownloads + blockedDownloads) + : null, + peakDailyBytes: exactCurrentBytes + ? Math.max(0, ...trafficTrend.map((row) => (row.uploadBytes ?? 0) + (row.downloadBytes ?? 0))) + : null, }, trafficTrend, - sourceBreakdown: percentRows([...sourceRows.values()], (row) => row.bytes), - issueStatus: percentRows( - [...statusRows.entries()].map(([status, countValue]) => ({ status, name: status, value: countValue })), - ).map(({ name, value, percent: pct }) => ({ status: name, count: value, percent: pct })), + sourceBreakdown: exactCurrentBytes ? percentRows([...sourceRows.values()], (row) => row.bytes) : [], + issueStatus: trafficLedgerComplete + ? percentRows( + [...statusRows.entries()].map(([status, countValue]) => ({ status, name: status, value: countValue })), + ).map(({ name, value, percent: pct }) => ({ status: name, count: value, percent: pct })) + : [], successTrend, - failureReasons: percentRows([...failureReasonRows.values()]), + failureReasons: trafficLedgerComplete ? percentRows([...failureReasonRows.values()]) : [], } } @@ -672,10 +737,10 @@ async function getDashboardSharingStats( ]) const landingDownloads = downloadSources.get('landing_share') ?? 0 const directDownloads = downloadSources.get('direct_share') ?? 0 - const sharingComparable = - comparable(coverage, comparisonCoverage) && - hasExactSharingHistory(dataQuality) && - hasExactSharingHistory(previousDataQuality) + const exactSharing = hasExactSharingHistory(dataQuality) + const exactPreviousSharing = hasExactSharingHistory(previousDataQuality) + const exactCurrentSharing = exactSharing + const sharingComparable = comparable(coverage, comparisonCoverage) && exactSharing && exactPreviousSharing const [viewsByDay, downloadsByDay, savesByDay] = await Promise.all([ getActivityMetricByDay(reader, metricSpec(['share_view']), 'count'), getActivityMetricByDay(reader, metricSpec(['share_download']), 'count'), @@ -683,14 +748,16 @@ async function getDashboardSharingStats( ]) const trend = createDateBuckets(effective).map((date) => ({ date, - views: viewsByDay.get(date) ?? 0, - downloads: downloadsByDay.get(date) ?? 0, + views: exactCurrentSharing ? (viewsByDay.get(date) ?? 0) : null, + downloads: exactCurrentSharing ? (downloadsByDay.get(date) ?? 0) : null, saves: savesByDay.get(date) ?? 0, })) - const topShares = await getTopSharesWithPercent(db, reader, { - totalViews: sharing.views, - totalDownloads: sharing.downloads, - }) + const topShares = exactCurrentSharing + ? await getTopSharesWithPercent(db, reader, { + totalViews: sharing.views, + totalDownloads: sharing.downloads, + }) + : [] return { ...statsFrame(now, effective, coverage, comparisonCoverage, snapshotCoverage, comparisonSnapshotCoverage), @@ -698,33 +765,43 @@ async function getDashboardSharingStats( summary: { activeShares: sharing.activeShares, createdShares: delta(createdInRange, createdPrevious, comparable(coverage, comparisonCoverage)), - views: delta(sharing.views, previousSharing.views, sharingComparable), - downloads: delta(sharing.downloads, previousSharing.downloads, sharingComparable), + views: delta( + exactCurrentSharing ? sharing.views : null, + exactPreviousSharing ? previousSharing.views : null, + sharingComparable, + ), + downloads: delta( + exactCurrentSharing ? sharing.downloads : null, + exactPreviousSharing ? previousSharing.downloads : null, + sharingComparable, + ), saves: delta(saveCount, previousSaveCount, comparable(coverage, comparisonCoverage)), - downloadsPer100Views: hasExactSharingHistory(dataQuality) - ? nullablePercent(landingDownloads, sharing.views) - : null, - savesPer100Views: hasExactSharingHistory(dataQuality) ? nullablePercent(saveCount, sharing.views) : null, + downloadsPer100Views: exactCurrentSharing ? nullablePercent(landingDownloads, sharing.views) : null, + savesPer100Views: exactCurrentSharing ? nullablePercent(saveCount, sharing.views) : null, passwordPasses: sharing.passwordPasses, }, trend, typeBreakdown: percentRows([...typeCounts.entries()].map(([name, value]) => ({ name, value }))), - sourceBreakdown: percentRows([ - { name: 'landing_share', value: landingDownloads }, - { name: 'direct_share', value: directDownloads }, - { name: 'save_to_drive', value: saveCount }, - ]), + sourceBreakdown: exactCurrentSharing + ? percentRows([ + { name: 'landing_share', value: landingDownloads }, + { name: 'direct_share', value: directDownloads }, + { name: 'save_to_drive', value: saveCount }, + ]) + : [], topShares, } } -async function getShareCreatedTotal(reader: AdminStatsHourlyReader): Promise { +async function getShareCreatedTotal(reader: AdminStatsHourlyReader): Promise { const rows = await reader.rows(ADMIN_STATS_METRICS.shareCreated) - return rows.filter((row) => row.dimensionKey === '').reduce((sum, row) => sum + row.count, 0) + const totals = rows.filter((row) => row.dimensionKey === '') + return totals.some((row) => row.lowerBound) ? null : totals.reduce((sum, row) => sum + row.count, 0) } async function getShareCreatedKinds(reader: AdminStatsHourlyReader): Promise> { const rows = await reader.rows(ADMIN_STATS_METRICS.shareCreated, ['kind']) + if (rows.some((row) => row.lowerBound)) return new Map() const result = new Map() for (const row of rows) { if (row.dimensionKey === 'kind') incrementMap(result, row.dimensionValue, row.count) @@ -796,6 +873,11 @@ function effectiveRange(range: AdminStatsDateRange, now: Date): AdminStatsDateRa } } +async function trafficLedgerCoversRange(db: Database, range: AdminStatsDateRange): Promise { + const opening = await createCloudTrafficReportRepo(db).getLedgerOpening() + return opening !== null && range.from >= trafficLedgerExactFrom(opening) +} + function delta(value: number | null, previousValue: number | null, canCompare = true): AdminStatsDelta { if (value === null || previousValue === null || !canCompare) { return { value, previousValue: canCompare ? previousValue : null, change: null, changePercent: null } @@ -955,22 +1037,31 @@ async function getRegistrationSources( ): Promise> { const counts = new Map() const rows = await reader.rows(ADMIN_STATS_METRICS.userSignup, ['provider']) + if (rows.some((row) => row.lowerBound)) return [] for (const row of rows) { if (row.dimensionKey === 'provider') incrementMap(counts, row.dimensionValue, row.count) } return percentRows([...counts.entries()].map(([name, value]) => ({ name, value }))) } -async function getSignupTotal(reader: AdminStatsHourlyReader): Promise { +async function getSignupTotal(reader: AdminStatsHourlyReader): Promise { const rows = await reader.rows(ADMIN_STATS_METRICS.userSignup) - return rows.filter((row) => row.dimensionKey === '').reduce((sum, row) => sum + row.count, 0) + const totals = rows.filter((row) => row.dimensionKey === '') + return totals.some((row) => row.lowerBound) ? null : totals.reduce((sum, row) => sum + row.count, 0) } -async function getSignupsByDay(reader: AdminStatsHourlyReader): Promise> { +async function getSignupsByDay(reader: AdminStatsHourlyReader): Promise> { const rows = await reader.rows(ADMIN_STATS_METRICS.userSignup) - const result = new Map() + const result = new Map() for (const row of rows) { - if (row.dimensionKey === '') incrementMap(result, reader.dayKey(row.bucketStart), row.count) + if (row.dimensionKey !== '') continue + const date = reader.dayKey(row.bucketStart) + if (row.lowerBound) { + result.set(date, null) + continue + } + const current = result.get(date) + if (current !== null) result.set(date, (current ?? 0) + row.count) } return result } @@ -1130,6 +1221,22 @@ async function getMissingTransferBytes(reader: AdminStatsHourlyReader): Promise< return result } +async function getMissingTransferBytesByDay( + reader: AdminStatsHourlyReader, +): Promise> { + const rows = await reader.rows(ADMIN_STATS_METRICS.statsMissingBytes, ['direction']) + const result = new Map() + for (const row of rows) { + if (row.dimensionKey !== 'direction') continue + const date = reader.dayKey(row.bucketStart) + const counts = result.get(date) ?? { upload: 0, download: 0 } + if (row.dimensionValue === 'upload') counts.upload += row.count + if (row.dimensionValue === 'download') counts.download += row.count + result.set(date, counts) + } + return result +} + async function getStorageUsedByDay(reader: AdminStatsHourlyReader): Promise> { return getLatestGaugeValueByDay(reader, ADMIN_STATS_METRICS.storageLedgerBalance, 'bytes', '', '') } diff --git a/server/adapters/repos/cloud-traffic-report.ts b/server/adapters/repos/cloud-traffic-report.ts index 2c1a329a..52dd45df 100644 --- a/server/adapters/repos/cloud-traffic-report.ts +++ b/server/adapters/repos/cloud-traffic-report.ts @@ -1,6 +1,7 @@ import { and, asc, eq, isNull, lte, or, sql } from 'drizzle-orm' import { nanoid } from 'nanoid' import { cloudTrafficReports } from '../../db/schema' +import { currentTrafficPeriod } from '../../domain/quota' import type { Database } from '../../platform/interface' import type { CloudTrafficReportRecord, @@ -9,6 +10,13 @@ import type { InsertCloudTrafficReportInput, } from '../../usecases/ports' +export const TRAFFIC_LEDGER_OPENING_EVENT_ID = 'traffic_ledger_opening_v1' +const HOUR_MS = 3_600_000 + +export function trafficLedgerExactFrom(opening: Date): Date { + return new Date(Math.ceil(opening.getTime() / HOUR_MS) * HOUR_MS) +} + function toRecord(row: typeof cloudTrafficReports.$inferSelect): CloudTrafficReportRecord { return { id: row.id, @@ -25,6 +33,7 @@ function toRecord(row: typeof cloudTrafficReports.$inferSelect): CloudTrafficRep error: row.error, attemptCount: row.attemptCount, nextRetryAt: row.nextRetryAt, + issuedAt: row.issuedAt, createdAt: row.createdAt, updatedAt: row.updatedAt, } @@ -32,6 +41,40 @@ function toRecord(row: typeof cloudTrafficReports.$inferSelect): CloudTrafficRep export function createCloudTrafficReportRepo(db: Database): CloudTrafficReportRepo { return { + async ensureLedgerOpening(now) { + await db + .insert(cloudTrafficReports) + .values({ + id: TRAFFIC_LEDGER_OPENING_EVENT_ID, + orgId: '', + period: currentTrafficPeriod(now), + source: 'object_download', + sourceId: TRAFFIC_LEDGER_OPENING_EVENT_ID, + eventId: TRAFFIC_LEDGER_OPENING_EVENT_ID, + bytes: 0, + storageId: null, + unitBytes: null, + creditsPerUnit: null, + status: 'ledger_opening', + error: null, + attemptCount: 0, + nextRetryAt: null, + issuedAt: null, + createdAt: now, + updatedAt: now, + }) + .onConflictDoNothing({ target: cloudTrafficReports.eventId }) + }, + + async getLedgerOpening() { + const rows = await db + .select({ createdAt: cloudTrafficReports.createdAt }) + .from(cloudTrafficReports) + .where(eq(cloudTrafficReports.eventId, TRAFFIC_LEDGER_OPENING_EVENT_ID)) + .limit(1) + return rows[0]?.createdAt ?? null + }, + async findByEventId(eventId) { const rows = await db.select().from(cloudTrafficReports).where(eq(cloudTrafficReports.eventId, eventId)).limit(1) return rows[0] ? toRecord(rows[0]) : undefined @@ -53,11 +96,30 @@ export function createCloudTrafficReportRepo(db: Database): CloudTrafficReportRe error: null, attemptCount: 0, nextRetryAt: null, + issuedAt: null, createdAt: input.now, updatedAt: input.now, }) }, + async markIssued(eventId, now) { + const rows = await db + .update(cloudTrafficReports) + .set({ issuedAt: now, updatedAt: now }) + .where(eq(cloudTrafficReports.eventId, eventId)) + .returning({ eventId: cloudTrafficReports.eventId }) + if (rows.length !== 1) throw new Error(`traffic_report_not_found:${eventId}`) + }, + + async reverse(eventId, now) { + const rows = await db + .update(cloudTrafficReports) + .set({ status: 'reversed', error: null, nextRetryAt: null, issuedAt: null, updatedAt: now }) + .where(eq(cloudTrafficReports.eventId, eventId)) + .returning({ eventId: cloudTrafficReports.eventId }) + if (rows.length !== 1) throw new Error(`traffic_report_not_found:${eventId}`) + }, + async updateStatus(eventId, status, error, now, retry) { await db .update(cloudTrafficReports) diff --git a/server/db/schema.ts b/server/db/schema.ts index 95b3ec1d..0dc60b35 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -112,6 +112,7 @@ export const cloudTrafficReports = sqliteTable( error: text('error'), attemptCount: integer('attempt_count').notNull().default(0), nextRetryAt: integer('next_retry_at', { mode: 'timestamp_ms' }), + issuedAt: integer('issued_at', { mode: 'timestamp_ms' }), createdAt: integer('created_at', { mode: 'timestamp_ms' }).notNull(), updatedAt: integer('updated_at', { mode: 'timestamp_ms' }).notNull(), }, @@ -120,6 +121,7 @@ export const cloudTrafficReports = sqliteTable( index('cloud_traffic_reports_org_period_idx').on(t.orgId, t.period), index('cloud_traffic_reports_status_idx').on(t.status), index('cloud_traffic_reports_retry_idx').on(t.status, t.nextRetryAt, t.createdAt), + index('cloud_traffic_reports_issued_idx').on(t.issuedAt), index('cloud_traffic_reports_updated_idx').on(t.updatedAt), ], ) diff --git a/server/http/admin-stats.integration.test.ts b/server/http/admin-stats.integration.test.ts index 70a52b33..dc095af4 100644 --- a/server/http/admin-stats.integration.test.ts +++ b/server/http/admin-stats.integration.test.ts @@ -3,6 +3,7 @@ import { sql } from 'drizzle-orm' import { describe, expect, it } from 'vitest' import { createAdminStatsRepo, metricSpec } from '../adapters/repos/admin-stats' import { captureAdminStatsSnapshot, rebuildAdminStatsHour } from '../adapters/repos/admin-stats-rollup' +import { createCloudTrafficReportRepo } from '../adapters/repos/cloud-traffic-report' import { currentTrafficPeriod } from '../domain/quota' import { adminHeaders, createTestApp, seedProLicense } from '../test/setup.js' @@ -189,6 +190,7 @@ describe('site stats routes', () => { await seedProLicense(db) const [{ id: orgId }] = await db.all<{ id: string }>(sql`SELECT id FROM organization LIMIT 1`) const at = Date.UTC(2026, 6, 1, 10) + await createCloudTrafficReportRepo(db).ensureLedgerOpening(new Date(at - 3_600_000)) await db.run(sql` INSERT INTO activity_events ( id, org_id, user_id, actor_type, action, target_type, target_id, target_name, metadata, created_at @@ -235,6 +237,7 @@ describe('site stats routes', () => { await seedProLicense(db) const [{ id: orgId }] = await db.all<{ id: string }>(sql`SELECT id FROM organization LIMIT 1`) const at = Date.UTC(2026, 6, 1, 10) + await createCloudTrafficReportRepo(db).ensureLedgerOpening(new Date(at - 3_600_000)) await db.run(sql` INSERT INTO stats_rollups_hourly (id, bucket_start, org_id, metric_key, dimension_key, dimension_value, @@ -268,6 +271,7 @@ describe('site stats routes', () => { typeBreakdown: Array<{ name: string; value: number }> } const traffic = (await trafficRes.json()) as { + summary: { requestCount: { value: number } } sourceBreakdown: Array<{ name: string; requests: number; bytes: number }> failureReasons: Array<{ name: string; value: number }> } @@ -279,9 +283,8 @@ describe('site stats routes', () => { expect(operations.backgroundJobOutcomes).toContainEqual(expect.objectContaining({ name: 'failed', value: 1 })) expect(sharing.summary.createdShares.value).toBe(1) expect(sharing.typeBreakdown).toContainEqual(expect.objectContaining({ name: 'landing', value: 1 })) - expect(traffic.sourceBreakdown).toContainEqual( - expect.objectContaining({ name: 'object_download', requests: 1, bytes: 10 }), - ) + expect(traffic.summary.requestCount.value).toBe(2) + expect(traffic.sourceBreakdown).toEqual([]) expect(traffic.failureReasons).toContainEqual(expect.objectContaining({ name: 'network', value: 1 })) }) @@ -478,6 +481,25 @@ describe('site stats routes', () => { ${bucketStart.getTime()} ) `) + await db.run(sql` + INSERT INTO matters + (id, org_id, alias, name, type, size, dirtype, parent, object, storage_id, status, created_at, updated_at) + VALUES ( + ${`quota-pressure-file-${index}`}, + ${`quota-pressure-org-${index}`}, + ${`quota-pressure-file-${index}`}, + ${`quota-pressure-file-${index}`}, + 'application/octet-stream', + ${index < 9 ? 90 : 110}, + 0, + '', + ${`quota-pressure-file-${index}`}, + 'stats-storage', + 'active', + ${Math.floor(bucketStart.getTime() / 1000)}, + ${Math.floor(bucketStart.getTime() / 1000)} + ) + `) } await db.run(sql` INSERT INTO organization (id, name, slug, metadata, created_at, updated_at) @@ -487,6 +509,25 @@ describe('site stats routes', () => { INSERT INTO org_quotas (id, org_id, quota, used, traffic_quota, traffic_used, traffic_period) VALUES ('quota-invalid', 'quota-invalid-org', 0, 1000, 0, 0, '2026-07') `) + await db.run(sql` + INSERT INTO matters + (id, org_id, alias, name, type, size, dirtype, parent, object, storage_id, status, created_at, updated_at) + VALUES ( + 'quota-invalid-file', + 'quota-invalid-org', + 'quota-invalid-file', + 'quota-invalid-file', + 'application/octet-stream', + 1000, + 0, + '', + 'quota-invalid-file', + 'stats-storage', + 'active', + ${Math.floor(bucketStart.getTime() / 1000)}, + ${Math.floor(bucketStart.getTime() / 1000)} + ) + `) await captureAdminStatsSnapshot(db, bucketStart, new Date(bucketStart.getTime() + 45 * 60_000)) await rebuildAdminStatsHour(db, bucketStart, new Date()) @@ -512,6 +553,34 @@ describe('site stats routes', () => { expect(body.topSpaces[0]).toMatchObject({ orgId: 'quota-invalid-org', utilization: null }) }) + it('omits quota-derived storage values when the usage counter has drifted', async () => { + const { app, db } = await createTestApp() + const headers = await adminHeaders(app) + await seedProLicense(db) + const { orgId, bucketStart } = await seedStatsFixture(db) + await db.run(sql`UPDATE org_quotas SET used = used + 1 WHERE org_id = ${orgId}`) + await captureAdminStatsSnapshot(db, bucketStart, new Date(bucketStart.getTime() + 45 * 60_000)) + await rebuildAdminStatsHour(db, bucketStart, new Date()) + + const res = await app.request('/api/site/stats/storage', { headers }) + const body = (await res.json()) as { + summary: { + storageUsedBytes: number | null + storageUtilization: number | null + nearQuotaSpaces: number | null + } + topSpaces: unknown[] + typeBreakdown: unknown[] + } + + expect(res.status).toBe(200) + expect(body.summary.storageUsedBytes).toBeNull() + expect(body.summary.storageUtilization).toBeNull() + expect(body.summary.nearQuotaSpaces).toBeNull() + expect(body.topSpaces).toEqual([]) + expect(body.typeBreakdown.length).toBeGreaterThan(0) + }) + it('reads historical active users from completed snapshots instead of raw activity', async () => { const { app, db } = await createTestApp() const headers = await adminHeaders(app) @@ -572,9 +641,10 @@ describe('site stats routes', () => { expect(emptyBody.successTrend).toEqual([{ date: '2000-01-01', uploadSuccessRate: null, downloadSuccessRate: null }]) }) - it('reports incomplete transfer byte metadata for the selected and comparison ranges', async () => { + it('omits incomplete transfer bytes while preserving exact request counts', async () => { const { app, db } = await createTestApp() const headers = await adminHeaders(app) + await seedProLicense(db) const { orgId, userId } = await seedStatsFixture(db) await db.run(sql` INSERT INTO activity_events @@ -588,18 +658,43 @@ describe('site stats routes', () => { await rebuildAdminStatsHour(db, new Date('2026-07-01T12:00:00.000Z'), new Date()) await rebuildAdminStatsHour(db, new Date('2026-06-30T12:00:00.000Z'), new Date()) - const res = await app.request('/api/site/stats/overview?from=2026-07-01&to=2026-07-01', { headers }) - const body = (await res.json()) as { dataQuality: AdminDashboardOverviewStats['dataQuality'] } + const [res, trafficRes, storageRes] = await Promise.all([ + app.request('/api/site/stats/overview?from=2026-07-01&to=2026-07-01', { headers }), + app.request('/api/site/stats/traffic?from=2026-07-01&to=2026-07-01', { headers }), + app.request('/api/site/stats/storage?from=2026-07-01&to=2026-07-01', { headers }), + ]) + const body = (await res.json()) as { + dataQuality: AdminDashboardOverviewStats['dataQuality'] + totals: { trafficBytes: { value: number | null }; uploadBytes: { value: number | null } } + trends: Array<{ uploadBytes: number | null }> + } + const traffic = (await trafficRes.json()) as { + summary: { totalBytes: { value: number | null }; requestCount: { value: number } } + sourceBreakdown: unknown[] + } + const storage = (await storageRes.json()) as { + summary: { newBytes: { value: number | null }; newFiles: { value: number } } + storageTrend: Array<{ newBytes: number | null; newFiles: number }> + } - expect(res.status).toBe(200) + expect([res.status, trafficRes.status, storageRes.status]).toEqual([200, 200, 200]) expect(body.dataQuality).toEqual({ missingBytesEvents: 1, - previousMissingBytesEvents: 1, + previousMissingBytesEvents: 0, missingUploadBytesEvents: 1, previousMissingUploadBytesEvents: 0, missingDownloadBytesEvents: 0, - previousMissingDownloadBytesEvents: 1, + previousMissingDownloadBytesEvents: 0, }) + expect(body.totals.trafficBytes.value).toBeNull() + expect(body.totals.uploadBytes.value).toBeNull() + expect(body.trends).toContainEqual(expect.objectContaining({ uploadBytes: null })) + expect(traffic.summary.totalBytes.value).toBeNull() + expect(traffic.summary.requestCount.value).toBe(1) + expect(traffic.sourceBreakdown).toEqual([]) + expect(storage.summary.newBytes.value).toBeNull() + expect(storage.summary.newFiles.value).toBe(1) + expect(storage.storageTrend).toContainEqual(expect.objectContaining({ newBytes: null, newFiles: 1 })) }) it('excludes expired and exhausted shares from the active-share total', async () => { @@ -622,7 +717,7 @@ describe('site stats routes', () => { expect(body.summary.activeShares).toBe(1) }) - it('returns traffic dashboard stats from audit-backed download events for Pro admins', async () => { + it('returns traffic dashboard stats from the confirmed traffic ledger for Pro admins', async () => { const { app, db } = await createTestApp() const headers = await adminHeaders(app) await seedProLicense(db) @@ -714,7 +809,7 @@ describe('site stats routes', () => { expect(mutableCounters).toBe(0) }) - it('applies dashboard ranges to sharing drill-down data', async () => { + it('omits sharing metrics when historical events cannot be located', async () => { const { app, db } = await createTestApp() const headers = await adminHeaders(app) await seedProLicense(db) @@ -724,8 +819,8 @@ describe('site stats routes', () => { const currentSharing = (await currentSharingRes.json()) as { dataQuality: { unlocatedViews: number; unlocatedDownloads: number; unlocatedEvents: number } summary: { - views: { value: number; change: number | null } - downloads: { value: number; change: number | null } + views: { value: number | null; change: number | null } + downloads: { value: number | null; change: number | null } downloadsPer100Views: number | null } topShares: Array<{ @@ -738,25 +833,19 @@ describe('site stats routes', () => { } const oldSharingRes = await app.request('/api/site/stats/sharing?from=2000-01-01&to=2000-01-02', { headers }) const oldSharing = (await oldSharingRes.json()) as { - summary: { views: { value: number }; downloads: { value: number } } + summary: { views: { value: number | null }; downloads: { value: number | null } } topShares: unknown[] } expect(currentSharingRes.status).toBe(200) - expect(currentSharing.summary.views.value).toBe(1) - expect(currentSharing.summary.downloads.value).toBe(1) + expect(currentSharing.summary.views.value).toBeNull() + expect(currentSharing.summary.downloads.value).toBeNull() expect(currentSharing.dataQuality).toEqual({ unlocatedViews: 11, unlocatedDownloads: 3, unlocatedEvents: 14 }) expect(currentSharing.summary.views.change).toBeNull() expect(currentSharing.summary.downloads.change).toBeNull() expect(currentSharing.summary.downloadsPer100Views).toBeNull() - expect(currentSharing.topShares[0]).toMatchObject({ - token: 'share-token-1', - views: 1, - downloads: 1, - viewPercent: 100, - downloadPercent: 100, - }) - expect(oldSharing.summary.views.value).toBe(0) - expect(oldSharing.summary.downloads.value).toBe(0) + expect(currentSharing.topShares).toEqual([]) + expect(oldSharing.summary.views.value).toBeNull() + expect(oldSharing.summary.downloads.value).toBeNull() expect(oldSharing.topShares).toEqual([]) }) @@ -764,7 +853,8 @@ describe('site stats routes', () => { const { app, db } = await createTestApp() const headers = await adminHeaders(app) await seedProLicense(db) - await seedStatsFixture(db) + const { bucketStart } = await seedStatsFixture(db) + await makeSharingHistoryExact(db, bucketStart) await db.run(sql`DELETE FROM shares WHERE id = 'share-1'`) const res = await app.request('/api/site/stats/sharing', { headers }) @@ -801,7 +891,7 @@ describe('site stats routes', () => { ('activity-download-heavy-2', ${orgId}, NULL, 'anonymous', 'share_download', 'share', 'share-download-heavy', 'report.pdf', '{"bytes":512,"source":"landing_share","anonymous":true}', ${eventSec}), ('activity-download-heavy-3', ${orgId}, NULL, 'anonymous', 'share_download', 'share', 'share-download-heavy', 'report.pdf', '{"bytes":512,"source":"landing_share","anonymous":true}', ${eventSec}) `) - await rebuildAdminStatsHour(db, bucketStart, new Date()) + await makeSharingHistoryExact(db, bucketStart) await db.run(sql` INSERT INTO stats_rollups_hourly (id, bucket_start, org_id, metric_key, dimension_key, dimension_value, @@ -836,7 +926,7 @@ describe('site stats routes', () => { '{"source":"landing_share"}', ${eventSec}) `) } - await rebuildAdminStatsHour(db, bucketStart, new Date()) + await makeSharingHistoryExact(db, bucketStart) const res = await app.request('/api/site/stats/sharing', { headers }) const body = (await res.json()) as { topShares: Array<{ viewPercent: number }> } @@ -847,6 +937,15 @@ describe('site stats routes', () => { }) }) +async function makeSharingHistoryExact( + db: Awaited>['db'], + bucketStart: Date, +): Promise { + await db.run(sql`UPDATE shares SET views = 1, downloads = 1 WHERE id = 'share-1'`) + await captureAdminStatsSnapshot(db, bucketStart, new Date(bucketStart.getTime() + 45 * 60_000)) + await rebuildAdminStatsHour(db, bucketStart, new Date()) +} + async function seedStatsFixture(db: Awaited>['db']) { const generatedAt = Date.now() const bucketStart = new Date(Math.floor(generatedAt / 3_600_000) * 3_600_000 - 3_600_000) @@ -859,6 +958,7 @@ async function seedStatsFixture(db: Awaited>['d sql`SELECT id FROM organization WHERE metadata LIKE '%"type":"personal"%' LIMIT 1`, ) const [{ id: userId }] = await db.all<{ id: string }>(sql`SELECT id FROM user LIMIT 1`) + await createCloudTrafficReportRepo(db).ensureLedgerOpening(new Date(generatedAt - 60 * 24 * 60 * 60 * 1000)) await db.run(sql`UPDATE user SET created_at = ${now}, updated_at = ${now} WHERE id = ${userId}`) await db.run(sql`UPDATE account SET created_at = ${now}, updated_at = ${now} WHERE user_id = ${userId}`) @@ -902,6 +1002,15 @@ async function seedStatsFixture(db: Awaited>['d ('stats-fixture-task-failed', ${orgId}, NULL, 'system', 'stats_remote_download_finished', 'remote_download', 'task-2', 'task-2', '{"category":"direct","outcome":"failed","bytes":0,"statsQuality":"exact"}', ${nowSec}), ('stats-fixture-job-failed', ${orgId}, NULL, 'system', 'stats_background_job_finished', 'background_job', 'job-1', 'job-1', '{"jobType":"extract","outcome":"failed","statsQuality":"exact"}', ${nowSec}) `) + await db.run(sql` + INSERT INTO cloud_traffic_reports ( + id, org_id, period, source, source_id, event_id, bytes, status, issued_at, created_at, updated_at + ) VALUES + ('stats-traffic-share', ${orgId}, ${period}, 'landing_share', 'share-1', 'stats-traffic-share', 512, + 'not_required', ${now}, ${now}, ${now}), + ('stats-traffic-object', ${orgId}, ${period}, 'object_download', 'stats-file', 'stats-traffic-object', 256, + 'not_required', ${now}, ${now}, ${now}) + `) await db.run(sql` UPDATE activity_events SET created_at = ${nowSec}, metadata = '{"provider":"credential","statsQuality":"exact"}' diff --git a/server/http/downloads/download-tasks.integration.test.ts b/server/http/downloads/download-tasks.integration.test.ts index 8e7bb7cb..ebfe6e82 100644 --- a/server/http/downloads/download-tasks.integration.test.ts +++ b/server/http/downloads/download-tasks.integration.test.ts @@ -1320,6 +1320,16 @@ describe('Download tasks API integration', () => { phase: 'metadata', trackerCount: 1, }) + const actors = await db.all<{ action: string; actorType: string; actorRef: string | null }>(sql` + SELECT action, actor_type AS actorType, actor_ref AS actorRef + FROM activity_events + WHERE target_type = 'download_task' AND target_id = ${task.id} + `) + expect(actors.find((event) => event.action === 'download_task_created')).toMatchObject({ actorType: 'user' }) + const assignedActor = actors.find((event) => event.action === 'download_task_assigned') + const lifecycleActor = actors.find((event) => event.action === 'download_resolve_started') + expect(assignedActor).toMatchObject({ actorType: 'downloader', actorRef: expect.any(String) }) + expect(lifecycleActor).toMatchObject({ actorType: 'downloader', actorRef: assignedActor?.actorRef }) }) it('returns storage failure details when multipart upload session creation fails [spec: download-tasks/upload-session-failure]', async () => { diff --git a/server/http/objects.integration.test.ts b/server/http/objects.integration.test.ts index 7d21f31f..b6b3e1e2 100644 --- a/server/http/objects.integration.test.ts +++ b/server/http/objects.integration.test.ts @@ -916,15 +916,9 @@ describe('Objects API', () => { const body = (await res.json()) as Record expect(body.downloadUrl).toBe('https://presigned-download.example.com') expect(fetch).toHaveBeenCalledTimes(1) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ - { - orgId, - source: 'object_download', - sourceId: 'm1', - bytes: 100, - status: 'reported', - }, - ]) + await expect( + db.select().from(cloudTrafficReports).where(eq(cloudTrafficReports.orgId, orgId)), + ).resolves.toMatchObject([{ orgId, source: 'object_download', sourceId: 'm1', bytes: 100, status: 'reported' }]) }) }) diff --git a/server/http/store/traffic-metering.integration.test.ts b/server/http/store/traffic-metering.integration.test.ts index 84d15a92..fbd0be96 100644 --- a/server/http/store/traffic-metering.integration.test.ts +++ b/server/http/store/traffic-metering.integration.test.ts @@ -1,6 +1,7 @@ -import { sql } from 'drizzle-orm' +import { ne, sql } from 'drizzle-orm' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { S3Service } from '../../adapters/gateways/s3' +import { TRAFFIC_LEDGER_OPENING_EVENT_ID } from '../../adapters/repos/cloud-traffic-report' import { createShareRepo } from '../../adapters/repos/share' import { cloudTrafficReports } from '../../db/schema' import { currentTrafficPeriod } from '../../domain/quota' @@ -65,6 +66,10 @@ async function getUserId(db: Database): Promise { return rows[0].id } +function trafficReports(db: Database) { + return db.select().from(cloudTrafficReports).where(ne(cloudTrafficReports.eventId, TRAFFIC_LEDGER_OPENING_EVENT_ID)) +} + async function insertFile(db: Database, orgId: string, id: string) { const now = Date.now() await db.run(sql` @@ -113,7 +118,7 @@ describe('object download cloud traffic reporting', () => { expect(res.status).toBe(200) expect(fetch).toHaveBeenCalledTimes(1) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + await expect(trafficReports(db)).resolves.toMatchObject([ { orgId, source: 'object_download', sourceId: 'm-cloud-report-ok', bytes: 100, status: 'reported' }, ]) }) @@ -148,9 +153,7 @@ describe('object download cloud traffic reporting', () => { sql`SELECT traffic_used AS trafficUsed FROM org_quotas WHERE org_id = ${orgId}`, ) expect(rows[0].trafficUsed).toBe(25) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ - { status: 'blocked', error: 'insufficient_credits' }, - ]) + await expect(trafficReports(db)).resolves.toMatchObject([{ status: 'blocked', error: 'insufficient_credits' }]) }) it('records a reported report without refunding local traffic when Cloud returns its own usage event id', async () => { @@ -174,7 +177,7 @@ describe('object download cloud traffic reporting', () => { sql`SELECT traffic_used AS trafficUsed FROM org_quotas WHERE org_id = ${orgId}`, ) expect(rows[0].trafficUsed).toBe(125) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'reported', error: null }]) + await expect(trafficReports(db)).resolves.toMatchObject([{ status: 'reported', error: null }]) }) it('keeps the pre-presign Cloud report and refunds local traffic when presign fails', async () => { @@ -196,7 +199,7 @@ describe('object download cloud traffic reporting', () => { sql`SELECT traffic_used AS trafficUsed FROM org_quotas WHERE org_id = ${orgId}`, ) expect(rows[0].trafficUsed).toBe(25) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'reported' }]) + await expect(trafficReports(db)).resolves.toMatchObject([{ status: 'reversed', issuedAt: null }]) }) }) @@ -220,7 +223,7 @@ describe('public redirect cloud traffic reporting', () => { const res = await app.request(`/r/${share.token}`, { redirect: 'manual' }) expect(res.status).toBe(302) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + await expect(trafficReports(db)).resolves.toMatchObject([ { source: 'direct_share', sourceId: share.id, bytes: 100, status: 'reported' }, ]) }) @@ -265,9 +268,7 @@ describe('public redirect cloud traffic reporting', () => { WHERE s.id = ${share.id} `) expect(rows[0]).toEqual({ downloads: 0, trafficUsed: 25 }) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ - { status: 'blocked', error: 'insufficient_credits' }, - ]) + await expect(trafficReports(db)).resolves.toMatchObject([{ status: 'blocked', error: 'insufficient_credits' }]) }) it('reports landing share downloads to Cloud', async () => { @@ -290,7 +291,7 @@ describe('public redirect cloud traffic reporting', () => { const res = await app.request(`/api/shares/${share.token}/objects/${ref}?downloadUrl=1`, { redirect: 'manual' }) expect(res.status).toBe(200) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + await expect(trafficReports(db)).resolves.toMatchObject([ { source: 'landing_share', sourceId: share.id, bytes: 100, status: 'reported' }, ]) }) @@ -336,9 +337,7 @@ describe('public redirect cloud traffic reporting', () => { WHERE s.id = ${share.id} `) expect(rows[0]).toEqual({ downloads: 0, trafficUsed: 25 }) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ - { status: 'blocked', error: 'insufficient_credits' }, - ]) + await expect(trafficReports(db)).resolves.toMatchObject([{ status: 'blocked', error: 'insufficient_credits' }]) }) it('fails and refunds landing share traffic when audit recording fails', async () => { @@ -386,7 +385,7 @@ describe('public redirect cloud traffic reporting', () => { const res = await app.request('/r/ih_cloudtoken', { redirect: 'manual' }) expect(res.status).toBe(302) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + await expect(trafficReports(db)).resolves.toMatchObject([ { source: 'image_hosting', sourceId: 'ih-cloud-token', bytes: 100, status: 'reported' }, ]) }) @@ -424,7 +423,7 @@ describe('public redirect cloud traffic reporting', () => { expect(res.status).toBe(500) expect(fetch).not.toHaveBeenCalled() - await expect(db.select().from(cloudTrafficReports)).resolves.toHaveLength(0) + await expect(trafficReports(db)).resolves.toHaveLength(0) }) it('reports custom-domain image-hosting redirects to Cloud', async () => { @@ -443,7 +442,7 @@ describe('public redirect cloud traffic reporting', () => { }) expect(res.status).toBe(302) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + await expect(trafficReports(db)).resolves.toMatchObject([ { source: 'custom_domain_image', sourceId: 'ih-cloud-domain', bytes: 100, status: 'reported' }, ]) }) diff --git a/server/http/store/traffic-metering.ts b/server/http/store/traffic-metering.ts index 35d3983b..04205dad 100644 --- a/server/http/store/traffic-metering.ts +++ b/server/http/store/traffic-metering.ts @@ -64,6 +64,7 @@ export async function reportTrafficForDownload( storage: DownloadTrafficStorage source: TrafficReportSource sourceId: string + eventId?: string onRejected?: () => Promise }, ): Promise { @@ -74,6 +75,7 @@ export async function reportTrafficForDownload( storage: params.storage, source: params.source, sourceId: params.sourceId, + eventId: params.eventId, onRejected: params.onRejected, }) // reportDownloadEgress never consumes quota, so it cannot return quota_exceeded. diff --git a/server/http/webdav.integration.test.ts b/server/http/webdav.integration.test.ts index db02d744..b36f36e5 100644 --- a/server/http/webdav.integration.test.ts +++ b/server/http/webdav.integration.test.ts @@ -682,7 +682,9 @@ describe('WebDAV API', () => { FROM cloud_traffic_reports WHERE org_id = ${workspace.id} `) - expect(reports).toEqual([]) + expect(reports).toEqual([ + { source: 'webdav_download', sourceId: 'traffic-report', bytes: 12, status: 'not_required' }, + ]) }) it('GET supports valid byte ranges and rejects invalid ranges [spec: webdav/get-range]', async () => { diff --git a/server/http/webdav.ts b/server/http/webdav.ts index ebfc301f..6e9efc33 100644 --- a/server/http/webdav.ts +++ b/server/http/webdav.ts @@ -766,7 +766,11 @@ async function readFile(c: DavContext, auth: DavAuth): Promise { }) return new Response(fixedLengthResponseBody(body, size), { headers }) } catch (e) { - await refundWebDavTraffic(c.get('deps'), { orgId: workspace.id, bytes: size }) + await refundWebDavTraffic(c.get('deps'), { + orgId: workspace.id, + bytes: size, + trafficEventId: reservation.trafficEventId, + }) throw e } } @@ -794,7 +798,11 @@ async function readFile(c: DavContext, auth: DavAuth): Promise { trafficEventId: reservation.trafficEventId, }) } catch (error) { - await refundWebDavTraffic(c.get('deps'), { orgId: workspace.id, bytes: trafficBytes }) + await refundWebDavTraffic(c.get('deps'), { + orgId: workspace.id, + bytes: trafficBytes, + trafficEventId: reservation.trafficEventId, + }) throw error } return new Response(fixedLengthResponseBody(body, contentLength), { status: 206, headers }) @@ -810,7 +818,11 @@ async function readFile(c: DavContext, auth: DavAuth): Promise { range: `bytes=${range.start}-${range.end}`, }) } catch (e) { - await refundWebDavTraffic(c.get('deps'), { orgId: workspace.id, bytes: contentLength }) + await refundWebDavTraffic(c.get('deps'), { + orgId: workspace.id, + bytes: contentLength, + trafficEventId: reservation.trafficEventId, + }) throw e } headers.set('Content-Length', String(contentLength)) @@ -826,7 +838,11 @@ async function readFile(c: DavContext, auth: DavAuth): Promise { trafficEventId: reservation.trafficEventId, }) } catch (error) { - await refundWebDavTraffic(c.get('deps'), { orgId: workspace.id, bytes: contentLength }) + await refundWebDavTraffic(c.get('deps'), { + orgId: workspace.id, + bytes: contentLength, + trafficEventId: reservation.trafficEventId, + }) throw error } return new Response(fixedLengthResponseBody(body, contentLength), { status: 206, headers }) diff --git a/server/middleware/image-hosting-domain.ts b/server/middleware/image-hosting-domain.ts index cb69d2e9..fc02a495 100644 --- a/server/middleware/image-hosting-domain.ts +++ b/server/middleware/image-hosting-domain.ts @@ -3,6 +3,8 @@ import { PRESIGN_TTL_SECS } from '../http/share-utils' import { reportTrafficForDownload } from '../http/store/traffic-metering' import type { Env } from '../middleware/platform' import { forbidden, notFound, quotaExceeded, storageNotFound } from '../usecases/ports' +import { confirmDownloadTraffic, reverseDownloadTraffic } from '../usecases/store/traffic-metering' +import { createTrafficEventId } from '../usecases/transfer-activity' function stripPort(host: string): string { const lastColon = host.lastIndexOf(':') @@ -65,14 +67,26 @@ async function handleImageByPath(c: Context, orgId: string, virtualPath: st throw e } + const trafficEventId = createTrafficEventId() const trafficReportError = await reportTrafficForDownload(c, { orgId: image.orgId, bytes: image.size, storage, source: 'custom_domain_image', sourceId: image.id, + eventId: trafficEventId, }) if (trafficReportError) return trafficReportError + try { + await confirmDownloadTraffic(c.get('deps'), { eventId: trafficEventId }) + } catch (error) { + await reverseDownloadTraffic(c.get('deps'), { + orgId: image.orgId, + bytes: image.size, + eventId: trafficEventId, + }) + throw error + } try { await c.get('deps').imageHosting.incrementAccessCount(image.id) diff --git a/server/scripts/backfill-admin-stats.test.ts b/server/scripts/backfill-admin-stats.test.ts index 81b933de..ad0aa12d 100644 --- a/server/scripts/backfill-admin-stats.test.ts +++ b/server/scripts/backfill-admin-stats.test.ts @@ -57,9 +57,10 @@ describe('admin stats backfill', () => { metadata TEXT, created_at INTEGER NOT NULL ); CREATE TABLE cloud_traffic_reports ( - id TEXT PRIMARY KEY, org_id TEXT NOT NULL, source TEXT NOT NULL, source_id TEXT NOT NULL, + id TEXT PRIMARY KEY, org_id TEXT NOT NULL, period TEXT NOT NULL, source TEXT NOT NULL, source_id TEXT NOT NULL, event_id TEXT NOT NULL UNIQUE, bytes INTEGER NOT NULL, storage_id TEXT, unit_bytes INTEGER, - credits_per_unit INTEGER, status TEXT NOT NULL, error TEXT, + credits_per_unit INTEGER, status TEXT NOT NULL, error TEXT, attempt_count INTEGER NOT NULL DEFAULT 0, + next_retry_at INTEGER, issued_at INTEGER, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL ); CREATE TABLE download_tasks ( @@ -114,10 +115,13 @@ describe('admin stats backfill', () => { ('task-completed', 'o1', 'u1', 'user', NULL, 'download_task_completed', 'remote_download', 't1', 'task', NULL, ${eventSec + 2}), ('blocked-download', 'o1', 'u1', 'user', NULL, 'download_failed', 'file', 'f1', 'file.bin', '{"bytes":512,"source":"object_download","reason":"quota_exceeded","trafficEventId":"traffic-3"}', ${eventSec}); - INSERT INTO cloud_traffic_reports VALUES - ('r1', 'o1', 'direct_share', 's1', 'traffic-1', 512, NULL, NULL, NULL, 'reported', NULL, ${eventMs}, ${eventMs}), - ('r2', 'o1', 'image_hosting', 'img1', 'traffic-2', 128, NULL, NULL, NULL, 'reported', NULL, ${eventMs}, ${eventMs}), - ('r3', 'o1', 'object_download', 'f1', 'traffic-3', 512, NULL, NULL, NULL, 'blocked', 'quota_exceeded', ${eventMs}, ${eventMs}); + INSERT INTO cloud_traffic_reports ( + id, org_id, period, source, source_id, event_id, bytes, storage_id, unit_bytes, credits_per_unit, + status, error, attempt_count, next_retry_at, issued_at, created_at, updated_at + ) VALUES + ('r1', 'o1', '2026-07', 'direct_share', 's1', 'traffic-1', 512, NULL, NULL, NULL, 'reported', NULL, 0, NULL, NULL, ${eventMs}, ${eventMs}), + ('r2', 'o1', '2026-07', 'image_hosting', 'img1', 'traffic-2', 128, NULL, NULL, NULL, 'reported', NULL, 0, NULL, NULL, ${eventMs}, ${eventMs}), + ('r3', 'o1', '2026-07', 'object_download', 'f1', 'traffic-3', 512, NULL, NULL, NULL, 'blocked', 'quota_exceeded', 0, NULL, NULL, ${eventMs}, ${eventMs}); INSERT INTO download_tasks VALUES ('t1', 'o1', 'video', 'url', 'd1', 'completed', 512, ${eventMs}, ${eventMs}), ('t2', 'o1', NULL, 'url', NULL, 'canceled', 0, ${eventMs}, ${eventMs}); diff --git a/server/test/setup.ts b/server/test/setup.ts index 00a4dd78..de2112be 100644 --- a/server/test/setup.ts +++ b/server/test/setup.ts @@ -206,6 +206,7 @@ const APP_SCHEMA_SQL = ` error TEXT, attempt_count INTEGER NOT NULL DEFAULT 0, next_retry_at INTEGER, + issued_at INTEGER, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL ); @@ -213,6 +214,7 @@ const APP_SCHEMA_SQL = ` CREATE INDEX IF NOT EXISTS cloud_traffic_reports_org_period_idx ON cloud_traffic_reports(org_id, period); CREATE INDEX IF NOT EXISTS cloud_traffic_reports_status_idx ON cloud_traffic_reports(status); CREATE INDEX IF NOT EXISTS cloud_traffic_reports_retry_idx ON cloud_traffic_reports(status, next_retry_at, created_at); + CREATE INDEX IF NOT EXISTS cloud_traffic_reports_issued_idx ON cloud_traffic_reports(issued_at); CREATE TABLE IF NOT EXISTS org_quota_entitlements ( id TEXT PRIMARY KEY, org_id TEXT NOT NULL, diff --git a/server/usecases/downloads/downloads.ts b/server/usecases/downloads/downloads.ts index f0245dcd..0f0d52f8 100644 --- a/server/usecases/downloads/downloads.ts +++ b/server/usecases/downloads/downloads.ts @@ -15,6 +15,7 @@ import { ZPAN_CLOUD_URL_DEFAULT } from '../../../shared/constants' import { hasFeature } from '../../domain/licensing' import type { Platform } from '../../platform/interface' import type { + ActivityActorType, ActivityEvent, ActivityRepo, DownloaderRepo, @@ -288,6 +289,7 @@ export async function createDownloadTask( sourceUri: input.source.uri, }, action: 'download_task_created', + actorType: 'user', metadata: { sourceType: input.source.type, targetFolder }, }) return deps.downloadTasks.get(orgId, id) @@ -468,6 +470,8 @@ export async function updateDownloadTask( runtime: nextRuntime, lifecycleFields, billingStatus, + actorType: actor.downloaderId ? 'downloader' : 'user', + actorRef: actor.downloaderId ?? null, }) return deps.downloadTasks.get(task.orgId, id) @@ -512,10 +516,10 @@ export async function performDownloadTaskAction( }), updatedAt: now, }) - await recordTaskActivity(deps, { task, action: 'download_task_deleted' }) + await recordTaskActivity(deps, { task, action: 'download_task_deleted', actorType: 'user' }) return { id, deleted: true } } - await recordTaskActivity(deps, { task, action: 'download_task_deleted' }) + await recordTaskActivity(deps, { task, action: 'download_task_deleted', actorType: 'user' }) await deps.downloadTasks.delete(id) return { id, deleted: true } } @@ -534,6 +538,7 @@ export async function performDownloadTaskAction( await recordTaskActivity(deps, { task, action: status === 'pausing' ? 'download_task_pause_requested' : 'download_task_paused', + actorType: 'user', }) return deps.downloadTasks.get(orgId, id) } @@ -551,7 +556,7 @@ export async function performDownloadTaskAction( runtime: clearTaskRuntimeMessageJson(task.runtime), updatedAt: now, }) - await recordTaskActivity(deps, { task, action: 'download_task_resume_requested' }) + await recordTaskActivity(deps, { task, action: 'download_task_resume_requested', actorType: 'user' }) return deps.downloadTasks.get(orgId, id) } @@ -574,6 +579,7 @@ export async function performDownloadTaskAction( await recordTaskActivity(deps, { task, action: status === 'canceling' ? 'download_task_cancel_requested' : 'download_task_canceled', + actorType: 'user', }) return deps.downloadTasks.get(orgId, id) } @@ -596,7 +602,7 @@ export async function performDownloadTaskAction( finishedAt: null, updatedAt: now, }) - await recordTaskActivity(deps, { task, action: 'download_task_retry_requested' }) + await recordTaskActivity(deps, { task, action: 'download_task_retry_requested', actorType: 'user' }) return deps.downloadTasks.get(orgId, id) } @@ -623,7 +629,7 @@ export async function performDownloadTaskAction( finishedAt: null, updatedAt: now, }) - await recordTaskActivity(deps, { task, action: 'download_task_restart_requested' }) + await recordTaskActivity(deps, { task, action: 'download_task_restart_requested', actorType: 'user' }) return deps.downloadTasks.get(orgId, id) } @@ -665,6 +671,8 @@ async function claimQueuedTasksForDownloader( await recordTaskActivity(deps, { task, action: 'download_task_assigned', + actorType: 'downloader', + actorRef: params.id, metadata: { downloaderId: params.id }, }) } @@ -902,6 +910,8 @@ async function recordUpdateActivity( runtime: DownloadTaskRuntime | null lifecycleFields: TaskLifecycleFields billingStatus: string + actorType: ActivityActorType + actorRef: string | null }, ): Promise { for (const field of LIFECYCLE_FIELDS) { @@ -909,6 +919,8 @@ async function recordUpdateActivity( await recordTaskActivity(deps, { task, action: lifecycleAction(field), + actorType: params.actorType, + actorRef: params.actorRef, metadata: runtimeMetadata(params.runtime), }) } @@ -917,6 +929,8 @@ async function recordUpdateActivity( await recordTaskActivity(deps, { task, action: statusAction(params.status), + actorType: params.actorType, + actorRef: params.actorRef, metadata: { from: params.previousStatus, to: params.status, @@ -929,6 +943,8 @@ async function recordUpdateActivity( await recordTaskActivity(deps, { task, action: 'download_task_error', + actorType: params.actorType, + actorRef: params.actorRef, metadata: { message: params.input.errorMessage }, }) } @@ -937,6 +953,8 @@ async function recordUpdateActivity( await recordTaskActivity(deps, { task, action: 'download_task_billing_suspended', + actorType: params.actorType, + actorRef: params.actorRef, metadata: { reason: 'insufficient_credits' }, }) } @@ -972,12 +990,16 @@ async function recordTaskActivity( input: { task: Pick action: string + actorType?: ActivityActorType + actorRef?: string | null metadata?: Record }, ): Promise { await deps.activity.record({ orgId: input.task.orgId, userId: input.task.createdByUserId, + actorType: input.actorType ?? 'system', + actorRef: input.actorRef ?? (input.actorType ? null : 'download-task-service'), action: input.action, targetType: DOWNLOAD_TASK_TARGET_TYPE, targetId: input.task.id, diff --git a/server/usecases/object.test.ts b/server/usecases/object.test.ts index ad19d5f6..62fb028c 100644 --- a/server/usecases/object.test.ts +++ b/server/usecases/object.test.ts @@ -46,7 +46,11 @@ import { meterDownloadTraffic } from './store/traffic-metering' // refund); object.getObject only orchestrates around it. Mock it and assert the // outcome mapping + presign rollback here; cloud-traffic-metering.test.ts owns // the metering internals. -vi.mock('./store/traffic-metering', () => ({ meterDownloadTraffic: vi.fn() })) +vi.mock('./store/traffic-metering', () => ({ + meterDownloadTraffic: vi.fn(), + confirmDownloadTraffic: vi.fn(async () => {}), + reverseDownloadTraffic: vi.fn(async (deps, params) => deps.quota.refundTraffic(params.orgId, params.bytes)), +})) const storage = { id: 'st-1', diff --git a/server/usecases/object.ts b/server/usecases/object.ts index 0676cac3..e2c2f012 100644 --- a/server/usecases/object.ts +++ b/server/usecases/object.ts @@ -46,7 +46,7 @@ import { storageNotFound, } from './ports' import { StorageQuotaExceededError, withStorageUsageReservation } from './storage-usage' -import { meterDownloadTraffic } from './store/traffic-metering' +import { confirmDownloadTraffic, meterDownloadTraffic, reverseDownloadTraffic } from './store/traffic-metering' import { createTrafficEventId, recordDownloadFailed, recordDownloadIssued } from './transfer-activity' export { ObjectUploadSessionError } from './ports' @@ -528,7 +528,7 @@ export async function getObject( try { downloadUrl = await deps.s3.presignDownload(storage, matter.object, matter.name) } catch (error) { - await deps.quota.refundTraffic(params.orgId, bytes) + await reverseDownloadTraffic(deps, { orgId: params.orgId, bytes, eventId: trafficEventId }) await recordDownloadFailed(deps.activity, { orgId: params.orgId, userId: params.actorId, @@ -556,8 +556,9 @@ export async function getObject( trafficEventId, metadata: { matterId: matter.id, storageId: matter.storageId, matterType: matter.type }, }) + await confirmDownloadTraffic(deps, { eventId: trafficEventId }) } catch (error) { - await deps.quota.refundTraffic(params.orgId, bytes) + await reverseDownloadTraffic(deps, { orgId: params.orgId, bytes, eventId: trafficEventId }) throw error } return { ok: true, matter, downloadUrl } diff --git a/server/usecases/ports/cloud-traffic-report.ts b/server/usecases/ports/cloud-traffic-report.ts index b9996064..9f2cd587 100644 --- a/server/usecases/ports/cloud-traffic-report.ts +++ b/server/usecases/ports/cloud-traffic-report.ts @@ -6,7 +6,16 @@ export type TrafficReportSource = | 'custom_domain_image' | 'webdav_download' -export type CloudTrafficReportStatus = 'pending' | 'reported' | 'skipped_unbound' | 'blocked' | 'failed' | 'dead_letter' +export type CloudTrafficReportStatus = + | 'pending' + | 'reported' + | 'not_required' + | 'skipped_unbound' + | 'blocked' + | 'failed' + | 'dead_letter' + | 'reversed' + | 'ledger_opening' export interface CloudTrafficReportRecord { id: string @@ -23,6 +32,7 @@ export interface CloudTrafficReportRecord { error: string | null attemptCount: number nextRetryAt: Date | null + issuedAt: Date | null createdAt: Date updatedAt: Date } @@ -34,16 +44,20 @@ export interface InsertCloudTrafficReportInput { sourceId: string eventId: string bytes: number - storageId: string - unitBytes: number - creditsPerUnit: number + storageId: string | null + unitBytes: number | null + creditsPerUnit: number | null status: CloudTrafficReportStatus now: Date } export interface CloudTrafficReportRepo { + ensureLedgerOpening(now: Date): Promise + getLedgerOpening(): Promise findByEventId(eventId: string): Promise insert(input: InsertCloudTrafficReportInput): Promise + markIssued(eventId: string, now: Date): Promise + reverse(eventId: string, now: Date): Promise updateStatus( eventId: string, status: CloudTrafficReportStatus, diff --git a/server/usecases/redirect.test.ts b/server/usecases/redirect.test.ts index 4b08bf71..ad9e99aa 100644 --- a/server/usecases/redirect.test.ts +++ b/server/usecases/redirect.test.ts @@ -40,6 +40,8 @@ vi.mock('./store/traffic-metering', async (importOriginal) => ({ ...(await importOriginal()), meterDownloadTraffic: vi.fn(), reportDownloadEgress: vi.fn(), + confirmDownloadTraffic: vi.fn(async () => {}), + reverseDownloadTraffic: vi.fn(async (deps, params) => deps.quota.refundTraffic(params.orgId, params.bytes)), })) const PRESIGNED_DOWNLOAD = 'https://presigned-download.example.com/file' diff --git a/server/usecases/redirect.ts b/server/usecases/redirect.ts index 2516284d..f4cb15b0 100644 --- a/server/usecases/redirect.ts +++ b/server/usecases/redirect.ts @@ -24,7 +24,13 @@ import { storageNotFound, } from './ports' import { PRESIGN_TTL_SECS } from './share' -import { type CloudTrafficMeteringDeps, meterDownloadTraffic, reportDownloadEgress } from './store/traffic-metering' +import { + type CloudTrafficMeteringDeps, + confirmDownloadTraffic, + meterDownloadTraffic, + reportDownloadEgress, + reverseDownloadTraffic, +} from './store/traffic-metering' import { createTrafficEventId, recordDownloadFailed, recordDownloadIssued } from './transfer-activity' // The metering usecases need the cloud-report ports plus quota; the redirect @@ -109,7 +115,7 @@ export async function resolveDirectShareDownload( try { url = await deps.s3.presignDownload(storage, matter.object, matter.name, PRESIGN_TTL_SECS) } catch (e) { - await deps.quota.refundTraffic(share.orgId, bytes) + await reverseDownloadTraffic(deps, { orgId: share.orgId, bytes, eventId: trafficEventId }) await deps.share.decrementDownloads(share.id) await recordDownloadFailed(deps.activity, { orgId: share.orgId, @@ -139,8 +145,9 @@ export async function resolveDirectShareDownload( trafficEventId, metadata: { shareId: share.id, matterId: matter.id, storageId: matter.storageId, kind: share.kind }, }) + await confirmDownloadTraffic(deps, { eventId: trafficEventId }) } catch (error) { - await deps.quota.refundTraffic(share.orgId, bytes) + await reverseDownloadTraffic(deps, { orgId: share.orgId, bytes, eventId: trafficEventId }) await deps.share.decrementDownloads(share.id) throw error } @@ -256,8 +263,9 @@ export async function resolveImageHostingDownload( trafficEventId, metadata: { imageId: image.id, storageId: image.storageId, mime: image.mime }, }) + await confirmDownloadTraffic(deps, { eventId: trafficEventId }) } catch (error) { - await deps.quota.refundTraffic(image.orgId, image.size) + await reverseDownloadTraffic(deps, { orgId: image.orgId, bytes: image.size, eventId: trafficEventId }) throw error } diff --git a/server/usecases/share.test.ts b/server/usecases/share.test.ts index 15357d5f..58cd814f 100644 --- a/server/usecases/share.test.ts +++ b/server/usecases/share.test.ts @@ -43,6 +43,8 @@ import { type DownloadTrafficOutcome, meterDownloadTraffic } from './store/traff vi.mock('./store/traffic-metering', async (importOriginal) => ({ ...(await importOriginal()), meterDownloadTraffic: vi.fn(), + confirmDownloadTraffic: vi.fn(async () => {}), + reverseDownloadTraffic: vi.fn(async (deps, params) => deps.quota.refundTraffic(params.orgId, params.bytes)), })) vi.mock('./object', () => ({ saveShareToDrive: vi.fn() })) diff --git a/server/usecases/share.ts b/server/usecases/share.ts index cb87f734..cd4b7623 100644 --- a/server/usecases/share.ts +++ b/server/usecases/share.ts @@ -44,7 +44,7 @@ import { storageNotFound, } from './ports' import type { CloudTrafficMeteringDeps } from './store/traffic-metering' -import { meterDownloadTraffic } from './store/traffic-metering' +import { confirmDownloadTraffic, meterDownloadTraffic, reverseDownloadTraffic } from './store/traffic-metering' import { createTrafficEventId, recordDownloadFailed, recordDownloadIssued } from './transfer-activity' // The ports + sub-usecase deps this resource touches. `c.get('deps')` (the full @@ -411,7 +411,7 @@ export async function downloadShareObject( try { url = await deps.s3.presignDownload(storage, targetMatter.object, targetMatter.name, PRESIGN_TTL_SECS) } catch (e) { - await deps.quota.refundTraffic(share.orgId, bytes) + await reverseDownloadTraffic(deps, { orgId: share.orgId, bytes, eventId: trafficEventId }) await deps.share.decrementDownloads(share.id) await recordDownloadFailed(deps.activity, { orgId: share.orgId, @@ -451,8 +451,9 @@ export async function downloadShareObject( kind: share.kind, }, }) + await confirmDownloadTraffic(deps, { eventId: trafficEventId }) } catch (error) { - await deps.quota.refundTraffic(share.orgId, bytes) + await reverseDownloadTraffic(deps, { orgId: share.orgId, bytes, eventId: trafficEventId }) await deps.share.decrementDownloads(share.id) throw error } diff --git a/server/usecases/store/traffic-metering.test.ts b/server/usecases/store/traffic-metering.test.ts index 02fb6f34..84a37a27 100644 --- a/server/usecases/store/traffic-metering.test.ts +++ b/server/usecases/store/traffic-metering.test.ts @@ -1,7 +1,11 @@ +import { ne } from 'drizzle-orm' import { afterEach, describe, expect, it, vi } from 'vitest' import { ZPAN_CLOUD_URL_DEFAULT } from '../../../shared/constants' import { createLicensingCloudGateway } from '../../adapters/gateways/licensing-cloud' -import { createCloudTrafficReportRepo } from '../../adapters/repos/cloud-traffic-report' +import { + createCloudTrafficReportRepo, + TRAFFIC_LEDGER_OPENING_EVENT_ID, +} from '../../adapters/repos/cloud-traffic-report' import { createLicenseBindingRepo } from '../../adapters/repos/license-binding' import { cloudTrafficReports } from '../../db/schema' import type { Database, Platform } from '../../platform/interface' @@ -61,6 +65,10 @@ function syncPendingCloudTrafficReports(params: { db: Database; cloudBaseUrl: st return syncPendingCloudTrafficReportsUsecase(meteringDeps(db), rest) } +function trafficReports(db: Database) { + return db.select().from(cloudTrafficReports).where(ne(cloudTrafficReports.eventId, TRAFFIC_LEDGER_OPENING_EVENT_ID)) +} + function makeResponse(body: unknown, status = 200): Response { return { ok: status >= 200 && status < 300, @@ -123,7 +131,7 @@ describe('cloud traffic metering', () => { expect(result).toMatchObject({ status: 'reported', eventId: 'evt_1', duplicate: false }) expect(fetch).toHaveBeenCalledTimes(1) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'reported' }]) + await expect(trafficReports(db)).resolves.toMatchObject([{ status: 'reported' }]) }) it('syncs pending traffic reports to Cloud outside the request path', async () => { @@ -160,10 +168,10 @@ describe('cloud traffic metering', () => { usageContext: { storageId: 'storage_1' }, pricing: { unitQuantity: 100 * 1024 ** 2, creditsPerUnit: 1 }, }) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'reported' }]) + await expect(trafficReports(db)).resolves.toMatchObject([{ status: 'reported' }]) }) - it('ignores zero-byte reports without writing local state', async () => { + it('records zero-byte requests locally without reporting them to Cloud', async () => { const { db, platform } = await createTestApp() vi.stubGlobal('fetch', vi.fn()) @@ -175,9 +183,9 @@ describe('cloud traffic metering', () => { sourceId: 'matter_1', }) - expect(result).toMatchObject({ status: 'reported', eventId: '', duplicate: false }) + expect(result).toMatchObject({ status: 'not_required', duplicate: false }) expect(fetch).not.toHaveBeenCalled() - await expect(db.select().from(cloudTrafficReports)).resolves.toHaveLength(0) + await expect(trafficReports(db)).resolves.toMatchObject([{ bytes: 0, status: 'not_required' }]) }) it('keeps idempotent reports local after the first reported request', async () => { @@ -203,7 +211,7 @@ describe('cloud traffic metering', () => { expect(first.duplicate).toBe(false) expect(second).toMatchObject({ duplicate: true, eventId: 'evt_dup', status: 'reported' }) expect(fetch).toHaveBeenCalledTimes(1) - await expect(db.select().from(cloudTrafficReports)).resolves.toHaveLength(1) + await expect(trafficReports(db)).resolves.toHaveLength(1) }) it('rejects idempotency conflicts for reused event ids', async () => { @@ -254,7 +262,7 @@ describe('cloud traffic metering', () => { }), ).rejects.toThrow(CloudTrafficBlockedError) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + await expect(trafficReports(db)).resolves.toMatchObject([ { eventId: 'evt_blocked', status: 'blocked', error: 'insufficient_credits' }, ]) await expect( @@ -308,9 +316,7 @@ describe('cloud traffic metering', () => { expect(result).toEqual({ attempted: 0, reported: 0, blocked: 0, failed: 0, deadLetter: 0 }) expect(fetch).toHaveBeenCalledTimes(2) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ - { status: 'reported', error: null, period: '2026-04' }, - ]) + await expect(trafficReports(db)).resolves.toMatchObject([{ status: 'reported', error: null, period: '2026-04' }]) }) it('selects new pending reports before retryable failures', async () => { @@ -356,7 +362,7 @@ describe('cloud traffic metering', () => { ...meteredStorage, }), ).resolves.toMatchObject({ status: 'dead_letter', eventId: 'evt_terminal' }) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + await expect(trafficReports(db)).resolves.toMatchObject([ { eventId: 'evt_terminal', status: 'dead_letter', @@ -394,7 +400,7 @@ describe('cloud traffic metering', () => { }) expect(result).toMatchObject({ status: 'reported', eventId: 'evt_mismatch', duplicate: false }) - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + await expect(trafficReports(db)).resolves.toMatchObject([ { eventId: 'evt_mismatch', status: 'reported', error: null }, ]) }) @@ -414,9 +420,9 @@ describe('cloud traffic metering', () => { ...meteredStorage, }) - expect(result.status).toBe('reported') + expect(result.status).toBe('not_required') expect(fetch).not.toHaveBeenCalled() - await expect(db.select().from(cloudTrafficReports)).resolves.toHaveLength(0) + await expect(trafficReports(db)).resolves.toMatchObject([{ eventId: 'evt_unbound', status: 'not_required' }]) }) it('keeps unbound report replays local', async () => { @@ -437,9 +443,9 @@ describe('cloud traffic metering', () => { const second = await reportTrafficEgress(input) expect(first.duplicate).toBe(false) - expect(second).toMatchObject({ status: 'reported', eventId: 'evt_unbound_dup', duplicate: false }) + expect(second).toMatchObject({ status: 'not_required', eventId: 'evt_unbound_dup', duplicate: true }) expect(fetch).not.toHaveBeenCalled() - await expect(db.select().from(cloudTrafficReports)).resolves.toHaveLength(0) + await expect(trafficReports(db)).resolves.toHaveLength(1) }) }) @@ -496,7 +502,7 @@ describe('download metering', () => { expect(out).toEqual({ ok: true }) expect(refundTraffic).not.toHaveBeenCalled() - await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'reported' }]) + await expect(trafficReports(db)).resolves.toMatchObject([{ status: 'reported' }]) }) it('refunds the quota and runs onRejected when the egress report throws, then rethrows', async () => { diff --git a/server/usecases/store/traffic-metering.ts b/server/usecases/store/traffic-metering.ts index d9686a07..a261edf7 100644 --- a/server/usecases/store/traffic-metering.ts +++ b/server/usecases/store/traffic-metering.ts @@ -54,16 +54,15 @@ export async function reportTrafficEgress( }, ): Promise<{ status: CloudTrafficReportStatus; eventId: string; duplicate: boolean }> { const { orgId, bytes, source, sourceId, now = new Date() } = params - if (bytes <= 0) return { status: 'reported', eventId: params.eventId ?? '', duplicate: false } - if (!params.egressCreditBillingEnabled) return { status: 'reported', eventId: params.eventId ?? '', duplicate: false } - if (!hasFeature('quota_store', await loadBindingState(deps))) { - return { status: 'reported', eventId: params.eventId ?? '', duplicate: false } - } - if (!params.storageId || !params.egressCreditUnitBytes || !params.egressCreditPerUnit) { + if (bytes < 0) throw new Error('traffic_bytes_invalid') + const cloudBillingEnabled = + bytes > 0 && params.egressCreditBillingEnabled && hasFeature('quota_store', await loadBindingState(deps)) + if (cloudBillingEnabled && (!params.storageId || !params.egressCreditUnitBytes || !params.egressCreditPerUnit)) { throw new Error('storage_egress_pricing_missing') } const eventId = params.eventId ?? `traffic_${nanoid()}` + await deps.cloudTrafficReports.ensureLedgerOpening(now) const existing = await deps.cloudTrafficReports.findByEventId(eventId) const period = existing?.period ?? currentTrafficPeriod(now) if (existing) { @@ -73,10 +72,11 @@ export async function reportTrafficEgress( source, sourceId, bytes, - storageId: params.storageId, - unitBytes: params.egressCreditUnitBytes, - creditsPerUnit: params.egressCreditPerUnit, + storageId: params.storageId ?? null, + unitBytes: params.egressCreditUnitBytes ?? null, + creditsPerUnit: params.egressCreditPerUnit ?? null, }) + if (existing.status === 'reversed') throw new Error('traffic_report_reversed') if (existing.status !== 'blocked') { return { status: existing.status, eventId, duplicate: true } } @@ -89,14 +89,16 @@ export async function reportTrafficEgress( sourceId, eventId, bytes, - storageId: params.storageId, - unitBytes: params.egressCreditUnitBytes, - creditsPerUnit: params.egressCreditPerUnit, - status: 'pending', + storageId: params.storageId ?? null, + unitBytes: params.egressCreditUnitBytes ?? null, + creditsPerUnit: params.egressCreditPerUnit ?? null, + status: cloudBillingEnabled ? 'pending' : 'not_required', now, }) } + if (!cloudBillingEnabled) return { status: 'not_required', eventId, duplicate: false } + const binding = await deps.licenseBinding.loadActiveLicenseBinding() if (!binding?.refreshToken || !binding.cloudStoreId) { await deps.cloudTrafficReports.updateStatus(eventId, 'skipped_unbound', null, now) @@ -220,9 +222,9 @@ function assertSameReport( source: TrafficReportSource sourceId: string bytes: number - storageId: string - unitBytes: number - creditsPerUnit: number + storageId: string | null + unitBytes: number | null + creditsPerUnit: number | null }, ) { if ( @@ -239,6 +241,21 @@ function assertSameReport( } } +export async function reverseDownloadTraffic( + deps: Pick, + params: { orgId: string; bytes: number; eventId: string; now?: Date }, +): Promise { + await deps.quota.refundTraffic(params.orgId, params.bytes, params.now) + await deps.cloudTrafficReports.reverse(params.eventId, params.now ?? new Date()) +} + +export async function confirmDownloadTraffic( + deps: Pick, + params: { eventId: string; now?: Date }, +): Promise { + await deps.cloudTrafficReports.markIssued(params.eventId, params.now ?? new Date()) +} + // ─── Download metering (egress quota + cloud report) ───────────────────────── // The end-to-end download meter, lifted out of the former http traffic helper: // consume the org's traffic quota, then report egress to Cloud (refunding the diff --git a/server/usecases/webdav.test.ts b/server/usecases/webdav.test.ts index e9c855a7..a8b6759f 100644 --- a/server/usecases/webdav.test.ts +++ b/server/usecases/webdav.test.ts @@ -45,7 +45,11 @@ import { // refund); the webdav usecase only orchestrates around it. Mock it and assert // the wiring + outcome mapping here; cloud-traffic-metering.test.ts owns the // internals. -vi.mock('./store/traffic-metering', () => ({ meterDownloadTraffic: vi.fn() })) +vi.mock('./store/traffic-metering', () => ({ + meterDownloadTraffic: vi.fn(), + confirmDownloadTraffic: vi.fn(async () => {}), + reverseDownloadTraffic: vi.fn(async (deps, params) => deps.quota.refundTraffic(params.orgId, params.bytes)), +})) const storage = { id: 'st-1', @@ -366,7 +370,8 @@ describe('webdav usecase', () => { }) describe('meterWebDavDownload', () => { - it('skips metering for zero-byte reads (HEAD / empty range)', async () => { + it('records zero-byte reads without consuming quota bytes', async () => { + vi.mocked(meterDownloadTraffic).mockResolvedValue({ ok: true }) const deps = makeDeps() const out = await meterWebDavDownload(deps, { cloudBaseUrl: 'https://cloud', @@ -379,7 +384,7 @@ describe('webdav usecase', () => { trafficEventId: 'traffic-1', }) expect(out).toEqual({ ok: true }) - expect(meterDownloadTraffic).not.toHaveBeenCalled() + expect(meterDownloadTraffic).toHaveBeenCalledWith(deps, expect.objectContaining({ bytes: 0 })) }) it('meters with the webdav_download source and forwards the outcome', async () => { @@ -446,7 +451,7 @@ describe('webdav usecase', () => { it('refundWebDavTraffic calls quota.refundTraffic', async () => { const refundTraffic = vi.fn(async () => {}) const deps = makeDeps({ quota: { refundTraffic } }) - await refundWebDavTraffic(deps, { orgId: 'ws-1', bytes: 250 }) + await refundWebDavTraffic(deps, { orgId: 'ws-1', bytes: 250, trafficEventId: 'traffic-1' }) expect(refundTraffic).toHaveBeenCalledWith('ws-1', 250) }) }) diff --git a/server/usecases/webdav.ts b/server/usecases/webdav.ts index 2b66bda6..b54b13f0 100644 --- a/server/usecases/webdav.ts +++ b/server/usecases/webdav.ts @@ -31,7 +31,13 @@ import { type WebDavWorkspace, } from './ports' import { withStorageUsageReservation } from './storage-usage' -import { type DownloadTrafficStorage, meterDownloadTraffic, type TrafficReportSource } from './store/traffic-metering' +import { + confirmDownloadTraffic, + type DownloadTrafficStorage, + meterDownloadTraffic, + reverseDownloadTraffic, + type TrafficReportSource, +} from './store/traffic-metering' import { recordDownloadFailed, recordDownloadIssued } from './transfer-activity' // ─── Auth resolution ────────────────────────────────────────────────────────── @@ -185,9 +191,7 @@ type WebDavTrafficDeps = Pick { - if (params.bytes <= 0) return { ok: true } const outcome = await meterDownloadTraffic(deps, { cloudBaseUrl: params.cloudBaseUrl, orgId: params.orgId, @@ -230,8 +233,8 @@ export async function meterWebDavDownload( const WEBDAV_DOWNLOAD_SOURCE: TrafficReportSource = 'webdav_download' -export function recordWebDavDownloadIssued( - deps: { activity: Pick }, +export async function recordWebDavDownloadIssued( + deps: Pick, params: { orgId: string userId: string @@ -242,7 +245,7 @@ export function recordWebDavDownloadIssued( trafficEventId: string }, ): Promise { - return recordDownloadIssued(deps.activity, { + await recordDownloadIssued(deps.activity, { orgId: params.orgId, userId: params.userId, action: 'webdav_download', @@ -254,14 +257,19 @@ export function recordWebDavDownloadIssued( trafficEventId: params.trafficEventId, metadata: { matterId: params.matterId, storageId: params.storageId }, }) + await confirmDownloadTraffic(deps, { eventId: params.trafficEventId }) } // Refunds previously-consumed traffic quota when an S3 read throws after metering. export function refundWebDavTraffic( - deps: Pick, - params: { orgId: string; bytes: number }, + deps: Pick, + params: { orgId: string; bytes: number; trafficEventId: string }, ): Promise { - return deps.quota.refundTraffic(params.orgId, params.bytes) + return reverseDownloadTraffic(deps, { + orgId: params.orgId, + bytes: params.bytes, + eventId: params.trafficEventId, + }) } // Returns the S3 body for streaming (optionally a byte range). The handler wraps diff --git a/shared/types/admin-overview.ts b/shared/types/admin-overview.ts index a1d7b5cd..99620104 100644 --- a/shared/types/admin-overview.ts +++ b/shared/types/admin-overview.ts @@ -33,7 +33,7 @@ export interface AdminOverviewStatistics { users: { total: number | null active30Days: number | null - new7Days: number + new7Days: number | null activity: { today: number | null last7Days: number | null @@ -44,7 +44,7 @@ export interface AdminOverviewStatistics { date: string totalUsers: number | null activeUsers: number | null - newUsers: number + newUsers: number | null }> topUsage: AdminOverviewUserUsage[] } diff --git a/shared/types/admin-stats.ts b/shared/types/admin-stats.ts index 25593965..3d4c6665 100644 --- a/shared/types/admin-stats.ts +++ b/shared/types/admin-stats.ts @@ -86,11 +86,11 @@ export interface AdminDashboardOverviewStats extends AdminStatsRange { } trends: Array<{ date: string - newUsers: number + newUsers: number | null activeUsers: number | null storageUsedBytes: number | null - uploadBytes: number - downloadBytes: number + uploadBytes: number | null + downloadBytes: number | null }> } @@ -105,7 +105,7 @@ export interface AdminDashboardGrowthStats extends AdminStatsRange { activeUserRate: number | null silentUserRate: number | null } - userScaleTrend: Array<{ date: string; newUsers: number; totalUsers: number | null }> + userScaleTrend: Array<{ date: string; newUsers: number | null; totalUsers: number | null }> activeUserTrend: Array<{ date: string; dau: number | null; wau: number | null; mau: number | null }> userStatus: Array<{ name: string; value: number; percent: number }> registrationSources: Array<{ name: string; value: number; percent: number }> @@ -128,7 +128,7 @@ export interface AdminDashboardStorageStats extends AdminStatsRange { overQuotaSpaces: number | null invalidQuotaSpaces: number | null } - storageTrend: Array<{ date: string; usedBytes: number | null; newBytes: number; newFiles: number }> + storageTrend: Array<{ date: string; usedBytes: number | null; newBytes: number | null; newFiles: number | null }> typeBreakdown: Array<{ type: string; bytes: number; files: number; percent: number }> sizeBreakdown: Array<{ name: string; bytes: number; files: number; percent: number }> ageBreakdown: Array<{ name: string; bytes: number; files: number; percent: number }> @@ -140,12 +140,17 @@ export interface AdminDashboardTrafficStats extends AdminStatsRange { summary: { totalBytes: AdminStatsDelta requestCount: AdminStatsDelta - issuedDownloads: number - blockedDownloads: number + issuedDownloads: number | null + blockedDownloads: number | null downloadIssueSuccessRate: number | null - peakDailyBytes: number + peakDailyBytes: number | null } - trafficTrend: Array<{ date: string; uploadBytes: number; downloadBytes: number; requests: number }> + trafficTrend: Array<{ + date: string + uploadBytes: number | null + downloadBytes: number | null + requests: number | null + }> sourceBreakdown: Array<{ name: string; bytes: number; requests: number; percent: number }> issueStatus: Array<{ status: string; count: number; percent: number }> successTrend: Array<{ date: string; uploadSuccessRate: number | null; downloadSuccessRate: number | null }> @@ -162,9 +167,9 @@ export interface AdminDashboardSharingStats extends AdminStatsRange { saves: AdminStatsDelta downloadsPer100Views: number | null savesPer100Views: number | null - passwordPasses: number + passwordPasses: number | null } - trend: Array<{ date: string; views: number; downloads: number; saves: number }> + trend: Array<{ date: string; views: number | null; downloads: number | null; saves: number | null }> typeBreakdown: Array<{ name: string; value: number; percent: number }> sourceBreakdown: Array<{ name: string; value: number; percent: number }> topShares: Array diff --git a/src/routes/_authenticated/admin/analytics.test.tsx b/src/routes/_authenticated/admin/analytics.test.tsx index 1b2b8be6..5fc4d567 100644 --- a/src/routes/_authenticated/admin/analytics.test.tsx +++ b/src/routes/_authenticated/admin/analytics.test.tsx @@ -1,14 +1,12 @@ -import type { AdminDashboardOperationsStats, AdminDashboardOverviewStats } from '@shared/types' +import type { AdminDashboardGrowthStats, AdminStatsCoverage } from '@shared/types' import { QueryClient, QueryClientProvider } from '@tanstack/react-query' -import { cleanup, render, screen } from '@testing-library/react' +import { cleanup, render, screen, waitFor } from '@testing-library/react' import userEvent from '@testing-library/user-event' import type { ReactNode } from 'react' import { afterEach, describe, expect, it, vi } from 'vitest' import { useEntitlement } from '@/hooks/useEntitlement' import { getAdminDashboardGrowthStats, - getAdminDashboardOperationsStats, - getAdminDashboardOverviewStats, getAdminDashboardSharingStats, getAdminDashboardStorageStats, getAdminDashboardTrafficStats, @@ -33,94 +31,12 @@ vi.mock('@/hooks/useEntitlement', () => ({ })) vi.mock('@/lib/api', () => ({ - getAdminDashboardOverviewStats: vi.fn(), - getAdminDashboardOperationsStats: vi.fn(), getAdminDashboardGrowthStats: vi.fn(), getAdminDashboardStorageStats: vi.fn(), getAdminDashboardTrafficStats: vi.fn(), getAdminDashboardSharingStats: vi.fn(), })) -const overviewStats: AdminDashboardOverviewStats = { - generatedAt: '2026-07-09T00:00:00.000Z', - from: '2026-07-01T00:00:00.000Z', - to: '2026-07-09T00:00:00.000Z', - timeZone: 'UTC', - coverage: { - status: 'complete', - expectedBuckets: 192, - completedBuckets: 192, - lowerBoundBuckets: 0, - quality: 'exact', - dataThrough: '2026-07-09T00:00:00.000Z', - }, - dataQuality: { - missingUploadBytesEvents: 0, - previousMissingUploadBytesEvents: 0, - missingDownloadBytesEvents: 0, - previousMissingDownloadBytesEvents: 0, - missingBytesEvents: 0, - previousMissingBytesEvents: 0, - }, - totals: { - users: 42, - newUsers: { value: 5, previousValue: 3, change: 2, changePercent: 66.7 }, - activeUsers: { value: 18, previousValue: 12, change: 6, changePercent: 50 }, - activeUserRate: 42.9, - storageUsedBytes: 1024, - storageQuotaBytes: 4096, - storageUtilization: 25, - trafficBytes: { value: 512, previousValue: 256, change: 256, changePercent: 100 }, - uploadBytes: { value: 128, previousValue: 64, change: 64, changePercent: 100 }, - downloadBytes: { value: 384, previousValue: 192, change: 192, changePercent: 100 }, - activeShares: 4, - shareViews: { value: 120, previousValue: 60, change: 60, changePercent: 100 }, - shareDownloads: { value: 30, previousValue: 15, change: 15, changePercent: 100 }, - }, - trends: [ - { - date: '2026-07-09', - newUsers: 5, - activeUsers: 18, - storageUsedBytes: 1024, - uploadBytes: 128, - downloadBytes: 384, - }, - ], -} - -const operationsStats: AdminDashboardOperationsStats = { - generatedAt: '2026-07-09T00:00:00.000Z', - from: '2026-07-01T00:00:00.000Z', - to: '2026-07-09T00:00:00.000Z', - timeZone: 'UTC', - coverage: { - status: 'complete', - expectedBuckets: 192, - completedBuckets: 192, - lowerBoundBuckets: 0, - quality: 'exact', - dataThrough: '2026-07-09T00:00:00.000Z', - }, - summary: { - activeBackgroundJobs: 2, - activeRemoteDownloads: 3, - onlineDownloaders: 4, - offlineDownloaders: 1, - backgroundJobFailureRate: 5, - remoteDownloadSuccessRate: 95, - cloudReportBacklog: 6, - cloudReportDeadLetters: 0, - webhookFailures: 7, - alertCount: 13, - }, - trend: [], - backgroundJobOutcomes: [], - remoteDownloadOutcomes: [], - downloaderStatus: [], - cloudReportStatus: [], -} - function renderOverviewPage() { const queryClient = new QueryClient({ defaultOptions: { @@ -136,13 +52,43 @@ function renderOverviewPage() { ) } +function sectionToggle(title: string): HTMLButtonElement { + const button = screen.getByText(title).closest('button') + if (!(button instanceof HTMLButtonElement)) throw new Error(`section toggle not found: ${title}`) + return button +} + +function growthStats(coverage: AdminStatsCoverage): AdminDashboardGrowthStats { + return { + generatedAt: '2026-07-21T04:00:00.000Z', + from: '2026-06-22T00:00:00.000Z', + to: '2026-07-21T03:59:59.999Z', + timeZone: 'UTC', + coverage, + summary: { + totalUsers: 10, + newUsers: { value: 2, previousValue: null, change: null, changePercent: null }, + activeUsers: { value: 4, previousValue: null, change: null, changePercent: null }, + verifiedUsers: 8, + bannedUsers: 0, + silentUsers: 6, + activeUserRate: 40, + silentUserRate: 60, + }, + userScaleTrend: [], + activeUserTrend: [], + userStatus: [], + registrationSources: [], + } +} + afterEach(() => { cleanup() vi.clearAllMocks() }) describe('Admin overview dashboard', () => { - it('renders overview stats without querying Pro sections when analytics is locked', async () => { + it('shows one page-level Pro badge and removes overview and operations sections', () => { vi.mocked(useEntitlement).mockReturnValue({ bound: false, active: false, @@ -153,72 +99,25 @@ describe('Admin overview dashboard', () => { isLoading: false, isError: false, }) - vi.mocked(getAdminDashboardOverviewStats).mockResolvedValue(overviewStats) renderOverviewPage() - expect(await screen.findByText('42')).toBeTruthy() - expect(screen.getByText('所选范围的离线结果完整。')).toBeTruthy() + expect(screen.getByText('站点统计')).toBeTruthy() + expect(screen.getAllByText('Pro')).toHaveLength(1) + expect(screen.queryByText('站点概览')).toBeNull() + expect(screen.queryByText('运行状态')).toBeNull() expect(screen.getByText('用户与增长')).toBeTruthy() + expect(screen.getByText('存储与文件')).toBeTruthy() + expect(screen.getByText('流量与传输')).toBeTruthy() + expect(screen.getByText('分享与访问')).toBeTruthy() + expect(screen.getByTestId('upgrade-hint')).toBeTruthy() expect(getAdminDashboardGrowthStats).not.toHaveBeenCalled() expect(getAdminDashboardStorageStats).not.toHaveBeenCalled() expect(getAdminDashboardTrafficStats).not.toHaveBeenCalled() expect(getAdminDashboardSharingStats).not.toHaveBeenCalled() - expect(getAdminDashboardOperationsStats).not.toHaveBeenCalled() }) - it('warns when historical transfer bytes are incomplete', async () => { - vi.mocked(useEntitlement).mockReturnValue({ - bound: false, - active: false, - edition: null, - licenseId: null, - cloudDashboardUrl: null, - hasFeature: () => false, - isLoading: false, - isError: false, - }) - vi.mocked(getAdminDashboardOverviewStats).mockResolvedValue({ - ...overviewStats, - dataQuality: { ...overviewStats.dataQuality, missingUploadBytesEvents: 3, missingBytesEvents: 3 }, - }) - - renderOverviewPage() - - expect(await screen.findByText('历史数据不完整')).toBeTruthy() - expect(screen.getByText(/当前区间有 3 条/)).toBeTruthy() - }) - - it('warns when the comparison range is missing offline result buckets', async () => { - vi.mocked(useEntitlement).mockReturnValue({ - bound: false, - active: false, - edition: null, - licenseId: null, - cloudDashboardUrl: null, - hasFeature: () => false, - isLoading: false, - isError: false, - }) - vi.mocked(getAdminDashboardOverviewStats).mockResolvedValue({ - ...overviewStats, - comparisonCoverage: { - status: 'partial', - expectedBuckets: 192, - completedBuckets: 144, - lowerBoundBuckets: 0, - quality: 'exact', - dataThrough: '2026-06-30T00:00:00.000Z', - }, - }) - - renderOverviewPage() - - expect(await screen.findByText('对比区间存在缺失的小时结果，环比数据不完整。')).toBeTruthy() - expect(screen.getByText(/对比 144\/192 小时/)).toBeTruthy() - }) - - it('loads the operations dashboard only when an entitled admin expands it', async () => { + it('loads growth by default and keeps the other Pro sections lazy', async () => { const user = userEvent.setup() vi.mocked(useEntitlement).mockReturnValue({ bound: true, @@ -230,15 +129,75 @@ describe('Admin overview dashboard', () => { isLoading: false, isError: false, }) - vi.mocked(getAdminDashboardOverviewStats).mockResolvedValue(overviewStats) - vi.mocked(getAdminDashboardOperationsStats).mockResolvedValue(operationsStats) + vi.mocked(getAdminDashboardGrowthStats).mockReturnValue(new Promise(() => {})) + vi.mocked(getAdminDashboardStorageStats).mockReturnValue(new Promise(() => {})) renderOverviewPage() - expect(getAdminDashboardOperationsStats).not.toHaveBeenCalled() - await user.click(screen.getByText('运行状态')) + await waitFor(() => expect(getAdminDashboardGrowthStats).toHaveBeenCalledTimes(1)) + expect(sectionToggle('用户与增长').getAttribute('aria-expanded')).toBe('true') + expect(screen.getAllByRole('button', { expanded: true })).toHaveLength(1) + expect(getAdminDashboardStorageStats).not.toHaveBeenCalled() + await user.click(sectionToggle('存储与文件')) + await waitFor(() => expect(getAdminDashboardStorageStats).toHaveBeenCalledTimes(1)) + expect(sectionToggle('用户与增长').getAttribute('aria-expanded')).toBe('false') + expect(sectionToggle('存储与文件').getAttribute('aria-expanded')).toBe('true') + expect(screen.getAllByRole('button', { expanded: true })).toHaveLength(1) + }) - expect(await screen.findByText('后台任务')).toBeTruthy() - expect(getAdminDashboardOperationsStats).toHaveBeenCalledTimes(1) + it('renders exact metrics without exposing coverage or lower-bound diagnostics', async () => { + vi.mocked(useEntitlement).mockReturnValue({ + bound: true, + active: true, + edition: 'pro', + licenseId: 'license-1', + cloudDashboardUrl: null, + hasFeature: (feature) => feature === 'analytics', + isLoading: false, + isError: false, + }) + vi.mocked(getAdminDashboardGrowthStats).mockResolvedValue( + growthStats({ + status: 'complete', + expectedBuckets: 700, + completedBuckets: 700, + lowerBoundBuckets: 150, + quality: 'lower_bound', + dataThrough: '2026-07-21T04:00:00.000Z', + }), + ) + + renderOverviewPage() + + expect(await screen.findByText('总用户数')).toBeTruthy() + expect(screen.queryByText(/部分小时|数据下限|当前 700\/700|快照采样/)).toBeNull() + }) + + it('shows no module data when the selected range is incomplete', async () => { + vi.mocked(useEntitlement).mockReturnValue({ + bound: true, + active: true, + edition: 'pro', + licenseId: 'license-1', + cloudDashboardUrl: null, + hasFeature: (feature) => feature === 'analytics', + isLoading: false, + isError: false, + }) + vi.mocked(getAdminDashboardGrowthStats).mockResolvedValue( + growthStats({ + status: 'partial', + expectedBuckets: 700, + completedBuckets: 699, + lowerBoundBuckets: 0, + quality: 'exact', + dataThrough: '2026-07-21T03:00:00.000Z', + }), + ) + + renderOverviewPage() + + expect(await screen.findByText('暂无统计数据')).toBeTruthy() + expect(screen.queryByText('总用户数')).toBeNull() }) }) diff --git a/src/routes/_authenticated/admin/analytics.tsx b/src/routes/_authenticated/admin/analytics.tsx index 97f4948e..eb26f8f8 100644 --- a/src/routes/_authenticated/admin/analytics.tsx +++ b/src/routes/_authenticated/admin/analytics.tsx @@ -1,20 +1,15 @@ import type { AdminDashboardGrowthStats, - AdminDashboardOperationsStats, - AdminDashboardOverviewStats, AdminDashboardSharingStats, AdminDashboardStorageStats, AdminDashboardTrafficStats, - AdminSharingDataQuality, AdminStatsRange, - AdminTransferDataQuality, } from '@shared/types' import { useQuery } from '@tanstack/react-query' import { createFileRoute } from '@tanstack/react-router' import { endOfDay, endOfMonth, format, startOfDay, startOfMonth, subDays, subMonths } from 'date-fns' import { Activity, - BarChart3, CalendarDays, ChevronDown, Database, @@ -50,8 +45,9 @@ import { XAxis, YAxis, } from 'recharts' +import { AdminPageHeader } from '@/components/admin/admin-page-header' +import { ProBadge } from '@/components/ProBadge' import { UpgradeHint } from '@/components/UpgradeHint' -import { Badge } from '@/components/ui/badge' import { Button } from '@/components/ui/button' import { Calendar } from '@/components/ui/calendar' import { Card, CardContent, CardTitle } from '@/components/ui/card' @@ -63,8 +59,6 @@ import { useEntitlement } from '@/hooks/useEntitlement' import { type AdminStatsRangeFilter, getAdminDashboardGrowthStats, - getAdminDashboardOperationsStats, - getAdminDashboardOverviewStats, getAdminDashboardSharingStats, getAdminDashboardStorageStats, getAdminDashboardTrafficStats, @@ -76,7 +70,7 @@ export const Route = createFileRoute('/_authenticated/admin/analytics')({ component: OverviewPage, }) -type SectionId = 'overview' | 'growth' | 'storage' | 'traffic' | 'sharing' | 'operations' +type SectionId = 'growth' | 'storage' | 'traffic' | 'sharing' const COLORS = { blue: '#0369a1', @@ -114,153 +108,103 @@ const SECTION_META: Array<{ id: SectionId title: string description: string - badges: string[] - icon: typeof BarChart3 + icon: typeof Users }> = [ - { - id: 'overview', - title: '站点概览', - description: '判断站点规模、活跃、存储占用和传输流量是否健康。', - badges: [], - icon: BarChart3, - }, { id: 'growth', title: '用户与增长', description: '观察用户规模、活跃度、账号状态和注册方式结构。', - badges: ['Pro+'], icon: Users, }, { id: 'storage', title: '存储与文件', description: '观察容量水位、文件结构、文件年龄和大对象占用。', - badges: ['Pro+'], icon: HardDrive, }, { id: 'traffic', title: '流量与传输', description: '观察确认上传、下载签发、失败率和计量状态。', - badges: ['Pro+'], icon: UploadCloud, }, { id: 'sharing', title: '分享与访问', description: '观察分享创建、访问、下载签发和转存行为。', - badges: ['Pro+'], icon: Share2, }, - { - id: 'operations', - title: '运行状态', - description: '检查后台任务、远程下载、下载器和计量上报健康。', - badges: ['Pro+'], - icon: Network, - }, ] export function OverviewPage() { const today = useMemo(() => utcCalendarDate(new Date()), []) - const [openSections, setOpenSections] = useState>(() => new Set(['overview'])) + const [openSection, setOpenSection] = useState('growth') const [range, setRange] = useState(() => ({ from: startOfDay(subDays(today, 29)), to: endOfDay(today) })) const { hasFeature, isLoading: entitlementLoading } = useEntitlement() const hasAnalytics = hasFeature('analytics') - const overviewQuery = useQuery({ - queryKey: ['admin', 'dashboard', 'overview', rangeKey(range)], - queryFn: () => getAdminDashboardOverviewStats(toRangeFilter(range)), - staleTime: 30_000, - }) - const growthQuery = useDashboardSectionQuery( - 'growth', - range, - openSections, - hasAnalytics, - getAdminDashboardGrowthStats, - ) + const growthQuery = useDashboardSectionQuery('growth', range, openSection, hasAnalytics, getAdminDashboardGrowthStats) const storageQuery = useDashboardSectionQuery( 'storage', range, - openSections, + openSection, hasAnalytics, getAdminDashboardStorageStats, ) const trafficQuery = useDashboardSectionQuery( 'traffic', range, - openSections, + openSection, hasAnalytics, getAdminDashboardTrafficStats, ) const sharingQuery = useDashboardSectionQuery( 'sharing', range, - openSections, + openSection, hasAnalytics, getAdminDashboardSharingStats, ) - const operationsQuery = useDashboardSectionQuery( - 'operations', - range, - openSections, - hasAnalytics, - getAdminDashboardOperationsStats, - ) - function toggleSection(section: SectionId) { - if (section !== 'overview' && !hasAnalytics) return - setOpenSections((current) => { - const next = new Set(current) - if (next.has(section)) next.delete(section) - else next.add(section) - return next - }) + if (!hasAnalytics) return + setOpenSection((current) => (current === section ? null : section)) } return ( -
-
-
-
-

站点统计

-

统一使用 UTC 日期与已完成的离线小时结果。

-
- -
-
- {SECTION_META.map((section) => ( - toggleSection(section.id)} - > - {section.id === 'overview' ? ( - {(data) => } - ) : !hasAnalytics ? ( - - ) : section.id === 'growth' ? ( - {(data) => } - ) : section.id === 'storage' ? ( - {(data) => } - ) : section.id === 'traffic' ? ( - {(data) => } - ) : section.id === 'sharing' ? ( - {(data) => } - ) : ( - {(data) => } - )} - - ))} -
+
+ } + action={} + /> +
+ {SECTION_META.map((section) => ( + toggleSection(section.id)} + > + {!hasAnalytics ? ( + + ) : section.id === 'growth' ? ( + {(data) => } + ) : section.id === 'storage' ? ( + {(data) => } + ) : section.id === 'traffic' ? ( + {(data) => } + ) : section.id === 'sharing' ? ( + {(data) => } + ) : null} + + ))}
) @@ -269,14 +213,14 @@ export function OverviewPage() { function useDashboardSectionQuery( section: SectionId, range: DateRange, - openSections: Set, + openSection: SectionId | null, enabled: boolean, queryFn: (filter: AdminStatsRangeFilter) => Promise, ) { return useQuery({ queryKey: ['admin', 'dashboard', section, rangeKey(range)], queryFn: () => queryFn(toRangeFilter(range)), - enabled: enabled && openSections.has(section), + enabled: enabled && openSection === section, staleTime: 30_000, }) } @@ -311,11 +255,6 @@ function DashboardSection({ {section.title} - {section.badges.map((badge) => ( - - {badge} - - ))} {locked && } {section.description} @@ -414,144 +353,6 @@ function DateRangePicker({ value, onChange }: { value: DateRange; onChange: (ran ) } -function OverviewSection({ stats }: { stats: AdminDashboardOverviewStats }) { - const cards = [ - { - label: '期末注册用户', - value: formatNumber(stats.totals.users), - delta: formatDelta(stats.totals.newUsers), - icon: Users, - metrics: [ - { label: '新增用户', value: formatNumber(stats.totals.newUsers.value) }, - { label: '30 日活跃用户', value: formatNumber(stats.totals.activeUsers.value) }, - ], - }, - { - label: '30 日活跃用户', - value: formatNumber(stats.totals.activeUsers.value), - delta: formatDelta(stats.totals.activeUsers), - icon: TrendingUp, - metrics: [ - { label: '30 日活跃率', value: formatPercent(stats.totals.activeUserRate) }, - { label: '环比人数', value: formatNumber(stats.totals.activeUsers.previousValue) }, - ], - }, - { - label: '期末存储占用', - value: formatSize(stats.totals.storageUsedBytes), - delta: formatDelta(stats.totals.uploadBytes, formatSize), - icon: Database, - metrics: [ - { label: '使用率', value: formatPercent(stats.totals.storageUtilization) }, - { label: '区间新增', value: formatSize(stats.totals.uploadBytes.value) }, - ], - }, - { - label: '确认/签发字节', - value: formatSize(stats.totals.trafficBytes.value), - delta: formatDelta(stats.totals.trafficBytes, formatSize), - icon: Network, - metrics: [ - { label: '上传', value: formatSize(stats.totals.uploadBytes.value) }, - { label: '下载', value: formatSize(stats.totals.downloadBytes.value) }, - ], - }, - ] - - return ( -
-
- {cards.map((card) => ( - - ))} -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- ) -} - function GrowthSection({ stats }: { stats: AdminDashboardGrowthStats }) { return (
@@ -718,8 +519,6 @@ function StorageSection({ stats }: { stats: AdminDashboardStorageStats }) { ]} />
- -
-
-
@@ -1202,88 +999,6 @@ function SharingSection({ stats }: { stats: AdminDashboardSharingStats }) { ) } -function OperationsSection({ stats }: { stats: AdminDashboardOperationsStats }) { - return ( -
-
- - - - -
- - - - - - - - - - - - - - - -
- - - - -
-
- ) -} - function StatCard({ label, value, @@ -1522,84 +1237,8 @@ function QueryState({
) if (!query.data) return - return ( -
- - {children(query.data)} -
- ) -} - -function StatsCoverageNotice({ stats }: { stats: AdminStatsRange }) { - const { coverage } = stats - const comparisonCoverage = stats.comparisonCoverage - const snapshotCoverage = stats.snapshotCoverage - const comparisonSnapshotCoverage = stats.comparisonSnapshotCoverage - const through = coverage.dataThrough - ? new Date(coverage.dataThrough).toISOString().replace('T', ' ').slice(0, 16) - : null - const snapshotThrough = snapshotCoverage?.dataThrough - ? new Date(snapshotCoverage.dataThrough).toISOString().replace('T', ' ').slice(0, 16) - : null - const comparisonIncomplete = comparisonCoverage ? comparisonCoverage.status !== 'complete' : false - const lowerBound = - coverage.quality === 'lower_bound' || - comparisonCoverage?.quality === 'lower_bound' || - snapshotCoverage?.quality === 'lower_bound' || - comparisonSnapshotCoverage?.quality === 'lower_bound' - const snapshotIncomplete = snapshotCoverage ? snapshotCoverage.status !== 'complete' : false - const comparisonSnapshotIncomplete = comparisonSnapshotCoverage - ? comparisonSnapshotCoverage.status !== 'complete' - : false - const incomplete = - coverage.status !== 'complete' || - comparisonIncomplete || - snapshotIncomplete || - comparisonSnapshotIncomplete || - lowerBound - return ( -
- - {lowerBound - ? '部分小时只有可验证的数据下限；页面不会把这些数值表述为完整事实。' - : coverage.status === 'empty' - ? '所选范围还没有可用的离线结果。' - : coverage.status === 'partial' - ? '所选范围存在缺失的小时结果，当前数据不完整。' - : comparisonCoverage?.status === 'empty' - ? '对比区间还没有可用的离线结果，环比不可对账。' - : comparisonCoverage?.status === 'partial' - ? '对比区间存在缺失的小时结果，环比数据不完整。' - : snapshotCoverage?.status === 'empty' - ? '所选范围没有可用的状态快照；状态类指标显示为 —，不会伪装成 0。' - : snapshotCoverage?.status === 'partial' - ? '所选范围的事件结果完整，但状态快照仅覆盖部分小时。' - : comparisonSnapshotCoverage?.status === 'empty' - ? '对比区间没有状态快照，状态类环比不可对账。' - : comparisonSnapshotCoverage?.status === 'partial' - ? '对比区间的状态快照不完整，状态类环比不可对账。' - : '所选范围的离线结果完整。'} - - - {through ? `数据截至 ${through} UTC · ` : ''} - 当前 {coverage.completedBuckets}/{coverage.expectedBuckets} 小时 - {coverage.lowerBoundBuckets > 0 ? ` · 下限 ${coverage.lowerBoundBuckets} 小时` : ''} - {comparisonCoverage - ? ` · 对比 ${comparisonCoverage.completedBuckets}/${comparisonCoverage.expectedBuckets} 小时` - : ''} - {snapshotCoverage - ? ` · 快照 ${snapshotCoverage.completedBuckets}/${snapshotCoverage.expectedBuckets} 小时` - : ''} - {snapshotThrough ? ` · 快照采样于 ${snapshotThrough} UTC` : ''} - -
- ) + if (query.data.coverage.status !== 'complete') return + return children(query.data) } function SectionSkeleton() { @@ -1615,71 +1254,6 @@ function SectionSkeleton() { ) } -function TransferDataQualityNotice({ quality }: { quality: AdminTransferDataQuality }) { - const currentMissing = quality.missingBytesEvents - const previousMissing = quality.previousMissingBytesEvents - if (currentMissing === 0 && previousMissing === 0) return null - - return ( -
- - 历史数据不完整 - - - 当前区间有 {formatNumber(currentMissing)} 条、对比区间有 {formatNumber(previousMissing)} - 条传输事件缺少可恢复的字节数；流量与新增容量仅代表已知下限，事件数量不受影响。 - -
- ) -} - -function SharingDataQualityNotice({ quality }: { quality: AdminSharingDataQuality }) { - if (quality.unlocatedEvents === null || quality.unlocatedEvents === 0) return null - return ( -
- - 历史事件无法定位 - - - 现存分享累计计数中有 {formatNumber(quality.unlocatedViews)} 次访问和 {formatNumber(quality.unlocatedDownloads)}{' '} - 次下载没有可信时间戳；区间值仅为已定位下限，相关环比和每百次访问比例已隐藏。 - -
- ) -} - -function StorageUsageDataQualityNotice({ quality }: { quality: AdminDashboardStorageStats['dataQuality'] }) { - const usageDrift = quality.usageDriftSpaces !== null && quality.usageDriftSpaces > 0 - const ledgerDrift = quality.ledgerDriftSpaces !== null && quality.ledgerDriftSpaces > 0 - if (!usageDrift && !ledgerDrift) return null - return ( -
- - 占用账本异常 - - - {usageDrift - ? `${formatNumber(quality.usageDriftSpaces)} 个空间的实时占用计数与可计费对象相差 ${formatSize(quality.usageDriftBytes)}` - : ''} - {usageDrift && ledgerDrift ? '；' : ''} - {ledgerDrift - ? `${formatNumber(quality.ledgerDriftSpaces)} 个空间的存储流水与可计费对象相差 ${formatSize(quality.ledgerDriftBytes)}` - : ''} - ；修复前不要把对应的存储数据作为准确事实。 - -
- ) -} - function EmptyState() { return (
diff --git a/src/routes/_authenticated/admin/dashboard.tsx b/src/routes/_authenticated/admin/dashboard.tsx index 5997a4bb..0fdf6f36 100644 --- a/src/routes/_authenticated/admin/dashboard.tsx +++ b/src/routes/_authenticated/admin/dashboard.tsx @@ -129,7 +129,7 @@ function OverviewContent({ overview }: { overview: AdminOverview }) { } label={t('admin.overview.metrics.new7Days')} - value={overview.users.new7Days.toLocaleString()} + value={formatCount(overview.users.new7Days)} detail={t('admin.overview.metrics.new7DaysDetail')} />