From ec5d08b2332ef604d8ee09fcf69977f0eef0e8ba Mon Sep 17 00:00:00 2001 From: Jasper Van Date: Fri, 8 May 2026 20:31:48 -0400 Subject: [PATCH] Report traffic egress to Cloud metering (#384) * feat(api): report traffic egress to cloud metering Agent-Profile: https://agent-kanban.dev/agents/a318237412dd8b98 * test(api): cover cloud traffic redirect reporting Agent-Profile: https://agent-kanban.dev/agents/a318237412dd8b98 * test(api): cover cloud traffic failure branches Agent-Profile: https://agent-kanban.dev/agents/a318237412dd8b98 * test(api): cover final traffic metering branches Agent-Profile: https://agent-kanban.dev/agents/a318237412dd8b98 * test(api): cover cloud metering rollback paths Agent-Profile: https://agent-kanban.dev/agents/a318237412dd8b98 --- migrations/0030_add-cloud-traffic-reports.sql | 17 + migrations/meta/0030_snapshot.json | 2656 +++++++++++++++++ migrations/meta/_journal.json | 7 + server/db/schema.ts | 22 + server/middleware/image-hosting-domain.ts | 16 +- ...cloud-traffic-metering.integration.test.ts | 410 +++ server/routes/objects.ts | 9 + server/routes/redirect.ts | 24 +- server/routes/shares.ts | 23 +- server/routes/traffic-metering-utils.ts | 37 + .../services/cloud-traffic-metering.test.ts | 261 ++ server/services/cloud-traffic-metering.ts | 151 + server/test/setup.ts | 16 + 13 files changed, 3643 insertions(+), 6 deletions(-) create mode 100644 migrations/0030_add-cloud-traffic-reports.sql create mode 100644 migrations/meta/0030_snapshot.json create mode 100644 server/routes/cloud-traffic-metering.integration.test.ts create mode 100644 server/routes/traffic-metering-utils.ts create mode 100644 server/services/cloud-traffic-metering.test.ts create mode 100644 server/services/cloud-traffic-metering.ts diff --git a/migrations/0030_add-cloud-traffic-reports.sql b/migrations/0030_add-cloud-traffic-reports.sql new file mode 100644 index 00000000..9845a0e1 --- /dev/null +++ b/migrations/0030_add-cloud-traffic-reports.sql @@ -0,0 +1,17 @@ +CREATE TABLE `cloud_traffic_reports` ( + `id` text PRIMARY KEY NOT NULL, + `org_id` text NOT NULL, + `period` text NOT NULL, + `source` text NOT NULL, + `source_id` text NOT NULL, + `event_id` text NOT NULL, + `bytes` integer NOT NULL, + `status` text NOT NULL, + `error` text, + `created_at` integer NOT NULL, + `updated_at` integer NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX `cloud_traffic_reports_event_uniq` ON `cloud_traffic_reports` (`event_id`);--> statement-breakpoint +CREATE INDEX `cloud_traffic_reports_org_period_idx` ON `cloud_traffic_reports` (`org_id`,`period`);--> statement-breakpoint +CREATE INDEX `cloud_traffic_reports_status_idx` ON `cloud_traffic_reports` (`status`); \ No newline at end of file diff --git a/migrations/meta/0030_snapshot.json b/migrations/meta/0030_snapshot.json new file mode 100644 index 00000000..5126f45b --- /dev/null +++ b/migrations/meta/0030_snapshot.json @@ -0,0 +1,2656 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "5b582181-3f03-4c31-bcd1-6db6d55f34ba", + "prevId": "b6129246-61ab-4f3f-b984-d329557b0f0b", + "tables": { + "activity_events": { + "name": "activity_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_name": { + "name": "target_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "announcements": { + "name": "announcements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "announcements_status_priority_idx": { + "name": "announcements_status_priority_idx", + "columns": [ + "status", + "priority" + ], + "isUnique": false + }, + "announcements_published_idx": { + "name": "announcements_published_idx", + "columns": [ + "published_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cloud_traffic_reports": { + "name": "cloud_traffic_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "period": { + "name": "period", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "cloud_traffic_reports_event_uniq": { + "name": "cloud_traffic_reports_event_uniq", + "columns": [ + "event_id" + ], + "isUnique": true + }, + "cloud_traffic_reports_org_period_idx": { + "name": "cloud_traffic_reports_org_period_idx", + "columns": [ + "org_id", + "period" + ], + "isUnique": false + }, + "cloud_traffic_reports_status_idx": { + "name": "cloud_traffic_reports_status_idx", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "image_hosting_configs": { + "name": "image_hosting_configs", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "custom_domain": { + "name": "custom_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cf_hostname_id": { + "name": "cf_hostname_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "domain_verified_at": { + "name": "domain_verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "referer_allowlist": { + "name": "referer_allowlist", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "image_hosting_configs_custom_domain_unique": { + "name": "image_hosting_configs_custom_domain_unique", + "columns": [ + "custom_domain" + ], + "isUnique": true + } + }, + "foreignKeys": { + "image_hosting_configs_org_id_organization_id_fk": { + "name": "image_hosting_configs_org_id_organization_id_fk", + "tableFrom": "image_hosting_configs", + "tableTo": "organization", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "image_hostings": { + "name": "image_hostings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime": { + "name": "mime", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "access_count": { + "name": "access_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_accessed_at": { + "name": "last_accessed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "image_hostings_token_unique": { + "name": "image_hostings_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "image_hostings_org_path_uniq": { + "name": "image_hostings_org_path_uniq", + "columns": [ + "org_id", + "path" + ], + "isUnique": true + }, + "image_hostings_org_created_idx": { + "name": "image_hostings_org_created_idx", + "columns": [ + "org_id", + "created_at" + ], + "isUnique": false + }, + "image_hostings_token_idx": { + "name": "image_hostings_token_idx", + "columns": [ + "token" + ], + "isUnique": false + } + }, + "foreignKeys": { + "image_hostings_org_id_organization_id_fk": { + "name": "image_hostings_org_id_organization_id_fk", + "tableFrom": "image_hostings", + "tableTo": "organization", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "image_hostings_storage_id_storages_id_fk": { + "name": "image_hostings_storage_id_storages_id_fk", + "tableFrom": "image_hostings", + "tableTo": "storages", + "columnsFrom": [ + "storage_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invite_codes": { + "name": "invite_codes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "used_by": { + "name": "used_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "used_at": { + "name": "used_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invite_codes_code_unique": { + "name": "invite_codes_code_unique", + "columns": [ + "code" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "license_bindings": { + "name": "license_bindings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "cloud_binding_id": { + "name": "cloud_binding_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cloud_store_id": { + "name": "cloud_store_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cloud_account_id": { + "name": "cloud_account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cloud_account_email": { + "name": "cloud_account_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cached_certificate": { + "name": "cached_certificate", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cached_certificate_expires_at": { + "name": "cached_certificate_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bound_at": { + "name": "bound_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "disconnected_at": { + "name": "disconnected_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refresh_at": { + "name": "last_refresh_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refresh_error": { + "name": "last_refresh_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "license_bindings_active_uniq": { + "name": "license_bindings_active_uniq", + "columns": [ + "status" + ], + "isUnique": true, + "where": "status = 'active'" + }, + "license_bindings_cloud_binding_idx": { + "name": "license_bindings_cloud_binding_idx", + "columns": [ + "cloud_binding_id" + ], + "isUnique": false + }, + "license_bindings_instance_idx": { + "name": "license_bindings_instance_idx", + "columns": [ + "instance_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "matters": { + "name": "matters", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "alias": { + "name": "alias", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "dirtype": { + "name": "dirtype", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "parent": { + "name": "parent", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "object": { + "name": "object", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "trashed_at": { + "name": "trashed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "matters_alias_unique": { + "name": "matters_alias_unique", + "columns": [ + "alias" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notifications": { + "name": "notifications", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "ref_type": { + "name": "ref_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_id": { + "name": "ref_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "read_at": { + "name": "read_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "notifications_user_created_idx": { + "name": "notifications_user_created_idx", + "columns": [ + "user_id", + "created_at" + ], + "isUnique": false + }, + "notifications_user_read_idx": { + "name": "notifications_user_read_idx", + "columns": [ + "user_id", + "read_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_quota_entitlements": { + "name": "org_quota_entitlements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "starts_at": { + "name": "starts_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "org_quota_entitlements_org_resource_idx": { + "name": "org_quota_entitlements_org_resource_idx", + "columns": [ + "org_id", + "resource_type", + "status" + ], + "isUnique": false + }, + "org_quota_entitlements_source_resource_uniq": { + "name": "org_quota_entitlements_source_resource_uniq", + "columns": [ + "source", + "source_id", + "resource_type" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_quotas": { + "name": "org_quotas", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "quota": { + "name": "quota", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "used": { + "name": "used", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "traffic_quota": { + "name": "traffic_quota", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "traffic_used": { + "name": "traffic_used", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "traffic_period": { + "name": "traffic_period", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01'" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "share_recipients": { + "name": "share_recipients", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "share_id": { + "name": "share_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "recipient_user_id": { + "name": "recipient_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "recipient_email": { + "name": "recipient_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "share_recipients_share_id_idx": { + "name": "share_recipients_share_id_idx", + "columns": [ + "share_id" + ], + "isUnique": false + }, + "share_recipients_user_id_idx": { + "name": "share_recipients_user_id_idx", + "columns": [ + "recipient_user_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "shares": { + "name": "shares", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "matter_id": { + "name": "matter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "creator_id": { + "name": "creator_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "download_limit": { + "name": "download_limit", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "views": { + "name": "views", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "downloads": { + "name": "downloads", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "shares_token_unique": { + "name": "shares_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "shares_creator_status_created_idx": { + "name": "shares_creator_status_created_idx", + "columns": [ + "creator_id", + "status", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "site_invitations": { + "name": "site_invitations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_by": { + "name": "accepted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "accepted_at": { + "name": "accepted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "site_invitations_token_unique": { + "name": "site_invitations_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "site_invitations_email_idx": { + "name": "site_invitations_email_idx", + "columns": [ + "email" + ], + "isUnique": false + }, + "site_invitations_created_idx": { + "name": "site_invitations_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + }, + "site_invitations_expires_idx": { + "name": "site_invitations_expires_idx", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storages": { + "name": "storages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bucket": { + "name": "bucket", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'auto'" + }, + "access_key": { + "name": "access_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret_key": { + "name": "secret_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "custom_host": { + "name": "custom_host", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "''" + }, + "capacity": { + "name": "capacity", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "used": { + "name": "used", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "system_options": { + "name": "system_options", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "public": { + "name": "public", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "team_invite_links": { + "name": "team_invite_links", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "team_invite_links_token_unique": { + "name": "team_invite_links_token_unique", + "columns": [ + "token" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhook_events": { + "name": "webhook_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'cloud'" + }, + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'order.quota_changed'" + }, + "payload_hash": { + "name": "payload_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "raw_payload": { + "name": "raw_payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhook_events_source_event_uniq": { + "name": "webhook_events_source_event_uniq", + "columns": [ + "source", + "event_id" + ], + "isUnique": true + }, + "webhook_events_source_created_idx": { + "name": "webhook_events_source_created_idx", + "columns": [ + "source", + "created_at" + ], + "isUnique": false + }, + "webhook_events_status_idx": { + "name": "webhook_events_status_idx", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_request": { + "name": "last_request", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apikey_config_id_idx": { + "name": "apikey_config_id_idx", + "columns": [ + "config_id" + ], + "isUnique": false + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": [ + "reference_id" + ], + "isUnique": false + }, + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": [ + "key" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invitation": { + "name": "invitation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invitation_organizationId_idx": { + "name": "invitation_organizationId_idx", + "columns": [ + "organization_id" + ], + "isUnique": false + }, + "invitation_email_idx": { + "name": "invitation_email_idx", + "columns": [ + "email" + ], + "isUnique": false + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": [ + "inviter_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "member": { + "name": "member", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "member_organizationId_idx": { + "name": "member_organizationId_idx", + "columns": [ + "organization_id" + ], + "isUnique": false + }, + "member_userId_idx": { + "name": "member_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organization": { + "name": "organization", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "banned": { + "name": "banned", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "display_username": { + "name": "display_username", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + }, + "user_username_unique": { + "name": "user_username_unique", + "columns": [ + "username" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/migrations/meta/_journal.json b/migrations/meta/_journal.json index 371f99a7..bdd0389d 100644 --- a/migrations/meta/_journal.json +++ b/migrations/meta/_journal.json @@ -204,6 +204,13 @@ "when": 1778277914288, "tag": "0029_wet_betty_brant", "breakpoints": true + }, + { + "idx": 30, + "version": "6", + "when": 1778282818664, + "tag": "0030_add-cloud-traffic-reports", + "breakpoints": true } ] } \ No newline at end of file diff --git a/server/db/schema.ts b/server/db/schema.ts index 4d945f3b..21f01d0f 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -47,6 +47,28 @@ export const orgQuotas = sqliteTable('org_quotas', { trafficPeriod: text('traffic_period').notNull().default('1970-01'), }) +export const cloudTrafficReports = sqliteTable( + 'cloud_traffic_reports', + { + id: text('id').primaryKey(), + orgId: text('org_id').notNull(), + period: text('period').notNull(), + source: text('source').notNull(), + sourceId: text('source_id').notNull(), + eventId: text('event_id').notNull(), + bytes: integer('bytes').notNull(), + status: text('status').notNull(), + error: text('error'), + createdAt: integer('created_at', { mode: 'timestamp_ms' }).notNull(), + updatedAt: integer('updated_at', { mode: 'timestamp_ms' }).notNull(), + }, + (t) => [ + uniqueIndex('cloud_traffic_reports_event_uniq').on(t.eventId), + index('cloud_traffic_reports_org_period_idx').on(t.orgId, t.period), + index('cloud_traffic_reports_status_idx').on(t.status), + ], +) + export const orgQuotaEntitlements = sqliteTable( 'org_quota_entitlements', { diff --git a/server/middleware/image-hosting-domain.ts b/server/middleware/image-hosting-domain.ts index d0af374d..7865beef 100644 --- a/server/middleware/image-hosting-domain.ts +++ b/server/middleware/image-hosting-domain.ts @@ -4,6 +4,7 @@ import type { Storage as S3Storage } from '../../shared/types' import { imageHostingConfigs } from '../db/schema' import type { Env } from '../middleware/platform' import { PRESIGN_TTL_SECS, s3 } from '../routes/share-utils' +import { reportTrafficForDownload } from '../routes/traffic-metering-utils' import { consumeTrafficIfQuotaAllows, refundTraffic } from '../services/effective-quota' import { getImageByOrgPath, incrementAccessCount, resolveCustomDomain } from '../services/image-hosting' import { getStorage } from '../services/storage' @@ -70,11 +71,24 @@ async function handleImageByPath(c: Context, orgId: string, virtualPath: st let url: string try { url = await s3.presignInline(storage, image.storageKey, image.mime, PRESIGN_TTL_SECS) - await incrementAccessCount(db, image.id) } catch (e) { await refundTraffic(db, image.orgId, image.size) throw e } + + const trafficReportError = await reportTrafficForDownload(c, { + orgId: image.orgId, + bytes: image.size, + source: 'custom_domain_image', + sourceId: image.id, + }) + if (trafficReportError) return trafficReportError + + try { + await incrementAccessCount(db, image.id) + } catch (error) { + console.error('[image-hosting-domain] incrementAccessCount failed:', error) + } const res = c.redirect(url, 302) res.headers.set('Cache-Control', 'no-store') return res diff --git a/server/routes/cloud-traffic-metering.integration.test.ts b/server/routes/cloud-traffic-metering.integration.test.ts new file mode 100644 index 00000000..ae2e4333 --- /dev/null +++ b/server/routes/cloud-traffic-metering.integration.test.ts @@ -0,0 +1,410 @@ +import { sql } from 'drizzle-orm' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { cloudTrafficReports } from '../db/schema' +import { createLicenseBinding } from '../licensing/license-state' +import type { Database } from '../platform/interface' +import { currentTrafficPeriod } from '../services/effective-quota' +import { S3Service } from '../services/s3' +import { createShare } from '../services/share' +import { authedHeaders, createTestApp } from '../test/setup' +import { encodeChildRef } from './share-utils' + +const STORAGE_ID = 'st-cloud-traffic-test' + +beforeEach(() => { + vi.restoreAllMocks() + vi.spyOn(S3Service.prototype, 'presignDownload').mockResolvedValue('https://presigned-download.example.com') + vi.spyOn(S3Service.prototype, 'presignInline').mockResolvedValue('https://presigned-inline.example.com') +}) + +afterEach(() => { + vi.unstubAllGlobals() +}) + +function makeCloudResponse(body: unknown, status = 200): Response { + return { + ok: status >= 200 && status < 300, + status, + json: async () => body, + text: async () => JSON.stringify(body), + } as unknown as Response +} + +function acceptedUsageResponse(_url: string, init?: RequestInit): Response { + const body = JSON.parse(init?.body as string) as { eventId: string } + return makeCloudResponse({ data: { accepted: true, duplicate: false, eventId: body.eventId } }) +} + +async function seedTrafficBinding(db: Database) { + await createLicenseBinding(db, { + cloudBindingId: 'test-binding', + cloudStoreId: 'store-test-binding', + instanceId: 'test-instance', + cloudAccountId: 'test-account', + refreshToken: 'test-refresh-token', + cachedCert: 'test-certificate', + cachedExpiresAt: Math.floor(Date.now() / 1000) + 3600, + lastRefreshAt: Math.floor(Date.now() / 1000), + }) +} + +async function insertStorage(db: Database) { + const now = Date.now() + await db.run(sql` + INSERT INTO storages (id, title, mode, bucket, endpoint, region, access_key, secret_key, file_path, custom_host, capacity, used, status, created_at, updated_at) + VALUES (${STORAGE_ID}, 'Cloud Traffic S3', 'private', 'test-bucket', 'https://s3.amazonaws.com', 'us-east-1', 'AK', 'SK', '', '', 0, 0, 'active', ${now}, ${now}) + `) +} + +async function getOrgId(db: Database): Promise { + const rows = await db.all<{ id: string }>( + sql`SELECT id FROM organization WHERE metadata LIKE '%"type":"personal"%' LIMIT 1`, + ) + return rows[0].id +} + +async function getUserId(db: Database): Promise { + const rows = await db.all<{ id: string }>(sql`SELECT id FROM user LIMIT 1`) + return rows[0].id +} + +async function insertFile(db: Database, orgId: string, id: string) { + const now = Date.now() + await db.run(sql` + INSERT INTO matters (id, org_id, alias, name, type, size, dirtype, parent, object, storage_id, status, created_at, updated_at) + VALUES (${id}, ${orgId}, ${`${id}-alias`}, 'download.txt', 'text/plain', 100, 0, '', 'some/key.txt', ${STORAGE_ID}, 'active', ${now}, ${now}) + `) +} + +async function insertImage(db: Database, orgId: string, id: string, token: string, path = `blog/${id}.png`) { + const now = Date.now() + await db.run(sql` + INSERT INTO image_hostings (id, org_id, token, path, storage_id, storage_key, size, mime, status, access_count, created_at) + VALUES (${id}, ${orgId}, ${token}, ${path}, ${STORAGE_ID}, ${`ih/${orgId}/${id}.png`}, 100, 'image/png', 'active', 0, ${now}) + `) +} + +async function insertImageConfig(db: Database, orgId: string, customDomain?: string) { + const now = Date.now() + const verifiedAt = customDomain ? now : null + await db.run(sql` + INSERT OR REPLACE INTO image_hosting_configs (org_id, custom_domain, domain_verified_at, referer_allowlist, created_at, updated_at) + VALUES (${orgId}, ${customDomain ?? null}, ${verifiedAt}, null, ${now}, ${now}) + `) +} + +async function setTrafficQuota(db: Database, orgId: string) { + await db.run(sql` + UPDATE org_quotas + SET traffic_quota = 500, traffic_used = 25, traffic_period = ${currentTrafficPeriod()} + WHERE org_id = ${orgId} + `) +} + +describe('object download cloud traffic reporting', () => { + it('reports successful object downloads to Cloud after presigning', async () => { + const { app, db } = await createTestApp({ ZPAN_CLOUD_URL: 'https://cloud.example' }) + await seedTrafficBinding(db) + vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse)) + const headers = await authedHeaders(app) + await insertStorage(db) + const orgId = await getOrgId(db) + await insertFile(db, orgId, 'm-cloud-report-ok') + await setTrafficQuota(db, orgId) + + const res = await app.request('/api/objects/m-cloud-report-ok', { headers }) + + expect(res.status).toBe(200) + expect(fetch).toHaveBeenCalledTimes(1) + const [, init] = vi.mocked(fetch).mock.calls[0] as [string, RequestInit] + expect(JSON.parse(init.body as string)).toMatchObject({ resource: 'traffic_egress', bytes: 100, endUserId: orgId }) + await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + { orgId, source: 'object_download', sourceId: 'm-cloud-report-ok', bytes: 100, status: 'reported' }, + ]) + }) + + it('refunds local traffic and denies the download when Cloud blocks usage', async () => { + const { app, db } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(makeCloudResponse({ error: { code: 'overage_cap_exceeded' } }, 429)), + ) + const headers = await authedHeaders(app) + await insertStorage(db) + const orgId = await getOrgId(db) + await insertFile(db, orgId, 'm-cloud-report-blocked') + await setTrafficQuota(db, orgId) + + const res = await app.request('/api/objects/m-cloud-report-blocked', { headers }) + + expect(res.status).toBe(429) + await expect(res.json()).resolves.toEqual({ error: 'Cloud traffic overage cap exceeded' }) + const rows = await db.all<{ trafficUsed: number }>( + sql`SELECT traffic_used AS trafficUsed FROM org_quotas WHERE org_id = ${orgId}`, + ) + expect(rows[0].trafficUsed).toBe(25) + await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'blocked' }]) + }) + + it('refunds local traffic when Cloud returns a mismatched event id', async () => { + const { app, db } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(makeCloudResponse({ data: { accepted: true, duplicate: false, eventId: 'wrong' } })), + ) + const headers = await authedHeaders(app) + await insertStorage(db) + const orgId = await getOrgId(db) + await insertFile(db, orgId, 'm-cloud-report-mismatch') + await setTrafficQuota(db, orgId) + + const res = await app.request('/api/objects/m-cloud-report-mismatch', { headers }) + + expect(res.status).toBe(500) + const rows = await db.all<{ trafficUsed: number }>( + sql`SELECT traffic_used AS trafficUsed FROM org_quotas WHERE org_id = ${orgId}`, + ) + expect(rows[0].trafficUsed).toBe(25) + await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'failed' }]) + }) + + it('does not report usage when presign fails and local traffic is refunded', async () => { + const { app, db } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse)) + vi.mocked(S3Service.prototype.presignDownload).mockRejectedValueOnce(new Error('sign failed')) + const headers = await authedHeaders(app) + await insertStorage(db) + const orgId = await getOrgId(db) + await insertFile(db, orgId, 'm-cloud-report-presign-fail') + await setTrafficQuota(db, orgId) + + const res = await app.request('/api/objects/m-cloud-report-presign-fail', { headers }) + + expect(res.status).toBe(500) + expect(fetch).not.toHaveBeenCalled() + const rows = await db.all<{ trafficUsed: number }>( + sql`SELECT traffic_used AS trafficUsed FROM org_quotas WHERE org_id = ${orgId}`, + ) + expect(rows[0].trafficUsed).toBe(25) + await expect(db.select().from(cloudTrafficReports)).resolves.toHaveLength(0) + }) +}) + +describe('public redirect cloud traffic reporting', () => { + it('reports direct share redirects to Cloud', async () => { + const { app, db } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse)) + await authedHeaders(app) + await insertStorage(db) + const orgId = await getOrgId(db) + const creatorId = await getUserId(db) + await insertFile(db, orgId, 'm-cloud-direct-share') + const share = await createShare(db, { matterId: 'm-cloud-direct-share', orgId, creatorId, kind: 'direct' }) + + const res = await app.request(`/r/${share.token}`, { redirect: 'manual' }) + + expect(res.status).toBe(302) + await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + { source: 'direct_share', sourceId: share.id, bytes: 100, status: 'reported' }, + ]) + }) + + it('refunds direct share traffic and downloads when Cloud blocks usage', async () => { + const { app, db } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(makeCloudResponse({ error: { code: 'overage_cap_exceeded' } }, 429)), + ) + await authedHeaders(app) + await insertStorage(db) + const orgId = await getOrgId(db) + const creatorId = await getUserId(db) + await insertFile(db, orgId, 'm-cloud-direct-blocked') + await setTrafficQuota(db, orgId) + const share = await createShare(db, { matterId: 'm-cloud-direct-blocked', orgId, creatorId, kind: 'direct' }) + + const res = await app.request(`/r/${share.token}`, { redirect: 'manual' }) + + expect(res.status).toBe(429) + await expect(res.json()).resolves.toEqual({ error: 'Cloud traffic overage cap exceeded' }) + const rows = await db.all<{ downloads: number; trafficUsed: number }>(sql` + SELECT s.downloads, q.traffic_used AS trafficUsed + FROM shares s + INNER JOIN org_quotas q ON q.org_id = s.org_id + WHERE s.id = ${share.id} + `) + expect(rows[0]).toEqual({ downloads: 0, trafficUsed: 25 }) + await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'blocked' }]) + }) + + it('reports landing share downloads to Cloud', async () => { + const { app, db } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse)) + await authedHeaders(app) + await insertStorage(db) + const orgId = await getOrgId(db) + const creatorId = await getUserId(db) + await insertFile(db, orgId, 'm-cloud-landing-share') + const share = await createShare(db, { matterId: 'm-cloud-landing-share', orgId, creatorId, kind: 'landing' }) + const ref = encodeChildRef(share.token, 'm-cloud-landing-share') + + const res = await app.request(`/api/shares/${share.token}/objects/${ref}?downloadUrl=1`, { redirect: 'manual' }) + + expect(res.status).toBe(200) + await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + { source: 'landing_share', sourceId: share.id, bytes: 100, status: 'reported' }, + ]) + }) + + it('refunds landing share traffic and downloads when Cloud blocks usage', async () => { + const { app, db } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(makeCloudResponse({ error: { code: 'overage_cap_exceeded' } }, 429)), + ) + await authedHeaders(app) + await insertStorage(db) + const orgId = await getOrgId(db) + const creatorId = await getUserId(db) + await insertFile(db, orgId, 'm-cloud-landing-blocked') + await setTrafficQuota(db, orgId) + const share = await createShare(db, { matterId: 'm-cloud-landing-blocked', orgId, creatorId, kind: 'landing' }) + const ref = encodeChildRef(share.token, 'm-cloud-landing-blocked') + + const res = await app.request(`/api/shares/${share.token}/objects/${ref}?downloadUrl=1`, { redirect: 'manual' }) + + expect(res.status).toBe(429) + await expect(res.json()).resolves.toEqual({ error: 'Cloud traffic overage cap exceeded' }) + const rows = await db.all<{ downloads: number; trafficUsed: number }>(sql` + SELECT s.downloads, q.traffic_used AS trafficUsed + FROM shares s + INNER JOIN org_quotas q ON q.org_id = s.org_id + WHERE s.id = ${share.id} + `) + expect(rows[0]).toEqual({ downloads: 0, trafficUsed: 25 }) + await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'blocked' }]) + }) + + it('still returns landing share URLs when audit recording fails after Cloud report', async () => { + const { app, db } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse)) + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined) + await authedHeaders(app) + await insertStorage(db) + const orgId = await getOrgId(db) + const creatorId = await getUserId(db) + await insertFile(db, orgId, 'm-cloud-landing-audit-fail') + const share = await createShare(db, { matterId: 'm-cloud-landing-audit-fail', orgId, creatorId, kind: 'landing' }) + const ref = encodeChildRef(share.token, 'm-cloud-landing-audit-fail') + await db.run(sql`DROP TABLE activity_events`) + + const res = await app.request(`/api/shares/${share.token}/objects/${ref}?downloadUrl=1`, { redirect: 'manual' }) + + expect(res.status).toBe(200) + expect(consoleError).toHaveBeenCalled() + }) + + it('reports token image-hosting redirects to Cloud', async () => { + const { app, db } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse)) + await authedHeaders(app) + await insertStorage(db) + const orgId = await getOrgId(db) + await insertImage(db, orgId, 'ih-cloud-token', 'ih_cloudtoken') + await insertImageConfig(db, orgId) + + const res = await app.request('/r/ih_cloudtoken', { redirect: 'manual' }) + + expect(res.status).toBe(302) + await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + { source: 'image_hosting', sourceId: 'ih-cloud-token', bytes: 100, status: 'reported' }, + ]) + }) + + it('still redirects token images when access-count recording fails after Cloud report', async () => { + const { app, db } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse)) + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined) + await authedHeaders(app) + await insertStorage(db) + const orgId = await getOrgId(db) + await insertImage(db, orgId, 'ih-cloud-log-fail', 'ih_cloudlogfail') + await insertImageConfig(db, orgId) + vi.spyOn(db, 'run').mockRejectedValue(new Error('access failed')) + + const res = await app.request('/r/ih_cloudlogfail', { redirect: 'manual' }) + + expect(res.status).toBe(302) + expect(consoleError).toHaveBeenCalled() + }) + + it('does not report token image usage when inline presigning fails', async () => { + const { app, db } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse)) + vi.mocked(S3Service.prototype.presignInline).mockRejectedValueOnce(new Error('inline sign failed')) + await authedHeaders(app) + await insertStorage(db) + const orgId = await getOrgId(db) + await insertImage(db, orgId, 'ih-cloud-presign-fail', 'ih_cloudpresignfail') + await insertImageConfig(db, orgId) + + const res = await app.request('/r/ih_cloudpresignfail', { redirect: 'manual' }) + + expect(res.status).toBe(500) + expect(fetch).not.toHaveBeenCalled() + await expect(db.select().from(cloudTrafficReports)).resolves.toHaveLength(0) + }) + + it('reports custom-domain image-hosting redirects to Cloud', async () => { + const { app, db } = await createTestApp({ PUBLIC_APP_HOST: 'zpan.example.com' }) + await seedTrafficBinding(db) + vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse)) + await authedHeaders(app) + await insertStorage(db) + const orgId = await getOrgId(db) + await insertImage(db, orgId, 'ih-cloud-domain', 'ih_clouddomain', 'blog/domain.png') + await insertImageConfig(db, orgId, 'img.example.com') + + const res = await app.request('https://img.example.com/blog/domain.png', { + headers: { host: 'img.example.com' }, + redirect: 'manual', + }) + + expect(res.status).toBe(302) + await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + { source: 'custom_domain_image', sourceId: 'ih-cloud-domain', bytes: 100, status: 'reported' }, + ]) + }) + + it('still redirects custom-domain images when access-count recording fails after Cloud report', async () => { + const { app, db } = await createTestApp({ PUBLIC_APP_HOST: 'zpan.example.com' }) + await seedTrafficBinding(db) + vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse)) + const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined) + await authedHeaders(app) + await insertStorage(db) + const orgId = await getOrgId(db) + await insertImage(db, orgId, 'ih-cloud-domain-log-fail', 'ih_clouddomainlogfail', 'blog/domain-log-fail.png') + await insertImageConfig(db, orgId, 'img.example.com') + vi.spyOn(db, 'run').mockRejectedValue(new Error('access failed')) + + const res = await app.request('https://img.example.com/blog/domain-log-fail.png', { + headers: { host: 'img.example.com' }, + redirect: 'manual', + }) + + expect(res.status).toBe(302) + expect(consoleError).toHaveBeenCalled() + }) +}) diff --git a/server/routes/objects.ts b/server/routes/objects.ts index e81c6db3..312f4050 100644 --- a/server/routes/objects.ts +++ b/server/routes/objects.ts @@ -34,6 +34,7 @@ import { buildObjectKey } from '../services/path-template' import { purgeRecursively } from '../services/purge' import { S3Service } from '../services/s3' import { getStorage, selectStorage } from '../services/storage' +import { reportTrafficForDownload } from './traffic-metering-utils' const s3 = new S3Service() @@ -206,6 +207,14 @@ const app = new Hono() throw e } + const trafficReportError = await reportTrafficForDownload(c, { + orgId, + bytes: matter.size ?? 0, + source: 'object_download', + sourceId: matter.id, + }) + if (trafficReportError) return trafficReportError + return c.json({ ...matter, downloadUrl }) }) .patch('/:id', requireTeamRole('editor'), zValidator('json', patchMatterSchema), async (c) => { diff --git a/server/routes/redirect.ts b/server/routes/redirect.ts index a9ec3946..ab3e473d 100644 --- a/server/routes/redirect.ts +++ b/server/routes/redirect.ts @@ -13,6 +13,7 @@ import { } from '../services/share' import { getStorage } from '../services/storage' import { PRESIGN_TTL_SECS, s3 } from './share-utils' +import { reportTrafficForDownload } from './traffic-metering-utils' // Strip optional file extension from token (e.g. "ih_aB3xK9.png" → "ih_aB3xK9") function stripExtension(token: string): string { @@ -69,6 +70,15 @@ async function handleDirectShare(c: Context, db: Database, token: string): throw e } + const trafficReportError = await reportTrafficForDownload(c, { + orgId: share.orgId, + bytes: matter.size ?? 0, + source: 'direct_share', + sourceId: share.id, + onRejected: () => decrementDownloads(db, share.id), + }) + if (trafficReportError) return trafficReportError + const res = c.redirect(url, 302) res.headers.set('Cache-Control', 'no-store') return res @@ -98,12 +108,24 @@ async function handleImageHosting(c: Context, db: Database, token: string): let url: string try { url = await s3.presignInline(storage, image.storageKey, image.mime, PRESIGN_TTL_SECS) - await incrementAccessCount(db, image.id) } catch (e) { await refundTraffic(db, image.orgId, image.size) throw e } + const trafficReportError = await reportTrafficForDownload(c, { + orgId: image.orgId, + bytes: image.size, + source: 'image_hosting', + sourceId: image.id, + }) + if (trafficReportError) return trafficReportError + + try { + await incrementAccessCount(db, image.id) + } catch (error) { + console.error('[redirect] incrementAccessCount failed:', error) + } const res = c.redirect(url, 302) res.headers.set('Cache-Control', 'no-store') return res diff --git a/server/routes/shares.ts b/server/routes/shares.ts index f67bc7e1..04ce0a9b 100644 --- a/server/routes/shares.ts +++ b/server/routes/shares.ts @@ -47,6 +47,7 @@ import { s3, viewCookieName, } from './share-utils' +import { reportTrafficForDownload } from './traffic-metering-utils' const ROLE_LEVELS: Record = { owner: 3, editor: 2, viewer: 1, member: 1 } @@ -289,6 +290,22 @@ export const publicShares = new Hono() let url: string try { url = await s3.presignDownload(storage, targetMatter.object, targetMatter.name, PRESIGN_TTL_SECS) + } catch (e) { + await refundTraffic(db, share.orgId, targetMatter.size ?? 0) + await decrementDownloads(db, share.id) + throw e + } + + const trafficReportError = await reportTrafficForDownload(c, { + orgId: share.orgId, + bytes: targetMatter.size ?? 0, + source: 'landing_share', + sourceId: share.id, + onRejected: () => decrementDownloads(db, share.id), + }) + if (trafficReportError) return trafficReportError + + try { await recordActivity(db, { orgId: share.orgId, userId: actorId, @@ -298,10 +315,8 @@ export const publicShares = new Hono() targetName: targetMatter.name, metadata: { anonymous: !viewerId }, }) - } catch (e) { - await refundTraffic(db, share.orgId, targetMatter.size ?? 0) - await decrementDownloads(db, share.id) - throw e + } catch (error) { + console.error('[shares] recordActivity failed:', error) } if (returnUrl) { diff --git a/server/routes/traffic-metering-utils.ts b/server/routes/traffic-metering-utils.ts new file mode 100644 index 00000000..6a56d3ee --- /dev/null +++ b/server/routes/traffic-metering-utils.ts @@ -0,0 +1,37 @@ +import type { Context } from 'hono' +import type { Env } from '../middleware/platform' +import { + CloudTrafficBlockedError, + reportTrafficEgress, + type TrafficReportSource, +} from '../services/cloud-traffic-metering' +import { refundTraffic } from '../services/effective-quota' + +export async function reportTrafficForDownload( + c: Context, + params: { + orgId: string + bytes: number + source: TrafficReportSource + sourceId: string + onRejected?: () => Promise + }, +): Promise { + try { + await reportTrafficEgress({ + platform: c.get('platform'), + orgId: params.orgId, + bytes: params.bytes, + source: params.source, + sourceId: params.sourceId, + }) + return null + } catch (error) { + await refundTraffic(c.get('platform').db, params.orgId, params.bytes) + await params.onRejected?.() + if (error instanceof CloudTrafficBlockedError) { + return c.json({ error: 'Cloud traffic overage cap exceeded' }, 429) + } + throw error + } +} diff --git a/server/services/cloud-traffic-metering.test.ts b/server/services/cloud-traffic-metering.test.ts new file mode 100644 index 00000000..fac28bb3 --- /dev/null +++ b/server/services/cloud-traffic-metering.test.ts @@ -0,0 +1,261 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { cloudTrafficReports } from '../db/schema' +import { createLicenseBinding } from '../licensing/license-state' +import type { Database } from '../platform/interface' +import { createTestApp } from '../test/setup' +import { CloudTrafficBlockedError, reportTrafficEgress } from './cloud-traffic-metering' + +function makeResponse(body: unknown, status = 200): Response { + return { + ok: status >= 200 && status < 300, + status, + json: async () => body, + text: async () => JSON.stringify(body), + } as unknown as Response +} + +async function seedTrafficBinding(db: Database) { + await createLicenseBinding(db, { + cloudBindingId: 'test-binding', + cloudStoreId: 'store-test-binding', + instanceId: 'test-instance', + cloudAccountId: 'test-account', + refreshToken: 'test-refresh-token', + cachedCert: 'test-certificate', + cachedExpiresAt: Math.floor(Date.now() / 1000) + 3600, + lastRefreshAt: Math.floor(Date.now() / 1000), + }) +} + +describe('cloud traffic metering', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('reports traffic egress through the active license binding', async () => { + const { db, platform } = await createTestApp({ ZPAN_CLOUD_URL: 'https://cloud.example' }) + await seedTrafficBinding(db) + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(makeResponse({ data: { accepted: true, duplicate: false, eventId: 'evt_1' } })), + ) + + await reportTrafficEgress({ + platform, + orgId: 'org_1', + bytes: 1024, + source: 'object_download', + sourceId: 'matter_1', + eventId: 'evt_1', + }) + + expect(fetch).toHaveBeenCalledTimes(1) + const [url, init] = vi.mocked(fetch).mock.calls[0] as [string, RequestInit] + expect(url).toBe('https://cloud.example/api/usage-events') + expect(init.headers).toMatchObject({ Authorization: 'Bearer test-refresh-token' }) + expect(JSON.parse(init.body as string)).toMatchObject({ + resource: 'traffic_egress', + bytes: 1024, + eventId: 'evt_1', + idempotencyKey: 'evt_1', + endUserId: 'org_1', + }) + await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'reported' }]) + }) + + it('ignores zero-byte reports without writing local state', async () => { + const { db, platform } = await createTestApp() + vi.stubGlobal('fetch', vi.fn()) + + const result = await reportTrafficEgress({ + platform, + orgId: 'org_1', + bytes: 0, + source: 'object_download', + sourceId: 'matter_1', + }) + + expect(result).toMatchObject({ status: 'reported', eventId: '', duplicate: false }) + expect(fetch).not.toHaveBeenCalled() + await expect(db.select().from(cloudTrafficReports)).resolves.toHaveLength(0) + }) + + it('keeps idempotent reports local after the first successful report', async () => { + const { db, platform } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(makeResponse({ data: { accepted: true, duplicate: false, eventId: 'evt_dup' } })), + ) + + const input = { + platform, + orgId: 'org_1', + bytes: 1024, + source: 'direct_share' as const, + sourceId: 'share_1', + eventId: 'evt_dup', + } + const first = await reportTrafficEgress(input) + const second = await reportTrafficEgress(input) + + expect(first.duplicate).toBe(false) + expect(second).toMatchObject({ duplicate: true, eventId: 'evt_dup', status: 'reported' }) + expect(fetch).toHaveBeenCalledTimes(1) + await expect(db.select().from(cloudTrafficReports)).resolves.toHaveLength(1) + }) + + it('rejects idempotency conflicts for reused event ids', async () => { + const { db, platform } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(makeResponse({ data: { accepted: true, duplicate: false, eventId: 'evt_conflict' } })), + ) + + await reportTrafficEgress({ + platform, + orgId: 'org_1', + bytes: 1024, + source: 'direct_share', + sourceId: 'share_1', + eventId: 'evt_conflict', + }) + + await expect( + reportTrafficEgress({ + platform, + orgId: 'org_1', + bytes: 2048, + source: 'direct_share', + sourceId: 'share_1', + eventId: 'evt_conflict', + }), + ).rejects.toThrow('traffic_report_idempotency_conflict') + }) + + it('surfaces Cloud cap rejection and records a blocked report', async () => { + const { db, platform } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(makeResponse({ error: { code: 'overage_cap_exceeded' } }, 429))) + + await expect( + reportTrafficEgress({ + platform, + orgId: 'org_1', + bytes: 1024, + source: 'landing_share', + sourceId: 'share_1', + eventId: 'evt_blocked', + }), + ).rejects.toThrow(CloudTrafficBlockedError) + + await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + { eventId: 'evt_blocked', status: 'blocked', error: 'overage_cap_exceeded' }, + ]) + await expect( + reportTrafficEgress({ + platform, + orgId: 'org_1', + bytes: 1024, + source: 'landing_share', + sourceId: 'share_1', + eventId: 'evt_blocked', + }), + ).rejects.toThrow(CloudTrafficBlockedError) + }) + + it('retries failed report ids instead of treating them as completed duplicates', async () => { + const { db, platform } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockRejectedValueOnce(new Error('offline')) + .mockResolvedValueOnce(makeResponse({ data: { accepted: true, duplicate: false, eventId: 'evt_retry' } })), + ) + + const input = { + platform, + orgId: 'org_1', + bytes: 1024, + source: 'object_download' as const, + sourceId: 'matter_1', + eventId: 'evt_retry', + now: new Date('2026-04-30T23:59:00.000Z'), + } + await expect(reportTrafficEgress(input)).rejects.toThrow('offline') + + const result = await reportTrafficEgress({ ...input, now: new Date('2026-05-01T00:01:00.000Z') }) + + expect(result).toMatchObject({ status: 'reported', duplicate: false }) + expect(fetch).toHaveBeenCalledTimes(2) + await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + { status: 'reported', error: null, period: '2026-04' }, + ]) + }) + + it('fails when Cloud does not accept the same event id', async () => { + const { db, platform } = await createTestApp() + await seedTrafficBinding(db) + vi.stubGlobal( + 'fetch', + vi.fn().mockResolvedValue(makeResponse({ data: { accepted: true, duplicate: false, eventId: 'other_evt' } })), + ) + + await expect( + reportTrafficEgress({ + platform, + orgId: 'org_1', + bytes: 1024, + source: 'object_download', + sourceId: 'matter_1', + eventId: 'evt_mismatch', + }), + ).rejects.toThrow('cloud_usage_report_rejected') + + await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([ + { eventId: 'evt_mismatch', status: 'failed', error: 'cloud_usage_report_rejected' }, + ]) + }) + + it('records skipped reporting when no active binding exists', async () => { + const { db, platform } = await createTestApp() + vi.stubGlobal('fetch', vi.fn()) + + const result = await reportTrafficEgress({ + platform, + orgId: 'org_1', + bytes: 1024, + source: 'image_hosting', + sourceId: 'image_1', + eventId: 'evt_unbound', + }) + + expect(result.status).toBe('skipped_unbound') + expect(fetch).not.toHaveBeenCalled() + await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'skipped_unbound' }]) + }) + + it('keeps unbound report replays local', async () => { + const { db, platform } = await createTestApp() + vi.stubGlobal('fetch', vi.fn()) + + const input = { + platform, + orgId: 'org_1', + bytes: 1024, + source: 'image_hosting' as const, + sourceId: 'image_1', + eventId: 'evt_unbound_dup', + } + const first = await reportTrafficEgress(input) + const second = await reportTrafficEgress(input) + + expect(first.duplicate).toBe(false) + expect(second).toMatchObject({ status: 'skipped_unbound', eventId: 'evt_unbound_dup', duplicate: true }) + expect(fetch).not.toHaveBeenCalled() + await expect(db.select().from(cloudTrafficReports)).resolves.toHaveLength(1) + }) +}) diff --git a/server/services/cloud-traffic-metering.ts b/server/services/cloud-traffic-metering.ts new file mode 100644 index 00000000..38ed6066 --- /dev/null +++ b/server/services/cloud-traffic-metering.ts @@ -0,0 +1,151 @@ +import { eq } from 'drizzle-orm' +import { nanoid } from 'nanoid' +import { z } from 'zod' +import { ZPAN_CLOUD_URL_DEFAULT } from '../../shared/constants' +import { cloudTrafficReports } from '../db/schema' +import { loadActiveLicenseBinding } from '../licensing/license-state' +import type { Platform } from '../platform/interface' +import { currentTrafficPeriod } from './effective-quota' +import { postBoundCloudJson } from './licensing-cloud' + +export type TrafficReportSource = + | 'object_download' + | 'direct_share' + | 'landing_share' + | 'image_hosting' + | 'custom_domain_image' + +export class CloudTrafficBlockedError extends Error { + constructor() { + super('cloud_traffic_blocked') + this.name = 'CloudTrafficBlockedError' + } +} + +const usageResponseSchema = z.object({ + accepted: z.boolean(), + duplicate: z.boolean(), + eventId: z.string().min(1), +}) + +type ReportStatus = 'pending' | 'reported' | 'skipped_unbound' | 'blocked' | 'failed' + +export async function reportTrafficEgress(params: { + platform: Platform + orgId: string + bytes: number + source: TrafficReportSource + sourceId: string + eventId?: string + now?: Date +}): Promise<{ status: ReportStatus; eventId: string; duplicate: boolean }> { + const { platform, orgId, bytes, source, sourceId, now = new Date() } = params + if (bytes <= 0) return { status: 'reported', eventId: params.eventId ?? '', duplicate: false } + + const eventId = params.eventId ?? `traffic_${nanoid()}` + const existing = await loadTrafficReport(platform.db, eventId) + const period = existing?.period ?? currentTrafficPeriod(now) + if (existing) { + assertSameReport(existing, { orgId, period, source, sourceId, bytes }) + if (existing.status === 'reported' || existing.status === 'skipped_unbound') { + return { status: existing.status as ReportStatus, eventId, duplicate: true } + } + if (existing.status === 'blocked') throw new CloudTrafficBlockedError() + } else { + await insertTrafficReport(platform, { orgId, period, source, sourceId, eventId, bytes, status: 'pending', now }) + } + + const binding = await loadActiveLicenseBinding(platform.db) + if (!binding?.refreshToken) { + await updateTrafficReport(platform, eventId, 'skipped_unbound', null, now) + return { status: 'skipped_unbound', eventId, duplicate: false } + } + + try { + const data = await postBoundCloudJson( + platform.getEnv('ZPAN_CLOUD_URL') ?? ZPAN_CLOUD_URL_DEFAULT, + '/api/usage-events', + binding.refreshToken, + { + resource: 'traffic_egress', + bytes, + eventId, + idempotencyKey: eventId, + endUserId: orgId, + }, + ) + const response = usageResponseSchema.parse(data) + if (!response.accepted || response.eventId !== eventId) throw new Error('cloud_usage_report_rejected') + await updateTrafficReport(platform, eventId, 'reported', null, now) + return { status: 'reported', eventId, duplicate: false } + } catch (error) { + const message = error instanceof Error ? error.message : 'cloud_usage_report_failed' + if (message === 'overage_cap_exceeded') { + await updateTrafficReport(platform, eventId, 'blocked', message, now) + throw new CloudTrafficBlockedError() + } + await updateTrafficReport(platform, eventId, 'failed', message, now) + throw error + } +} + +async function loadTrafficReport(db: Platform['db'], eventId: string) { + const rows = await db.select().from(cloudTrafficReports).where(eq(cloudTrafficReports.eventId, eventId)).limit(1) + return rows[0] +} + +function assertSameReport( + report: typeof cloudTrafficReports.$inferSelect, + params: { orgId: string; period: string; source: TrafficReportSource; sourceId: string; bytes: number }, +) { + if ( + report.orgId !== params.orgId || + report.period !== params.period || + report.source !== params.source || + report.sourceId !== params.sourceId || + report.bytes !== params.bytes + ) { + throw new Error('traffic_report_idempotency_conflict') + } +} + +async function insertTrafficReport( + platform: Platform, + params: { + orgId: string + period: string + source: TrafficReportSource + sourceId: string + eventId: string + bytes: number + status: ReportStatus + now: Date + }, +) { + await platform.db.insert(cloudTrafficReports).values({ + id: nanoid(), + orgId: params.orgId, + period: params.period, + source: params.source, + sourceId: params.sourceId, + eventId: params.eventId, + bytes: params.bytes, + status: params.status, + error: null, + createdAt: params.now, + updatedAt: params.now, + }) +} + +async function updateTrafficReport( + platform: Platform, + eventId: string, + status: ReportStatus, + error: string | null, + now: Date, +) { + await platform.db + .update(cloudTrafficReports) + .set({ status, error, updatedAt: now }) + .where(eq(cloudTrafficReports.eventId, eventId)) +} diff --git a/server/test/setup.ts b/server/test/setup.ts index 121a24b5..7e60071d 100644 --- a/server/test/setup.ts +++ b/server/test/setup.ts @@ -137,6 +137,22 @@ const APP_SCHEMA_SQL = ` traffic_used INTEGER NOT NULL DEFAULT 0, traffic_period TEXT NOT NULL DEFAULT '1970-01' ); + CREATE TABLE IF NOT EXISTS cloud_traffic_reports ( + id TEXT PRIMARY KEY, + org_id TEXT NOT NULL, + period TEXT NOT NULL, + source TEXT NOT NULL, + source_id TEXT NOT NULL, + event_id TEXT NOT NULL, + bytes INTEGER NOT NULL, + status TEXT NOT NULL, + error TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS cloud_traffic_reports_event_uniq ON cloud_traffic_reports(event_id); + CREATE INDEX IF NOT EXISTS cloud_traffic_reports_org_period_idx ON cloud_traffic_reports(org_id, period); + CREATE INDEX IF NOT EXISTS cloud_traffic_reports_status_idx ON cloud_traffic_reports(status); CREATE TABLE IF NOT EXISTS org_quota_entitlements ( id TEXT PRIMARY KEY, org_id TEXT NOT NULL,