From 3abf4f231e66fdd35f9b7428aa8a334aa2ef171b Mon Sep 17 00:00:00 2001 From: saltbo Date: Tue, 21 Jul 2026 00:29:27 -0400 Subject: [PATCH] fix(storage): preserve purge history and track usage deltas Retain purged matter and image metadata while hiding tombstones from live reads. Record exact storage deltas and build hourly trends only after the opening balance. --- docs/design/admin-analytics.md | 11 +- migrations/0059_storage-usage-ledger.sql | 21 + migrations/meta/0059_snapshot.json | 4366 +++++++++++++++++ migrations/meta/_journal.json | 7 + scripts/backfill-admin-stats.ts | 55 +- .../admin-stats-rollup.integration.test.ts | 3 +- server/adapters/repos/admin-stats-rollup.ts | 73 +- server/adapters/repos/admin-stats.ts | 20 +- .../adapters/repos/archive-target-folder.ts | 1 + server/adapters/repos/image-hosting.ts | 102 +- .../adapters/repos/matter.integration.test.ts | 10 + server/adapters/repos/matter.ts | 184 +- server/adapters/repos/share.ts | 11 +- server/adapters/repos/storage-usage-ledger.ts | 230 + server/adapters/repos/storage-usage.ts | 4 + server/adapters/repos/storage.ts | 7 +- server/adapters/repos/webdav-path.ts | 2 + server/adapters/repos/zip.ts | 3 +- server/db/schema.ts | 25 +- server/db/transaction.ts | 18 + server/domain/admin-stats-metrics.ts | 2 + server/http/admin-stats.integration.test.ts | 4 +- .../image-hosting/images.integration.test.ts | 18 +- server/http/objects.integration.test.ts | 35 +- server/http/webdav.integration.test.ts | 6 + server/scripts/backfill-admin-stats.test.ts | 5 + server/test/setup.ts | 22 +- server/usecases/archive-processing.test.ts | 4 +- .../downloads/download-folders.test.ts | 1 + server/usecases/object.integration.test.ts | 1 + server/usecases/object.test.ts | 1 + server/usecases/object.ts | 11 +- server/usecases/ports/image-hosting.ts | 1 + server/usecases/ports/matter.ts | 2 +- server/usecases/webdav.test.ts | 1 + shared/types/admin-stats.ts | 2 + src/routes/_authenticated/admin/analytics.tsx | 14 +- 37 files changed, 5182 insertions(+), 101 deletions(-) create mode 100644 migrations/0059_storage-usage-ledger.sql create mode 100644 migrations/meta/0059_snapshot.json create mode 100644 server/adapters/repos/storage-usage-ledger.ts diff --git a/docs/design/admin-analytics.md b/docs/design/admin-analytics.md index f9d3ed8a..9556c6de 100644 --- a/docs/design/admin-analytics.md +++ b/docs/design/admin-analytics.md @@ -15,13 +15,15 @@ The dashboard is read-only and does not support report or CSV export. Metrics derived from activity facts use a validated producer contract. New transfer and sharing facts fail fast when required fields such as `bytes`, `source`, `trafficEventId`, or `shareId` are missing. User signup, share creation, background-job completion, and remote-download completion each write a deterministic immutable stats fact in the same transaction as the business change. Deleting the mutable user/share/job/task row therefore does not rewrite history. Historical rows that predate this contract remain diagnosable through quality metrics. +Storage accounting uses the typed `storage_usage_ledger`, not activity-event JSON. Before the first mutation, each organization/storage pair receives an exact opening balance from currently billable matter and image rows. Activating, resizing, or purging content writes a signed byte delta with the resource id and reason alongside the resource state change. Purged files and images keep a `purged_at` tombstone; normal readers exclude them, while their identity and accounting history remain available for audit. + A deduplicated share view updates `shares.views` and inserts its `share_view` fact in one database transaction. Download flows record an issued fact only after quota/metering and presigning succeed; a failed fact records the stable reason and traffic event id. If the issued-fact write fails, the flow refunds the counters it already reserved. ## Result writes Metric and dimension combinations are declared in `server/domain/admin-stats-metrics.ts`: -- Counters are additive event totals for a closed hour. +- Counter-scope results are rebuilt from immutable facts for a closed hour. Most are additive event totals; `storage.ledger_balance` is the exact cumulative balance at the hour boundary. - Gauges are point-in-time snapshots captured during the current open hour and continuously replaced in that same bucket. - High-volume source rows are grouped by the database before the Worker builds result rows; the scheduler does not load a whole hour of raw events into memory. @@ -62,10 +64,11 @@ Request-time work is bounded and result-only. Charts and totals read only the ag - Download bytes and counts mean issued downloads, not client-completed transfers. - Cloud traffic report status is a closed-hour snapshot of the current metering queue and belongs to Operations, not download success. Mutable report rows are never modeled as additive counters. - Remote-download outcomes count terminal run transitions. A failed run followed by a successful retry contributes one failure and one completion; deleting the task does not erase either outcome. -- User, storage, share lifecycle, active job/task, downloader, report, and webhook state comes from closed-hour snapshots, not live request-time reads. +- User, storage inventory, share lifecycle, active job/task, downloader, report, and webhook state comes from closed-hour snapshots, not live request-time reads. - Storage inventory includes non-trashed normal files and image-hosting objects. - Quota usage includes files retained in trash until purge. `storage.trash_snapshot` exposes that file and byte total separately, so active-file inventory is not expected to equal the quota waterline by itself. -- `stats.data_quality_snapshot/storage_usage_drift` compares the quota usage ledger with all currently billable database objects. A non-zero space or byte drift is shown as an error instead of silently presenting the waterline as authoritative. +- `storage.ledger_balance` is the exact end-of-hour sum of the opening balance and immutable byte deltas. The storage trend reads this rebuildable result; it remains empty before the opening balance instead of estimating missing history. +- `stats.data_quality_snapshot/storage_usage_drift` compares the live quota counter with all currently billable database objects. `storage_ledger_drift` independently compares the immutable ledger with those objects. A non-zero space or byte drift is shown as an error instead of silently presenting either source as authoritative. - Workspace storage quota is the active paid plan, or the active Free baseline when no paid plan is valid, plus active non-plan grants. Missing or zero effective storage entitlement is invalid and always fails closed; it never means unlimited space. - The ten-minute maintenance cycle idempotently restores missing or accidentally revoked Free baseline entitlements before capturing quota snapshots. It preserves each existing baseline size and uses the configured finite defaults only when a baseline row is absent. - Files older than 90 days are an age cohort, not proven cold data; the UI labels this explicitly. @@ -79,7 +82,7 @@ Legacy share counters can exceed the number of timestamped activity facts. The s Run `pnpm stats:backfill -- --apply ...` after the migration in each environment. The script repairs recoverable historical fact metadata, creates deterministic lower-bound facts for currently recoverable signup/share/job/task history, removes incompatible result versions, and rebuilds historical counters idempotently. It writes a continuous `scope=counters` marker for every closed UTC hour from the first available immutable fact through the latest closed hour, including hours with zero activity, and rejects missing or open-hour markers during validation. Incompatible older snapshots are removed rather than relabeled as current-version facts. -The script does not fabricate historical inventory or active-user snapshots. Snapshot-backed views therefore report partial or empty coverage before full rollups began, while event-backed views can report complete historical coverage. Facts whose original byte size can no longer be recovered remain visible as `quality=lower_bound` instead of being guessed. +The script does not fabricate historical inventory, active-user snapshots, or storage balances from before the ledger opening. Snapshot-backed views therefore report partial or empty coverage before full rollups began, while event-backed views can report complete historical coverage. After the opening balance, storage balances are rebuilt exactly from the immutable ledger. Facts whose original byte size can no longer be recovered remain visible as `quality=lower_bound` instead of being guessed. Validation must cover: diff --git a/migrations/0059_storage-usage-ledger.sql b/migrations/0059_storage-usage-ledger.sql new file mode 100644 index 00000000..f8fd3c87 --- /dev/null +++ b/migrations/0059_storage-usage-ledger.sql @@ -0,0 +1,21 @@ +CREATE TABLE `storage_usage_ledger` ( + `id` text PRIMARY KEY NOT NULL, + `event_key` text NOT NULL, + `org_id` text NOT NULL, + `storage_id` text NOT NULL, + `resource_type` text NOT NULL, + `resource_id` text NOT NULL, + `delta_bytes` integer NOT NULL, + `reason` text NOT NULL, + `occurred_at` integer NOT NULL, + `created_at` integer NOT NULL +); +--> statement-breakpoint +CREATE UNIQUE INDEX `storage_usage_ledger_event_key_unique` ON `storage_usage_ledger` (`event_key`);--> statement-breakpoint +CREATE INDEX `storage_usage_ledger_occurred_idx` ON `storage_usage_ledger` (`occurred_at`);--> statement-breakpoint +CREATE INDEX `storage_usage_ledger_org_occurred_idx` ON `storage_usage_ledger` (`org_id`,`occurred_at`);--> statement-breakpoint +CREATE INDEX `storage_usage_ledger_storage_occurred_idx` ON `storage_usage_ledger` (`storage_id`,`occurred_at`);--> statement-breakpoint +DROP INDEX `image_hostings_org_path_uniq`;--> statement-breakpoint +ALTER TABLE `image_hostings` ADD `purged_at` integer;--> statement-breakpoint +CREATE UNIQUE INDEX `image_hostings_org_path_uniq` ON `image_hostings` (`org_id`,`path`) WHERE "image_hostings"."purged_at" IS NULL;--> statement-breakpoint +ALTER TABLE `matters` ADD `purged_at` integer; \ No newline at end of file diff --git a/migrations/meta/0059_snapshot.json b/migrations/meta/0059_snapshot.json new file mode 100644 index 00000000..161fed40 --- /dev/null +++ b/migrations/meta/0059_snapshot.json @@ -0,0 +1,4366 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "b6d63ed5-2212-40d2-90f8-304d37ac1309", + "prevId": "a5f85406-bcd4-4a88-8915-0dd23bf2a271", + "tables": { + "activity_events": { + "name": "activity_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_name": { + "name": "target_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "actor_type": { + "name": "actor_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_ref": { + "name": "actor_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "activity_events_org_created_idx": { + "name": "activity_events_org_created_idx", + "columns": [ + "org_id", + "created_at" + ], + "isUnique": false + }, + "activity_events_user_created_idx": { + "name": "activity_events_user_created_idx", + "columns": [ + "user_id", + "created_at" + ], + "isUnique": false + }, + "activity_events_action_created_idx": { + "name": "activity_events_action_created_idx", + "columns": [ + "action", + "created_at" + ], + "isUnique": false + }, + "activity_events_target_created_idx": { + "name": "activity_events_target_created_idx", + "columns": [ + "target_type", + "target_id", + "created_at" + ], + "isUnique": false + }, + "activity_events_created_idx": { + "name": "activity_events_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "announcements": { + "name": "announcements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "announcements_status_priority_idx": { + "name": "announcements_status_priority_idx", + "columns": [ + "status", + "priority" + ], + "isUnique": false + }, + "announcements_published_idx": { + "name": "announcements_published_idx", + "columns": [ + "published_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "background_jobs": { + "name": "background_jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "target_folder": { + "name": "target_folder", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_path": { + "name": "target_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "input_bytes": { + "name": "input_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "output_bytes": { + "name": "output_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processed_bytes": { + "name": "processed_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "current_filename": { + "name": "current_filename", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "result_metadata": { + "name": "result_metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "retryable": { + "name": "retryable", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "cancelable": { + "name": "cancelable", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "retried_from_job_id": { + "name": "retried_from_job_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "background_jobs_org_created_idx": { + "name": "background_jobs_org_created_idx", + "columns": [ + "org_id", + "created_at" + ], + "isUnique": false + }, + "background_jobs_org_status_idx": { + "name": "background_jobs_org_status_idx", + "columns": [ + "org_id", + "status" + ], + "isUnique": false + }, + "background_jobs_org_type_idx": { + "name": "background_jobs_org_type_idx", + "columns": [ + "org_id", + "type" + ], + "isUnique": false + }, + "background_jobs_created_idx": { + "name": "background_jobs_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cloud_traffic_reports": { + "name": "cloud_traffic_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "period": { + "name": "period", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "unit_bytes": { + "name": "unit_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "credits_per_unit": { + "name": "credits_per_unit", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "next_retry_at": { + "name": "next_retry_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "cloud_traffic_reports_event_uniq": { + "name": "cloud_traffic_reports_event_uniq", + "columns": [ + "event_id" + ], + "isUnique": true + }, + "cloud_traffic_reports_org_period_idx": { + "name": "cloud_traffic_reports_org_period_idx", + "columns": [ + "org_id", + "period" + ], + "isUnique": false + }, + "cloud_traffic_reports_status_idx": { + "name": "cloud_traffic_reports_status_idx", + "columns": [ + "status" + ], + "isUnique": false + }, + "cloud_traffic_reports_retry_idx": { + "name": "cloud_traffic_reports_retry_idx", + "columns": [ + "status", + "next_retry_at", + "created_at" + ], + "isUnique": false + }, + "cloud_traffic_reports_updated_idx": { + "name": "cloud_traffic_reports_updated_idx", + "columns": [ + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "download_tasks": { + "name": "download_tasks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by_user_id": { + "name": "created_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_type": { + "name": "source_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_uri": { + "name": "source_uri", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "target_folder": { + "name": "target_folder", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "assigned_downloader_id": { + "name": "assigned_downloader_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "billing_authorized_bytes": { + "name": "billing_authorized_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "billing_charged_bytes": { + "name": "billing_charged_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "billing_charged_credits": { + "name": "billing_charged_credits", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "billing_status": { + "name": "billing_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'none'" + }, + "error_code": { + "name": "error_code", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "result_object_id": { + "name": "result_object_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "runtime": { + "name": "runtime", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolve_started_at": { + "name": "resolve_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolve_completed_at": { + "name": "resolve_completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "download_completed_at": { + "name": "download_completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ingest_started_at": { + "name": "ingest_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ingest_completed_at": { + "name": "ingest_completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "seeding_started_at": { + "name": "seeding_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "seeding_stopped_at": { + "name": "seeding_stopped_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "assigned_at": { + "name": "assigned_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "download_tasks_org_created_idx": { + "name": "download_tasks_org_created_idx", + "columns": [ + "org_id", + "created_at" + ], + "isUnique": false + }, + "download_tasks_org_status_idx": { + "name": "download_tasks_org_status_idx", + "columns": [ + "org_id", + "status" + ], + "isUnique": false + }, + "download_tasks_org_category_idx": { + "name": "download_tasks_org_category_idx", + "columns": [ + "org_id", + "category" + ], + "isUnique": false + }, + "download_tasks_org_tags_idx": { + "name": "download_tasks_org_tags_idx", + "columns": [ + "org_id", + "tags" + ], + "isUnique": false + }, + "download_tasks_downloader_idx": { + "name": "download_tasks_downloader_idx", + "columns": [ + "assigned_downloader_id", + "status" + ], + "isUnique": false + }, + "download_tasks_created_idx": { + "name": "download_tasks_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + }, + "download_tasks_finished_idx": { + "name": "download_tasks_finished_idx", + "columns": [ + "finished_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "downloaders": { + "name": "downloaders", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token_jti": { + "name": "token_jti", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'offline'" + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "version": { + "name": "version", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "hostname": { + "name": "hostname", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "platform": { + "name": "platform", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "arch": { + "name": "arch", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "engine": { + "name": "engine", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'http'" + }, + "capabilities": { + "name": "capabilities", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "max_concurrent_tasks": { + "name": "max_concurrent_tasks", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "current_tasks": { + "name": "current_tasks", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "download_bps": { + "name": "download_bps", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "upload_bps": { + "name": "upload_bps", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "free_disk_bytes": { + "name": "free_disk_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "remote_download_credit_billing_enabled": { + "name": "remote_download_credit_billing_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "remote_download_credit_unit_bytes": { + "name": "remote_download_credit_unit_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 104857600 + }, + "remote_download_credit_per_unit": { + "name": "remote_download_credit_per_unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "last_heartbeat_at": { + "name": "last_heartbeat_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "downloaders_token_jti_unique": { + "name": "downloaders_token_jti_unique", + "columns": [ + "token_jti" + ], + "isUnique": true + }, + "downloaders_status_idx": { + "name": "downloaders_status_idx", + "columns": [ + "status" + ], + "isUnique": false + }, + "downloaders_enabled_idx": { + "name": "downloaders_enabled_idx", + "columns": [ + "enabled" + ], + "isUnique": false + }, + "downloaders_created_idx": { + "name": "downloaders_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "image_hosting_configs": { + "name": "image_hosting_configs", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "custom_domain": { + "name": "custom_domain", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cf_hostname_id": { + "name": "cf_hostname_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "domain_verified_at": { + "name": "domain_verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "referer_allowlist": { + "name": "referer_allowlist", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "image_hosting_configs_custom_domain_unique": { + "name": "image_hosting_configs_custom_domain_unique", + "columns": [ + "custom_domain" + ], + "isUnique": true + } + }, + "foreignKeys": { + "image_hosting_configs_org_id_organization_id_fk": { + "name": "image_hosting_configs_org_id_organization_id_fk", + "tableFrom": "image_hosting_configs", + "tableTo": "organization", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "image_hostings": { + "name": "image_hostings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "path": { + "name": "path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime": { + "name": "mime", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "purged_at": { + "name": "purged_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_count": { + "name": "access_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_accessed_at": { + "name": "last_accessed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "image_hostings_token_unique": { + "name": "image_hostings_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "image_hostings_org_path_uniq": { + "name": "image_hostings_org_path_uniq", + "columns": [ + "org_id", + "path" + ], + "isUnique": true, + "where": "\"image_hostings\".\"purged_at\" IS NULL" + }, + "image_hostings_org_created_idx": { + "name": "image_hostings_org_created_idx", + "columns": [ + "org_id", + "created_at" + ], + "isUnique": false + }, + "image_hostings_token_idx": { + "name": "image_hostings_token_idx", + "columns": [ + "token" + ], + "isUnique": false + } + }, + "foreignKeys": { + "image_hostings_org_id_organization_id_fk": { + "name": "image_hostings_org_id_organization_id_fk", + "tableFrom": "image_hostings", + "tableTo": "organization", + "columnsFrom": [ + "org_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "image_hostings_storage_id_storages_id_fk": { + "name": "image_hostings_storage_id_storages_id_fk", + "tableFrom": "image_hostings", + "tableTo": "storages", + "columnsFrom": [ + "storage_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invite_codes": { + "name": "invite_codes", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "used_by": { + "name": "used_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "used_at": { + "name": "used_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invite_codes_code_unique": { + "name": "invite_codes_code_unique", + "columns": [ + "code" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "license_bindings": { + "name": "license_bindings", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "cloud_binding_id": { + "name": "cloud_binding_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cloud_store_id": { + "name": "cloud_store_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "instance_id": { + "name": "instance_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cloud_account_id": { + "name": "cloud_account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "cloud_account_email": { + "name": "cloud_account_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cached_certificate": { + "name": "cached_certificate", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cached_certificate_expires_at": { + "name": "cached_certificate_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bound_at": { + "name": "bound_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "disconnected_at": { + "name": "disconnected_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refresh_at": { + "name": "last_refresh_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refresh_error": { + "name": "last_refresh_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "license_bindings_active_uniq": { + "name": "license_bindings_active_uniq", + "columns": [ + "status" + ], + "isUnique": true, + "where": "status = 'active'" + }, + "license_bindings_cloud_binding_idx": { + "name": "license_bindings_cloud_binding_idx", + "columns": [ + "cloud_binding_id" + ], + "isUnique": false + }, + "license_bindings_instance_idx": { + "name": "license_bindings_instance_idx", + "columns": [ + "instance_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "matters": { + "name": "matters", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "alias": { + "name": "alias", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "dirtype": { + "name": "dirtype", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "parent": { + "name": "parent", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "object": { + "name": "object", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'draft'" + }, + "trashed_at": { + "name": "trashed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "purged_at": { + "name": "purged_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "matters_alias_unique": { + "name": "matters_alias_unique", + "columns": [ + "alias" + ], + "isUnique": true + }, + "matters_status_dir_created_idx": { + "name": "matters_status_dir_created_idx", + "columns": [ + "status", + "dirtype", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "notifications": { + "name": "notifications", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "body": { + "name": "body", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "ref_type": { + "name": "ref_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_id": { + "name": "ref_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "read_at": { + "name": "read_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "notifications_user_created_idx": { + "name": "notifications_user_created_idx", + "columns": [ + "user_id", + "created_at" + ], + "isUnique": false + }, + "notifications_user_read_idx": { + "name": "notifications_user_read_idx", + "columns": [ + "user_id", + "read_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "object_upload_sessions": { + "name": "object_upload_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_id": { + "name": "object_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "upload_id": { + "name": "upload_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "part_size": { + "name": "part_size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "on_conflict": { + "name": "on_conflict", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'fail'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "object_upload_sessions_object_idx": { + "name": "object_upload_sessions_object_idx", + "columns": [ + "org_id", + "object_id" + ], + "isUnique": false + }, + "object_upload_sessions_expires_idx": { + "name": "object_upload_sessions_expires_idx", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_quota_entitlements": { + "name": "org_quota_entitlements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "entitlement_type": { + "name": "entitlement_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'grant'" + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "starts_at": { + "name": "starts_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "org_quota_entitlements_org_resource_idx": { + "name": "org_quota_entitlements_org_resource_idx", + "columns": [ + "org_id", + "resource_type", + "status" + ], + "isUnique": false + }, + "org_quota_entitlements_org_type_idx": { + "name": "org_quota_entitlements_org_type_idx", + "columns": [ + "org_id", + "resource_type", + "entitlement_type", + "status" + ], + "isUnique": false + }, + "org_quota_entitlements_active_plan_uniq": { + "name": "org_quota_entitlements_active_plan_uniq", + "columns": [ + "org_id", + "resource_type", + "entitlement_type" + ], + "isUnique": true, + "where": "status = 'active' AND entitlement_type = 'plan' AND source <> 'free_plan'" + }, + "org_quota_entitlements_source_resource_uniq": { + "name": "org_quota_entitlements_source_resource_uniq", + "columns": [ + "source", + "source_id", + "resource_type" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "org_quotas": { + "name": "org_quotas", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "quota": { + "name": "quota", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "used": { + "name": "used", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "traffic_quota": { + "name": "traffic_quota", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "traffic_used": { + "name": "traffic_used", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "traffic_period": { + "name": "traffic_period", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'1970-01'" + } + }, + "indexes": { + "org_quotas_org_uniq": { + "name": "org_quotas_org_uniq", + "columns": [ + "org_id" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "remote_download_usage_reports": { + "name": "remote_download_usage_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "downloader_id": { + "name": "downloader_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "task_id": { + "name": "task_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unit_index": { + "name": "unit_index", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unit_bytes": { + "name": "unit_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "credits_per_unit": { + "name": "credits_per_unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "remote_download_usage_reports_event_id_unique": { + "name": "remote_download_usage_reports_event_id_unique", + "columns": [ + "event_id" + ], + "isUnique": true + }, + "remote_download_usage_task_unit_uniq": { + "name": "remote_download_usage_task_unit_uniq", + "columns": [ + "task_id", + "unit_index" + ], + "isUnique": true + }, + "remote_download_usage_org_idx": { + "name": "remote_download_usage_org_idx", + "columns": [ + "org_id" + ], + "isUnique": false + }, + "remote_download_usage_status_idx": { + "name": "remote_download_usage_status_idx", + "columns": [ + "status" + ], + "isUnique": false + }, + "remote_download_usage_created_idx": { + "name": "remote_download_usage_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "share_recipients": { + "name": "share_recipients", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "share_id": { + "name": "share_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "recipient_user_id": { + "name": "recipient_user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "recipient_email": { + "name": "recipient_email", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "share_recipients_share_id_idx": { + "name": "share_recipients_share_id_idx", + "columns": [ + "share_id" + ], + "isUnique": false + }, + "share_recipients_user_id_idx": { + "name": "share_recipients_user_id_idx", + "columns": [ + "recipient_user_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "shares": { + "name": "shares", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "matter_id": { + "name": "matter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "creator_id": { + "name": "creator_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "password_hash": { + "name": "password_hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "download_limit": { + "name": "download_limit", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "views": { + "name": "views", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "downloads": { + "name": "downloads", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "shares_token_unique": { + "name": "shares_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "shares_creator_status_created_idx": { + "name": "shares_creator_status_created_idx", + "columns": [ + "creator_id", + "status", + "created_at" + ], + "isUnique": false + }, + "shares_created_idx": { + "name": "shares_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "site_invitations": { + "name": "site_invitations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invited_by": { + "name": "invited_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accepted_by": { + "name": "accepted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "accepted_at": { + "name": "accepted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_by": { + "name": "revoked_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "site_invitations_token_unique": { + "name": "site_invitations_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "site_invitations_email_idx": { + "name": "site_invitations_email_idx", + "columns": [ + "email" + ], + "isUnique": false + }, + "site_invitations_created_idx": { + "name": "site_invitations_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + }, + "site_invitations_expires_idx": { + "name": "site_invitations_expires_idx", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "stats_rollups_hourly": { + "name": "stats_rollups_hourly", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "bucket_start": { + "name": "bucket_start", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "metric_key": { + "name": "metric_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dimension_key": { + "name": "dimension_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "dimension_value": { + "name": "dimension_value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "unique_count": { + "name": "unique_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "stats_rollups_hourly_bucket_metric_dim_uniq": { + "name": "stats_rollups_hourly_bucket_metric_dim_uniq", + "columns": [ + "bucket_start", + "org_id", + "metric_key", + "dimension_key", + "dimension_value" + ], + "isUnique": true + }, + "stats_rollups_hourly_metric_bucket_idx": { + "name": "stats_rollups_hourly_metric_bucket_idx", + "columns": [ + "metric_key", + "bucket_start" + ], + "isUnique": false + }, + "stats_rollups_hourly_dimension_bucket_idx": { + "name": "stats_rollups_hourly_dimension_bucket_idx", + "columns": [ + "metric_key", + "dimension_key", + "bucket_start" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_usage_ledger": { + "name": "storage_usage_ledger", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "event_key": { + "name": "event_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_id": { + "name": "storage_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_type": { + "name": "resource_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_id": { + "name": "resource_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "delta_bytes": { + "name": "delta_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "storage_usage_ledger_event_key_unique": { + "name": "storage_usage_ledger_event_key_unique", + "columns": [ + "event_key" + ], + "isUnique": true + }, + "storage_usage_ledger_occurred_idx": { + "name": "storage_usage_ledger_occurred_idx", + "columns": [ + "occurred_at" + ], + "isUnique": false + }, + "storage_usage_ledger_org_occurred_idx": { + "name": "storage_usage_ledger_org_occurred_idx", + "columns": [ + "org_id", + "occurred_at" + ], + "isUnique": false + }, + "storage_usage_ledger_storage_occurred_idx": { + "name": "storage_usage_ledger_storage_occurred_idx", + "columns": [ + "storage_id", + "occurred_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storages": { + "name": "storages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "bucket": { + "name": "bucket", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'auto'" + }, + "access_key": { + "name": "access_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret_key": { + "name": "secret_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_path": { + "name": "file_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "custom_host": { + "name": "custom_host", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "''" + }, + "capacity": { + "name": "capacity", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "egress_credit_billing_enabled": { + "name": "egress_credit_billing_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "egress_credit_unit_bytes": { + "name": "egress_credit_unit_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 104857600 + }, + "egress_credit_per_unit": { + "name": "egress_credit_per_unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "force_path_style": { + "name": "force_path_style", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "used": { + "name": "used", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "system_options": { + "name": "system_options", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "team_invite_links": { + "name": "team_invite_links", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "team_invite_links_token_unique": { + "name": "team_invite_links_token_unique", + "columns": [ + "token" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webdav_dead_properties": { + "name": "webdav_dead_properties", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_path": { + "name": "resource_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "namespace": { + "name": "namespace", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "webdav_dead_properties_resource_prop_uniq": { + "name": "webdav_dead_properties_resource_prop_uniq", + "columns": [ + "org_id", + "resource_path", + "namespace", + "name" + ], + "isUnique": true + }, + "webdav_dead_properties_resource_idx": { + "name": "webdav_dead_properties_resource_idx", + "columns": [ + "org_id", + "resource_path" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webdav_locks": { + "name": "webdav_locks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resource_path": { + "name": "resource_path", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "depth": { + "name": "depth", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'infinity'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "webdav_locks_token_unique": { + "name": "webdav_locks_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "webdav_locks_resource_idx": { + "name": "webdav_locks_resource_idx", + "columns": [ + "org_id", + "resource_path" + ], + "isUnique": false + }, + "webdav_locks_expires_idx": { + "name": "webdav_locks_expires_idx", + "columns": [ + "expires_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhook_events": { + "name": "webhook_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'cloud'" + }, + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'order.quota_changed'" + }, + "payload_hash": { + "name": "payload_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "raw_payload": { + "name": "raw_payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "processed_at": { + "name": "processed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhook_events_source_event_uniq": { + "name": "webhook_events_source_event_uniq", + "columns": [ + "source", + "event_id" + ], + "isUnique": true + }, + "webhook_events_source_created_idx": { + "name": "webhook_events_source_created_idx", + "columns": [ + "source", + "created_at" + ], + "isUnique": false + }, + "webhook_events_status_idx": { + "name": "webhook_events_status_idx", + "columns": [ + "status" + ], + "isUnique": false + }, + "webhook_events_processed_idx": { + "name": "webhook_events_processed_idx", + "columns": [ + "processed_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_request": { + "name": "last_request", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apikey_config_id_idx": { + "name": "apikey_config_id_idx", + "columns": [ + "config_id" + ], + "isUnique": false + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": [ + "reference_id" + ], + "isUnique": false + }, + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": [ + "key" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "deviceCode": { + "name": "deviceCode", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "device_code": { + "name": "device_code", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_code": { + "name": "user_code", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_polled_at": { + "name": "last_polled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "polling_interval": { + "name": "polling_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "deviceCode_device_code_idx": { + "name": "deviceCode_device_code_idx", + "columns": [ + "device_code" + ], + "isUnique": false + }, + "deviceCode_user_code_idx": { + "name": "deviceCode_user_code_idx", + "columns": [ + "user_code" + ], + "isUnique": false + }, + "deviceCode_status_idx": { + "name": "deviceCode_status_idx", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invitation": { + "name": "invitation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invitation_organizationId_idx": { + "name": "invitation_organizationId_idx", + "columns": [ + "organization_id" + ], + "isUnique": false + }, + "invitation_email_idx": { + "name": "invitation_email_idx", + "columns": [ + "email" + ], + "isUnique": false + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": [ + "inviter_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "member": { + "name": "member", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "member_organizationId_idx": { + "name": "member_organizationId_idx", + "columns": [ + "organization_id" + ], + "isUnique": false + }, + "member_userId_idx": { + "name": "member_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organization": { + "name": "organization", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "impersonated_by": { + "name": "impersonated_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + "user_id" + ], + "isUnique": false + }, + "session_created_idx": { + "name": "session_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "banned": { + "name": "banned", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + }, + "ban_reason": { + "name": "ban_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ban_expires": { + "name": "ban_expires", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "username": { + "name": "username", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "display_username": { + "name": "display_username", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + }, + "user_username_unique": { + "name": "user_username_unique", + "columns": [ + "username" + ], + "isUnique": true + }, + "user_created_idx": { + "name": "user_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(cast(unixepoch('subsecond') * 1000 as integer))" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/migrations/meta/_journal.json b/migrations/meta/_journal.json index edcc19e6..f1257076 100644 --- a/migrations/meta/_journal.json +++ b/migrations/meta/_journal.json @@ -407,6 +407,13 @@ "when": 1784572374065, "tag": "0058_drop-system-options-public", "breakpoints": true + }, + { + "idx": 59, + "version": "6", + "when": 1784606180765, + "tag": "0059_storage-usage-ledger", + "breakpoints": true } ] } \ No newline at end of file diff --git a/scripts/backfill-admin-stats.ts b/scripts/backfill-admin-stats.ts index afe91eb9..e2eddc89 100644 --- a/scripts/backfill-admin-stats.ts +++ b/scripts/backfill-admin-stats.ts @@ -516,7 +516,7 @@ function buildHourlyBackfillSql(now: Date): string { ] const statements = sources.flatMap((source) => hourlyStatements(source, currentHour)) - statements.push(userSignupBackfillSql(currentHour), rollupMarkerBackfillSql(now)) + statements.push(userSignupBackfillSql(currentHour), storageLedgerBalanceBackfillSql(currentHour), rollupMarkerBackfillSql(now)) return statements.join('\n\n') } @@ -539,7 +539,7 @@ function purgeCounterRollupsSql(): string { WHERE metric_key IN ( 'background_job.finished', 'remote_download.task_finished', 'share.created', 'share.download_issued', 'share.password_passed', 'share.saved', 'share.view', - 'stats.quality_missing_bytes', 'traffic.report_sync', 'transfer.download_failed', + 'stats.quality_missing_bytes', 'storage.ledger_balance', 'traffic.report_sync', 'transfer.download_failed', 'transfer.download_issued', 'transfer.upload', 'user.signup' ) OR ( @@ -628,6 +628,46 @@ function userSignupBackfillSql(before: number): string { }, before).join('\n\n') } +function storageLedgerBalanceBackfillSql(currentHour: number): string { + return `WITH +digits(n) AS (VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9)), +numbers(n) AS ( + SELECT ones.n + tens.n * 10 + hundreds.n * 100 + thousands.n * 1000 + ten_thousands.n * 10000 + FROM digits ones + CROSS JOIN digits tens + CROSS JOIN digits hundreds + CROSS JOIN digits thousands + CROSS JOIN digits ten_thousands +), +opening AS ( + SELECT CAST(MIN(occurred_at) / 3600000 AS INTEGER) * 3600000 AS start_at + FROM storage_usage_ledger + WHERE reason = 'opening_balance_complete' +), +buckets AS ( + SELECT opening.start_at + numbers.n * 3600000 AS bucket_start + FROM opening + JOIN numbers ON opening.start_at + numbers.n * 3600000 < ${currentHour} + WHERE opening.start_at IS NOT NULL +) +INSERT INTO stats_rollups_hourly ( + id, bucket_start, org_id, metric_key, dimension_key, dimension_value, + count, bytes, unique_count, metadata, updated_at +) +SELECT + CAST(bucket_start AS TEXT) || ':global:storage.ledger_balance:all:all', + bucket_start, '', 'storage.ledger_balance', '', '', 0, + (SELECT COALESCE(SUM(delta_bytes), 0) FROM storage_usage_ledger WHERE occurred_at < bucket_start + 3600000), + 0, json_object('version', 3, 'scope', 'counters', 'quality', 'exact'), bucket_start + 3600000 +FROM buckets +WHERE true +ON CONFLICT(bucket_start, org_id, metric_key, dimension_key, dimension_value) +DO UPDATE SET count = excluded.count, bytes = excluded.bytes, unique_count = excluded.unique_count, + metadata = excluded.metadata, updated_at = excluded.updated_at +WHERE count <> excluded.count OR bytes <> excluded.bytes OR unique_count <> excluded.unique_count + OR metadata <> excluded.metadata OR updated_at <> excluded.updated_at;` +} + function rollupMarkerBackfillSql(now: Date): string { const currentHour = Math.floor(now.getTime() / 3_600_000) * 3_600_000 const latestClosedHour = currentHour - 3_600_000 @@ -730,9 +770,8 @@ WHERE count <> excluded.count OR bytes <> excluded.bytes OR unique_count <> excl } function statsHistoryStartSql(): string { - const missing = '9223372036854775807' - return `NULLIF(COALESCE(( - SELECT CAST((MIN(ae.created_at) * 1000) / 3600000 AS INTEGER) * 3600000 + return `(SELECT MIN(start_at) FROM ( + SELECT CAST((MIN(ae.created_at) * 1000) / 3600000 AS INTEGER) * 3600000 AS start_at FROM activity_events ae WHERE ae.created_at >= ${MIN_VALID_TIMESTAMP_SECONDS} AND ae.action IN ( @@ -742,7 +781,11 @@ function statsHistoryStartSql(): string { 'stats_user_signup', 'stats_share_created', 'stats_remote_download_finished', 'stats_background_job_finished' ) - ), ${missing}), ${missing})` + UNION ALL + SELECT CAST(MIN(occurred_at) / 3600000 AS INTEGER) * 3600000 AS start_at + FROM storage_usage_ledger + WHERE reason = 'opening_balance_complete' + ) history_starts WHERE start_at IS NOT NULL)` } export function buildValidationSql(now = new Date()): string { diff --git a/server/adapters/repos/admin-stats-rollup.integration.test.ts b/server/adapters/repos/admin-stats-rollup.integration.test.ts index 436878f5..86ae110e 100644 --- a/server/adapters/repos/admin-stats-rollup.integration.test.ts +++ b/server/adapters/repos/admin-stats-rollup.integration.test.ts @@ -78,7 +78,7 @@ describe('admin hourly stats rollup', () => { VALUES ('rollup-file', ${orgId}, 'rollup-file', 'video.mp4', 'video/mp4', 200, 0, '', 'video.mp4', 'rollup-storage', 'active', NULL, ${atSec}, ${atSec}), ('rollup-dir', ${orgId}, 'rollup-dir', 'folder', '', 0, 1, '', '', 'rollup-storage', 'active', NULL, ${atSec}, ${atSec}), - ('rollup-trashed', ${orgId}, 'rollup-trashed', 'old.bin', 'application/octet-stream', 800, 0, '', 'old.bin', 'rollup-storage', 'trashed', ${atMs}, ${atSec}, ${atSec}) + ('rollup-trashed', ${orgId}, 'rollup-trashed', 'old.bin', 'application/octet-stream', 800, 0, '', 'old.bin', 'rollup-storage', 'active', ${atMs}, ${atSec}, ${atSec}) `) await db.run(sql` INSERT INTO image_hostings @@ -226,6 +226,7 @@ describe('admin hourly stats rollup', () => { expect(row(M.remoteDownloadTaskFinished)).toMatchObject({ count: 1, bytes: 60 }) expect(row(M.backgroundJobFinished)).toMatchObject({ count: 1 }) expect(row(M.storageInventory)).toMatchObject({ count: 2, bytes: 500 }) + expect(row(M.storageLedgerBalance, '', '', '')).toMatchObject({ bytes: 1300 }) expect(row(M.storageUsed)).toMatchObject({ bytes: 1300 }) expect(row(M.storageQuota)).toMatchObject({ bytes: 1000 }) expect(row(M.storageQuota, 'status', 'over', '')).toMatchObject({ count: 1 }) diff --git a/server/adapters/repos/admin-stats-rollup.ts b/server/adapters/repos/admin-stats-rollup.ts index 15402f63..243f57a6 100644 --- a/server/adapters/repos/admin-stats-rollup.ts +++ b/server/adapters/repos/admin-stats-rollup.ts @@ -1,4 +1,4 @@ -import { and, eq, inArray, isNull, sql } from 'drizzle-orm' +import { and, eq, inArray, isNull, lt, sql } from 'drizzle-orm' import { organization } from '../../db/auth-schema' import { backgroundJobs, @@ -10,6 +10,7 @@ import { orgQuotas, shares, statsRollupsHourly, + storageUsageLedger, webhookEvents, } from '../../db/schema' import { type AtomicQuery, executeWriteTransaction } from '../../db/transaction' @@ -23,6 +24,7 @@ import { } from '../../domain/admin-stats-metrics' import type { Database } from '../../platform/interface' import { getEffectiveQuotasByOrg } from './quota' +import { ensureStorageUsageOpeningBalances } from './storage-usage-ledger' const HOUR_MS = 3_600_000 const D1_MAX_BOUND_PARAMS = 100 @@ -39,6 +41,7 @@ const COUNTER_METRICS: AdminStatsMetric[] = [ M.shareView, M.statsMissingBytes, M.statsRollupRun, + M.storageLedgerBalance, M.transferDownloadFailed, M.transferDownloadIssued, M.transferUpload, @@ -91,9 +94,11 @@ export async function rebuildAdminStatsHour( const rollups = new RollupAccumulator() const capturedSnapshot = await compatibleSnapshotMarker(db, bucketStart) + await ensureStorageUsageOpeningBalances(db, generatedAt) await addEventMetrics(db, rollups, bucketStart, bucketEnd) await addUserMetrics(db, rollups, bucketStart, bucketEnd) await addOperationalMetrics(db, rollups, bucketStart, bucketEnd) + await addStorageLedgerBalance(db, rollups, bucketEnd) const values = rollups.values() const lowerBoundRows = values.filter((row) => row.lowerBound).length rollups.add(M.statsRollupRun, '', 1, 0, { outcome: 'success' }, lowerBoundRows > 0) @@ -163,6 +168,7 @@ export async function captureAdminStatsSnapshot( } const bucketEnd = new Date(bucketStart.getTime() + HOUR_MS) const rollups = new RollupAccumulator() + await ensureStorageUsageOpeningBalances(db, observedAt) await addSnapshotMetrics(db, rollups, observedAt) rollups.add(M.statsRollupRun, '', 1, 0, { outcome: 'success' }) const rows = rollups.values().map((row) => ({ @@ -439,6 +445,29 @@ async function addOperationalMetrics(db: Database, rollups: RollupAccumulator, f } } +async function addStorageLedgerBalance(db: Database, rollups: RollupAccumulator, bucketEnd: Date): Promise { + const complete = await queryStage( + 'storage-ledger-opening', + db + .select({ id: storageUsageLedger.id }) + .from(storageUsageLedger) + .where( + and(eq(storageUsageLedger.reason, 'opening_balance_complete'), lt(storageUsageLedger.occurredAt, bucketEnd)), + ) + .limit(1), + ) + if (complete.length === 0) return + + const rows = await queryStage( + 'storage-ledger-balance', + db + .select({ bytes: sql`COALESCE(SUM(${storageUsageLedger.deltaBytes}), 0)` }) + .from(storageUsageLedger) + .where(lt(storageUsageLedger.occurredAt, bucketEnd)), + ) + rollups.setGauge(M.storageLedgerBalance, '', 0, Number(rows[0]?.bytes ?? 0)) +} + async function addSnapshotMetrics(db: Database, rollups: RollupAccumulator, now: Date): Promise { const [ userRows, @@ -570,6 +599,14 @@ async function addSnapshotMetrics(db: Database, rollups: RollupAccumulator, now: 'kind', 'storage_usage_drift', ) + rollups.setGauge( + M.statsDataQualitySnapshot, + '', + dataQualityRows.storageLedgerDriftSpaces, + dataQualityRows.storageLedgerDriftBytes, + 'kind', + 'storage_ledger_drift', + ) rollups.setGauge(M.backgroundJobSnapshot, '', 0, 0) rollups.setGauge(M.remoteDownloadTaskSnapshot, '', 0, 0) rollups.setGauge(M.downloaderSnapshot, '', 0, 0) @@ -647,12 +684,16 @@ async function dataQualitySnapshot(db: Database): Promise<{ unlocatedShareDownloads: number storageUsageDriftSpaces: number storageUsageDriftBytes: number + storageLedgerDriftSpaces: number + storageLedgerDriftBytes: number }> { const rows = await db.all<{ unlocatedShareViews: number unlocatedShareDownloads: number storageUsageDriftSpaces: number storageUsageDriftBytes: number + storageLedgerDriftSpaces: number + storageLedgerDriftBytes: number }>(sql` WITH event_counts AS ( SELECT @@ -668,11 +709,11 @@ async function dataQualitySnapshot(db: Database): Promise<{ FROM ( SELECT org_id, COALESCE(size, 0) AS bytes FROM matters - WHERE status IN ('active', 'trashed') AND dirtype = 0 + WHERE status IN ('active', 'trashed') AND dirtype = 0 AND purged_at IS NULL UNION ALL SELECT org_id, COALESCE(size, 0) AS bytes FROM image_hostings - WHERE status = 'active' + WHERE status = 'active' AND purged_at IS NULL ) inventory GROUP BY org_id ), @@ -682,12 +723,27 @@ async function dataQualitySnapshot(db: Database): Promise<{ JOIN organization o ON o.id = q.org_id LEFT JOIN billable_storage ON billable_storage.org_id = q.org_id WHERE q.used <> COALESCE(billable_storage.bytes, 0) + ), + ledger_storage AS ( + SELECT org_id, SUM(delta_bytes) AS bytes + FROM storage_usage_ledger + WHERE org_id <> '' + GROUP BY org_id + ), + ledger_drift AS ( + SELECT ABS(COALESCE(ledger_storage.bytes, 0) - COALESCE(billable_storage.bytes, 0)) AS bytes + FROM organization o + LEFT JOIN billable_storage ON billable_storage.org_id = o.id + LEFT JOIN ledger_storage ON ledger_storage.org_id = o.id + WHERE COALESCE(ledger_storage.bytes, 0) <> COALESCE(billable_storage.bytes, 0) ) SELECT COALESCE(SUM(MAX(${shares.views} - COALESCE(event_counts.views, 0), 0)), 0) AS unlocatedShareViews, COALESCE(SUM(MAX(${shares.downloads} - COALESCE(event_counts.downloads, 0), 0)), 0) AS unlocatedShareDownloads, (SELECT COUNT(*) FROM storage_drift) AS storageUsageDriftSpaces, - (SELECT COALESCE(SUM(bytes), 0) FROM storage_drift) AS storageUsageDriftBytes + (SELECT COALESCE(SUM(bytes), 0) FROM storage_drift) AS storageUsageDriftBytes, + (SELECT COUNT(*) FROM ledger_drift) AS storageLedgerDriftSpaces, + (SELECT COALESCE(SUM(bytes), 0) FROM ledger_drift) AS storageLedgerDriftBytes FROM ${shares} LEFT JOIN event_counts ON event_counts.share_id = ${shares.id} `) @@ -699,6 +755,8 @@ async function dataQualitySnapshot(db: Database): Promise<{ unlocatedShareDownloads, storageUsageDriftSpaces: Number(rows[0]?.storageUsageDriftSpaces ?? 0), storageUsageDriftBytes: Number(rows[0]?.storageUsageDriftBytes ?? 0), + storageLedgerDriftSpaces: Number(rows[0]?.storageLedgerDriftSpaces ?? 0), + storageLedgerDriftBytes: Number(rows[0]?.storageLedgerDriftBytes ?? 0), } } @@ -717,6 +775,7 @@ async function trashInventoryGroups( and( inArray(matters.status, ['active', 'trashed']), sql`${matters.trashedAt} IS NOT NULL`, + isNull(matters.purgedAt), eq(matters.dirtype, 0), ), ) @@ -813,7 +872,9 @@ async function inventoryGroups( dimensionValue: dimensionSql, }) .from(matters) - .where(and(eq(matters.status, 'active'), isNull(matters.trashedAt), eq(matters.dirtype, 0))) + .where( + and(eq(matters.status, 'active'), isNull(matters.trashedAt), isNull(matters.purgedAt), eq(matters.dirtype, 0)), + ) .groupBy(matters.orgId, dimensionSql) const imageDimensionSql = dimension === 'file_type_group' @@ -831,7 +892,7 @@ async function inventoryGroups( dimensionValue: imageDimensionSql, }) .from(imageHostings) - .where(eq(imageHostings.status, 'active')) + .where(and(eq(imageHostings.status, 'active'), isNull(imageHostings.purgedAt))) .groupBy(imageHostings.orgId, imageDimensionSql) const result: InventoryGroup[] = matterRows.map((row) => ({ orgId: row.orgId, diff --git a/server/adapters/repos/admin-stats.ts b/server/adapters/repos/admin-stats.ts index 775b877e..772d4db1 100644 --- a/server/adapters/repos/admin-stats.ts +++ b/server/adapters/repos/admin-stats.ts @@ -1131,7 +1131,7 @@ async function getMissingTransferBytes(reader: AdminStatsHourlyReader): Promise< } async function getStorageUsedByDay(reader: AdminStatsHourlyReader): Promise> { - return getLatestGaugeValueByDay(reader, ADMIN_STATS_METRICS.storageUsed, 'bytes', '', '') + return getLatestGaugeValueByDay(reader, ADMIN_STATS_METRICS.storageLedgerBalance, 'bytes', '', '') } async function getStorageInventory(reader: AdminStatsHourlyReader): Promise<{ files: number; bytes: number } | null> { @@ -1150,15 +1150,25 @@ async function getStorageTrashInventory( async function getStorageDataQuality( reader: AdminStatsHourlyReader, -): Promise> { +): Promise< + Pick +> { const rows = await reader.latestRows(ADMIN_STATS_METRICS.statsDataQualitySnapshot, ['', 'kind']) if (!rows.some((row) => row.orgId === '' && row.dimensionKey === '')) { - return { usageDriftSpaces: null, usageDriftBytes: null } + return { usageDriftSpaces: null, usageDriftBytes: null, ledgerDriftSpaces: null, ledgerDriftBytes: null } } - const drift = rows.find( + const usageDrift = rows.find( (row) => row.orgId === '' && row.dimensionKey === 'kind' && row.dimensionValue === 'storage_usage_drift', ) - return { usageDriftSpaces: drift?.count ?? 0, usageDriftBytes: drift?.bytes ?? 0 } + const ledgerDrift = rows.find( + (row) => row.orgId === '' && row.dimensionKey === 'kind' && row.dimensionValue === 'storage_ledger_drift', + ) + return { + usageDriftSpaces: usageDrift?.count ?? 0, + usageDriftBytes: usageDrift?.bytes ?? 0, + ledgerDriftSpaces: ledgerDrift?.count ?? 0, + ledgerDriftBytes: ledgerDrift?.bytes ?? 0, + } } async function getLatestInventoryBreakdown( diff --git a/server/adapters/repos/archive-target-folder.ts b/server/adapters/repos/archive-target-folder.ts index 24b063a9..94c42ab4 100644 --- a/server/adapters/repos/archive-target-folder.ts +++ b/server/adapters/repos/archive-target-folder.ts @@ -20,6 +20,7 @@ async function requireTargetFolder(db: Database, orgId: string, targetFolder: st eq(matters.name, name), eq(matters.status, 'active'), isNull(matters.trashedAt), + isNull(matters.purgedAt), ), ) .limit(1) diff --git a/server/adapters/repos/image-hosting.ts b/server/adapters/repos/image-hosting.ts index 5668bff1..02fe93ef 100644 --- a/server/adapters/repos/image-hosting.ts +++ b/server/adapters/repos/image-hosting.ts @@ -1,6 +1,7 @@ -import { and, asc, eq, gt, isNotNull, like, sql } from 'drizzle-orm' +import { and, asc, eq, gt, isNotNull, isNull, like, sql } from 'drizzle-orm' import { nanoid } from 'nanoid' import { imageHostingConfigs, imageHostings } from '../../db/schema' +import { executeWriteTransaction, executeWriteTransactionWithResults } from '../../db/transaction' import { mimeToExt } from '../../lib/mime-utils' import type { Database } from '../../platform/interface' import type { @@ -9,6 +10,11 @@ import type { ImageHostingRepo, ImageResolution, } from '../../usecases/ports' +import { + imageActivationLedgerQuery, + imagePurgeLedgerQuery, + storageUsageOpeningBalanceQuery, +} from './storage-usage-ledger' type ImageHostingRow = typeof imageHostings.$inferSelect @@ -39,7 +45,7 @@ export function createImageHostingRepo(db: Database): ImageHostingRepo { const rows = await db .select({ path: imageHostings.path }) .from(imageHostings) - .where(and(eq(imageHostings.orgId, orgId), eq(imageHostings.path, requestedPath))) + .where(and(eq(imageHostings.orgId, orgId), eq(imageHostings.path, requestedPath), isNull(imageHostings.purgedAt))) if (rows.length === 0) return requestedPath @@ -53,7 +59,7 @@ export function createImageHostingRepo(db: Database): ImageHostingRepo { const conflict = await db .select({ path: imageHostings.path }) .from(imageHostings) - .where(and(eq(imageHostings.orgId, orgId), eq(imageHostings.path, candidate))) + .where(and(eq(imageHostings.orgId, orgId), eq(imageHostings.path, candidate), isNull(imageHostings.purgedAt))) if (conflict.length === 0) return candidate } @@ -63,7 +69,11 @@ export function createImageHostingRepo(db: Database): ImageHostingRepo { return { async resolveActiveByToken(token): Promise { - const rows = await db.select().from(imageHostings).where(eq(imageHostings.token, token)).limit(1) + const rows = await db + .select() + .from(imageHostings) + .where(and(eq(imageHostings.token, token), isNull(imageHostings.purgedAt))) + .limit(1) const row = rows[0] if (!row || row.status !== 'active') return null @@ -92,7 +102,14 @@ export function createImageHostingRepo(db: Database): ImageHostingRepo { const rows = await db .select() .from(imageHostings) - .where(and(eq(imageHostings.orgId, orgId), eq(imageHostings.path, path), eq(imageHostings.status, 'active'))) + .where( + and( + eq(imageHostings.orgId, orgId), + eq(imageHostings.path, path), + eq(imageHostings.status, 'active'), + isNull(imageHostings.purgedAt), + ), + ) .limit(1) if (!rows[0]) return null @@ -111,7 +128,7 @@ export function createImageHostingRepo(db: Database): ImageHostingRepo { async incrementAccessCount(id) { await db.run( - sql`UPDATE image_hostings SET access_count = access_count + 1, last_accessed_at = ${Date.now()} WHERE id = ${id}`, + sql`UPDATE image_hostings SET access_count = access_count + 1, last_accessed_at = ${Date.now()} WHERE id = ${id} AND purged_at IS NULL`, ) }, @@ -136,6 +153,7 @@ export function createImageHostingRepo(db: Database): ImageHostingRepo { width: null, height: null, status: input.status, + purgedAt: null, accessCount: 0, lastAccessedAt: null, createdAt: now, @@ -149,12 +167,16 @@ export function createImageHostingRepo(db: Database): ImageHostingRepo { const rows = await db .select() .from(imageHostings) - .where(and(eq(imageHostings.id, id), eq(imageHostings.orgId, orgId))) + .where(and(eq(imageHostings.id, id), eq(imageHostings.orgId, orgId), isNull(imageHostings.purgedAt))) return rows[0] ? toRecord(rows[0]) : null }, async list(orgId, opts) { - const conditions = [eq(imageHostings.orgId, orgId), eq(imageHostings.status, 'active')] + const conditions = [ + eq(imageHostings.orgId, orgId), + eq(imageHostings.status, 'active'), + isNull(imageHostings.purgedAt), + ] if (opts.pathPrefix) { conditions.push(like(imageHostings.path, `${opts.pathPrefix}%`)) @@ -191,16 +213,74 @@ export function createImageHostingRepo(db: Database): ImageHostingRepo { }, async setActive(id, orgId) { - const updated = await db + const existing = await db + .select() + .from(imageHostings) + .where( + and( + eq(imageHostings.id, id), + eq(imageHostings.orgId, orgId), + eq(imageHostings.status, 'draft'), + isNull(imageHostings.purgedAt), + ), + ) + .limit(1) + const row = existing[0] + if (!row) return false + const now = new Date() + await executeWriteTransaction(db, [storageUsageOpeningBalanceQuery(db, orgId, row.storageId, now)]) + const activateQuery = db .update(imageHostings) .set({ status: 'active' }) - .where(and(eq(imageHostings.id, id), eq(imageHostings.orgId, orgId), eq(imageHostings.status, 'draft'))) + .where( + and( + eq(imageHostings.id, id), + eq(imageHostings.orgId, orgId), + eq(imageHostings.status, 'draft'), + isNull(imageHostings.purgedAt), + ), + ) .returning({ id: imageHostings.id }) + const results = await executeWriteTransactionWithResults( + db, + [activateQuery, imageActivationLedgerQuery(db, orgId, id, now)], + [0], + ) + const updated = results[0] as { id: string }[] return updated.length > 0 }, async delete(id, orgId) { - await db.delete(imageHostings).where(and(eq(imageHostings.id, id), eq(imageHostings.orgId, orgId))) + const existing = await db + .select() + .from(imageHostings) + .where(and(eq(imageHostings.id, id), eq(imageHostings.orgId, orgId), isNull(imageHostings.purgedAt))) + .limit(1) + const row = existing[0] + if (!row) return + if (row.status === 'draft') { + await db + .delete(imageHostings) + .where( + and( + eq(imageHostings.id, id), + eq(imageHostings.orgId, orgId), + eq(imageHostings.status, 'draft'), + isNull(imageHostings.purgedAt), + ), + ) + return + } + + const now = new Date() + await executeWriteTransaction(db, [ + storageUsageOpeningBalanceQuery(db, orgId, row.storageId, now), + imagePurgeLedgerQuery(db, orgId, id, now), + db + .update(imageHostings) + .set({ purgedAt: now.getTime() }) + .where(and(eq(imageHostings.id, id), eq(imageHostings.orgId, orgId), isNull(imageHostings.purgedAt))), + ]) }, } } diff --git a/server/adapters/repos/matter.integration.test.ts b/server/adapters/repos/matter.integration.test.ts index db063696..f93a6104 100644 --- a/server/adapters/repos/matter.integration.test.ts +++ b/server/adapters/repos/matter.integration.test.ts @@ -297,15 +297,19 @@ describe('reserveStorageUsage', () => { VALUES ('active-file', ${orgId}, 'active-file-alias', 'active.txt', 'text/plain', 100, 0, '', 'active-key', ${storageId}, 'active', NULL, ${now}, ${now}), ('trashed-file', ${orgId}, 'trashed-file-alias', 'trashed.txt', 'text/plain', 200, 0, '', 'trashed-key', ${storageId}, 'active', ${now}, ${now}, ${now}), + ('purged-file', ${orgId}, 'purged-file-alias', 'purged.txt', 'text/plain', 999, 0, '', 'purged-key', ${storageId}, 'active', ${now}, ${now}, ${now}), ('draft-file', ${orgId}, 'draft-file-alias', 'draft.txt', 'text/plain', 300, 0, '', 'draft-key', ${storageId}, 'draft', NULL, ${now}, ${now}), ('active-folder', ${orgId}, 'active-folder-alias', 'Folder', 'folder', 400, 1, '', '', ${storageId}, 'active', NULL, ${now}, ${now}) `) + await db.run(sql`UPDATE matters SET purged_at = ${now} WHERE id = 'purged-file'`) await db.run(sql` INSERT INTO image_hostings (id, org_id, token, path, storage_id, storage_key, size, mime, status, access_count, created_at) VALUES ('active-image', ${orgId}, 'ih_active', 'active.png', ${storageId}, 'ih/active.png', 50, 'image/png', 'active', 0, ${now}), + ('purged-image', ${orgId}, 'ih_purged', 'purged.png', ${storageId}, 'ih/purged.png', 888, 'image/png', 'active', 0, ${now}), ('draft-image', ${orgId}, 'ih_draft', 'draft.png', ${storageId}, 'ih/draft.png', 70, 'image/png', 'draft', 0, ${now}) `) + await db.run(sql`UPDATE image_hostings SET purged_at = ${now} WHERE id = 'purged-image'`) await createStorageUsageRepo(db).reconcile(orgId, [storageId]) @@ -336,6 +340,12 @@ describe('confirmUpload', () => { expect(storageRows[0].used).toBe(500) const quotaRows = await db.all<{ used: number }>(sql`SELECT used FROM org_quotas WHERE org_id = ${orgId}`) expect(quotaRows[0].used).toBe(500) + const ledgerRows = await db.all<{ bytes: number; events: number }>(sql` + SELECT COALESCE(SUM(delta_bytes), 0) AS bytes, COUNT(*) AS events + FROM storage_usage_ledger + WHERE org_id = ${orgId} AND storage_id = ${storageId} + `) + expect(ledgerRows[0]).toEqual({ bytes: 500, events: 2 }) }) it('returns { matter } and does not increment usage when file size is 0', async () => { diff --git a/server/adapters/repos/matter.ts b/server/adapters/repos/matter.ts index ff3ea101..0261277f 100644 --- a/server/adapters/repos/matter.ts +++ b/server/adapters/repos/matter.ts @@ -3,6 +3,7 @@ import type { SQL } from 'drizzle-orm' import { and, asc, count, desc, eq, inArray, isNotNull, isNull, like, lt, ne, or, sql } from 'drizzle-orm' import { nanoid } from 'nanoid' import { matters } from '../../db/schema' +import { executeWriteTransaction, executeWriteTransactionWithResults } from '../../db/transaction' import { suggestRenamed } from '../../domain/matter-name-conflict' import type { Database } from '../../platform/interface' import type { @@ -19,6 +20,13 @@ import type { } from '../../usecases/ports' import { NameConflictError } from '../../usecases/ports' import { createActivityRepo } from './activity' +import { + matterActivationLedgerQuery, + matterPurgeLedgerQuery, + matterResizeLedgerQuery, + storageUsageMutationQuery, + storageUsageOpeningBalanceQuery, +} from './storage-usage-ledger' type MatterRow = typeof matters.$inferSelect @@ -62,7 +70,7 @@ export function createMatterRepo(db: Database): MatterRepo { const rows = await db .select() .from(matters) - .where(and(eq(matters.id, id), eq(matters.orgId, orgId))) + .where(and(eq(matters.id, id), eq(matters.orgId, orgId), isNull(matters.purgedAt))) return rows[0] ? toMatter(rows[0]) : null } @@ -70,14 +78,14 @@ export function createMatterRepo(db: Database): MatterRepo { return db .select() .from(matters) - .where(and(eq(matters.orgId, orgId), descendantParentCondition(folderPath))) + .where(and(eq(matters.orgId, orgId), isNull(matters.purgedAt), descendantParentCondition(folderPath))) } function getDirectChildren(orgId: string, folderPath: string): Promise { return db .select() .from(matters) - .where(and(eq(matters.orgId, orgId), eq(matters.parent, folderPath))) + .where(and(eq(matters.orgId, orgId), eq(matters.parent, folderPath), isNull(matters.purgedAt))) } async function cascadeParentPath(orgId: string, oldPath: string, newPath: string): Promise { @@ -85,7 +93,7 @@ export function createMatterRepo(db: Database): MatterRepo { await db .update(matters) .set({ parent: newPath, updatedAt: new Date() }) - .where(and(eq(matters.orgId, orgId), eq(matters.parent, oldPath))) + .where(and(eq(matters.orgId, orgId), eq(matters.parent, oldPath), isNull(matters.purgedAt))) // Deeper descendants: parent starts with 'oldPath/' → replace prefix. await db @@ -94,7 +102,7 @@ export function createMatterRepo(db: Database): MatterRepo { parent: sql`${newPath} || SUBSTR(${matters.parent}, LENGTH(${oldPath}) + 1)`, updatedAt: new Date(), }) - .where(and(eq(matters.orgId, orgId), descendantParentCondition(oldPath))) + .where(and(eq(matters.orgId, orgId), isNull(matters.purgedAt), descendantParentCondition(oldPath))) } /** @@ -115,6 +123,7 @@ export function createMatterRepo(db: Database): MatterRepo { eq(matters.parent, parent), eq(matters.status, ObjectStatus.ACTIVE), isNull(matters.trashedAt), + isNull(matters.purgedAt), sql`lower(${matters.name}) = lower(${name})`, ] if (excludeId) conditions.push(ne(matters.id, excludeId)) @@ -178,7 +187,7 @@ export function createMatterRepo(db: Database): MatterRepo { await db .update(matters) .set({ trashedAt: now.getTime(), updatedAt: now }) - .where(and(eq(matters.id, existing.id), eq(matters.orgId, orgId))) + .where(and(eq(matters.id, existing.id), eq(matters.orgId, orgId), isNull(matters.purgedAt))) if (userId) { await activity.record({ @@ -267,11 +276,34 @@ export function createMatterRepo(db: Database): MatterRepo { storageId: input.storageId, status: input.status, trashedAt: null, + purgedAt: null, createdAt: now, updatedAt: now, } - await db.insert(matters).values(row) + const isBillable = row.status === ObjectStatus.ACTIVE && row.dirtype === DirType.FILE + if (isBillable) { + await executeWriteTransaction(db, [ + storageUsageOpeningBalanceQuery(db, row.orgId, row.storageId, now), + db.insert(matters).values(row), + ...(row.size && row.size > 0 + ? [ + storageUsageMutationQuery(db, { + eventKey: `matter:${row.id}:activated`, + orgId: row.orgId, + storageId: row.storageId, + resourceType: 'matter', + resourceId: row.id, + deltaBytes: row.size, + reason: 'matter_activated', + occurredAt: now, + }), + ] + : []), + ]) + } else { + await db.insert(matters).values(row) + } if (input.userId) { await activity.record({ @@ -291,7 +323,12 @@ export function createMatterRepo(db: Database): MatterRepo { const offset = (filters.page - 1) * filters.pageSize // Live objects only. Trashed rows are status='active' too, so exclude them // by trashedAt; the recycle bin is served by listTrashedRoots. - const conditions = [eq(matters.orgId, orgId), eq(matters.status, ObjectStatus.ACTIVE), isNull(matters.trashedAt)] + const conditions = [ + eq(matters.orgId, orgId), + eq(matters.status, ObjectStatus.ACTIVE), + isNull(matters.trashedAt), + isNull(matters.purgedAt), + ] const typeCond = filters.typeFilter ? typeFilterCondition(filters.typeFilter) : undefined if (filters.search) { conditions.push(like(matters.name, `%${filters.search}%`)) @@ -326,7 +363,7 @@ export function createMatterRepo(db: Database): MatterRepo { const rows = await db .select() .from(matters) - .where(and(eq(matters.orgId, orgId), inArray(matters.id, ids))) + .where(and(eq(matters.orgId, orgId), inArray(matters.id, ids), isNull(matters.purgedAt))) return rows.map(toMatter) }, @@ -370,7 +407,7 @@ export function createMatterRepo(db: Database): MatterRepo { await db .update(matters) .set({ name: newName, parent: newParent, updatedAt: now }) - .where(and(eq(matters.id, id), eq(matters.orgId, orgId))) + .where(and(eq(matters.id, id), eq(matters.orgId, orgId), isNull(matters.purgedAt))) const updated = { ...existing, name: newName, parent: newParent, updatedAt: now } @@ -435,11 +472,29 @@ export function createMatterRepo(db: Database): MatterRepo { storageId: source.storageId, status: 'active', trashedAt: null, + purgedAt: null, createdAt: now, updatedAt: now, } - await db.insert(matters).values(row) + await executeWriteTransaction(db, [ + storageUsageOpeningBalanceQuery(db, row.orgId, row.storageId, now), + db.insert(matters).values(row), + ...(row.dirtype === DirType.FILE && row.size && row.size > 0 + ? [ + storageUsageMutationQuery(db, { + eventKey: `matter:${row.id}:activated`, + orgId: row.orgId, + storageId: row.storageId, + resourceType: 'matter', + resourceId: row.id, + deltaBytes: row.size, + reason: 'matter_activated', + occurredAt: now, + }), + ] + : []), + ]) if (opts.userId) { await activity.record({ @@ -456,19 +511,13 @@ export function createMatterRepo(db: Database): MatterRepo { return toMatter(row) }, - async delete(id, orgId): Promise { - const existing = await getMatter(id, orgId) - if (!existing) return null - - await db.delete(matters).where(and(eq(matters.id, id), eq(matters.orgId, orgId))) - return existing - }, - async cancelDraft(id, orgId, userId?): Promise { const existing = await getMatter(id, orgId) if (!existing || existing.status !== 'draft') return null - await db.delete(matters).where(and(eq(matters.id, id), eq(matters.orgId, orgId), eq(matters.status, 'draft'))) + await db + .delete(matters) + .where(and(eq(matters.id, id), eq(matters.orgId, orgId), eq(matters.status, 'draft'), isNull(matters.purgedAt))) if (userId) { await activity.record({ @@ -522,6 +571,7 @@ export function createMatterRepo(db: Database): MatterRepo { eq(matters.orgId, orgId), eq(matters.status, ObjectStatus.ACTIVE), isNull(matters.trashedAt), + isNull(matters.purgedAt), ), ) } @@ -574,7 +624,7 @@ export function createMatterRepo(db: Database): MatterRepo { await db .update(matters) .set({ name: finalName, updatedAt: now }) - .where(and(eq(matters.id, existing.id), eq(matters.orgId, orgId))) + .where(and(eq(matters.id, existing.id), eq(matters.orgId, orgId), isNull(matters.purgedAt))) if (isFolder) { const oldPath = buildPath(existing.parent, existing.name) const newPath = buildPath(existing.parent, finalName) @@ -587,7 +637,14 @@ export function createMatterRepo(db: Database): MatterRepo { await db .update(matters) .set({ trashedAt: null, updatedAt: now }) - .where(and(eq(matters.id, targetId), eq(matters.orgId, orgId), isNotNull(matters.trashedAt))) + .where( + and( + eq(matters.id, targetId), + eq(matters.orgId, orgId), + isNotNull(matters.trashedAt), + isNull(matters.purgedAt), + ), + ) } const restored = { ...existing, name: finalName, trashedAt: null, updatedAt: now } @@ -609,9 +666,40 @@ export function createMatterRepo(db: Database): MatterRepo { collectForPurge, async purge(orgId, ids): Promise { - for (const id of ids) { - await db.delete(matters).where(and(eq(matters.id, id), eq(matters.orgId, orgId))) - } + if (ids.length === 0) return + const rows = await db + .select() + .from(matters) + .where(and(eq(matters.orgId, orgId), inArray(matters.id, ids), isNull(matters.purgedAt))) + if (rows.length === 0) return + + const now = new Date() + const storageIds = new Set( + rows + .filter((row) => row.dirtype === DirType.FILE && row.status === ObjectStatus.ACTIVE) + .map((row) => row.storageId), + ) + await executeWriteTransaction(db, [ + ...[...storageIds].map((storageId) => storageUsageOpeningBalanceQuery(db, orgId, storageId, now)), + ...rows.map((row) => matterPurgeLedgerQuery(db, orgId, row.id, now)), + db + .update(matters) + .set({ + trashedAt: sql`COALESCE(${matters.trashedAt}, ${now.getTime()})`, + purgedAt: now.getTime(), + updatedAt: now, + }) + .where( + and( + eq(matters.orgId, orgId), + inArray( + matters.id, + rows.map((row) => row.id), + ), + isNull(matters.purgedAt), + ), + ), + ]) }, async listActiveDescendants(orgId, parentPath): Promise { @@ -625,6 +713,7 @@ export function createMatterRepo(db: Database): MatterRepo { eq(matters.orgId, orgId), eq(matters.status, ObjectStatus.ACTIVE), isNull(matters.trashedAt), + isNull(matters.purgedAt), descendantParentCondition(parentPath), ), ) @@ -636,7 +725,7 @@ export function createMatterRepo(db: Database): MatterRepo { await db .update(matters) .set({ trashedAt: Date.now(), updatedAt: new Date() }) - .where(and(eq(matters.orgId, orgId), inArray(matters.id, ids))) + .where(and(eq(matters.orgId, orgId), inArray(matters.id, ids), isNull(matters.purgedAt))) }, async restoreActiveByIds(orgId, ids): Promise { @@ -644,28 +733,42 @@ export function createMatterRepo(db: Database): MatterRepo { await db .update(matters) .set({ trashedAt: null, updatedAt: new Date() }) - .where(and(eq(matters.orgId, orgId), inArray(matters.id, ids))) + .where(and(eq(matters.orgId, orgId), inArray(matters.id, ids), isNull(matters.purgedAt))) }, async touch(orgId, id): Promise { await db .update(matters) .set({ updatedAt: new Date() }) - .where(and(eq(matters.id, id), eq(matters.orgId, orgId))) + .where(and(eq(matters.id, id), eq(matters.orgId, orgId), isNull(matters.purgedAt))) }, async applyUpload(orgId, id, fields): Promise { - await db - .update(matters) - .set({ type: fields.type, size: fields.size, object: fields.object, updatedAt: new Date() }) - .where(and(eq(matters.id, id), eq(matters.orgId, orgId))) + const existing = await getMatter(id, orgId) + if (!existing) return + const now = new Date() + await executeWriteTransaction(db, [ + storageUsageOpeningBalanceQuery(db, orgId, existing.storageId, now), + matterResizeLedgerQuery(db, orgId, id, fields.size, now), + db + .update(matters) + .set({ type: fields.type, size: fields.size, object: fields.object, updatedAt: now }) + .where(and(eq(matters.id, id), eq(matters.orgId, orgId), isNull(matters.purgedAt))), + ]) }, async listTrashedRoots(orgId): Promise { const all = await db .select() .from(matters) - .where(and(eq(matters.orgId, orgId), eq(matters.status, ObjectStatus.ACTIVE), isNotNull(matters.trashedAt))) + .where( + and( + eq(matters.orgId, orgId), + eq(matters.status, ObjectStatus.ACTIVE), + isNotNull(matters.trashedAt), + isNull(matters.purgedAt), + ), + ) const trashedPaths = new Set(all.map((m) => buildPath(m.parent, m.name))) return all @@ -683,7 +786,7 @@ export function createMatterRepo(db: Database): MatterRepo { const rows = await db .selectDistinct({ orgId: matters.orgId }) .from(matters) - .where(and(isNotNull(matters.trashedAt), lt(matters.trashedAt, cutoff))) + .where(and(isNotNull(matters.trashedAt), isNull(matters.purgedAt), lt(matters.trashedAt, cutoff))) return rows.map((r) => r.orgId) }, @@ -704,11 +807,20 @@ export function createMatterRepo(db: Database): MatterRepo { }, async activateDraft(id, orgId, finalName, now): Promise { - const updated = await db + const existing = await getMatter(id, orgId) + if (!existing || existing.status !== 'draft') return false + await executeWriteTransaction(db, [storageUsageOpeningBalanceQuery(db, orgId, existing.storageId, now)]) + const activateQuery = db .update(matters) .set({ name: finalName, status: 'active', updatedAt: now }) - .where(and(eq(matters.id, id), eq(matters.orgId, orgId), eq(matters.status, 'draft'))) + .where(and(eq(matters.id, id), eq(matters.orgId, orgId), eq(matters.status, 'draft'), isNull(matters.purgedAt))) .returning({ id: matters.id }) + const results = await executeWriteTransactionWithResults( + db, + [activateQuery, matterActivationLedgerQuery(db, orgId, id, now)], + [0], + ) + const updated = results[0] as { id: string }[] return updated.length > 0 }, } diff --git a/server/adapters/repos/share.ts b/server/adapters/repos/share.ts index 88403fcf..9c4b01b6 100644 --- a/server/adapters/repos/share.ts +++ b/server/adapters/repos/share.ts @@ -40,7 +40,7 @@ export function createShareRepo(db: Database): ShareRepo { const matter = await db .select() .from(matters) - .where(and(eq(matters.id, input.matterId), eq(matters.orgId, input.orgId))) + .where(and(eq(matters.id, input.matterId), eq(matters.orgId, input.orgId), isNull(matters.purgedAt))) .then((rows) => rows[0] ?? null) if (!matter) throw new CreateShareError('MATTER_NOT_FOUND') @@ -321,6 +321,7 @@ export function createShareRepo(db: Database): ShareRepo { eq(matters.orgId, matter.orgId), eq(matters.status, 'active'), isNull(matters.trashedAt), + isNull(matters.purgedAt), eq(matters.dirtype, DirType.FILE), or(eq(matters.parent, folderPath), like(matters.parent, `${folderPath}/%`)), ), @@ -338,6 +339,7 @@ export function createShareRepo(db: Database): ShareRepo { eq(matters.parent, folderPath), eq(matters.status, 'active'), isNull(matters.trashedAt), + isNull(matters.purgedAt), ), ) }, @@ -357,7 +359,11 @@ export function createShareRepo(db: Database): ShareRepo { }, async getMatterName(matterId: string): Promise { - const rows = await db.select({ name: matters.name }).from(matters).where(eq(matters.id, matterId)).limit(1) + const rows = await db + .select({ name: matters.name }) + .from(matters) + .where(and(eq(matters.id, matterId), isNull(matters.purgedAt))) + .limit(1) return rows[0]?.name ?? null }, @@ -376,6 +382,7 @@ export function createShareRepo(db: Database): ShareRepo { eq(matters.orgId, rootMatter.orgId), eq(matters.status, 'active'), isNull(matters.trashedAt), + isNull(matters.purgedAt), or(eq(matters.parent, root), sql`${matters.parent} LIKE ${likePattern} ESCAPE '\\'`), ), ) diff --git a/server/adapters/repos/storage-usage-ledger.ts b/server/adapters/repos/storage-usage-ledger.ts new file mode 100644 index 00000000..bf813204 --- /dev/null +++ b/server/adapters/repos/storage-usage-ledger.ts @@ -0,0 +1,230 @@ +import { DirType } from '@shared/constants' +import { and, eq, isNull, sql } from 'drizzle-orm' +import { nanoid } from 'nanoid' +import { imageHostings, matters, storageUsageLedger } from '../../db/schema' +import { type AtomicQuery, executeWriteTransaction } from '../../db/transaction' +import type { Database } from '../../platform/interface' + +export type StorageUsageResourceType = 'matter' | 'image_hosting' | 'storage' + +export type StorageUsageReason = + | 'opening_balance' + | 'opening_balance_complete' + | 'matter_activated' + | 'matter_resized' + | 'matter_purged' + | 'image_activated' + | 'image_purged' + +export interface StorageUsageLedgerMutation { + eventKey?: string + orgId: string + storageId: string + resourceType: StorageUsageResourceType + resourceId: string + deltaBytes: number + reason: StorageUsageReason + occurredAt: Date +} + +function openingEventKey(orgId: string, storageId: string): string { + return `opening:${orgId}:${storageId}` +} + +export function storageUsageOpeningBalanceQuery( + db: Database, + orgId: string, + storageId: string, + occurredAt: Date, +): AtomicQuery { + const eventKey = openingEventKey(orgId, storageId) + return db + .insert(storageUsageLedger) + .values({ + id: eventKey, + eventKey, + orgId, + storageId, + resourceType: 'storage', + resourceId: storageId, + deltaBytes: sql`COALESCE(( + SELECT SUM(${matters.size}) + FROM ${matters} + WHERE ${matters.orgId} = ${orgId} + AND ${matters.storageId} = ${storageId} + AND ${matters.dirtype} = ${DirType.FILE} + AND ${matters.status} = 'active' + AND ${matters.purgedAt} IS NULL + ), 0) + COALESCE(( + SELECT SUM(${imageHostings.size}) + FROM ${imageHostings} + WHERE ${imageHostings.orgId} = ${orgId} + AND ${imageHostings.storageId} = ${storageId} + AND ${imageHostings.status} = 'active' + AND ${imageHostings.purgedAt} IS NULL + ), 0)`, + reason: 'opening_balance', + occurredAt, + createdAt: occurredAt, + }) + .onConflictDoNothing({ target: storageUsageLedger.eventKey }) +} + +export function storageUsageMutationQuery(db: Database, mutation: StorageUsageLedgerMutation): AtomicQuery { + const eventKey = mutation.eventKey ?? `mutation:${nanoid()}` + return db + .insert(storageUsageLedger) + .values({ + id: nanoid(), + eventKey, + orgId: mutation.orgId, + storageId: mutation.storageId, + resourceType: mutation.resourceType, + resourceId: mutation.resourceId, + deltaBytes: mutation.deltaBytes, + reason: mutation.reason, + occurredAt: mutation.occurredAt, + createdAt: mutation.occurredAt, + }) + .onConflictDoNothing({ target: storageUsageLedger.eventKey }) +} + +function conditionalMutationQuery(db: Database, selection: ReturnType): AtomicQuery { + return db.insert(storageUsageLedger).select(selection).onConflictDoNothing({ target: storageUsageLedger.eventKey }) +} + +export function matterActivationLedgerQuery( + db: Database, + orgId: string, + matterId: string, + occurredAt: Date, +): AtomicQuery { + const eventKey = `matter:${matterId}:activated` + return conditionalMutationQuery( + db, + sql`SELECT + ${nanoid()}, ${eventKey}, ${matters.orgId}, ${matters.storageId}, 'matter', ${matters.id}, + ${matters.size}, 'matter_activated', ${occurredAt.getTime()}, ${occurredAt.getTime()} + FROM ${matters} + WHERE ${matters.id} = ${matterId} + AND ${matters.orgId} = ${orgId} + AND ${matters.status} = 'active' + AND ${matters.dirtype} = ${DirType.FILE} + AND ${matters.purgedAt} IS NULL + AND COALESCE(${matters.size}, 0) > 0`, + ) +} + +export function matterResizeLedgerQuery( + db: Database, + orgId: string, + matterId: string, + nextSize: number, + occurredAt: Date, +): AtomicQuery { + return conditionalMutationQuery( + db, + sql`SELECT + ${nanoid()}, ${`matter:${matterId}:resized:${nanoid()}`}, ${matters.orgId}, ${matters.storageId}, 'matter', + ${matters.id}, ${nextSize} - COALESCE(${matters.size}, 0), 'matter_resized', + ${occurredAt.getTime()}, ${occurredAt.getTime()} + FROM ${matters} + WHERE ${matters.id} = ${matterId} + AND ${matters.orgId} = ${orgId} + AND ${matters.status} = 'active' + AND ${matters.dirtype} = ${DirType.FILE} + AND ${matters.purgedAt} IS NULL + AND ${nextSize} <> COALESCE(${matters.size}, 0)`, + ) +} + +export function matterPurgeLedgerQuery(db: Database, orgId: string, matterId: string, occurredAt: Date): AtomicQuery { + const eventKey = `matter:${matterId}:purged` + return conditionalMutationQuery( + db, + sql`SELECT + ${nanoid()}, ${eventKey}, ${matters.orgId}, ${matters.storageId}, 'matter', ${matters.id}, + -COALESCE(${matters.size}, 0), 'matter_purged', ${occurredAt.getTime()}, ${occurredAt.getTime()} + FROM ${matters} + WHERE ${matters.id} = ${matterId} + AND ${matters.orgId} = ${orgId} + AND ${matters.status} = 'active' + AND ${matters.dirtype} = ${DirType.FILE} + AND ${matters.purgedAt} IS NULL + AND COALESCE(${matters.size}, 0) > 0`, + ) +} + +export function imageActivationLedgerQuery( + db: Database, + orgId: string, + imageId: string, + occurredAt: Date, +): AtomicQuery { + const eventKey = `image:${imageId}:activated` + return conditionalMutationQuery( + db, + sql`SELECT + ${nanoid()}, ${eventKey}, ${imageHostings.orgId}, ${imageHostings.storageId}, 'image_hosting', + ${imageHostings.id}, ${imageHostings.size}, 'image_activated', ${occurredAt.getTime()}, ${occurredAt.getTime()} + FROM ${imageHostings} + WHERE ${imageHostings.id} = ${imageId} + AND ${imageHostings.orgId} = ${orgId} + AND ${imageHostings.status} = 'active' + AND ${imageHostings.purgedAt} IS NULL + AND ${imageHostings.size} > 0`, + ) +} + +export function imagePurgeLedgerQuery(db: Database, orgId: string, imageId: string, occurredAt: Date): AtomicQuery { + const eventKey = `image:${imageId}:purged` + return conditionalMutationQuery( + db, + sql`SELECT + ${nanoid()}, ${eventKey}, ${imageHostings.orgId}, ${imageHostings.storageId}, 'image_hosting', + ${imageHostings.id}, -${imageHostings.size}, 'image_purged', ${occurredAt.getTime()}, ${occurredAt.getTime()} + FROM ${imageHostings} + WHERE ${imageHostings.id} = ${imageId} + AND ${imageHostings.orgId} = ${orgId} + AND ${imageHostings.status} = 'active' + AND ${imageHostings.purgedAt} IS NULL + AND ${imageHostings.size} > 0`, + ) +} + +export async function ensureStorageUsageOpeningBalances(db: Database, occurredAt: Date): Promise { + const [matterPairs, imagePairs] = await Promise.all([ + db + .selectDistinct({ orgId: matters.orgId, storageId: matters.storageId }) + .from(matters) + .where(and(eq(matters.dirtype, DirType.FILE), eq(matters.status, 'active'), isNull(matters.purgedAt))), + db + .selectDistinct({ orgId: imageHostings.orgId, storageId: imageHostings.storageId }) + .from(imageHostings) + .where(and(eq(imageHostings.status, 'active'), isNull(imageHostings.purgedAt))), + ]) + + const pairs = new Map() + for (const pair of [...matterPairs, ...imagePairs]) { + pairs.set(`${pair.orgId}\u0000${pair.storageId}`, pair) + } + + const queries = [...pairs.values()].map(({ orgId, storageId }) => + storageUsageOpeningBalanceQuery(db, orgId, storageId, occurredAt), + ) + for (let offset = 0; offset < queries.length; offset += 50) { + await executeWriteTransaction(db, queries.slice(offset, offset + 50)) + } + await executeWriteTransaction(db, [ + storageUsageMutationQuery(db, { + eventKey: 'opening:complete', + orgId: '', + storageId: '', + resourceType: 'storage', + resourceId: 'global', + deltaBytes: 0, + reason: 'opening_balance_complete', + occurredAt, + }), + ]) +} diff --git a/server/adapters/repos/storage-usage.ts b/server/adapters/repos/storage-usage.ts index 80da6087..24a696cf 100644 --- a/server/adapters/repos/storage-usage.ts +++ b/server/adapters/repos/storage-usage.ts @@ -41,11 +41,13 @@ export function createStorageUsageRepo(db: Database): StorageUsageRepo { WHERE ${matters.orgId} = ${orgId} AND ${matters.dirtype} = ${DirType.FILE} AND ${matters.status} = 'active' + AND ${matters.purgedAt} IS NULL ), 0) + COALESCE(( SELECT SUM(${imageHostings.size}) FROM ${imageHostings} WHERE ${imageHostings.orgId} = ${orgId} AND ${imageHostings.status} = 'active' + AND ${imageHostings.purgedAt} IS NULL ), 0)`, }) .where(eq(orgQuotas.orgId, orgId)) @@ -60,11 +62,13 @@ export function createStorageUsageRepo(db: Database): StorageUsageRepo { WHERE ${matters.storageId} = ${storageId} AND ${matters.dirtype} = ${DirType.FILE} AND ${matters.status} = 'active' + AND ${matters.purgedAt} IS NULL ), 0) + COALESCE(( SELECT SUM(${imageHostings.size}) FROM ${imageHostings} WHERE ${imageHostings.storageId} = ${storageId} AND ${imageHostings.status} = 'active' + AND ${imageHostings.purgedAt} IS NULL ), 0)`, }) .where(eq(storages.id, storageId)) diff --git a/server/adapters/repos/storage.ts b/server/adapters/repos/storage.ts index 1a54394a..34b7e87b 100644 --- a/server/adapters/repos/storage.ts +++ b/server/adapters/repos/storage.ts @@ -1,4 +1,4 @@ -import { and, asc, count, eq, lt, or } from 'drizzle-orm' +import { and, asc, count, eq, isNull, lt, or } from 'drizzle-orm' import { nanoid } from 'nanoid' import { matters, storages } from '../../db/schema' import type { Database } from '../../platform/interface' @@ -88,7 +88,10 @@ export function createStorageRepo(db: Database): StorageRepo { const existing = await getRow(id) if (!existing) return 'not_found' - const refs = await db.select({ count: count() }).from(matters).where(eq(matters.storageId, id)) + const refs = await db + .select({ count: count() }) + .from(matters) + .where(and(eq(matters.storageId, id), isNull(matters.purgedAt))) if ((refs[0]?.count ?? 0) > 0) return 'in_use' await db.delete(storages).where(eq(storages.id, id)) diff --git a/server/adapters/repos/webdav-path.ts b/server/adapters/repos/webdav-path.ts index 93c5af78..f5c2c920 100644 --- a/server/adapters/repos/webdav-path.ts +++ b/server/adapters/repos/webdav-path.ts @@ -54,6 +54,7 @@ export function createWebDavPathRepo(db: Database): WebDavPathRepo { eq(matters.parent, parent), eq(matters.status, ObjectStatus.ACTIVE), isNull(matters.trashedAt), + isNull(matters.purgedAt), ), ) .orderBy(desc(matters.dirtype), asc(matters.name)) @@ -145,6 +146,7 @@ async function findMatterByPath(db: Database, orgId: string, parent: string, nam eq(matters.name, name), eq(matters.status, ObjectStatus.ACTIVE), isNull(matters.trashedAt), + isNull(matters.purgedAt), ), ) .limit(1) diff --git a/server/adapters/repos/zip.ts b/server/adapters/repos/zip.ts index ccbfa892..360ff05d 100644 --- a/server/adapters/repos/zip.ts +++ b/server/adapters/repos/zip.ts @@ -25,7 +25,7 @@ async function collectCompressionPlan( const roots = await db .select() .from(matters) - .where(and(eq(matters.orgId, orgId), or(...uniqueIds.map((id) => eq(matters.id, id))))) + .where(and(eq(matters.orgId, orgId), isNull(matters.purgedAt), or(...uniqueIds.map((id) => eq(matters.id, id))))) if (roots.length !== uniqueIds.length) throw new Error('Some archive source IDs do not belong to this organization') if (roots.some((matter) => matter.status !== 'active' || matter.trashedAt != null)) throw new Error('Only active matters can be archived') @@ -97,6 +97,7 @@ async function collectRootEntries( eq(matters.orgId, orgId), eq(matters.status, 'active'), isNull(matters.trashedAt), + isNull(matters.purgedAt), or(eq(matters.parent, rootPath), like(matters.parent, `${rootPath}/%`)), ), ) diff --git a/server/db/schema.ts b/server/db/schema.ts index ecea3b9d..95b3ec1d 100644 --- a/server/db/schema.ts +++ b/server/db/schema.ts @@ -17,6 +17,7 @@ export const matters = sqliteTable( storageId: text('storage_id').notNull(), status: text('status').notNull().default('draft'), // draft, active trashedAt: integer('trashed_at'), // null = live, epoch ms = in trash (soft delete) + purgedAt: integer('purged_at'), // null = retained/billable, epoch ms = content permanently removed createdAt: integer('created_at', { mode: 'timestamp' }).notNull(), updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull(), }, @@ -505,6 +506,27 @@ export const statsRollupsHourly = sqliteTable( ], ) +export const storageUsageLedger = sqliteTable( + 'storage_usage_ledger', + { + id: text('id').primaryKey(), + eventKey: text('event_key').notNull().unique(), + orgId: text('org_id').notNull(), + storageId: text('storage_id').notNull(), + resourceType: text('resource_type').notNull(), + resourceId: text('resource_id').notNull(), + deltaBytes: integer('delta_bytes').notNull(), + reason: text('reason').notNull(), + occurredAt: integer('occurred_at', { mode: 'timestamp_ms' }).notNull(), + createdAt: integer('created_at', { mode: 'timestamp_ms' }).notNull(), + }, + (t) => [ + index('storage_usage_ledger_occurred_idx').on(t.occurredAt), + index('storage_usage_ledger_org_occurred_idx').on(t.orgId, t.occurredAt), + index('storage_usage_ledger_storage_occurred_idx').on(t.storageId, t.occurredAt), + ], +) + export const shares = sqliteTable( 'shares', { @@ -575,12 +597,13 @@ export const imageHostings = sqliteTable( width: integer('width'), height: integer('height'), status: text('status').notNull().default('draft'), // 'draft' | 'active' + purgedAt: integer('purged_at'), accessCount: integer('access_count').notNull().default(0), lastAccessedAt: integer('last_accessed_at', { mode: 'timestamp_ms' }), createdAt: integer('created_at', { mode: 'timestamp_ms' }).notNull(), }, (t) => [ - uniqueIndex('image_hostings_org_path_uniq').on(t.orgId, t.path), + uniqueIndex('image_hostings_org_path_uniq').on(t.orgId, t.path).where(sql`${t.purgedAt} IS NULL`), index('image_hostings_org_created_idx').on(t.orgId, t.createdAt), index('image_hostings_token_idx').on(t.token), ], diff --git a/server/db/transaction.ts b/server/db/transaction.ts index 0e4f55db..e56f09d1 100644 --- a/server/db/transaction.ts +++ b/server/db/transaction.ts @@ -35,3 +35,21 @@ export async function executeWriteTransaction(db: Database, queries: AtomicQuery } }) } + +export async function executeWriteTransactionWithResults( + db: Database, + queries: AtomicQuery[], + returningQueryIndexes: readonly number[], +): Promise { + const batch = (db as unknown as { batch?: (queries: AtomicQuery[]) => Promise }).batch + if (batch) return await batch.call(db, queries) + + if (!isSyncDatabase(db)) throw new Error('db_transaction_unavailable') + + return (db as unknown as { transaction(fn: () => T): T }).transaction(() => + queries.map((query, index) => { + if (returningQueryIndexes.includes(index)) return query.all?.() + return query.run?.() + }), + ) +} diff --git a/server/domain/admin-stats-metrics.ts b/server/domain/admin-stats-metrics.ts index 4da5c7b3..d7f380f6 100644 --- a/server/domain/admin-stats-metrics.ts +++ b/server/domain/admin-stats-metrics.ts @@ -54,6 +54,7 @@ export const ADMIN_STATS_METRICS = { statsMissingBytes: 'stats.quality_missing_bytes', statsRollupRun: 'stats.rollup_run', storageInventory: 'storage.inventory', + storageLedgerBalance: 'storage.ledger_balance', storageQuota: 'storage.quota', storageTrashSnapshot: 'storage.trash_snapshot', storageUsed: 'storage.used', @@ -94,6 +95,7 @@ export const ADMIN_STATS_METRIC_REGISTRY = { [M.statsMissingBytes]: counter(['direction', 'source']), [M.statsRollupRun]: counter(['outcome']), [M.storageInventory]: gauge(['age_bucket', 'file_type_group', 'size_bucket', 'storage_id'], 'entities', true), + [M.storageLedgerBalance]: counter(['storage_id'], true), [M.storageQuota]: gauge(['status'], null, true), [M.storageTrashSnapshot]: gauge(['storage_id'], 'entities', true), [M.storageUsed]: gauge(['storage_id'], null, true), diff --git a/server/http/admin-stats.integration.test.ts b/server/http/admin-stats.integration.test.ts index 7522666c..70a52b33 100644 --- a/server/http/admin-stats.integration.test.ts +++ b/server/http/admin-stats.integration.test.ts @@ -169,8 +169,8 @@ describe('site stats routes', () => { 'storage-marker-2026-01-01', ${Date.UTC(2026, 0, 1)}, '', 'stats.rollup_run', '', '', 1, 0, 0, '{"version":3,"scope":"full","quality":"exact"}', ${Date.UTC(2026, 0, 2)} ), ( - 'storage-used-2026-01-01', ${Date.UTC(2026, 0, 1)}, '', 'storage.used', '', '', - 0, 4096, 0, '{"version":3,"scope":"snapshots","quality":"exact"}', ${Date.UTC(2026, 0, 2)} + 'storage-used-2026-01-01', ${Date.UTC(2026, 0, 1)}, '', 'storage.ledger_balance', '', '', + 0, 4096, 0, '{"version":3,"scope":"counters","quality":"exact"}', ${Date.UTC(2026, 0, 2)} ) `) diff --git a/server/http/image-hosting/images.integration.test.ts b/server/http/image-hosting/images.integration.test.ts index 8317b160..ca24f47d 100644 --- a/server/http/image-hosting/images.integration.test.ts +++ b/server/http/image-hosting/images.integration.test.ts @@ -1162,7 +1162,7 @@ describe('DELETE /api/image-hosting/images/:id', () => { expect(res.status).toBe(404) }) - it('removes S3 object and DB row; deleteObject called once', async () => { + it('removes the S3 object and retains a hidden DB tombstone', async () => { const { app, db } = await createTestApp() await insertStorage(db) const headers = await authedHeaders(app) @@ -1179,12 +1179,22 @@ describe('DELETE /api/image-hosting/images/:id', () => { expect(S3Service.prototype.deleteObject).toHaveBeenCalledTimes(1) - // Row gone — verify via GET + // Public reads hide the row; the tombstone and exact storage ledger remain. const checkRes = await app.request(`/api/image-hosting/images/${id}`, { headers }) expect(checkRes.status).toBe(404) + const tombstone = await db.all<{ purgedAt: number | null }>( + sql`SELECT purged_at AS purgedAt FROM image_hostings WHERE id = ${id}`, + ) + expect(tombstone[0]?.purgedAt).toEqual(expect.any(Number)) + const ledger = await db.all<{ bytes: number }>(sql` + SELECT COALESCE(SUM(delta_bytes), 0) AS bytes + FROM storage_usage_ledger + WHERE org_id = ${orgId} AND storage_id = ${validStorage.id} + `) + expect(ledger[0].bytes).toBe(0) }) - it('still deletes DB row when storage record is missing (S3 delete skipped)', async () => { + it('still tombstones the DB row when the storage record is missing (S3 delete skipped)', async () => { const { app, db } = await createTestApp() await insertStorage(db) const headers = await authedHeaders(app) @@ -1206,7 +1216,7 @@ describe('DELETE /api/image-hosting/images/:id', () => { // S3 deleteObject should NOT have been called (storage was null) expect(S3Service.prototype.deleteObject).not.toHaveBeenCalled() - // DB row must be gone + // Public reads hide the retained tombstone. const checkRes = await app.request(`/api/image-hosting/images/${id}`, { headers }) expect(checkRes.status).toBe(404) }) diff --git a/server/http/objects.integration.test.ts b/server/http/objects.integration.test.ts index c95684b0..7d21f31f 100644 --- a/server/http/objects.integration.test.ts +++ b/server/http/objects.integration.test.ts @@ -49,8 +49,12 @@ function copyMatter( ) { return createMatterRepo(db).copy(source, targetParent, newObject, opts) } -function deleteMatter(db: TestDbForMatter, id: string, orgId: string) { - return createMatterRepo(db).delete(id, orgId) +async function deleteMatter(db: TestDbForMatter, id: string, orgId: string) { + const repo = createMatterRepo(db) + const existing = await repo.get(id, orgId) + if (!existing) return null + await repo.purge(orgId, [id]) + return existing } function confirmUpload(db: TestDbForMatter, id: string, orgId: string, opts: ConfirmUploadOptions = {}) { return confirmUploadUsecase( @@ -600,6 +604,19 @@ describe('Objects API', () => { expect(await getMatter(db, 'movie-folder', orgId)).toBeNull() expect(await getMatter(db, 'movie-file', orgId)).toBeNull() + const tombstones = await db.all<{ id: string; purgedAt: number | null }>(sql` + SELECT id, purged_at AS purgedAt FROM matters + WHERE id IN ('movie-folder', 'movie-file') + ORDER BY id + `) + expect(tombstones).toHaveLength(2) + expect(tombstones.every((row) => typeof row.purgedAt === 'number')).toBe(true) + const ledger = await db.all<{ bytes: number }>(sql` + SELECT COALESCE(SUM(delta_bytes), 0) AS bytes + FROM storage_usage_ledger + WHERE org_id = ${orgId} AND storage_id = ${validStorage.id} + `) + expect(ledger[0].bytes).toBe(0) }) it('DELETE /api/objects/:id is idempotent for an already-trashed object → 204', async () => { @@ -1029,7 +1046,7 @@ describe('Matter service', () => { expect(result).toBeNull() }) - it('deleteMatter removes and returns the record', async () => { + it('deleteMatter hides and retains the purged record', async () => { const { db } = await createTestApp() const now = Date.now() await db.run(sql` @@ -1051,6 +1068,10 @@ describe('Matter service', () => { const check = await getMatter(db, matter.id, 'org-1') expect(check).toBeNull() + const retained = await db.all<{ purgedAt: number | null }>( + sql`SELECT purged_at AS purgedAt FROM matters WHERE id = ${matter.id}`, + ) + expect(retained[0]?.purgedAt).toEqual(expect.any(Number)) }) it('deleteMatter returns null for missing record', async () => { @@ -2128,9 +2149,11 @@ describe('Objects API — quota enforcement', () => { const completeRes = await complete(app, headers, ref) expect(completeRes.status).toBe(200) - // Incumbent purged (overwritten, not trashed) and usage unchanged. - const incumbent = await db.all(sql`SELECT id FROM matters WHERE id = 'incumbent'`) - expect(incumbent).toHaveLength(0) + // Incumbent content is purged but its tombstone is retained; usage is unchanged. + const incumbent = await db.all<{ purgedAt: number | null }>( + sql`SELECT purged_at AS purgedAt FROM matters WHERE id = 'incumbent'`, + ) + expect(incumbent[0]?.purgedAt).toEqual(expect.any(Number)) const quotaRows = await db.all<{ used: number }>(sql`SELECT used FROM org_quotas WHERE org_id = ${orgId}`) expect(quotaRows[0].used).toBe(100) }) diff --git a/server/http/webdav.integration.test.ts b/server/http/webdav.integration.test.ts index 7386a3fd..db02d744 100644 --- a/server/http/webdav.integration.test.ts +++ b/server/http/webdav.integration.test.ts @@ -1071,6 +1071,12 @@ describe('WebDAV API', () => { expect(update.status).toBe(204) const rows = await db.all<{ size: number; type: string }>(sql`SELECT size, type FROM matters WHERE id = 'existing'`) expect(rows[0]).toEqual({ size: 5, type: 'application/octet-stream' }) + const ledger = await db.all<{ bytes: number }>(sql` + SELECT COALESCE(SUM(delta_bytes), 0) AS bytes + FROM storage_usage_ledger + WHERE org_id = ${workspace.id} AND storage_id = ${storage.id} + `) + expect(ledger[0].bytes).toBe(5) const root = await app.request(`/dav/${workspace.slug}/`, { method: 'PUT', diff --git a/server/scripts/backfill-admin-stats.test.ts b/server/scripts/backfill-admin-stats.test.ts index 8e414be2..81b933de 100644 --- a/server/scripts/backfill-admin-stats.test.ts +++ b/server/scripts/backfill-admin-stats.test.ts @@ -79,6 +79,11 @@ describe('admin stats backfill', () => { id TEXT PRIMARY KEY, status TEXT NOT NULL, created_at INTEGER NOT NULL, processed_at INTEGER ); CREATE TABLE org_quotas (id TEXT PRIMARY KEY, used INTEGER NOT NULL); + CREATE TABLE storage_usage_ledger ( + id TEXT PRIMARY KEY, event_key TEXT NOT NULL UNIQUE, org_id TEXT NOT NULL, storage_id TEXT NOT NULL, + resource_type TEXT NOT NULL, resource_id TEXT NOT NULL, delta_bytes INTEGER NOT NULL, reason TEXT NOT NULL, + occurred_at INTEGER NOT NULL, created_at INTEGER NOT NULL + ); CREATE TABLE stats_rollups_hourly ( id TEXT PRIMARY KEY, bucket_start INTEGER NOT NULL, org_id TEXT NOT NULL, metric_key TEXT NOT NULL, dimension_key TEXT NOT NULL, dimension_value TEXT NOT NULL, diff --git a/server/test/setup.ts b/server/test/setup.ts index a9fe910b..00a4dd78 100644 --- a/server/test/setup.ts +++ b/server/test/setup.ts @@ -128,9 +128,13 @@ const APP_SCHEMA_SQL = ` storage_id TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'draft', trashed_at INTEGER, + purged_at INTEGER, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL ); + CREATE UNIQUE INDEX IF NOT EXISTS matters_active_name_uniq + ON matters(org_id, parent, LOWER(name)) + WHERE status = 'active' AND trashed_at IS NULL; CREATE TABLE IF NOT EXISTS webdav_dead_properties ( id TEXT PRIMARY KEY, org_id TEXT NOT NULL, @@ -320,6 +324,21 @@ const APP_SCHEMA_SQL = ` CREATE UNIQUE INDEX IF NOT EXISTS stats_rollups_hourly_bucket_metric_dim_uniq ON stats_rollups_hourly(bucket_start, org_id, metric_key, dimension_key, dimension_value); CREATE INDEX IF NOT EXISTS stats_rollups_hourly_metric_bucket_idx ON stats_rollups_hourly(metric_key, bucket_start); CREATE INDEX IF NOT EXISTS stats_rollups_hourly_dimension_bucket_idx ON stats_rollups_hourly(metric_key, dimension_key, bucket_start); + CREATE TABLE IF NOT EXISTS storage_usage_ledger ( + id TEXT PRIMARY KEY NOT NULL, + event_key TEXT NOT NULL UNIQUE, + org_id TEXT NOT NULL, + storage_id TEXT NOT NULL, + resource_type TEXT NOT NULL, + resource_id TEXT NOT NULL, + delta_bytes INTEGER NOT NULL, + reason TEXT NOT NULL, + occurred_at INTEGER NOT NULL, + created_at INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS storage_usage_ledger_occurred_idx ON storage_usage_ledger(occurred_at); + CREATE INDEX IF NOT EXISTS storage_usage_ledger_org_occurred_idx ON storage_usage_ledger(org_id, occurred_at); + CREATE INDEX IF NOT EXISTS storage_usage_ledger_storage_occurred_idx ON storage_usage_ledger(storage_id, occurred_at); CREATE TABLE IF NOT EXISTS shares ( id TEXT PRIMARY KEY, token TEXT NOT NULL UNIQUE, @@ -366,11 +385,12 @@ const APP_SCHEMA_SQL = ` width INTEGER, height INTEGER, status TEXT NOT NULL DEFAULT 'draft', + purged_at INTEGER, access_count INTEGER NOT NULL DEFAULT 0, last_accessed_at INTEGER, created_at INTEGER NOT NULL ); - CREATE UNIQUE INDEX IF NOT EXISTS image_hostings_org_path_uniq ON image_hostings(org_id, path); + CREATE UNIQUE INDEX IF NOT EXISTS image_hostings_org_path_uniq ON image_hostings(org_id, path) WHERE purged_at IS NULL; CREATE INDEX IF NOT EXISTS image_hostings_org_created_idx ON image_hostings(org_id, created_at); CREATE INDEX IF NOT EXISTS image_hostings_token_idx ON image_hostings(token); CREATE TABLE IF NOT EXISTS notifications ( diff --git a/server/usecases/archive-processing.test.ts b/server/usecases/archive-processing.test.ts index 53538ace..ce0e8ef7 100644 --- a/server/usecases/archive-processing.test.ts +++ b/server/usecases/archive-processing.test.ts @@ -855,7 +855,9 @@ async function seedMatter( async function activeMatterCount(db: TestDb): Promise { const rows = await db.all<{ count: number }>(sql` - SELECT COUNT(*) AS count FROM matters WHERE org_id = ${ORG_ID} AND status = 'active' + SELECT COUNT(*) AS count + FROM matters + WHERE org_id = ${ORG_ID} AND status = 'active' AND purged_at IS NULL `) return rows[0]?.count ?? 0 } diff --git a/server/usecases/downloads/download-folders.test.ts b/server/usecases/downloads/download-folders.test.ts index 7fee57d7..113a8000 100644 --- a/server/usecases/downloads/download-folders.test.ts +++ b/server/usecases/downloads/download-folders.test.ts @@ -18,6 +18,7 @@ const folder = (name: string, parent = ''): Matter => ({ storageId: 'storage-1', status: 'active', trashedAt: null, + purgedAt: null, createdAt: new Date(0), updatedAt: new Date(0), }) diff --git a/server/usecases/object.integration.test.ts b/server/usecases/object.integration.test.ts index ac1ea499..f7fe9b45 100644 --- a/server/usecases/object.integration.test.ts +++ b/server/usecases/object.integration.test.ts @@ -85,6 +85,7 @@ async function seedMatter( storageId: opts.storageId ?? STORAGE_ID, status: opts.status ?? 'active', trashedAt: null, + purgedAt: null, createdAt: now, updatedAt: now, } diff --git a/server/usecases/object.test.ts b/server/usecases/object.test.ts index 85246370..ad19d5f6 100644 --- a/server/usecases/object.test.ts +++ b/server/usecases/object.test.ts @@ -73,6 +73,7 @@ function file(id: string, overrides: Partial = {}): Matter { storageId: 'st-1', status: 'active', trashedAt: null, + purgedAt: null, createdAt: FIXED_DATE, updatedAt: FIXED_DATE, ...overrides, diff --git a/server/usecases/object.ts b/server/usecases/object.ts index cd4204cb..0676cac3 100644 --- a/server/usecases/object.ts +++ b/server/usecases/object.ts @@ -925,15 +925,10 @@ export type PurgeDeps = { export async function purgeRecursively(deps: PurgeDeps, orgId: string, matters: Matter[]): Promise { const keysByStorage = new Map() - const bytesByStorage = new Map() - let totalBytes = 0 + const affectedStorageIds = new Set() for (const m of matters) { - const size = m.size ?? 0 - if (m.dirtype === DirType.FILE && size > 0) { - bytesByStorage.set(m.storageId, (bytesByStorage.get(m.storageId) ?? 0) + size) - totalBytes += size - } + if (m.dirtype === DirType.FILE) affectedStorageIds.add(m.storageId) if (!m.object) continue let entry = keysByStorage.get(m.storageId) if (!entry) { @@ -956,7 +951,7 @@ export async function purgeRecursively(deps: PurgeDeps, orgId: string, matters: orgId, matters.map((m) => m.id), ) - if (totalBytes > 0) await deps.storageUsage.reconcile(orgId, bytesByStorage.keys()) + if (affectedStorageIds.size > 0) await deps.storageUsage.reconcile(orgId, affectedStorageIds) return matters.length } diff --git a/server/usecases/ports/image-hosting.ts b/server/usecases/ports/image-hosting.ts index 51579e76..977b301e 100644 --- a/server/usecases/ports/image-hosting.ts +++ b/server/usecases/ports/image-hosting.ts @@ -4,6 +4,7 @@ import type { ImageHosting } from '@shared/types' // Server-side record: the shared DTO, but timestamps stay as Date until the http // layer serializes them. Drizzle row types never cross this boundary. export type ImageHostingRecord = Omit & { + purgedAt: number | null lastAccessedAt: Date | null createdAt: Date } diff --git a/server/usecases/ports/matter.ts b/server/usecases/ports/matter.ts index 518bcd4a..6f4648a2 100644 --- a/server/usecases/ports/matter.ts +++ b/server/usecases/ports/matter.ts @@ -19,6 +19,7 @@ export interface Matter { storageId: string status: string trashedAt: number | null + purgedAt: number | null createdAt: Date updatedAt: Date } @@ -105,7 +106,6 @@ export interface MatterRepo { getMany(orgId: string, ids: string[]): Promise update(id: string, orgId: string, input: UpdateMatterInput, userId?: string): Promise copy(source: Matter, targetParent: string, newObject: string, opts?: CopyMatterOptions): Promise - delete(id: string, orgId: string): Promise cancelDraft(id: string, orgId: string, userId?: string): Promise trash(orgId: string, id: string, userId?: string): Promise restore(orgId: string, id: string, userId?: string, onConflict?: ConflictStrategy): Promise diff --git a/server/usecases/webdav.test.ts b/server/usecases/webdav.test.ts index 8a134b85..e9c855a7 100644 --- a/server/usecases/webdav.test.ts +++ b/server/usecases/webdav.test.ts @@ -75,6 +75,7 @@ function file(id: string, overrides: Partial = {}): Matter { storageId: 'st-1', status: 'active', trashedAt: null, + purgedAt: null, createdAt: new Date('2020-01-01'), updatedAt: new Date('2020-01-01'), ...overrides, diff --git a/shared/types/admin-stats.ts b/shared/types/admin-stats.ts index a142e038..25593965 100644 --- a/shared/types/admin-stats.ts +++ b/shared/types/admin-stats.ts @@ -63,6 +63,8 @@ export interface AdminSharingDataQuality { export interface AdminStorageDataQuality extends AdminTransferDataQuality { usageDriftSpaces: number | null usageDriftBytes: number | null + ledgerDriftSpaces: number | null + ledgerDriftBytes: number | null } export interface AdminDashboardOverviewStats extends AdminStatsRange { diff --git a/src/routes/_authenticated/admin/analytics.tsx b/src/routes/_authenticated/admin/analytics.tsx index c7d58e18..97f4948e 100644 --- a/src/routes/_authenticated/admin/analytics.tsx +++ b/src/routes/_authenticated/admin/analytics.tsx @@ -1655,7 +1655,9 @@ function SharingDataQualityNotice({ quality }: { quality: AdminSharingDataQualit } function StorageUsageDataQualityNotice({ quality }: { quality: AdminDashboardStorageStats['dataQuality'] }) { - if (quality.usageDriftSpaces === null || quality.usageDriftSpaces === 0) return null + const usageDrift = quality.usageDriftSpaces !== null && quality.usageDriftSpaces > 0 + const ledgerDrift = quality.ledgerDriftSpaces !== null && quality.ledgerDriftSpaces > 0 + if (!usageDrift && !ledgerDrift) return null return (
- {formatNumber(quality.usageDriftSpaces)} 个空间的配额占用账本与可计费对象相差{' '} - {formatSize(quality.usageDriftBytes)}；修复前不要把配额水位作为准确事实。 + {usageDrift + ? `${formatNumber(quality.usageDriftSpaces)} 个空间的实时占用计数与可计费对象相差 ${formatSize(quality.usageDriftBytes)}` + : ''} + {usageDrift && ledgerDrift ? '；' : ''} + {ledgerDrift + ? `${formatNumber(quality.ledgerDriftSpaces)} 个空间的存储流水与可计费对象相差 ${formatSize(quality.ledgerDriftBytes)}` + : ''} + ；修复前不要把对应的存储数据作为准确事实。
)