Report traffic egress to Cloud metering (#384)

* feat(api): report traffic egress to cloud metering

Agent-Profile: https://agent-kanban.dev/agents/a318237412dd8b98

* test(api): cover cloud traffic redirect reporting

Agent-Profile: https://agent-kanban.dev/agents/a318237412dd8b98

* test(api): cover cloud traffic failure branches

Agent-Profile: https://agent-kanban.dev/agents/a318237412dd8b98

* test(api): cover final traffic metering branches

Agent-Profile: https://agent-kanban.dev/agents/a318237412dd8b98

* test(api): cover cloud metering rollback paths

Agent-Profile: https://agent-kanban.dev/agents/a318237412dd8b98
This commit is contained in:
Jasper Van
2026-05-08 20:31:48 -04:00
committed by GitHub
parent bf8a4f5877
commit ec5d08b233
13 changed files with 3643 additions and 6 deletions
@@ -0,0 +1,17 @@
CREATE TABLE `cloud_traffic_reports` (
`id` text PRIMARY KEY NOT NULL,
`org_id` text NOT NULL,
`period` text NOT NULL,
`source` text NOT NULL,
`source_id` text NOT NULL,
`event_id` text NOT NULL,
`bytes` integer NOT NULL,
`status` text NOT NULL,
`error` text,
`created_at` integer NOT NULL,
`updated_at` integer NOT NULL
);
--> statement-breakpoint
CREATE UNIQUE INDEX `cloud_traffic_reports_event_uniq` ON `cloud_traffic_reports` (`event_id`);--> statement-breakpoint
CREATE INDEX `cloud_traffic_reports_org_period_idx` ON `cloud_traffic_reports` (`org_id`,`period`);--> statement-breakpoint
CREATE INDEX `cloud_traffic_reports_status_idx` ON `cloud_traffic_reports` (`status`);
File diff suppressed because it is too large Load Diff
+7
View File
@@ -204,6 +204,13 @@
"when": 1778277914288,
"tag": "0029_wet_betty_brant",
"breakpoints": true
},
{
"idx": 30,
"version": "6",
"when": 1778282818664,
"tag": "0030_add-cloud-traffic-reports",
"breakpoints": true
}
]
}
+22
View File
@@ -47,6 +47,28 @@ export const orgQuotas = sqliteTable('org_quotas', {
trafficPeriod: text('traffic_period').notNull().default('1970-01'),
})
export const cloudTrafficReports = sqliteTable(
'cloud_traffic_reports',
{
id: text('id').primaryKey(),
orgId: text('org_id').notNull(),
period: text('period').notNull(),
source: text('source').notNull(),
sourceId: text('source_id').notNull(),
eventId: text('event_id').notNull(),
bytes: integer('bytes').notNull(),
status: text('status').notNull(),
error: text('error'),
createdAt: integer('created_at', { mode: 'timestamp_ms' }).notNull(),
updatedAt: integer('updated_at', { mode: 'timestamp_ms' }).notNull(),
},
(t) => [
uniqueIndex('cloud_traffic_reports_event_uniq').on(t.eventId),
index('cloud_traffic_reports_org_period_idx').on(t.orgId, t.period),
index('cloud_traffic_reports_status_idx').on(t.status),
],
)
export const orgQuotaEntitlements = sqliteTable(
'org_quota_entitlements',
{
+15 -1
View File
@@ -4,6 +4,7 @@ import type { Storage as S3Storage } from '../../shared/types'
import { imageHostingConfigs } from '../db/schema'
import type { Env } from '../middleware/platform'
import { PRESIGN_TTL_SECS, s3 } from '../routes/share-utils'
import { reportTrafficForDownload } from '../routes/traffic-metering-utils'
import { consumeTrafficIfQuotaAllows, refundTraffic } from '../services/effective-quota'
import { getImageByOrgPath, incrementAccessCount, resolveCustomDomain } from '../services/image-hosting'
import { getStorage } from '../services/storage'
@@ -70,11 +71,24 @@ async function handleImageByPath(c: Context<Env>, orgId: string, virtualPath: st
let url: string
try {
url = await s3.presignInline(storage, image.storageKey, image.mime, PRESIGN_TTL_SECS)
await incrementAccessCount(db, image.id)
} catch (e) {
await refundTraffic(db, image.orgId, image.size)
throw e
}
const trafficReportError = await reportTrafficForDownload(c, {
orgId: image.orgId,
bytes: image.size,
source: 'custom_domain_image',
sourceId: image.id,
})
if (trafficReportError) return trafficReportError
try {
await incrementAccessCount(db, image.id)
} catch (error) {
console.error('[image-hosting-domain] incrementAccessCount failed:', error)
}
const res = c.redirect(url, 302)
res.headers.set('Cache-Control', 'no-store')
return res
@@ -0,0 +1,410 @@
import { sql } from 'drizzle-orm'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { cloudTrafficReports } from '../db/schema'
import { createLicenseBinding } from '../licensing/license-state'
import type { Database } from '../platform/interface'
import { currentTrafficPeriod } from '../services/effective-quota'
import { S3Service } from '../services/s3'
import { createShare } from '../services/share'
import { authedHeaders, createTestApp } from '../test/setup'
import { encodeChildRef } from './share-utils'
const STORAGE_ID = 'st-cloud-traffic-test'
beforeEach(() => {
vi.restoreAllMocks()
vi.spyOn(S3Service.prototype, 'presignDownload').mockResolvedValue('https://presigned-download.example.com')
vi.spyOn(S3Service.prototype, 'presignInline').mockResolvedValue('https://presigned-inline.example.com')
})
afterEach(() => {
vi.unstubAllGlobals()
})
function makeCloudResponse(body: unknown, status = 200): Response {
return {
ok: status >= 200 && status < 300,
status,
json: async () => body,
text: async () => JSON.stringify(body),
} as unknown as Response
}
function acceptedUsageResponse(_url: string, init?: RequestInit): Response {
const body = JSON.parse(init?.body as string) as { eventId: string }
return makeCloudResponse({ data: { accepted: true, duplicate: false, eventId: body.eventId } })
}
async function seedTrafficBinding(db: Database) {
await createLicenseBinding(db, {
cloudBindingId: 'test-binding',
cloudStoreId: 'store-test-binding',
instanceId: 'test-instance',
cloudAccountId: 'test-account',
refreshToken: 'test-refresh-token',
cachedCert: 'test-certificate',
cachedExpiresAt: Math.floor(Date.now() / 1000) + 3600,
lastRefreshAt: Math.floor(Date.now() / 1000),
})
}
async function insertStorage(db: Database) {
const now = Date.now()
await db.run(sql`
INSERT INTO storages (id, title, mode, bucket, endpoint, region, access_key, secret_key, file_path, custom_host, capacity, used, status, created_at, updated_at)
VALUES (${STORAGE_ID}, 'Cloud Traffic S3', 'private', 'test-bucket', 'https://s3.amazonaws.com', 'us-east-1', 'AK', 'SK', '', '', 0, 0, 'active', ${now}, ${now})
`)
}
async function getOrgId(db: Database): Promise<string> {
const rows = await db.all<{ id: string }>(
sql`SELECT id FROM organization WHERE metadata LIKE '%"type":"personal"%' LIMIT 1`,
)
return rows[0].id
}
async function getUserId(db: Database): Promise<string> {
const rows = await db.all<{ id: string }>(sql`SELECT id FROM user LIMIT 1`)
return rows[0].id
}
async function insertFile(db: Database, orgId: string, id: string) {
const now = Date.now()
await db.run(sql`
INSERT INTO matters (id, org_id, alias, name, type, size, dirtype, parent, object, storage_id, status, created_at, updated_at)
VALUES (${id}, ${orgId}, ${`${id}-alias`}, 'download.txt', 'text/plain', 100, 0, '', 'some/key.txt', ${STORAGE_ID}, 'active', ${now}, ${now})
`)
}
async function insertImage(db: Database, orgId: string, id: string, token: string, path = `blog/${id}.png`) {
const now = Date.now()
await db.run(sql`
INSERT INTO image_hostings (id, org_id, token, path, storage_id, storage_key, size, mime, status, access_count, created_at)
VALUES (${id}, ${orgId}, ${token}, ${path}, ${STORAGE_ID}, ${`ih/${orgId}/${id}.png`}, 100, 'image/png', 'active', 0, ${now})
`)
}
async function insertImageConfig(db: Database, orgId: string, customDomain?: string) {
const now = Date.now()
const verifiedAt = customDomain ? now : null
await db.run(sql`
INSERT OR REPLACE INTO image_hosting_configs (org_id, custom_domain, domain_verified_at, referer_allowlist, created_at, updated_at)
VALUES (${orgId}, ${customDomain ?? null}, ${verifiedAt}, null, ${now}, ${now})
`)
}
async function setTrafficQuota(db: Database, orgId: string) {
await db.run(sql`
UPDATE org_quotas
SET traffic_quota = 500, traffic_used = 25, traffic_period = ${currentTrafficPeriod()}
WHERE org_id = ${orgId}
`)
}
describe('object download cloud traffic reporting', () => {
it('reports successful object downloads to Cloud after presigning', async () => {
const { app, db } = await createTestApp({ ZPAN_CLOUD_URL: 'https://cloud.example' })
await seedTrafficBinding(db)
vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse))
const headers = await authedHeaders(app)
await insertStorage(db)
const orgId = await getOrgId(db)
await insertFile(db, orgId, 'm-cloud-report-ok')
await setTrafficQuota(db, orgId)
const res = await app.request('/api/objects/m-cloud-report-ok', { headers })
expect(res.status).toBe(200)
expect(fetch).toHaveBeenCalledTimes(1)
const [, init] = vi.mocked(fetch).mock.calls[0] as [string, RequestInit]
expect(JSON.parse(init.body as string)).toMatchObject({ resource: 'traffic_egress', bytes: 100, endUserId: orgId })
await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([
{ orgId, source: 'object_download', sourceId: 'm-cloud-report-ok', bytes: 100, status: 'reported' },
])
})
it('refunds local traffic and denies the download when Cloud blocks usage', async () => {
const { app, db } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal(
'fetch',
vi.fn().mockResolvedValue(makeCloudResponse({ error: { code: 'overage_cap_exceeded' } }, 429)),
)
const headers = await authedHeaders(app)
await insertStorage(db)
const orgId = await getOrgId(db)
await insertFile(db, orgId, 'm-cloud-report-blocked')
await setTrafficQuota(db, orgId)
const res = await app.request('/api/objects/m-cloud-report-blocked', { headers })
expect(res.status).toBe(429)
await expect(res.json()).resolves.toEqual({ error: 'Cloud traffic overage cap exceeded' })
const rows = await db.all<{ trafficUsed: number }>(
sql`SELECT traffic_used AS trafficUsed FROM org_quotas WHERE org_id = ${orgId}`,
)
expect(rows[0].trafficUsed).toBe(25)
await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'blocked' }])
})
it('refunds local traffic when Cloud returns a mismatched event id', async () => {
const { app, db } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal(
'fetch',
vi.fn().mockResolvedValue(makeCloudResponse({ data: { accepted: true, duplicate: false, eventId: 'wrong' } })),
)
const headers = await authedHeaders(app)
await insertStorage(db)
const orgId = await getOrgId(db)
await insertFile(db, orgId, 'm-cloud-report-mismatch')
await setTrafficQuota(db, orgId)
const res = await app.request('/api/objects/m-cloud-report-mismatch', { headers })
expect(res.status).toBe(500)
const rows = await db.all<{ trafficUsed: number }>(
sql`SELECT traffic_used AS trafficUsed FROM org_quotas WHERE org_id = ${orgId}`,
)
expect(rows[0].trafficUsed).toBe(25)
await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'failed' }])
})
it('does not report usage when presign fails and local traffic is refunded', async () => {
const { app, db } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse))
vi.mocked(S3Service.prototype.presignDownload).mockRejectedValueOnce(new Error('sign failed'))
const headers = await authedHeaders(app)
await insertStorage(db)
const orgId = await getOrgId(db)
await insertFile(db, orgId, 'm-cloud-report-presign-fail')
await setTrafficQuota(db, orgId)
const res = await app.request('/api/objects/m-cloud-report-presign-fail', { headers })
expect(res.status).toBe(500)
expect(fetch).not.toHaveBeenCalled()
const rows = await db.all<{ trafficUsed: number }>(
sql`SELECT traffic_used AS trafficUsed FROM org_quotas WHERE org_id = ${orgId}`,
)
expect(rows[0].trafficUsed).toBe(25)
await expect(db.select().from(cloudTrafficReports)).resolves.toHaveLength(0)
})
})
describe('public redirect cloud traffic reporting', () => {
it('reports direct share redirects to Cloud', async () => {
const { app, db } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse))
await authedHeaders(app)
await insertStorage(db)
const orgId = await getOrgId(db)
const creatorId = await getUserId(db)
await insertFile(db, orgId, 'm-cloud-direct-share')
const share = await createShare(db, { matterId: 'm-cloud-direct-share', orgId, creatorId, kind: 'direct' })
const res = await app.request(`/r/${share.token}`, { redirect: 'manual' })
expect(res.status).toBe(302)
await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([
{ source: 'direct_share', sourceId: share.id, bytes: 100, status: 'reported' },
])
})
it('refunds direct share traffic and downloads when Cloud blocks usage', async () => {
const { app, db } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal(
'fetch',
vi.fn().mockResolvedValue(makeCloudResponse({ error: { code: 'overage_cap_exceeded' } }, 429)),
)
await authedHeaders(app)
await insertStorage(db)
const orgId = await getOrgId(db)
const creatorId = await getUserId(db)
await insertFile(db, orgId, 'm-cloud-direct-blocked')
await setTrafficQuota(db, orgId)
const share = await createShare(db, { matterId: 'm-cloud-direct-blocked', orgId, creatorId, kind: 'direct' })
const res = await app.request(`/r/${share.token}`, { redirect: 'manual' })
expect(res.status).toBe(429)
await expect(res.json()).resolves.toEqual({ error: 'Cloud traffic overage cap exceeded' })
const rows = await db.all<{ downloads: number; trafficUsed: number }>(sql`
SELECT s.downloads, q.traffic_used AS trafficUsed
FROM shares s
INNER JOIN org_quotas q ON q.org_id = s.org_id
WHERE s.id = ${share.id}
`)
expect(rows[0]).toEqual({ downloads: 0, trafficUsed: 25 })
await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'blocked' }])
})
it('reports landing share downloads to Cloud', async () => {
const { app, db } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse))
await authedHeaders(app)
await insertStorage(db)
const orgId = await getOrgId(db)
const creatorId = await getUserId(db)
await insertFile(db, orgId, 'm-cloud-landing-share')
const share = await createShare(db, { matterId: 'm-cloud-landing-share', orgId, creatorId, kind: 'landing' })
const ref = encodeChildRef(share.token, 'm-cloud-landing-share')
const res = await app.request(`/api/shares/${share.token}/objects/${ref}?downloadUrl=1`, { redirect: 'manual' })
expect(res.status).toBe(200)
await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([
{ source: 'landing_share', sourceId: share.id, bytes: 100, status: 'reported' },
])
})
it('refunds landing share traffic and downloads when Cloud blocks usage', async () => {
const { app, db } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal(
'fetch',
vi.fn().mockResolvedValue(makeCloudResponse({ error: { code: 'overage_cap_exceeded' } }, 429)),
)
await authedHeaders(app)
await insertStorage(db)
const orgId = await getOrgId(db)
const creatorId = await getUserId(db)
await insertFile(db, orgId, 'm-cloud-landing-blocked')
await setTrafficQuota(db, orgId)
const share = await createShare(db, { matterId: 'm-cloud-landing-blocked', orgId, creatorId, kind: 'landing' })
const ref = encodeChildRef(share.token, 'm-cloud-landing-blocked')
const res = await app.request(`/api/shares/${share.token}/objects/${ref}?downloadUrl=1`, { redirect: 'manual' })
expect(res.status).toBe(429)
await expect(res.json()).resolves.toEqual({ error: 'Cloud traffic overage cap exceeded' })
const rows = await db.all<{ downloads: number; trafficUsed: number }>(sql`
SELECT s.downloads, q.traffic_used AS trafficUsed
FROM shares s
INNER JOIN org_quotas q ON q.org_id = s.org_id
WHERE s.id = ${share.id}
`)
expect(rows[0]).toEqual({ downloads: 0, trafficUsed: 25 })
await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'blocked' }])
})
it('still returns landing share URLs when audit recording fails after Cloud report', async () => {
const { app, db } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse))
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
await authedHeaders(app)
await insertStorage(db)
const orgId = await getOrgId(db)
const creatorId = await getUserId(db)
await insertFile(db, orgId, 'm-cloud-landing-audit-fail')
const share = await createShare(db, { matterId: 'm-cloud-landing-audit-fail', orgId, creatorId, kind: 'landing' })
const ref = encodeChildRef(share.token, 'm-cloud-landing-audit-fail')
await db.run(sql`DROP TABLE activity_events`)
const res = await app.request(`/api/shares/${share.token}/objects/${ref}?downloadUrl=1`, { redirect: 'manual' })
expect(res.status).toBe(200)
expect(consoleError).toHaveBeenCalled()
})
it('reports token image-hosting redirects to Cloud', async () => {
const { app, db } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse))
await authedHeaders(app)
await insertStorage(db)
const orgId = await getOrgId(db)
await insertImage(db, orgId, 'ih-cloud-token', 'ih_cloudtoken')
await insertImageConfig(db, orgId)
const res = await app.request('/r/ih_cloudtoken', { redirect: 'manual' })
expect(res.status).toBe(302)
await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([
{ source: 'image_hosting', sourceId: 'ih-cloud-token', bytes: 100, status: 'reported' },
])
})
it('still redirects token images when access-count recording fails after Cloud report', async () => {
const { app, db } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse))
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
await authedHeaders(app)
await insertStorage(db)
const orgId = await getOrgId(db)
await insertImage(db, orgId, 'ih-cloud-log-fail', 'ih_cloudlogfail')
await insertImageConfig(db, orgId)
vi.spyOn(db, 'run').mockRejectedValue(new Error('access failed'))
const res = await app.request('/r/ih_cloudlogfail', { redirect: 'manual' })
expect(res.status).toBe(302)
expect(consoleError).toHaveBeenCalled()
})
it('does not report token image usage when inline presigning fails', async () => {
const { app, db } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse))
vi.mocked(S3Service.prototype.presignInline).mockRejectedValueOnce(new Error('inline sign failed'))
await authedHeaders(app)
await insertStorage(db)
const orgId = await getOrgId(db)
await insertImage(db, orgId, 'ih-cloud-presign-fail', 'ih_cloudpresignfail')
await insertImageConfig(db, orgId)
const res = await app.request('/r/ih_cloudpresignfail', { redirect: 'manual' })
expect(res.status).toBe(500)
expect(fetch).not.toHaveBeenCalled()
await expect(db.select().from(cloudTrafficReports)).resolves.toHaveLength(0)
})
it('reports custom-domain image-hosting redirects to Cloud', async () => {
const { app, db } = await createTestApp({ PUBLIC_APP_HOST: 'zpan.example.com' })
await seedTrafficBinding(db)
vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse))
await authedHeaders(app)
await insertStorage(db)
const orgId = await getOrgId(db)
await insertImage(db, orgId, 'ih-cloud-domain', 'ih_clouddomain', 'blog/domain.png')
await insertImageConfig(db, orgId, 'img.example.com')
const res = await app.request('https://img.example.com/blog/domain.png', {
headers: { host: 'img.example.com' },
redirect: 'manual',
})
expect(res.status).toBe(302)
await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([
{ source: 'custom_domain_image', sourceId: 'ih-cloud-domain', bytes: 100, status: 'reported' },
])
})
it('still redirects custom-domain images when access-count recording fails after Cloud report', async () => {
const { app, db } = await createTestApp({ PUBLIC_APP_HOST: 'zpan.example.com' })
await seedTrafficBinding(db)
vi.stubGlobal('fetch', vi.fn().mockImplementation(acceptedUsageResponse))
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => undefined)
await authedHeaders(app)
await insertStorage(db)
const orgId = await getOrgId(db)
await insertImage(db, orgId, 'ih-cloud-domain-log-fail', 'ih_clouddomainlogfail', 'blog/domain-log-fail.png')
await insertImageConfig(db, orgId, 'img.example.com')
vi.spyOn(db, 'run').mockRejectedValue(new Error('access failed'))
const res = await app.request('https://img.example.com/blog/domain-log-fail.png', {
headers: { host: 'img.example.com' },
redirect: 'manual',
})
expect(res.status).toBe(302)
expect(consoleError).toHaveBeenCalled()
})
})
+9
View File
@@ -34,6 +34,7 @@ import { buildObjectKey } from '../services/path-template'
import { purgeRecursively } from '../services/purge'
import { S3Service } from '../services/s3'
import { getStorage, selectStorage } from '../services/storage'
import { reportTrafficForDownload } from './traffic-metering-utils'
const s3 = new S3Service()
@@ -206,6 +207,14 @@ const app = new Hono<Env>()
throw e
}
const trafficReportError = await reportTrafficForDownload(c, {
orgId,
bytes: matter.size ?? 0,
source: 'object_download',
sourceId: matter.id,
})
if (trafficReportError) return trafficReportError
return c.json({ ...matter, downloadUrl })
})
.patch('/:id', requireTeamRole('editor'), zValidator('json', patchMatterSchema), async (c) => {
+23 -1
View File
@@ -13,6 +13,7 @@ import {
} from '../services/share'
import { getStorage } from '../services/storage'
import { PRESIGN_TTL_SECS, s3 } from './share-utils'
import { reportTrafficForDownload } from './traffic-metering-utils'
// Strip optional file extension from token (e.g. "ih_aB3xK9.png" → "ih_aB3xK9")
function stripExtension(token: string): string {
@@ -69,6 +70,15 @@ async function handleDirectShare(c: Context<Env>, db: Database, token: string):
throw e
}
const trafficReportError = await reportTrafficForDownload(c, {
orgId: share.orgId,
bytes: matter.size ?? 0,
source: 'direct_share',
sourceId: share.id,
onRejected: () => decrementDownloads(db, share.id),
})
if (trafficReportError) return trafficReportError
const res = c.redirect(url, 302)
res.headers.set('Cache-Control', 'no-store')
return res
@@ -98,12 +108,24 @@ async function handleImageHosting(c: Context<Env>, db: Database, token: string):
let url: string
try {
url = await s3.presignInline(storage, image.storageKey, image.mime, PRESIGN_TTL_SECS)
await incrementAccessCount(db, image.id)
} catch (e) {
await refundTraffic(db, image.orgId, image.size)
throw e
}
const trafficReportError = await reportTrafficForDownload(c, {
orgId: image.orgId,
bytes: image.size,
source: 'image_hosting',
sourceId: image.id,
})
if (trafficReportError) return trafficReportError
try {
await incrementAccessCount(db, image.id)
} catch (error) {
console.error('[redirect] incrementAccessCount failed:', error)
}
const res = c.redirect(url, 302)
res.headers.set('Cache-Control', 'no-store')
return res
+19 -4
View File
@@ -47,6 +47,7 @@ import {
s3,
viewCookieName,
} from './share-utils'
import { reportTrafficForDownload } from './traffic-metering-utils'
const ROLE_LEVELS: Record<string, number> = { owner: 3, editor: 2, viewer: 1, member: 1 }
@@ -289,6 +290,22 @@ export const publicShares = new Hono<Env>()
let url: string
try {
url = await s3.presignDownload(storage, targetMatter.object, targetMatter.name, PRESIGN_TTL_SECS)
} catch (e) {
await refundTraffic(db, share.orgId, targetMatter.size ?? 0)
await decrementDownloads(db, share.id)
throw e
}
const trafficReportError = await reportTrafficForDownload(c, {
orgId: share.orgId,
bytes: targetMatter.size ?? 0,
source: 'landing_share',
sourceId: share.id,
onRejected: () => decrementDownloads(db, share.id),
})
if (trafficReportError) return trafficReportError
try {
await recordActivity(db, {
orgId: share.orgId,
userId: actorId,
@@ -298,10 +315,8 @@ export const publicShares = new Hono<Env>()
targetName: targetMatter.name,
metadata: { anonymous: !viewerId },
})
} catch (e) {
await refundTraffic(db, share.orgId, targetMatter.size ?? 0)
await decrementDownloads(db, share.id)
throw e
} catch (error) {
console.error('[shares] recordActivity failed:', error)
}
if (returnUrl) {
+37
View File
@@ -0,0 +1,37 @@
import type { Context } from 'hono'
import type { Env } from '../middleware/platform'
import {
CloudTrafficBlockedError,
reportTrafficEgress,
type TrafficReportSource,
} from '../services/cloud-traffic-metering'
import { refundTraffic } from '../services/effective-quota'
export async function reportTrafficForDownload(
c: Context<Env>,
params: {
orgId: string
bytes: number
source: TrafficReportSource
sourceId: string
onRejected?: () => Promise<void>
},
): Promise<Response | null> {
try {
await reportTrafficEgress({
platform: c.get('platform'),
orgId: params.orgId,
bytes: params.bytes,
source: params.source,
sourceId: params.sourceId,
})
return null
} catch (error) {
await refundTraffic(c.get('platform').db, params.orgId, params.bytes)
await params.onRejected?.()
if (error instanceof CloudTrafficBlockedError) {
return c.json({ error: 'Cloud traffic overage cap exceeded' }, 429)
}
throw error
}
}
@@ -0,0 +1,261 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { cloudTrafficReports } from '../db/schema'
import { createLicenseBinding } from '../licensing/license-state'
import type { Database } from '../platform/interface'
import { createTestApp } from '../test/setup'
import { CloudTrafficBlockedError, reportTrafficEgress } from './cloud-traffic-metering'
function makeResponse(body: unknown, status = 200): Response {
return {
ok: status >= 200 && status < 300,
status,
json: async () => body,
text: async () => JSON.stringify(body),
} as unknown as Response
}
async function seedTrafficBinding(db: Database) {
await createLicenseBinding(db, {
cloudBindingId: 'test-binding',
cloudStoreId: 'store-test-binding',
instanceId: 'test-instance',
cloudAccountId: 'test-account',
refreshToken: 'test-refresh-token',
cachedCert: 'test-certificate',
cachedExpiresAt: Math.floor(Date.now() / 1000) + 3600,
lastRefreshAt: Math.floor(Date.now() / 1000),
})
}
describe('cloud traffic metering', () => {
afterEach(() => {
vi.unstubAllGlobals()
})
it('reports traffic egress through the active license binding', async () => {
const { db, platform } = await createTestApp({ ZPAN_CLOUD_URL: 'https://cloud.example' })
await seedTrafficBinding(db)
vi.stubGlobal(
'fetch',
vi.fn().mockResolvedValue(makeResponse({ data: { accepted: true, duplicate: false, eventId: 'evt_1' } })),
)
await reportTrafficEgress({
platform,
orgId: 'org_1',
bytes: 1024,
source: 'object_download',
sourceId: 'matter_1',
eventId: 'evt_1',
})
expect(fetch).toHaveBeenCalledTimes(1)
const [url, init] = vi.mocked(fetch).mock.calls[0] as [string, RequestInit]
expect(url).toBe('https://cloud.example/api/usage-events')
expect(init.headers).toMatchObject({ Authorization: 'Bearer test-refresh-token' })
expect(JSON.parse(init.body as string)).toMatchObject({
resource: 'traffic_egress',
bytes: 1024,
eventId: 'evt_1',
idempotencyKey: 'evt_1',
endUserId: 'org_1',
})
await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'reported' }])
})
it('ignores zero-byte reports without writing local state', async () => {
const { db, platform } = await createTestApp()
vi.stubGlobal('fetch', vi.fn())
const result = await reportTrafficEgress({
platform,
orgId: 'org_1',
bytes: 0,
source: 'object_download',
sourceId: 'matter_1',
})
expect(result).toMatchObject({ status: 'reported', eventId: '', duplicate: false })
expect(fetch).not.toHaveBeenCalled()
await expect(db.select().from(cloudTrafficReports)).resolves.toHaveLength(0)
})
it('keeps idempotent reports local after the first successful report', async () => {
const { db, platform } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal(
'fetch',
vi.fn().mockResolvedValue(makeResponse({ data: { accepted: true, duplicate: false, eventId: 'evt_dup' } })),
)
const input = {
platform,
orgId: 'org_1',
bytes: 1024,
source: 'direct_share' as const,
sourceId: 'share_1',
eventId: 'evt_dup',
}
const first = await reportTrafficEgress(input)
const second = await reportTrafficEgress(input)
expect(first.duplicate).toBe(false)
expect(second).toMatchObject({ duplicate: true, eventId: 'evt_dup', status: 'reported' })
expect(fetch).toHaveBeenCalledTimes(1)
await expect(db.select().from(cloudTrafficReports)).resolves.toHaveLength(1)
})
it('rejects idempotency conflicts for reused event ids', async () => {
const { db, platform } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal(
'fetch',
vi.fn().mockResolvedValue(makeResponse({ data: { accepted: true, duplicate: false, eventId: 'evt_conflict' } })),
)
await reportTrafficEgress({
platform,
orgId: 'org_1',
bytes: 1024,
source: 'direct_share',
sourceId: 'share_1',
eventId: 'evt_conflict',
})
await expect(
reportTrafficEgress({
platform,
orgId: 'org_1',
bytes: 2048,
source: 'direct_share',
sourceId: 'share_1',
eventId: 'evt_conflict',
}),
).rejects.toThrow('traffic_report_idempotency_conflict')
})
it('surfaces Cloud cap rejection and records a blocked report', async () => {
const { db, platform } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal('fetch', vi.fn().mockResolvedValue(makeResponse({ error: { code: 'overage_cap_exceeded' } }, 429)))
await expect(
reportTrafficEgress({
platform,
orgId: 'org_1',
bytes: 1024,
source: 'landing_share',
sourceId: 'share_1',
eventId: 'evt_blocked',
}),
).rejects.toThrow(CloudTrafficBlockedError)
await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([
{ eventId: 'evt_blocked', status: 'blocked', error: 'overage_cap_exceeded' },
])
await expect(
reportTrafficEgress({
platform,
orgId: 'org_1',
bytes: 1024,
source: 'landing_share',
sourceId: 'share_1',
eventId: 'evt_blocked',
}),
).rejects.toThrow(CloudTrafficBlockedError)
})
it('retries failed report ids instead of treating them as completed duplicates', async () => {
const { db, platform } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal(
'fetch',
vi
.fn()
.mockRejectedValueOnce(new Error('offline'))
.mockResolvedValueOnce(makeResponse({ data: { accepted: true, duplicate: false, eventId: 'evt_retry' } })),
)
const input = {
platform,
orgId: 'org_1',
bytes: 1024,
source: 'object_download' as const,
sourceId: 'matter_1',
eventId: 'evt_retry',
now: new Date('2026-04-30T23:59:00.000Z'),
}
await expect(reportTrafficEgress(input)).rejects.toThrow('offline')
const result = await reportTrafficEgress({ ...input, now: new Date('2026-05-01T00:01:00.000Z') })
expect(result).toMatchObject({ status: 'reported', duplicate: false })
expect(fetch).toHaveBeenCalledTimes(2)
await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([
{ status: 'reported', error: null, period: '2026-04' },
])
})
it('fails when Cloud does not accept the same event id', async () => {
const { db, platform } = await createTestApp()
await seedTrafficBinding(db)
vi.stubGlobal(
'fetch',
vi.fn().mockResolvedValue(makeResponse({ data: { accepted: true, duplicate: false, eventId: 'other_evt' } })),
)
await expect(
reportTrafficEgress({
platform,
orgId: 'org_1',
bytes: 1024,
source: 'object_download',
sourceId: 'matter_1',
eventId: 'evt_mismatch',
}),
).rejects.toThrow('cloud_usage_report_rejected')
await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([
{ eventId: 'evt_mismatch', status: 'failed', error: 'cloud_usage_report_rejected' },
])
})
it('records skipped reporting when no active binding exists', async () => {
const { db, platform } = await createTestApp()
vi.stubGlobal('fetch', vi.fn())
const result = await reportTrafficEgress({
platform,
orgId: 'org_1',
bytes: 1024,
source: 'image_hosting',
sourceId: 'image_1',
eventId: 'evt_unbound',
})
expect(result.status).toBe('skipped_unbound')
expect(fetch).not.toHaveBeenCalled()
await expect(db.select().from(cloudTrafficReports)).resolves.toMatchObject([{ status: 'skipped_unbound' }])
})
it('keeps unbound report replays local', async () => {
const { db, platform } = await createTestApp()
vi.stubGlobal('fetch', vi.fn())
const input = {
platform,
orgId: 'org_1',
bytes: 1024,
source: 'image_hosting' as const,
sourceId: 'image_1',
eventId: 'evt_unbound_dup',
}
const first = await reportTrafficEgress(input)
const second = await reportTrafficEgress(input)
expect(first.duplicate).toBe(false)
expect(second).toMatchObject({ status: 'skipped_unbound', eventId: 'evt_unbound_dup', duplicate: true })
expect(fetch).not.toHaveBeenCalled()
await expect(db.select().from(cloudTrafficReports)).resolves.toHaveLength(1)
})
})
+151
View File
@@ -0,0 +1,151 @@
import { eq } from 'drizzle-orm'
import { nanoid } from 'nanoid'
import { z } from 'zod'
import { ZPAN_CLOUD_URL_DEFAULT } from '../../shared/constants'
import { cloudTrafficReports } from '../db/schema'
import { loadActiveLicenseBinding } from '../licensing/license-state'
import type { Platform } from '../platform/interface'
import { currentTrafficPeriod } from './effective-quota'
import { postBoundCloudJson } from './licensing-cloud'
export type TrafficReportSource =
| 'object_download'
| 'direct_share'
| 'landing_share'
| 'image_hosting'
| 'custom_domain_image'
export class CloudTrafficBlockedError extends Error {
constructor() {
super('cloud_traffic_blocked')
this.name = 'CloudTrafficBlockedError'
}
}
const usageResponseSchema = z.object({
accepted: z.boolean(),
duplicate: z.boolean(),
eventId: z.string().min(1),
})
type ReportStatus = 'pending' | 'reported' | 'skipped_unbound' | 'blocked' | 'failed'
export async function reportTrafficEgress(params: {
platform: Platform
orgId: string
bytes: number
source: TrafficReportSource
sourceId: string
eventId?: string
now?: Date
}): Promise<{ status: ReportStatus; eventId: string; duplicate: boolean }> {
const { platform, orgId, bytes, source, sourceId, now = new Date() } = params
if (bytes <= 0) return { status: 'reported', eventId: params.eventId ?? '', duplicate: false }
const eventId = params.eventId ?? `traffic_${nanoid()}`
const existing = await loadTrafficReport(platform.db, eventId)
const period = existing?.period ?? currentTrafficPeriod(now)
if (existing) {
assertSameReport(existing, { orgId, period, source, sourceId, bytes })
if (existing.status === 'reported' || existing.status === 'skipped_unbound') {
return { status: existing.status as ReportStatus, eventId, duplicate: true }
}
if (existing.status === 'blocked') throw new CloudTrafficBlockedError()
} else {
await insertTrafficReport(platform, { orgId, period, source, sourceId, eventId, bytes, status: 'pending', now })
}
const binding = await loadActiveLicenseBinding(platform.db)
if (!binding?.refreshToken) {
await updateTrafficReport(platform, eventId, 'skipped_unbound', null, now)
return { status: 'skipped_unbound', eventId, duplicate: false }
}
try {
const data = await postBoundCloudJson(
platform.getEnv('ZPAN_CLOUD_URL') ?? ZPAN_CLOUD_URL_DEFAULT,
'/api/usage-events',
binding.refreshToken,
{
resource: 'traffic_egress',
bytes,
eventId,
idempotencyKey: eventId,
endUserId: orgId,
},
)
const response = usageResponseSchema.parse(data)
if (!response.accepted || response.eventId !== eventId) throw new Error('cloud_usage_report_rejected')
await updateTrafficReport(platform, eventId, 'reported', null, now)
return { status: 'reported', eventId, duplicate: false }
} catch (error) {
const message = error instanceof Error ? error.message : 'cloud_usage_report_failed'
if (message === 'overage_cap_exceeded') {
await updateTrafficReport(platform, eventId, 'blocked', message, now)
throw new CloudTrafficBlockedError()
}
await updateTrafficReport(platform, eventId, 'failed', message, now)
throw error
}
}
async function loadTrafficReport(db: Platform['db'], eventId: string) {
const rows = await db.select().from(cloudTrafficReports).where(eq(cloudTrafficReports.eventId, eventId)).limit(1)
return rows[0]
}
function assertSameReport(
report: typeof cloudTrafficReports.$inferSelect,
params: { orgId: string; period: string; source: TrafficReportSource; sourceId: string; bytes: number },
) {
if (
report.orgId !== params.orgId ||
report.period !== params.period ||
report.source !== params.source ||
report.sourceId !== params.sourceId ||
report.bytes !== params.bytes
) {
throw new Error('traffic_report_idempotency_conflict')
}
}
async function insertTrafficReport(
platform: Platform,
params: {
orgId: string
period: string
source: TrafficReportSource
sourceId: string
eventId: string
bytes: number
status: ReportStatus
now: Date
},
) {
await platform.db.insert(cloudTrafficReports).values({
id: nanoid(),
orgId: params.orgId,
period: params.period,
source: params.source,
sourceId: params.sourceId,
eventId: params.eventId,
bytes: params.bytes,
status: params.status,
error: null,
createdAt: params.now,
updatedAt: params.now,
})
}
async function updateTrafficReport(
platform: Platform,
eventId: string,
status: ReportStatus,
error: string | null,
now: Date,
) {
await platform.db
.update(cloudTrafficReports)
.set({ status, error, updatedAt: now })
.where(eq(cloudTrafficReports.eventId, eventId))
}
+16
View File
@@ -137,6 +137,22 @@ const APP_SCHEMA_SQL = `
traffic_used INTEGER NOT NULL DEFAULT 0,
traffic_period TEXT NOT NULL DEFAULT '1970-01'
);
CREATE TABLE IF NOT EXISTS cloud_traffic_reports (
id TEXT PRIMARY KEY,
org_id TEXT NOT NULL,
period TEXT NOT NULL,
source TEXT NOT NULL,
source_id TEXT NOT NULL,
event_id TEXT NOT NULL,
bytes INTEGER NOT NULL,
status TEXT NOT NULL,
error TEXT,
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS cloud_traffic_reports_event_uniq ON cloud_traffic_reports(event_id);
CREATE INDEX IF NOT EXISTS cloud_traffic_reports_org_period_idx ON cloud_traffic_reports(org_id, period);
CREATE INDEX IF NOT EXISTS cloud_traffic_reports_status_idx ON cloud_traffic_reports(status);
CREATE TABLE IF NOT EXISTS org_quota_entitlements (
id TEXT PRIMARY KEY,
org_id TEXT NOT NULL,