mirror of
https://github.com/saltbo/zpan.git
synced 2026-08-28 15:51:29 +08:00
perf(webdav): cache verified auth bursts
This commit is contained in:
@@ -152,6 +152,9 @@
|
||||
"CACHE_KV": {
|
||||
"description": "Workers KV namespace for distributed application caches"
|
||||
},
|
||||
"WEBDAV_RATE_LIMITER": {
|
||||
"description": "Workers native rate limiter for WebDAV credentials"
|
||||
},
|
||||
"ZPAN_CACHE_MODE": {
|
||||
"description": "Cache mode: off, memory, or distributed"
|
||||
}
|
||||
|
||||
+21
@@ -84,6 +84,27 @@ describe('runtime cache', () => {
|
||||
expect(loader).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps memory-only policies out of the distributed backend', async () => {
|
||||
const { backend } = fakeBackend()
|
||||
const cache = createRuntimeCache({ mode: 'distributed', distributed: backend })
|
||||
const memoryOnlyPolicy = { ...stringPolicy, namespace: 'sensitive', distributed: false }
|
||||
|
||||
expect(await cache.getOrLoad(memoryOnlyPolicy, 'a', async () => 'source')).toMatchObject({
|
||||
value: 'source',
|
||||
tier: 'source',
|
||||
})
|
||||
expect(await cache.getOrLoad(memoryOnlyPolicy, 'a', async () => 'unexpected')).toMatchObject({
|
||||
value: 'source',
|
||||
tier: 'memory',
|
||||
})
|
||||
await cache.replace(memoryOnlyPolicy, 'b', 'replacement')
|
||||
await cache.invalidate(memoryOnlyPolicy, 'a')
|
||||
|
||||
expect(backend.get).not.toHaveBeenCalled()
|
||||
expect(backend.put).not.toHaveBeenCalled()
|
||||
expect(backend.delete).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects expired or invalid distributed envelopes and refreshes them from source', async () => {
|
||||
const { backend, values } = fakeBackend()
|
||||
values.set('zpan:v2:test:expired', JSON.stringify({ freshUntil: 999, value: 'old' }))
|
||||
|
||||
+8
-4
@@ -87,6 +87,10 @@ export function createRuntimeCache(options: RuntimeCacheOptions): CacheService {
|
||||
return `zpan:v${policy.version}:${policy.namespace}:${key}`
|
||||
}
|
||||
|
||||
function usesDistributed<T>(policy: CachePolicy<T>): boolean {
|
||||
return options.mode === 'distributed' && policy.distributed !== false
|
||||
}
|
||||
|
||||
async function distributedGet<T>(policy: CachePolicy<T>, key: string): Promise<T | undefined> {
|
||||
if (!options.distributed) return undefined
|
||||
try {
|
||||
@@ -147,7 +151,7 @@ export function createRuntimeCache(options: RuntimeCacheOptions): CacheService {
|
||||
const inMemory = memoryGet(policy, key)
|
||||
if (inMemory !== undefined) return observed(policy.namespace, 'memory', startedAt, inMemory)
|
||||
|
||||
if (options.mode === 'distributed') {
|
||||
if (usesDistributed(policy)) {
|
||||
const distributed = await distributedGet(policy, key)
|
||||
if (distributed !== undefined) return observed(policy.namespace, 'distributed', startedAt, distributed)
|
||||
}
|
||||
@@ -155,7 +159,7 @@ export function createRuntimeCache(options: RuntimeCacheOptions): CacheService {
|
||||
const value = await loader()
|
||||
const freshUntil = now() + ttlFor(policy, value)
|
||||
memoryPut(policy, key, value, freshUntil)
|
||||
if (options.mode === 'distributed') await distributedPut(policy, key, value, freshUntil)
|
||||
if (usesDistributed(policy)) await distributedPut(policy, key, value, freshUntil)
|
||||
return observed(policy.namespace, 'source', startedAt, value)
|
||||
},
|
||||
|
||||
@@ -163,12 +167,12 @@ export function createRuntimeCache(options: RuntimeCacheOptions): CacheService {
|
||||
if (options.mode === 'off') return
|
||||
const freshUntil = now() + ttlFor(policy, value)
|
||||
memoryPut(policy, key, value, freshUntil)
|
||||
if (options.mode === 'distributed') await distributedPut(policy, key, value, freshUntil)
|
||||
if (usesDistributed(policy)) await distributedPut(policy, key, value, freshUntil)
|
||||
},
|
||||
|
||||
async invalidate<T>(policy: CachePolicy<T>, key: string): Promise<void> {
|
||||
memoryStore(policy.namespace).delete(key)
|
||||
if (options.mode !== 'distributed' || !options.distributed) return
|
||||
if (!usesDistributed(policy) || !options.distributed) return
|
||||
try {
|
||||
await options.distributed.delete(cacheKey(policy, key))
|
||||
} catch (error) {
|
||||
|
||||
+3
-1
@@ -26,6 +26,7 @@ import {
|
||||
WEBDAV_API_KEY_PERMISSIONS,
|
||||
WEBDAV_API_KEY_RATE_LIMIT_MAX_REQUESTS,
|
||||
WEBDAV_API_KEY_RATE_LIMIT_WINDOW_MS,
|
||||
WEBDAV_RATE_LIMITER_BINDING,
|
||||
} from '../shared/api-key-templates'
|
||||
import { DEFAULT_ORG_QUOTA, DEFAULT_ORG_TRAFFIC_QUOTA, SignupMode } from '../shared/constants'
|
||||
import {
|
||||
@@ -327,6 +328,7 @@ export async function createAuth(
|
||||
const systemOptionsRepo = createSystemOptionsRepo(db)
|
||||
const email = createEmailGateway(systemOptionsRepo)
|
||||
const providerConfigs = await loadProviderConfigs(rawDb)
|
||||
const usesNativeWebDavRateLimit = Boolean(authPlatform.getBinding(WEBDAV_RATE_LIMITER_BINDING))
|
||||
const authOptions = {
|
||||
database: drizzleAdapter(db, { provider: 'sqlite', schema: authSchema }),
|
||||
secret,
|
||||
@@ -567,7 +569,7 @@ export async function createAuth(
|
||||
references: 'user',
|
||||
enableMetadata: true,
|
||||
rateLimit: {
|
||||
enabled: true,
|
||||
enabled: !usesNativeWebDavRateLimit,
|
||||
// Filesystem clients such as macOS Finder issue bursts of PROPFIND
|
||||
// and stat requests while browsing mounted folders.
|
||||
timeWindow: WEBDAV_API_KEY_RATE_LIMIT_WINDOW_MS,
|
||||
|
||||
@@ -5,6 +5,7 @@ import { serve } from '@hono/node-server'
|
||||
import { sql } from 'drizzle-orm'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { createClient } from 'webdav'
|
||||
import { WEBDAV_RATE_LIMITER_BINDING } from '../../shared/api-key-templates.js'
|
||||
import { S3Service } from '../adapters/gateways/s3.js'
|
||||
import { storages } from '../db/schema.js'
|
||||
import { currentTrafficPeriod } from '../domain/quota.js'
|
||||
@@ -160,6 +161,41 @@ async function folder(db: TestApp['db'], orgId: string, opts: { id: string; name
|
||||
}
|
||||
|
||||
describe('WebDAV API', () => {
|
||||
it('uses the native limiter for every request and reuses successful auth briefly', async () => {
|
||||
const limit = vi.fn(async () => ({ success: true }))
|
||||
const { app, db, auth, deps } = await createTestApp({}, { [WEBDAV_RATE_LIMITER_BINDING]: { limit } })
|
||||
await authedHeaders(app)
|
||||
const account = await userAccount(db)
|
||||
const key = await apiKey(auth, account.id, { webdav: ['read'] })
|
||||
const verify = vi.spyOn(deps.apiKeys, 'verifyApiKeyForPermission')
|
||||
const headers = basicHeaders(account.email, key, { Depth: '0' })
|
||||
|
||||
const first = await app.request('/dav/', { method: 'PROPFIND', headers })
|
||||
const second = await app.request('/dav/', { method: 'PROPFIND', headers })
|
||||
|
||||
expect(first.status).toBe(207)
|
||||
expect(second.status).toBe(207)
|
||||
expect(limit).toHaveBeenCalledTimes(2)
|
||||
expect(verify).toHaveBeenCalledTimes(1)
|
||||
expect(second.headers.get('Server-Timing')).toContain('webdav-auth:memory')
|
||||
})
|
||||
|
||||
it('rejects native WebDAV rate limits before API-key verification', async () => {
|
||||
const limit = vi.fn(async () => ({ success: false }))
|
||||
const { app, deps } = await createTestApp({}, { [WEBDAV_RATE_LIMITER_BINDING]: { limit } })
|
||||
const verify = vi.spyOn(deps.apiKeys, 'verifyApiKeyForPermission')
|
||||
|
||||
const response = await app.request('/dav/', {
|
||||
method: 'PROPFIND',
|
||||
headers: basicHeaders('user@example.com', 'secret'),
|
||||
})
|
||||
|
||||
expect(response.status).toBe(429)
|
||||
expect(response.headers.get('Retry-After')).toBe('60')
|
||||
expect(limit).toHaveBeenCalledTimes(1)
|
||||
expect(verify).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not count API-key WebDAV requests as Better Auth session activity', async () => {
|
||||
const { app, db, auth } = await createTestApp()
|
||||
await authedHeaders(app)
|
||||
|
||||
+75
-10
@@ -1,6 +1,6 @@
|
||||
import type { Context } from 'hono'
|
||||
import { Hono } from 'hono'
|
||||
import { ApiKeyTemplate } from '../../shared/api-key-templates'
|
||||
import { ApiKeyTemplate, WEBDAV_RATE_LIMITER_BINDING } from '../../shared/api-key-templates'
|
||||
import { DirType, ZPAN_CLOUD_URL_DEFAULT } from '../../shared/constants'
|
||||
import { encodeDavPathSegment, joinMatterPath, workspaceHref } from '../domain/webdav'
|
||||
import { WEBDAV_AUTH_CHALLENGE, type WebDavMountPath, webDavPublicUrl } from '../domain/webdav-public-url'
|
||||
@@ -26,6 +26,7 @@ import { isDownloadFailureStatus, transferAuditActor, transferFailureReason } fr
|
||||
import type { Env } from '../middleware/platform'
|
||||
import {
|
||||
ApiKeyRateLimitError,
|
||||
type CachePolicy,
|
||||
insufficientCredits,
|
||||
type RecordAuditEventInput,
|
||||
type StorageRecord,
|
||||
@@ -79,6 +80,41 @@ type DavAuth = {
|
||||
permissions: Record<string, string[]> | null
|
||||
}
|
||||
|
||||
type VerifiedWebDavAuth = Extract<Awaited<ReturnType<typeof resolveWebDavAuth>>, { ok: true }>
|
||||
|
||||
interface NativeRateLimiter {
|
||||
limit(options: { key: string }): Promise<{ success: boolean }>
|
||||
}
|
||||
|
||||
// The native limiter still counts every request. This only collapses repeated D1
|
||||
// verification inside a Finder burst; memory-only storage bounds revocation lag
|
||||
// to one second and keeps credential-derived keys out of KV.
|
||||
const WEBDAV_AUTH_CACHE_POLICY: CachePolicy<VerifiedWebDavAuth> = {
|
||||
namespace: 'webdav-auth',
|
||||
version: 1,
|
||||
ttlMs: 1_000,
|
||||
maxEntries: 256,
|
||||
distributed: false,
|
||||
validate(value): value is VerifiedWebDavAuth {
|
||||
if (typeof value !== 'object' || value === null) return false
|
||||
const auth = value as Partial<VerifiedWebDavAuth>
|
||||
return (
|
||||
auth.ok === true &&
|
||||
typeof auth.userId === 'string' &&
|
||||
typeof auth.keyId === 'string' &&
|
||||
typeof auth.configId === 'string' &&
|
||||
(auth.permissions === null ||
|
||||
(typeof auth.permissions === 'object' && auth.permissions !== null && !Array.isArray(auth.permissions)))
|
||||
)
|
||||
},
|
||||
}
|
||||
|
||||
class WebDavAuthRejected extends Error {
|
||||
constructor(readonly outcome: Exclude<Awaited<ReturnType<typeof resolveWebDavAuth>>, { ok: true }>) {
|
||||
super('WebDAV authentication rejected')
|
||||
}
|
||||
}
|
||||
|
||||
const cloudBaseUrl = (c: DavContext): string => c.get('platform').getEnv('ZPAN_CLOUD_URL') ?? ZPAN_CLOUD_URL_DEFAULT
|
||||
|
||||
async function requireWebDavApiKey(c: DavContext): Promise<DavAuth | Response> {
|
||||
@@ -89,16 +125,39 @@ async function requireWebDavApiKey(c: DavContext): Promise<DavAuth | Response> {
|
||||
const credentials = parseBasicAuth(c.req.raw.headers.get('Authorization'))
|
||||
if (!credentials) return unauthorized()
|
||||
|
||||
const nativeRateLimiter = c.get('platform').getBinding<NativeRateLimiter>(WEBDAV_RATE_LIMITER_BINDING)
|
||||
const credentialKey = nativeRateLimiter ? await webDavCredentialKey(credentials) : null
|
||||
if (nativeRateLimiter && credentialKey) {
|
||||
const { success } = await nativeRateLimiter.limit({ key: credentialKey })
|
||||
if (!success) {
|
||||
return rateLimited(new ApiKeyRateLimitError('Rate limit exceeded.', 60_000))
|
||||
}
|
||||
}
|
||||
|
||||
const startedAt = performance.now()
|
||||
const result = await resolveWebDavAuth(c.get('deps'), {
|
||||
auth: c.get('auth'),
|
||||
db: c.get('platform').db,
|
||||
username: credentials.username,
|
||||
password: credentials.password,
|
||||
resource: WEBDAV_RESOURCE,
|
||||
action,
|
||||
configId: ApiKeyTemplate.WEBDAV,
|
||||
})
|
||||
const loadAuth = async () => {
|
||||
const result = await resolveWebDavAuth(c.get('deps'), {
|
||||
auth: c.get('auth'),
|
||||
db: c.get('platform').db,
|
||||
username: credentials.username,
|
||||
password: credentials.password,
|
||||
resource: WEBDAV_RESOURCE,
|
||||
action,
|
||||
configId: ApiKeyTemplate.WEBDAV,
|
||||
})
|
||||
if (!result.ok) throw new WebDavAuthRejected(result)
|
||||
return result
|
||||
}
|
||||
let result: Awaited<ReturnType<typeof resolveWebDavAuth>>
|
||||
try {
|
||||
result =
|
||||
nativeRateLimiter && credentialKey
|
||||
? (await c.get('deps').cache.getOrLoad(WEBDAV_AUTH_CACHE_POLICY, `${credentialKey}:${action}`, loadAuth)).value
|
||||
: await loadAuth()
|
||||
} catch (error) {
|
||||
if (!(error instanceof WebDavAuthRejected)) throw error
|
||||
result = error.outcome
|
||||
}
|
||||
c.get('webDavTrace').push(`auth:${Math.round(performance.now() - startedAt)}`)
|
||||
if (!result.ok) {
|
||||
if (result.reason === 'rate_limited')
|
||||
@@ -119,6 +178,12 @@ async function requireWebDavApiKey(c: DavContext): Promise<DavAuth | Response> {
|
||||
return result
|
||||
}
|
||||
|
||||
async function webDavCredentialKey(credentials: { username: string; password: string }): Promise<string> {
|
||||
const bytes = new TextEncoder().encode(`${credentials.username}\0${credentials.password}`)
|
||||
const digest = new Uint8Array(await crypto.subtle.digest('SHA-256', bytes))
|
||||
return Array.from(digest, (byte) => byte.toString(16).padStart(2, '0')).join('')
|
||||
}
|
||||
|
||||
function unauthorized(): Response {
|
||||
return new Response('Unauthorized', { status: 401, headers: { 'WWW-Authenticate': WEBDAV_AUTH_CHALLENGE } })
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ export interface CachePolicy<T> {
|
||||
ttlMs: number
|
||||
negativeTtlMs?: number
|
||||
maxEntries: number
|
||||
distributed?: boolean
|
||||
validate(value: unknown): value is T
|
||||
}
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ export function parseApiKeyScope(metadata: unknown): ApiKeyScope | null {
|
||||
|
||||
export const WEBDAV_API_KEY_RATE_LIMIT_WINDOW_MS = 60_000
|
||||
export const WEBDAV_API_KEY_RATE_LIMIT_MAX_REQUESTS = 3600
|
||||
export const WEBDAV_RATE_LIMITER_BINDING = 'WEBDAV_RATE_LIMITER'
|
||||
|
||||
export const IHOST_API_KEY_PERMISSIONS = { ihost: ['upload'] } satisfies ApiKeyPermissions
|
||||
export const WEBDAV_API_KEY_PERMISSIONS = { webdav: ['read', 'write'] } satisfies ApiKeyPermissions
|
||||
|
||||
@@ -18,6 +18,14 @@ migrations_dir = "./migrations"
|
||||
binding = "CACHE_KV"
|
||||
id = "2721f9d7b47d433d8a0e02171e276f57"
|
||||
|
||||
[[ratelimits]]
|
||||
name = "WEBDAV_RATE_LIMITER"
|
||||
namespace_id = "2801"
|
||||
|
||||
[ratelimits.simple]
|
||||
limit = 3600
|
||||
period = 60
|
||||
|
||||
[vars]
|
||||
ZPAN_CACHE_MODE = "distributed"
|
||||
|
||||
@@ -74,6 +82,14 @@ migrations_dir = "./migrations"
|
||||
binding = "CACHE_KV"
|
||||
id = "7d8625bacbc54e23a46ca4464a9f64a0"
|
||||
|
||||
[[env.staging.ratelimits]]
|
||||
name = "WEBDAV_RATE_LIMITER"
|
||||
namespace_id = "2802"
|
||||
|
||||
[env.staging.ratelimits.simple]
|
||||
limit = 3600
|
||||
period = 60
|
||||
|
||||
[[env.staging.r2_buckets]]
|
||||
binding = "PUBLIC_IMAGES"
|
||||
bucket_name = "zpan-public-images-staging"
|
||||
|
||||
Reference in New Issue
Block a user