opensource/sim
3
0
Fork 0
mirror of https://github.com/simstudioai/sim.git synced 2026-10-30 19:32:46 +08:00
Code Issues Packages Projects Releases Wiki Activity

improvement(kbs): ownership bindings (#4833)

Browse Source 
* improvement(kbs): ownership association

* address comments

* more comments
This commit is contained in:
Vikhyath Mondreti
2026-06-01 15:46:41 -07:00
committed by GitHub
parent 2e0f506699
commit 9bed841173
27 changed files with 18945 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/sim/app/api/files/authorization.test.ts
+153
View File
@@ -0,0 +1,153 @@
/**
* Tests for KB file authorization (`verifyKBFileAccess` via `verifyFileAccess`).
*
* These lock in the security-critical contract: access is granted only when a
* trusted ownership binding names a workspace the caller can access AND an active
* document still references the exact key. A planted `document.fileUrl` (the
* reported vulnerability) can never grant access because ownership comes from the
* binding, not the document.
*
* @vitest-environment node
*/
import { dbChainMock, dbChainMockFns } from '@sim/testing'
import { beforeEach, describe, expect, it, vi } from 'vitest'
const { mockGetFileMetadataByKey, mockGetUserEntityPermissions } = vi.hoisted(() => ({
mockGetFileMetadataByKey: vi.fn(),
mockGetUserEntityPermissions: vi.fn(),
}))
vi.mock('@sim/db', () => dbChainMock)
vi.mock('@/lib/uploads', () => ({
getFileMetadata: vi.fn(),
}))
vi.mock('@/lib/uploads/config', () => ({
BLOB_CHAT_CONFIG: {},
S3_CHAT_CONFIG: {},
}))
vi.mock('@/lib/uploads/server/metadata', () => ({
getFileMetadataByKey: mockGetFileMetadataByKey,
}))
vi.mock('@/lib/uploads/utils/file-utils', () => ({
inferContextFromKey: vi.fn(() => 'knowledge-base'),
}))
vi.mock('@/lib/workspaces/permissions/utils', () => ({
getUserEntityPermissions: mockGetUserEntityPermissions,
}))
vi.mock('@/executor/constants', () => ({
isUuid: vi.fn(() => false),
}))
import { verifyFileAccess, verifyKBFileWriteAccess } from '@/app/api/files/authorization'
const CLOUD_KEY = 'kb/1780162789495-secret.txt'
const USER_ID = 'user-1'
function grantAccess(cloudKey: string) {
return verifyFileAccess(cloudKey, USER_ID, undefined, 'knowledge-base')
}
describe('verifyKBFileAccess (binding-only)', () => {
beforeEach(() => {
vi.clearAllMocks()
// Default liveness query result: one active document references the exact storage key.
dbChainMockFns.limit.mockResolvedValue([{ id: 'doc-1' }])
})
it('grants access when the binding owner workspace is accessible and an active document references the key', async () => {
mockGetFileMetadataByKey.mockResolvedValue({ workspaceId: 'ws-1', deletedAt: null })
mockGetUserEntityPermissions.mockResolvedValue('read')
await expect(grantAccess(CLOUD_KEY)).resolves.toBe(true)
expect(mockGetUserEntityPermissions).toHaveBeenCalledWith(USER_ID, 'workspace', 'ws-1')
})
it('denies when the caller lacks permission on the owner workspace (cross-tenant)', async () => {
mockGetFileMetadataByKey.mockResolvedValue({ workspaceId: 'victim-ws', deletedAt: null })
mockGetUserEntityPermissions.mockResolvedValue(null)
await expect(grantAccess(CLOUD_KEY)).resolves.toBe(false)
})
it('denies when there is no ownership binding (planted or un-backfilled key)', async () => {
mockGetFileMetadataByKey.mockResolvedValue(null)
await expect(grantAccess(CLOUD_KEY)).resolves.toBe(false)
// Authorization never consults workspace permissions without a binding.
expect(mockGetUserEntityPermissions).not.toHaveBeenCalled()
})
it('denies when the binding is soft-deleted', async () => {
mockGetFileMetadataByKey.mockResolvedValue({ workspaceId: 'ws-1', deletedAt: new Date() })
await expect(grantAccess(CLOUD_KEY)).resolves.toBe(false)
expect(mockGetUserEntityPermissions).not.toHaveBeenCalled()
})
it('denies when the binding has no workspace owner', async () => {
mockGetFileMetadataByKey.mockResolvedValue({ workspaceId: null, deletedAt: null })
await expect(grantAccess(CLOUD_KEY)).resolves.toBe(false)
expect(mockGetUserEntityPermissions).not.toHaveBeenCalled()
})
it('denies when no active document references the key (archived/soft-deleted KB liveness)', async () => {
mockGetFileMetadataByKey.mockResolvedValue({ workspaceId: 'ws-1', deletedAt: null })
mockGetUserEntityPermissions.mockResolvedValue('admin')
dbChainMockFns.limit.mockResolvedValue([])
await expect(grantAccess(CLOUD_KEY)).resolves.toBe(false)
})
it('fails closed when the binding lookup throws', async () => {
mockGetFileMetadataByKey.mockRejectedValue(new Error('db down'))
await expect(grantAccess(CLOUD_KEY)).resolves.toBe(false)
})
})
describe('verifyKBFileWriteAccess (binding-only delete authorization)', () => {
beforeEach(() => {
vi.clearAllMocks()
})
it('grants delete when the caller has write on the owner workspace', async () => {
mockGetFileMetadataByKey.mockResolvedValue({ workspaceId: 'ws-1', deletedAt: null })
mockGetUserEntityPermissions.mockResolvedValue('write')
await expect(verifyKBFileWriteAccess(CLOUD_KEY, USER_ID)).resolves.toBe(true)
})
it('grants delete when the caller is admin on the owner workspace', async () => {
mockGetFileMetadataByKey.mockResolvedValue({ workspaceId: 'ws-1', deletedAt: null })
mockGetUserEntityPermissions.mockResolvedValue('admin')
await expect(verifyKBFileWriteAccess(CLOUD_KEY, USER_ID)).resolves.toBe(true)
})
it('denies delete when the caller has only read on the owner workspace', async () => {
mockGetFileMetadataByKey.mockResolvedValue({ workspaceId: 'ws-1', deletedAt: null })
mockGetUserEntityPermissions.mockResolvedValue('read')
await expect(verifyKBFileWriteAccess(CLOUD_KEY, USER_ID)).resolves.toBe(false)
})
it('denies delete when there is no binding (no fallback)', async () => {
mockGetFileMetadataByKey.mockResolvedValue(null)
await expect(verifyKBFileWriteAccess(CLOUD_KEY, USER_ID)).resolves.toBe(false)
expect(mockGetUserEntityPermissions).not.toHaveBeenCalled()
})
it('fails closed when the binding lookup throws', async () => {
mockGetFileMetadataByKey.mockRejectedValue(new Error('db down'))
await expect(verifyKBFileWriteAccess(CLOUD_KEY, USER_ID)).resolves.toBe(false)
})
})
apps/sim/app/api/files/authorization.ts
+104 -47
View File
@@ -1,7 +1,7 @@
import { db } from '@sim/db'
import { document, knowledgeBase, workspaceFile } from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { and, eq, isNull, like, or } from 'drizzle-orm'
import { and, eq, isNull } from 'drizzle-orm'
import { NextResponse } from 'next/server'
import { getFileMetadata } from '@/lib/uploads'
import type { StorageContext } from '@/lib/uploads/config'
@@ -371,8 +371,46 @@ async function verifyCopilotFileAccess(
}
/**
* Verify access to KB files
* KB files: kb/filename
* Whether an active KB document (non-archived/excluded/deleted, in a
* non-deleted KB) in the owning workspace references exactly `cloudKey`, matched
* on the document's persisted canonical `storageKey`. This is an exact, indexed
* lookup — no URL parsing or wildcard matching at read time. It is a lifecycle
* signal only: it reflects whether the file is still part of a live KB, not who
* owns it (ownership comes from the binding).
*/
async function hasActiveKbDocumentForKey(cloudKey: string, workspaceId: string): Promise<boolean> {
const rows = await db
.select({ id: document.id })
.from(document)
.innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id))
.where(
and(
eq(knowledgeBase.workspaceId, workspaceId),
eq(document.storageKey, cloudKey),
eq(document.userExcluded, false),
isNull(document.archivedAt),
isNull(document.deletedAt),
isNull(knowledgeBase.deletedAt)
)
)
.limit(1)
return rows.length > 0
}
/**
* Verify access to KB files (`kb/<key>`).
*
* Authorization is determined entirely by clear state:
* 1. Ownership — the trusted `workspace_files` binding (exact key) names the
* owning workspace; the caller must have permission on it. Ownership is
* never inferred from an attacker-authorable `document.fileUrl`.
* 2. Liveness — an active document must still reference the exact key, so the
* retained bytes of an archived document or soft-deleted KB are not
* downloadable (the liveness document is not an authorization signal).
*
* A missing binding denies (the ownership backfill populates bindings for
* pre-existing objects before this path is deployed).
*/
async function verifyKBFileAccess(
cloudKey: string,
@@ -380,64 +418,83 @@ async function verifyKBFileAccess(
customConfig?: StorageConfig
): Promise<boolean> {
try {
const activeKbFileDocuments = await db
.select({
workspaceId: knowledgeBase.workspaceId,
})
.from(document)
.innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id))
.where(
and(
eq(document.userExcluded, false),
isNull(document.archivedAt),
isNull(document.deletedAt),
isNull(knowledgeBase.deletedAt),
or(
like(document.fileUrl, `%${cloudKey}%`),
like(document.fileUrl, `%${encodeURIComponent(cloudKey)}%`)
)
)
)
.limit(10)
for (const doc of activeKbFileDocuments) {
if (!doc.workspaceId) {
continue
}
const permission = await getUserEntityPermissions(userId, 'workspace', doc.workspaceId)
if (permission !== null) {
logger.debug('KB file access granted (active document lookup)', {
userId,
workspaceId: doc.workspaceId,
cloudKey,
})
return true
}
}
// KB file access must resolve through an active KB document. Metadata alone is not enough
// because parent archives intentionally keep the underlying file bytes around for history.
const fileRecord = await getFileMetadataByKey(cloudKey, 'knowledge-base', {
const binding = await getFileMetadataByKey(cloudKey, 'knowledge-base', {
includeDeleted: true,
})
if (fileRecord?.deletedAt) {
logger.warn('KB file access denied for deleted file metadata', { userId, cloudKey })
if (!binding) {
logger.warn('KB file access denied: no ownership binding', { userId, cloudKey })
return false
}
if (binding.deletedAt) {
logger.warn('KB file access denied for deleted file binding', { userId, cloudKey })
return false
}
if (!binding.workspaceId) {
logger.warn('KB file binding missing workspace owner', { userId, cloudKey })
return false
}
logger.warn('KB file access denied because no active KB document matched the file', {
cloudKey,
const permission = await getUserEntityPermissions(userId, 'workspace', binding.workspaceId)
if (permission === null) {
logger.warn('User does not have workspace access for KB file', {
userId,
workspaceId: binding.workspaceId,
cloudKey,
})
return false
}
if (!(await hasActiveKbDocumentForKey(cloudKey, binding.workspaceId))) {
logger.warn('KB file access denied: no active document references the file', {
userId,
cloudKey,
})
return false
}
logger.debug('KB file access granted (ownership binding)', {
userId,
workspaceId: binding.workspaceId,
cloudKey,
})
return false
return true
} catch (error) {
logger.error('Error verifying KB file access', { cloudKey, userId, error })
return false
}
}
/**
* Authorize a destructive operation (delete) on a KB file.
*
* Binding-only: resolves the owning workspace from the trusted ownership binding
* and requires write/admin permission. Never uses the transitional read fallback,
* so a not-yet-bound key cannot be deleted cross-tenant.
*/
export async function verifyKBFileWriteAccess(cloudKey: string, userId: string): Promise<boolean> {
try {
const binding = await getFileMetadataByKey(cloudKey, 'knowledge-base')
if (!binding?.workspaceId) {
logger.warn('KB file delete denied: no ownership binding', { userId, cloudKey })
return false
}
const permission = await getUserEntityPermissions(userId, 'workspace', binding.workspaceId)
if (permission !== 'write' && permission !== 'admin') {
logger.warn('KB file delete denied: write/admin required on owner workspace', {
userId,
workspaceId: binding.workspaceId,
cloudKey,
})
return false
}
return true
} catch (error) {
logger.error('Error verifying KB file write access', { cloudKey, userId, error })
return false
}
}
/**
* Verify access to chat files
* Chat files: chat/filename
apps/sim/app/api/files/delete/route.ts
+13 -8
View File
@@ -9,7 +9,7 @@ import type { StorageContext } from '@/lib/uploads/config'
import { deleteFile, hasCloudStorage } from '@/lib/uploads/core/storage-service'
import { deleteFileMetadata } from '@/lib/uploads/server/metadata'
import { extractStorageKey, inferContextFromKey } from '@/lib/uploads/utils/file-utils'
import { verifyFileAccess } from '@/app/api/files/authorization'
import { verifyFileAccess, verifyKBFileWriteAccess } from '@/app/api/files/authorization'
import {
createErrorResponse,
createSuccessResponse,
@@ -64,13 +64,18 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const storageContext: StorageContext = context || inferContextFromKey(key)
const hasAccess = await verifyFileAccess(
key,
userId,
undefined, // customConfig
storageContext, // context
!hasCloudStorage() // isLocal
)
// KB deletes are binding-only and require write/admin on the owning workspace;
// they never use the transitional read fallback that file serving allows.
const hasAccess =
storageContext === 'knowledge-base'
? await verifyKBFileWriteAccess(key, userId)
: await verifyFileAccess(
key,
userId,
undefined, // customConfig
storageContext, // context
!hasCloudStorage() // isLocal
)
if (!hasAccess) {
logger.warn('Unauthorized file delete attempt', { userId, key, context: storageContext })
apps/sim/app/api/files/multipart/route.ts
+52 -22
View File
@@ -18,11 +18,13 @@ import {
isUsingCloudStorage,
type StorageContext,
} from '@/lib/uploads'
import { deleteFile } from '@/lib/uploads/core/storage-service'
import {
signUploadToken,
type UploadTokenPayload,
verifyUploadToken,
} from '@/lib/uploads/core/upload-token'
import { insertFileMetadata } from '@/lib/uploads/server/metadata'
import { QUOTA_EXEMPT_STORAGE_CONTEXTS, type StorageConfig } from '@/lib/uploads/shared/types'
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
@@ -82,6 +84,29 @@ const verifyTokenForUser = (token: string | undefined, userId: string) => {
return result.payload
}
/**
* Record a trusted storage-key -> workspace ownership binding for completed
* knowledge-base uploads. KB file authorization resolves the owning workspace
* from this binding, so every KB object must have one. No-op for other contexts.
*/
const recordKnowledgeBaseOwnership = async (
payload: UploadTokenPayload,
key: string
): Promise<void> => {
if (payload.context !== 'knowledge-base' || !payload.workspaceId) {
return
}
await insertFileMetadata({
key,
userId: payload.userId,
workspaceId: payload.workspaceId,
context: 'knowledge-base',
originalName: payload.fileName ?? key.split('/').pop() ?? key,
contentType: payload.contentType ?? 'application/octet-stream',
size: typeof payload.fileSize === 'number' ? payload.fileSize : 0,
})
}
export const POST = withRouteHandler(async (request: NextRequest) => {
try {
const session = await getSession()
@@ -220,6 +245,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
userId,
workspaceId,
context: storageContext,
fileName,
contentType,
...(typeof fileSize === 'number' ? { fileSize } : {}),
})
logger.info(
@@ -301,6 +329,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const { uploadId, key, context } = payload
const config = getStorageConfig(context)
let completed: { location: string; path: string; key: string }
if (storageProvider === 's3' && s3Module) {
const { completeS3MultipartUpload } = s3Module
const s3Parts = parts.map((p) => {
@@ -309,40 +338,41 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
}
return { ETag: p.etag, PartNumber: p.partNumber }
})
const result = await completeS3MultipartUpload(
completed = await completeS3MultipartUpload(
key,
uploadId,
s3Parts,
buildS3CustomConfig(config)
)
return {
success: true as const,
location: result.location,
path: result.path,
key: result.key,
}
}
if (storageProvider === 'blob' && blobModule) {
} else if (storageProvider === 'blob' && blobModule) {
const { completeMultipartUpload, deriveBlobBlockId } = blobModule
const blobParts = parts.map((p) => ({
partNumber: p.partNumber,
blockId: deriveBlobBlockId(p.partNumber),
}))
const result = await completeMultipartUpload(
key,
blobParts,
buildBlobCustomConfig(config)
)
return {
success: true as const,
location: result.location,
path: result.path,
key: result.key,
}
completed = await completeMultipartUpload(key, blobParts, buildBlobCustomConfig(config))
} else {
throw new Error(`Unsupported storage provider: ${storageProvider}`)
}
throw new Error(`Unsupported storage provider: ${storageProvider}`)
try {
await recordKnowledgeBaseOwnership(payload, completed.key)
} catch (error) {
// The object is committed, but without an ownership binding a KB file
// is unreadable and undeletable via the KB paths. Remove the orphan
// best-effort and surface a retryable error so the client re-uploads.
if (payload.context === 'knowledge-base') {
await deleteFile({ key: completed.key, context: 'knowledge-base' }).catch(() => {})
}
throw error
}
return {
success: true as const,
location: completed.location,
path: completed.path,
key: completed.key,
}
}
if ('uploads' in data && Array.isArray(data.uploads)) {
apps/sim/app/api/files/presigned/batch/route.ts
+40 -2
View File
@@ -13,7 +13,9 @@ import {
generateBatchPresignedUploadUrls,
hasCloudStorage,
} from '@/lib/uploads/core/storage-service'
import { insertFileMetadataMany } from '@/lib/uploads/server/metadata'
import { validateFileType } from '@/lib/uploads/utils/validation'
import { getUserEntityPermissions } from '@/lib/workspaces/permissions/utils'
import { createErrorResponse } from '@/app/api/files/utils'
const logger = createLogger('BatchPresignedUploadAPI')
@@ -60,6 +62,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
const uploadType = uploadTypeResult.data as StorageContext
const sessionUserId = session.user.id
let knowledgeBaseWorkspaceId: string | null = null
if (uploadType === 'knowledge-base') {
for (const file of files) {
const fileValidationError = validateFileType(file.fileName, file.contentType)
@@ -74,9 +79,27 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
)
}
}
}
const sessionUserId = session.user.id
knowledgeBaseWorkspaceId = request.nextUrl.searchParams.get('workspaceId')
if (!knowledgeBaseWorkspaceId?.trim()) {
return NextResponse.json(
{ error: 'workspaceId query parameter is required for knowledge-base uploads' },
{ status: 400 }
)
}
const permission = await getUserEntityPermissions(
sessionUserId,
'workspace',
knowledgeBaseWorkspaceId
)
if (permission !== 'write' && permission !== 'admin') {
return NextResponse.json(
{ error: 'Write or Admin access required for knowledge-base uploads' },
{ status: 403 }
)
}
}
if (uploadType === 'copilot' && !sessionUserId?.trim()) {
return NextResponse.json(
@@ -126,6 +149,21 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
`Generated ${files.length} presigned URLs in ${duration}ms (avg ${Math.round(duration / files.length)}ms per file)`
)
if (uploadType === 'knowledge-base' && knowledgeBaseWorkspaceId) {
const ownerWorkspaceId = knowledgeBaseWorkspaceId
await insertFileMetadataMany(
presignedUrls.map((urlResponse, index) => ({
key: urlResponse.key,
userId: sessionUserId,
workspaceId: ownerWorkspaceId,
context: 'knowledge-base',
originalName: files[index].fileName,
contentType: files[index].contentType,
size: files[index].fileSize,
}))
)
}
const storagePrefix = USE_BLOB_STORAGE ? 'blob' : 's3'
return NextResponse.json({
apps/sim/app/api/files/presigned/route.test.ts
+41 -2
View File
@@ -352,7 +352,7 @@ describe('/api/files/presigned', () => {
})
const request = new NextRequest(
'http://localhost:3000/api/files/presigned?type=knowledge-base',
'http://localhost:3000/api/files/presigned?type=knowledge-base&workspaceId=ws-1',
{
method: 'POST',
body: JSON.stringify({
@@ -810,7 +810,7 @@ describe('/api/files/presigned', () => {
setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' })
const request = new NextRequest(
'http://localhost:3000/api/files/presigned?type=knowledge-base',
'http://localhost:3000/api/files/presigned?type=knowledge-base&workspaceId=ws-1',
{
method: 'POST',
body: JSON.stringify({
@@ -826,5 +826,44 @@ describe('/api/files/presigned', () => {
expect(mockValidateFileType).toHaveBeenCalledWith('doc.pdf', 'application/pdf')
expect(mockValidateAttachmentFileType).not.toHaveBeenCalled()
})
it('requires workspaceId for knowledge-base uploads', async () => {
setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' })
const request = new NextRequest(
'http://localhost:3000/api/files/presigned?type=knowledge-base',
{
method: 'POST',
body: JSON.stringify({
fileName: 'doc.pdf',
contentType: 'application/pdf',
fileSize: 4096,
}),
}
)
const response = await POST(request)
expect(response.status).toBe(400)
})
it('returns 403 when the user lacks write access to the workspace', async () => {
setupFileApiMocks({ cloudEnabled: true, storageProvider: 's3' })
mockGetUserEntityPermissions.mockResolvedValue('read')
const request = new NextRequest(
'http://localhost:3000/api/files/presigned?type=knowledge-base&workspaceId=ws-1',
{
method: 'POST',
body: JSON.stringify({
fileName: 'doc.pdf',
contentType: 'application/pdf',
fileSize: 4096,
}),
}
)
const response = await POST(request)
expect(response.status).toBe(403)
})
})
})
apps/sim/app/api/files/presigned/route.ts
+35
View File
@@ -252,6 +252,41 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
contentType,
size: fileSize,
})
} else if (uploadType === 'knowledge-base') {
const workspaceId = request.nextUrl.searchParams.get('workspaceId')
if (!workspaceId?.trim()) {
throw new ValidationError(
'workspaceId query parameter is required for knowledge-base uploads'
)
}
const permission = await getUserEntityPermissions(sessionUserId, 'workspace', workspaceId)
if (permission !== 'write' && permission !== 'admin') {
return NextResponse.json(
{ error: 'Write or Admin access required for knowledge-base uploads' },
{ status: 403 }
)
}
presignedUrlResponse = await generatePresignedUploadUrl({
fileName,
contentType,
fileSize,
context: 'knowledge-base',
userId: sessionUserId,
expirationSeconds: 3600,
metadata: { workspaceId },
})
await insertFileMetadata({
key: presignedUrlResponse.key,
userId: sessionUserId,
workspaceId,
context: 'knowledge-base',
originalName: fileName,
contentType,
size: fileSize,
})
} else {
if (uploadType === 'profile-pictures') {
if (!sessionUserId?.trim()) {
apps/sim/app/api/files/upload/route.ts
+10 -15
View File
@@ -141,18 +141,16 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
throw new InvalidRequestError(validationError.message)
}
if (workspaceId) {
const permission = await getUserEntityPermissions(
session.user.id,
'workspace',
workspaceId
if (!workspaceId) {
throw new InvalidRequestError('workspaceId is required for knowledge-base uploads')
}
const permission = await getUserEntityPermissions(session.user.id, 'workspace', workspaceId)
if (permission !== 'write' && permission !== 'admin') {
return NextResponse.json(
{ error: 'Write or Admin access required for knowledge-base uploads' },
{ status: 403 }
)
if (permission === null) {
return NextResponse.json(
{ error: 'Insufficient permissions for workspace' },
{ status: 403 }
)
}
}
logger.info(`Uploading knowledge-base file: ${originalName}`)
@@ -166,10 +164,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
uploadedAt: new Date().toISOString(),
purpose: 'knowledge-base',
userId: session.user.id,
}
if (workspaceId) {
metadata.workspaceId = workspaceId
workspaceId,
}
const fileInfo = await storageService.uploadFile({
apps/sim/app/api/knowledge/[id]/connectors/[connectorId]/route.ts
+7 -3
View File
@@ -359,10 +359,15 @@ export const DELETE = withRouteHandler(async (request: NextRequest, { params }:
return { deletedDocs: deleteDocuments ? docs : [], docCount: docs.length }
})
const kbWorkspaceId = writeCheck.knowledgeBase?.workspaceId ?? null
if (deleteDocuments) {
await Promise.all([
deletedDocs.length > 0
? deleteDocumentStorageFiles(deletedDocs, requestId)
? deleteDocumentStorageFiles(
deletedDocs.map((doc) => ({ ...doc, workspaceId: kbWorkspaceId })),
requestId
)
: Promise.resolve(),
cleanupUnusedTagDefinitions(knowledgeBaseId, requestId).catch((error) => {
logger.warn(`[${requestId}] Failed to cleanup tag definitions`, error)
@@ -374,13 +379,12 @@ export const DELETE = withRouteHandler(async (request: NextRequest, { params }:
`[${requestId}] Deleted connector ${connectorId}${deleteDocuments ? ` and ${docCount} documents` : `, kept ${docCount} documents`}`
)
const kbWorkspaceId = writeCheck.knowledgeBase.workspaceId ?? ''
captureServerEvent(
auth.userId,
'knowledge_base_connector_removed',
{
knowledge_base_id: knowledgeBaseId,
workspace_id: kbWorkspaceId,
workspace_id: kbWorkspaceId ?? '',
connector_type: existingConnector[0].connectorType,
documents_deleted: deleteDocuments ? docCount : 0,
},
apps/sim/app/api/knowledge/[id]/documents/route.test.ts
+1
View File
@@ -40,6 +40,7 @@ vi.mock('@/lib/knowledge/documents/service', () => ({
deleteDocument: vi.fn(),
markDocumentAsFailedTimeout: vi.fn(),
retryDocumentProcessing: vi.fn(),
KnowledgeBaseFileOwnershipError: class KnowledgeBaseFileOwnershipError extends Error {},
}))
vi.mock('@sim/audit', () => auditMock)
apps/sim/app/api/knowledge/[id]/documents/route.ts
+8
View File
@@ -20,6 +20,7 @@ import {
createSingleDocument,
getDocuments,
getProcessingConfig,
KnowledgeBaseFileOwnershipError,
processDocumentsWithQueue,
type TagFilterCondition,
} from '@/lib/knowledge/documents/service'
@@ -302,6 +303,13 @@ export const POST = withRouteHandler(
} catch (error) {
logger.error(`[${requestId}] Error creating document`, error)
if (error instanceof KnowledgeBaseFileOwnershipError) {
return NextResponse.json(
{ error: 'File URL does not reference a file owned by this knowledge base' },
{ status: 403 }
)
}
const errorMessage = getErrorMessage(error, 'Failed to create document')
const isStorageLimitError =
errorMessage.includes('Storage limit exceeded') || errorMessage.includes('storage limit')
apps/sim/app/api/knowledge/[id]/documents/upsert/route.test.ts
+1
View File
@@ -32,6 +32,7 @@ vi.mock('@/lib/knowledge/documents/service', () => ({
deleteDocument: vi.fn(),
getProcessingConfig: vi.fn().mockReturnValue({ maxConcurrentDocuments: 1, batchSize: 1 }),
processDocumentsWithQueue: vi.fn(),
KnowledgeBaseFileOwnershipError: class KnowledgeBaseFileOwnershipError extends Error {},
}))
import { createDocumentRecords, processDocumentsWithQueue } from '@/lib/knowledge/documents/service'
apps/sim/app/api/knowledge/[id]/documents/upsert/route.ts
+8
View File
@@ -15,6 +15,7 @@ import {
createDocumentRecords,
deleteDocument,
getProcessingConfig,
KnowledgeBaseFileOwnershipError,
processDocumentsWithQueue,
} from '@/lib/knowledge/documents/service'
import { checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils'
@@ -219,6 +220,13 @@ export const POST = withRouteHandler(
} catch (error) {
logger.error(`[${requestId}] Error upserting document`, error)
if (error instanceof KnowledgeBaseFileOwnershipError) {
return NextResponse.json(
{ error: 'File URL does not reference a file owned by this knowledge base' },
{ status: 403 }
)
}
const errorMessage = getErrorMessage(error, 'Failed to upsert document')
const isStorageLimitError =
errorMessage.includes('Storage limit exceeded') || errorMessage.includes('storage limit')
apps/sim/app/workspace/[workspaceId]/knowledge/hooks/use-knowledge-upload.ts
+11 -4
View File
@@ -105,7 +105,8 @@ interface BatchPresignedFile {
* single-PUT URL.
*/
const fetchBatchPresignedData = async (
files: File[]
files: File[],
workspaceId: string
): Promise<(PresignedUploadInfo | undefined)[]> => {
const result: (PresignedUploadInfo | undefined)[] = new Array(files.length).fill(undefined)
const smallFileIndices: number[] = []
@@ -114,6 +115,8 @@ const fetchBatchPresignedData = async (
}
if (smallFileIndices.length === 0) return result
const batchEndpoint = `${KB_BATCH_PRESIGNED_ENDPOINT}&workspaceId=${encodeURIComponent(workspaceId)}`
for (let start = 0; start < smallFileIndices.length; start += BATCH_REQUEST_SIZE) {
const batchIndices = smallFileIndices.slice(start, start + BATCH_REQUEST_SIZE)
const batchFiles = batchIndices.map((i) => files[i])
@@ -125,7 +128,7 @@ const fetchBatchPresignedData = async (
})),
}
const response = await fetch(KB_BATCH_PRESIGNED_ENDPOINT, {
const response = await fetch(batchEndpoint, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
@@ -264,7 +267,7 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) {
file,
workspaceId: options.workspaceId,
context: 'knowledge-base',
presignedEndpoint: '/api/files/presigned?type=knowledge-base',
presignedEndpoint: `/api/files/presigned?type=knowledge-base&workspaceId=${encodeURIComponent(options.workspaceId)}`,
presignedOverride: presigned,
onProgress,
})
@@ -292,6 +295,10 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) {
}
const uploadFilesInBatches = async (files: File[]): Promise<UploadedFile[]> => {
if (!options.workspaceId) {
throw new KnowledgeUploadError('workspaceId is required for upload', 'MISSING_WORKSPACE_ID')
}
const fileStatuses: FileUploadStatus[] = files.map((file) => ({
fileName: file.name,
fileSize: file.size,
@@ -303,7 +310,7 @@ export function useKnowledgeUpload(options: UseKnowledgeUploadOptions = {}) {
logger.info(`Starting batch upload of ${files.length} files`)
const presignedData = await fetchBatchPresignedData(files)
const presignedData = await fetchBatchPresignedData(files, options.workspaceId)
const settled = await runWithConcurrency(
files,
apps/sim/background/cleanup-soft-deletes.test.ts
+164
View File
@@ -0,0 +1,164 @@
/**
* @vitest-environment node
*/
import { beforeEach, describe, expect, it, vi } from 'vitest'
const {
mockBatchDeleteByWorkspaceAndTimestamp,
mockDeleteFileMetadata,
mockDeleteFiles,
mockDeleteRowsById,
mockIsUsingCloudStorage,
mockLimit,
mockOrderBy,
mockPrepareChatCleanup,
mockSelect,
mockSelectRowsByIdChunks,
mockTask,
mockWhere,
} = vi.hoisted(() => {
const mockLimit = vi.fn(async () => [] as Array<{ key: string }>)
const mockOrderBy = vi.fn(() => ({ limit: mockLimit }))
const mockWhere = vi.fn(() => ({ orderBy: mockOrderBy, limit: mockLimit }))
const mockFrom = vi.fn(() => ({
where: mockWhere,
leftJoin: vi.fn(() => ({ where: mockWhere })),
}))
const mockSelect = vi.fn(() => ({ from: mockFrom }))
return {
mockBatchDeleteByWorkspaceAndTimestamp: vi.fn(async () => ({ deleted: 0, failed: 0 })),
mockDeleteFileMetadata: vi.fn(async () => true),
mockDeleteFiles: vi.fn(async () => ({ deleted: 0, failed: [] as Array<{ key: string }> })),
mockDeleteRowsById: vi.fn(async () => ({ deleted: 0, failed: 0 })),
mockIsUsingCloudStorage: vi.fn(() => true),
mockLimit,
mockOrderBy,
mockPrepareChatCleanup: vi.fn(async () => ({ execute: vi.fn(async () => undefined) })),
mockSelect,
mockSelectRowsByIdChunks: vi.fn(async () => [] as unknown[]),
mockTask: vi.fn((config: unknown) => config),
mockWhere,
}
})
vi.mock('@sim/db', () => ({ db: { select: mockSelect } }))
vi.mock('@sim/db/schema', () => {
const table = (cols: string[]) =>
Object.fromEntries(cols.map((c) => [c, `col.${c}`])) as Record<string, string>
const wsFileCols = ['id', 'key', 'context', 'workspaceId', 'deletedAt', 'uploadedAt']
const softCols = ['id', 'archivedAt', 'deletedAt', 'workspaceId']
return {
a2aAgent: table(softCols),
copilotChats: table(['id', 'workflowId']),
document: table(['id', 'storageKey', 'knowledgeBaseId']),
knowledgeBase: table(softCols),
mcpServers: table(softCols),
memory: table(softCols),
userTableDefinitions: table(softCols),
workflow: table(softCols),
workflowFolder: table(softCols),
workflowMcpServer: table(softCols),
workspaceFile: table(wsFileCols),
workspaceFiles: table(wsFileCols),
}
})
vi.mock('@sim/logger', () => ({
createLogger: vi.fn(() => ({ error: vi.fn(), info: vi.fn(), warn: vi.fn() })),
}))
vi.mock('@trigger.dev/sdk', () => ({ task: mockTask }))
vi.mock('drizzle-orm', () => ({
and: vi.fn((...args: unknown[]) => ({ op: 'and', args })),
asc: vi.fn((column: unknown) => ({ op: 'asc', column })),
eq: vi.fn((...args: unknown[]) => ({ op: 'eq', args })),
inArray: vi.fn((...args: unknown[]) => ({ op: 'inArray', args })),
isNotNull: vi.fn((...args: unknown[]) => ({ op: 'isNotNull', args })),
isNull: vi.fn((...args: unknown[]) => ({ op: 'isNull', args })),
lt: vi.fn((...args: unknown[]) => ({ op: 'lt', args })),
sql: vi.fn((strings: TemplateStringsArray, ...values: unknown[]) => ({ strings, values })),
}))
vi.mock('@/lib/cleanup/batch-delete', () => ({
batchDeleteByWorkspaceAndTimestamp: mockBatchDeleteByWorkspaceAndTimestamp,
chunkArray: (items: string[], size: number) => {
const chunks: string[][] = []
for (let index = 0; index < items.length; index += size) {
chunks.push(items.slice(index, index + size))
}
return chunks
},
deleteRowsById: mockDeleteRowsById,
selectRowsByIdChunks: mockSelectRowsByIdChunks,
}))
vi.mock('@/lib/cleanup/chat-cleanup', () => ({ prepareChatCleanup: mockPrepareChatCleanup }))
vi.mock('@/lib/uploads', () => ({
isUsingCloudStorage: mockIsUsingCloudStorage,
StorageService: { deleteFiles: mockDeleteFiles },
}))
vi.mock('@/lib/uploads/server/metadata', () => ({ deleteFileMetadata: mockDeleteFileMetadata }))
import { runCleanupSoftDeletes } from '@/background/cleanup-soft-deletes'
const basePayload = {
label: 'free/1',
plan: 'free' as const,
retentionHours: 720,
workspaceIds: ['ws-1'],
}
describe('cleanup soft deletes — orphan KB binding sweep', () => {
beforeEach(() => {
vi.clearAllMocks()
mockIsUsingCloudStorage.mockReturnValue(true)
mockLimit.mockResolvedValue([])
})
it('soft-deletes abandoned KB bindings and removes their storage objects', async () => {
mockLimit
.mockResolvedValueOnce([{ key: 'kb/orphan-1' }, { key: 'kb/orphan-2' }])
.mockResolvedValueOnce([])
await runCleanupSoftDeletes(basePayload)
expect(mockDeleteFiles).toHaveBeenCalledWith(['kb/orphan-1', 'kb/orphan-2'], 'knowledge-base')
expect(mockDeleteFileMetadata).toHaveBeenCalledWith('kb/orphan-1')
expect(mockDeleteFileMetadata).toHaveBeenCalledWith('kb/orphan-2')
expect(mockDeleteFileMetadata).toHaveBeenCalledTimes(2)
})
it('still removes bindings but skips object deletion without cloud storage', async () => {
mockIsUsingCloudStorage.mockReturnValue(false)
mockLimit.mockResolvedValueOnce([{ key: 'kb/orphan-1' }]).mockResolvedValueOnce([])
await runCleanupSoftDeletes(basePayload)
expect(mockDeleteFiles).not.toHaveBeenCalled()
expect(mockDeleteFileMetadata).toHaveBeenCalledWith('kb/orphan-1')
})
it('stops the batch loop when binding deletion makes no progress', async () => {
mockLimit.mockResolvedValue([{ key: 'kb/stuck' }])
mockDeleteFileMetadata.mockRejectedValue(new Error('db down'))
await runCleanupSoftDeletes(basePayload)
// One batch attempted, then the no-progress guard breaks the loop.
expect(mockDeleteFileMetadata).toHaveBeenCalledTimes(1)
})
it('does not run the sweep when there are no workspaces', async () => {
await runCleanupSoftDeletes({ ...basePayload, workspaceIds: [] })
expect(mockSelect).not.toHaveBeenCalled()
expect(mockDeleteFiles).not.toHaveBeenCalled()
expect(mockDeleteFileMetadata).not.toHaveBeenCalled()
})
})
apps/sim/background/cleanup-soft-deletes.ts
+98 -2
View File
@@ -2,6 +2,7 @@ import { db } from '@sim/db'
import {
a2aAgent,
copilotChats,
document,
knowledgeBase,
mcpServers,
memory,
@@ -14,19 +15,31 @@ import {
} from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { task } from '@trigger.dev/sdk'
import { and, inArray, isNotNull, lt } from 'drizzle-orm'
import { and, asc, eq, inArray, isNotNull, isNull, lt, sql } from 'drizzle-orm'
import type { CleanupJobPayload } from '@/lib/billing/cleanup-dispatcher'
import {
batchDeleteByWorkspaceAndTimestamp,
chunkArray,
deleteRowsById,
selectRowsByIdChunks,
} from '@/lib/cleanup/batch-delete'
import { prepareChatCleanup } from '@/lib/cleanup/chat-cleanup'
import type { StorageContext } from '@/lib/uploads'
import { isUsingCloudStorage, StorageService } from '@/lib/uploads'
import { deleteFileMetadata } from '@/lib/uploads/server/metadata'
const logger = createLogger('CleanupSoftDeletes')
const KB_ORPHAN_BINDING_BATCH_SIZE = 500
const KB_ORPHAN_BINDING_TOTAL_LIMIT = 5_000
/**
* Grace window before an unreferenced KB binding is swept. Comfortably longer
* than any presign → upload → document-insert flow, so an in-flight upload is
* never mistaken for an abandoned one.
*/
const KB_ORPHAN_BINDING_GRACE_HOURS = 7 * 24
const KB_ORPHAN_BINDING_WORKSPACE_CHUNK = 50
interface WorkspaceFileScope {
/** Rows from `workspace_file` (singular, legacy workspace-context only). */
legacyRows: Array<{ id: string; key: string }>
@@ -162,6 +175,87 @@ const CLEANUP_TARGETS = [
},
] as const
/**
* Sweep abandoned knowledge-base ownership bindings. The presigned upload flow
* writes a `workspace_files` binding when it hands out an upload URL, before the
* object is stored and before any document is created. If the upload is never
* completed, that binding is orphaned — no `document.storageKey` ever references
* its key. Such bindings are inert (read access requires a live document, and
* the move re-point only follows referenced keys), but they accumulate, so we
* drop the best-effort object and soft-delete the binding once they are older
* than the grace window.
*/
async function cleanupOrphanedKnowledgeBaseBindings(
workspaceIds: string[],
label: string
): Promise<{ total: number; deleted: number; failed: number }> {
const stats = { total: 0, deleted: 0, failed: 0 }
if (workspaceIds.length === 0) return stats
const orphanCutoff = new Date(Date.now() - KB_ORPHAN_BINDING_GRACE_HOURS * 60 * 60 * 1000)
for (const chunkIds of chunkArray(workspaceIds, KB_ORPHAN_BINDING_WORKSPACE_CHUNK)) {
let attempted = 0
while (attempted < KB_ORPHAN_BINDING_TOTAL_LIMIT) {
const limit = Math.min(
KB_ORPHAN_BINDING_BATCH_SIZE,
KB_ORPHAN_BINDING_TOTAL_LIMIT - attempted
)
const rows = await db
.select({ key: workspaceFiles.key })
.from(workspaceFiles)
.where(
and(
inArray(workspaceFiles.workspaceId, chunkIds),
eq(workspaceFiles.context, 'knowledge-base'),
isNull(workspaceFiles.deletedAt),
lt(workspaceFiles.uploadedAt, orphanCutoff),
sql`NOT EXISTS (
SELECT 1 FROM ${document} AS doc
WHERE doc.storage_key = ${workspaceFiles.key}
)`
)
)
.orderBy(asc(workspaceFiles.uploadedAt), asc(workspaceFiles.key))
.limit(limit)
if (rows.length === 0) break
const keys = rows.map((row) => row.key)
stats.total += keys.length
attempted += keys.length
if (isUsingCloudStorage()) {
const result = await StorageService.deleteFiles(keys, 'knowledge-base')
stats.failed += result.failed.length
for (const { key, error } of result.failed) {
logger.error(`[${label}] Failed to delete orphan KB object ${key}:`, { error })
}
}
let deletedThisBatch = 0
for (const key of keys) {
try {
await deleteFileMetadata(key)
deletedThisBatch++
} catch (error) {
stats.failed++
logger.error(`[${label}] Failed to delete orphan KB binding ${key}:`, { error })
}
}
stats.deleted += deletedThisBatch
// No progress (every delete failed) — stop rather than reselect the same rows.
if (deletedThisBatch === 0) break
}
}
logger.info(
`[${label}/kb_orphan_bindings] Complete: ${stats.deleted}/${stats.total} bindings cleaned, ${stats.failed} failed`
)
return stats
}
export async function runCleanupSoftDeletes(payload: CleanupJobPayload): Promise<void> {
const startTime = Date.now()
const { workspaceIds, retentionHours, label } = payload
@@ -256,8 +350,10 @@ export async function runCleanupSoftDeletes(payload: CleanupJobPayload): Promise
totalDeleted += result.deleted
}
const orphanBindingStats = await cleanupOrphanedKnowledgeBaseBindings(workspaceIds, label)
logger.info(
`[${label}] Complete: ${totalDeleted} rows deleted, ${fileStats.filesDeleted} files cleaned`
`[${label}] Complete: ${totalDeleted} rows deleted, ${fileStats.filesDeleted} files cleaned, ${orphanBindingStats.deleted} orphan KB bindings cleaned`
)
// Clean up copilot backend + chat storage files after DB rows are gone
apps/sim/lib/knowledge/connectors/sync-engine.ts
+37 -2
View File
@@ -21,6 +21,7 @@ import {
} from '@/lib/knowledge/documents/service'
import { StorageService } from '@/lib/uploads'
import { deleteFile } from '@/lib/uploads/core/storage-service'
import { deleteFileMetadata } from '@/lib/uploads/server/metadata'
import { extractStorageKey } from '@/lib/uploads/utils/file-utils'
import { refreshAccessTokenIfNeeded } from '@/app/api/auth/oauth/utils'
import { knowledgeConnectorSync } from '@/background/knowledge-connector-sync'
@@ -314,7 +315,7 @@ export async function executeSync(
}
const kbRows = await db
.select({ userId: knowledgeBase.userId })
.select({ userId: knowledgeBase.userId, workspaceId: knowledgeBase.workspaceId })
.from(knowledgeBase)
.where(and(eq(knowledgeBase.id, connector.knowledgeBaseId), isNull(knowledgeBase.deletedAt)))
.limit(1)
@@ -336,6 +337,9 @@ export async function executeSync(
}
const userId = kbRows[0].userId
// Resolved once per sync and threaded into add/updateDocument so every synced
// kb/ object records a trusted ownership binding without an N+1 KB lookup.
const kbOwner: KnowledgeBaseOwner = { workspaceId: kbRows[0].workspaceId, userId }
const sourceConfig = connector.sourceConfig as Record<string, unknown>
const lockResult = await db
@@ -584,6 +588,7 @@ export async function executeSync(
connectorId,
connector.connectorType,
op.extDoc,
kbOwner,
sourceConfig
)
}
@@ -593,6 +598,7 @@ export async function executeSync(
connectorId,
connector.connectorType,
op.extDoc,
kbOwner,
sourceConfig
)
})
@@ -880,6 +886,26 @@ export async function executeSync(
}
}
/** Owning workspace + user for a knowledge base, resolved once per sync. */
interface KnowledgeBaseOwner {
workspaceId: string | null
userId: string
}
/**
* Build the storage `metadata` that records a trusted ownership binding for a
* synced `kb/` object. Returns `undefined` for legacy null-workspace KBs (no
* workspace-scoped ownership to bind), which `uploadFile` treats as "no binding".
*/
function kbOwnershipMetadata(
kbOwner: KnowledgeBaseOwner,
originalName: string
): { workspaceId: string; userId: string; originalName: string } | undefined {
return kbOwner.workspaceId
? { workspaceId: kbOwner.workspaceId, userId: kbOwner.userId, originalName }
: undefined
}
/**
* Upload content to storage as a .txt file, create a document record,
* and trigger processing via the existing pipeline.
@@ -889,6 +915,7 @@ async function addDocument(
connectorId: string,
connectorType: string,
extDoc: ExternalDocument,
kbOwner: KnowledgeBaseOwner,
sourceConfig?: Record<string, unknown>
): Promise<DocumentData> {
const documentId = generateId()
@@ -903,6 +930,7 @@ async function addDocument(
context: 'knowledge-base',
customKey,
preserveKey: true,
metadata: kbOwnershipMetadata(kbOwner, `${safeTitle}.txt`),
})
const fileUrl = `${getInternalApiBaseUrl()}${fileInfo.path}?context=knowledge-base`
@@ -925,6 +953,7 @@ async function addDocument(
knowledgeBaseId,
filename: extDoc.title,
fileUrl,
storageKey: fileInfo.key,
fileSize: contentBuffer.length,
mimeType: 'text/plain',
chunkCount: 0,
@@ -945,6 +974,7 @@ async function addDocument(
const storageKey = extractStorageKey(urlPath)
if (storageKey && storageKey !== urlPath) {
await deleteFile({ key: storageKey, context: 'knowledge-base' }).catch(() => undefined)
await deleteFileMetadata(storageKey).catch(() => undefined)
}
throw error
}
@@ -968,6 +998,7 @@ async function updateDocument(
connectorId: string,
connectorType: string,
extDoc: ExternalDocument,
kbOwner: KnowledgeBaseOwner,
sourceConfig?: Record<string, unknown>
): Promise<DocumentData> {
// Fetch old file URL before uploading replacement
@@ -989,6 +1020,7 @@ async function updateDocument(
context: 'knowledge-base',
customKey,
preserveKey: true,
metadata: kbOwnershipMetadata(kbOwner, `${safeTitle}.txt`),
})
const fileUrl = `${getInternalApiBaseUrl()}${fileInfo.path}?context=knowledge-base`
@@ -1011,6 +1043,7 @@ async function updateDocument(
.set({
filename: extDoc.title,
fileUrl,
storageKey: fileInfo.key,
fileSize: contentBuffer.length,
contentHash: extDoc.contentHash,
sourceUrl: extDoc.sourceUrl ?? null,
@@ -1037,17 +1070,19 @@ async function updateDocument(
const storageKey = extractStorageKey(urlPath)
if (storageKey && storageKey !== urlPath) {
await deleteFile({ key: storageKey, context: 'knowledge-base' }).catch(() => undefined)
await deleteFileMetadata(storageKey).catch(() => undefined)
}
throw error
}
// Clean up old storage file
// Clean up old storage file and its ownership binding
if (oldFileUrl) {
try {
const urlPath = new URL(oldFileUrl, 'http://localhost').pathname
const storageKey = extractStorageKey(urlPath)
if (storageKey && storageKey !== urlPath) {
await deleteFile({ key: storageKey, context: 'knowledge-base' })
await deleteFileMetadata(storageKey)
}
} catch (error) {
logger.warn('Failed to delete old storage file', {
apps/sim/lib/knowledge/documents/document-processor.ts
+4 -8
View File
@@ -654,20 +654,16 @@ async function processChunk(
const safeFileName = filename.replace(/[^a-zA-Z0-9.-]/g, '_')
const chunkKey = `kb/${timestamp}-${uniqueId}-chunk${chunkIndex + 1}-${safeFileName}`
const metadata: Record<string, string> = {
originalName: `${filename}_chunk${chunkIndex + 1}`,
uploadedAt: new Date().toISOString(),
purpose: 'knowledge-base',
...(userId && { userId }),
}
// No metadata: these chunks are ephemeral OCR artifacts (deleted in the
// finally below) that are fetched via a direct presigned URL, never through
// verifyKBFileAccess. Omitting metadata avoids writing an orphan ownership
// binding row per chunk.
const uploadResult = await StorageService.uploadFile({
file: chunk.buffer,
fileName: `${filename}_chunk${chunkIndex + 1}`,
contentType: 'application/pdf',
context: 'knowledge-base',
customKey: chunkKey,
metadata,
})
uploadedKey = uploadResult.key
apps/sim/lib/knowledge/documents/service.ts
+164 -5
View File
@@ -47,6 +47,7 @@ import {
import type { ProcessedDocumentTags } from '@/lib/knowledge/types'
import { estimateTokenCount } from '@/lib/tokenization/estimators'
import { deleteFile } from '@/lib/uploads/core/storage-service'
import { deleteFileMetadata, getFileMetadataByKeys } from '@/lib/uploads/server/metadata'
import { extractStorageKey } from '@/lib/uploads/utils/file-utils'
import type {
DocumentProcessingPayload,
@@ -56,6 +57,90 @@ import { calculateCost } from '@/providers/utils'
const logger = createLogger('DocumentService')
/**
* Thrown when a knowledge-base document's `fileUrl` references an internal `kb/`
* storage object that is not owned by the target knowledge base's workspace.
* Routes map this to a 403.
*/
export class KnowledgeBaseFileOwnershipError extends Error {
constructor(public readonly storageKey: string) {
super('Document file is not owned by this knowledge base')
this.name = 'KnowledgeBaseFileOwnershipError'
}
}
/**
* Guard document `fileUrl`s at creation time. When a URL points at an internal
* `kb/` storage object, require that the target knowledge base owns the object,
* resolved from the trusted `workspace_files` binding:
*
* - Workspace KB (`kbWorkspaceId` set): the binding's `workspaceId` must match.
* - Personal KB (`kbWorkspaceId` null): the binding's `userId` must be the KB
* owner. A key bound to another tenant is rejected; an unbound key (legacy /
* never reserved) passes since it carries no cross-tenant ownership.
*
* External `http(s)`/`data:` URLs (ingestion sources) and non-`kb/` internal keys
* pass through unchanged. This blocks a user from asserting ownership of another
* tenant's `kb/` key via a planted `fileUrl` — including in a personal KB, which
* otherwise could be moved into a workspace to launder the binding. All
* referenced bindings are resolved in one query (no N+1 inside the `FOR UPDATE`
* window). Single-document callers pass a one-element array.
*/
async function assertKnowledgeBaseFileUrlsOwnership(
fileUrls: string[],
kbWorkspaceId: string | null,
kbUserId: string,
requestId: string,
executor: DbExecutor = db
): Promise<void> {
const keys = [
...new Set(
fileUrls
.map((url) => getKnowledgeBaseStorageKey(url))
.filter((key): key is string => key !== null && key.startsWith('kb/'))
),
]
if (keys.length === 0) {
return
}
// Read bindings on the caller's transaction so the security check shares the
// same connection/lock context as the FOR UPDATE'd insert that follows.
const bindings = await getFileMetadataByKeys(keys, 'knowledge-base', executor)
const bindingByKey = new Map(bindings.map((binding) => [binding.key, binding]))
for (const key of keys) {
const binding = bindingByKey.get(key)
if (kbWorkspaceId) {
if (!binding || binding.workspaceId !== kbWorkspaceId) {
logger.warn(`[${requestId}] Rejected document referencing unowned knowledge-base file`, {
storageKey: key,
kbWorkspaceId,
bindingWorkspaceId: binding?.workspaceId ?? null,
})
throw new KnowledgeBaseFileOwnershipError(key)
}
continue
}
// Personal KB: reject a key whose binding belongs to a different user. An
// unbound key carries no ownership and is allowed (legacy personal files).
if (binding && binding.userId !== kbUserId) {
logger.warn(
`[${requestId}] Rejected personal-KB document referencing another tenant's file`,
{
storageKey: key,
kbUserId,
bindingUserId: binding.userId,
bindingWorkspaceId: binding.workspaceId ?? null,
}
)
throw new KnowledgeBaseFileOwnershipError(key)
}
}
}
const TIMEOUTS = {
OVERALL_PROCESSING: envNumber(env.KB_CONFIG_MAX_DURATION, 600) * 1000,
} as const
@@ -768,7 +853,11 @@ export async function createDocumentRecords(
await tx.execute(sql`SELECT 1 FROM knowledge_base WHERE id = ${knowledgeBaseId} FOR UPDATE`)
const kb = await tx
.select({ id: knowledgeBase.id })
.select({
id: knowledgeBase.id,
workspaceId: knowledgeBase.workspaceId,
userId: knowledgeBase.userId,
})
.from(knowledgeBase)
.where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt)))
.limit(1)
@@ -777,6 +866,15 @@ export async function createDocumentRecords(
throw new Error('Knowledge base not found')
}
const kbWorkspaceId = kb[0].workspaceId
await assertKnowledgeBaseFileUrlsOwnership(
documents.map((docData) => docData.fileUrl),
kbWorkspaceId,
kb[0].userId,
requestId,
tx
)
// One load per batch (was N+1); skip entirely if no doc carries tags.
const hasTaggedDocs = documents.some((d) => d.documentTagsData)
const tagDefinitions = hasTaggedDocs
@@ -812,6 +910,7 @@ export async function createDocumentRecords(
knowledgeBaseId,
filename: docData.filename,
fileUrl: docData.fileUrl,
storageKey: getKnowledgeBaseStorageKey(docData.fileUrl),
fileSize: docData.fileSize,
mimeType: docData.mimeType,
chunkCount: 0,
@@ -1307,6 +1406,7 @@ export async function createSingleDocument(
knowledgeBaseId,
filename: documentData.filename,
fileUrl: documentData.fileUrl,
storageKey: getKnowledgeBaseStorageKey(documentData.fileUrl),
fileSize: documentData.fileSize,
mimeType: documentData.mimeType,
chunkCount: 0,
@@ -1321,7 +1421,11 @@ export async function createSingleDocument(
await tx.execute(sql`SELECT 1 FROM knowledge_base WHERE id = ${knowledgeBaseId} FOR UPDATE`)
const kb = await tx
.select({ id: knowledgeBase.id })
.select({
id: knowledgeBase.id,
workspaceId: knowledgeBase.workspaceId,
userId: knowledgeBase.userId,
})
.from(knowledgeBase)
.where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt)))
.limit(1)
@@ -1330,6 +1434,14 @@ export async function createSingleDocument(
throw new Error('Knowledge base not found')
}
await assertKnowledgeBaseFileUrlsOwnership(
[documentData.fileUrl],
kb[0].workspaceId,
kb[0].userId,
requestId,
tx
)
await tx.insert(document).values(newDocument)
await tx
@@ -1848,18 +1960,63 @@ function getKnowledgeBaseStorageKey(fileUrl: string | null): string | null {
}
export async function deleteDocumentStorageFiles(
documentsToDelete: Array<{ id: string; fileUrl: string | null }>,
documentsToDelete: Array<{ id: string; fileUrl: string | null; workspaceId?: string | null }>,
requestId: string
): Promise<void> {
const entries = documentsToDelete.map((doc) => ({
doc,
storageKey: getKnowledgeBaseStorageKey(doc.fileUrl),
}))
// Resolve all kb/ ownership bindings in one query (avoids an N+1 across the
// delete fan-out below).
const kbKeys = [
...new Set(
entries
.map((entry) => entry.storageKey)
.filter((key): key is string => key !== null && key.startsWith('kb/'))
),
]
const ownerByKey = new Map<string, string | null>()
if (kbKeys.length > 0) {
const bindings = await getFileMetadataByKeys(kbKeys, 'knowledge-base')
for (const binding of bindings) {
ownerByKey.set(binding.key, binding.workspaceId)
}
}
await Promise.allSettled(
documentsToDelete.map(async (doc) => {
const storageKey = getKnowledgeBaseStorageKey(doc.fileUrl)
entries.map(async ({ doc, storageKey }) => {
if (!storageKey) {
return
}
// Only delete a kb/ object when its trusted ownership binding confirms the
// deleting document's workspace owns it. Prevents deleting another tenant's
// object via a document with a planted fileUrl.
if (storageKey.startsWith('kb/')) {
const bindingWorkspaceId = ownerByKey.get(storageKey)
if (!bindingWorkspaceId) {
logger.warn(`[${requestId}] Skipping storage delete: no ownership binding for key`, {
documentId: doc.id,
storageKey,
})
return
}
if (!doc.workspaceId || bindingWorkspaceId !== doc.workspaceId) {
logger.warn(`[${requestId}] Skipping storage delete: ownership binding mismatch`, {
documentId: doc.id,
storageKey,
bindingWorkspaceId,
documentWorkspaceId: doc.workspaceId ?? null,
})
return
}
}
try {
await deleteFile({ key: storageKey, context: 'knowledge-base' })
await deleteFileMetadata(storageKey)
} catch (error) {
logger.warn(`[${requestId}] Failed to delete document storage file`, {
documentId: doc.id,
@@ -1938,8 +2095,10 @@ export async function hardDeleteDocuments(
.select({
id: document.id,
fileUrl: document.fileUrl,
workspaceId: knowledgeBase.workspaceId,
})
.from(document)
.innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id))
.where(inArray(document.id, ids))
if (documentsToDelete.length === 0) {
apps/sim/lib/knowledge/service.test.ts
+82
View File
@@ -112,3 +112,85 @@ describe('updateKnowledgeBase — workspace transfer authorization', () => {
expect(dbChainMockFns.for).toHaveBeenCalledWith('update')
})
})
/**
* These tests guard the file-authorization follow-through: KB file ownership is
* resolved from the trusted `workspace_files` binding, so when a KB moves to a
* new workspace the bindings for its stored files must move with it. Otherwise
* the bindings stay frozen at the upload-time workspace and the KB's files
* become unreadable after a move.
*/
describe('updateKnowledgeBase — file ownership binding re-point on workspace change', () => {
beforeEach(() => {
vi.clearAllMocks()
dbChainMockFns.limit.mockReset()
resetDbChainMock()
})
// The mocked `@sim/db` cannot satisfy the post-transaction read-back select, so
// the call rejects after the transaction body commits. These tests assert the
// in-transaction binding statements, then swallow that read-back rejection.
const runIgnoringReadBack = (promise: Promise<unknown>) => promise.catch(() => undefined)
it('re-points file ownership bindings to the new workspace on a move', async () => {
dbChainMockFns.limit.mockResolvedValueOnce([{ workspaceId: 'ws-current', userId: 'u-1' }])
permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValueOnce('admin')
await runIgnoringReadBack(
updateKnowledgeBase('kb-1', { workspaceId: 'ws-target' }, 'req-1', { actorUserId: 'u-1' })
)
// Two updates inside the txn: the KB row, then the file bindings.
expect(dbChainMockFns.update).toHaveBeenCalledTimes(2)
expect(dbChainMockFns.set).toHaveBeenCalledWith({ workspaceId: 'ws-target' })
})
it('clears file ownership bindings when the KB is removed from its workspace', async () => {
dbChainMockFns.limit.mockResolvedValueOnce([{ workspaceId: 'ws-current', userId: 'owner' }])
await runIgnoringReadBack(
updateKnowledgeBase('kb-1', { workspaceId: null }, 'req-1', { actorUserId: 'owner' })
)
expect(dbChainMockFns.update).toHaveBeenCalledTimes(2)
expect(dbChainMockFns.set).toHaveBeenCalledWith({ workspaceId: null })
})
it('does not re-point bindings when promoting a personal (null-workspace) KB into a workspace', async () => {
// A null current workspace owns no bindings, so the move must not rewrite
// any binding — this prevents a key planted in a personal KB from being
// laundered into the destination workspace on move.
dbChainMockFns.limit.mockResolvedValueOnce([{ workspaceId: null, userId: 'owner' }])
permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValueOnce('admin')
await runIgnoringReadBack(
updateKnowledgeBase('kb-1', { workspaceId: 'ws-target' }, 'req-1', { actorUserId: 'owner' })
)
// Only the KB row is updated; the binding re-point is skipped entirely.
expect(dbChainMockFns.update).toHaveBeenCalledTimes(1)
expect(dbChainMockFns.set).not.toHaveBeenCalledWith({ workspaceId: 'ws-target' })
})
it('does not touch bindings when the workspace is unchanged', async () => {
dbChainMockFns.limit.mockResolvedValueOnce([{ workspaceId: 'ws-current', userId: 'u-1' }])
await runIgnoringReadBack(
updateKnowledgeBase('kb-1', { workspaceId: 'ws-current' }, 'req-1', { actorUserId: 'u-1' })
)
// Only the KB row is updated; no binding re-point statement runs.
expect(dbChainMockFns.update).toHaveBeenCalledTimes(1)
expect(dbChainMockFns.set).not.toHaveBeenCalledWith({ workspaceId: 'ws-current' })
})
it('does not touch bindings when no workspace change is requested', async () => {
dbChainMockFns.limit
.mockResolvedValueOnce([{ workspaceId: 'ws-current', userId: 'u-1' }]) // currentKb lock
.mockResolvedValueOnce([]) // duplicate-name check: none
await runIgnoringReadBack(updateKnowledgeBase('kb-1', { name: 'Renamed' }, 'req-1'))
expect(dbChainMockFns.update).toHaveBeenCalledTimes(1)
})
})
apps/sim/lib/knowledge/service.ts
+47 -2
View File
@@ -1,9 +1,16 @@
import { db } from '@sim/db'
import { document, knowledgeBase, knowledgeConnector, permissions, workspace } from '@sim/db/schema'
import {
document,
knowledgeBase,
knowledgeConnector,
permissions,
workspace,
workspaceFiles,
} from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { getPostgresErrorCode } from '@sim/utils/errors'
import { generateId } from '@sim/utils/id'
import { and, count, eq, inArray, isNotNull, isNull, ne, or, sql } from 'drizzle-orm'
import { and, count, eq, exists, inArray, isNotNull, isNull, ne, or, sql } from 'drizzle-orm'
import { generateRestoreName } from '@/lib/core/utils/restore-name'
import type {
ChunkingConfig,
@@ -333,6 +340,44 @@ export async function updateKnowledgeBase(
.update(knowledgeBase)
.set(updateData)
.where(and(eq(knowledgeBase.id, knowledgeBaseId), isNull(knowledgeBase.deletedAt)))
// When a KB changes workspace, re-point the ownership bindings for its
// stored files so file authorization (which resolves the owning workspace
// from the trusted binding, not from document.fileUrl) follows the KB to
// its new workspace. Only bindings the KB's *current* workspace already
// owns are moved: this scopes the update to this KB's own files and
// prevents a document referencing another tenant's key (e.g. one planted
// while the KB had no workspace) from hijacking that key's binding on
// move. A null current workspace owns no bindings, so nothing is moved.
if (updates.workspaceId !== undefined) {
const currentWorkspaceId = currentKb.workspaceId ?? null
const targetWorkspaceId = updates.workspaceId ?? null
if (currentWorkspaceId && targetWorkspaceId !== currentWorkspaceId) {
await tx
.update(workspaceFiles)
.set({ workspaceId: targetWorkspaceId })
.where(
and(
eq(workspaceFiles.context, 'knowledge-base'),
eq(workspaceFiles.workspaceId, currentWorkspaceId),
isNull(workspaceFiles.deletedAt),
exists(
tx
.select({ one: sql`1` })
.from(document)
.where(
and(
eq(document.knowledgeBaseId, knowledgeBaseId),
isNotNull(document.storageKey),
eq(document.storageKey, workspaceFiles.key)
)
)
)
)
)
}
}
})
} catch (error: unknown) {
if (getPostgresErrorCode(error) === '23505' && updates.name !== undefined) {
apps/sim/lib/uploads/core/upload-token.ts
+9
View File
@@ -9,6 +9,12 @@ export interface UploadTokenPayload {
userId: string
workspaceId: string
context: StorageContext
/** Original file name, carried so the completion handler can record ownership metadata. */
fileName?: string
/** File MIME type, carried for ownership metadata at completion. */
contentType?: string
/** File size in bytes, carried for ownership metadata at completion. */
fileSize?: number
}
interface SignedPayload extends UploadTokenPayload {
@@ -82,6 +88,9 @@ export function verifyUploadToken(token: string): UploadTokenVerification {
userId: parsed.userId,
workspaceId: parsed.workspaceId,
context: parsed.context as StorageContext,
...(typeof parsed.fileName === 'string' ? { fileName: parsed.fileName } : {}),
...(typeof parsed.contentType === 'string' ? { contentType: parsed.contentType } : {}),
...(typeof parsed.fileSize === 'number' ? { fileSize: parsed.fileSize } : {}),
},
}
}
apps/sim/lib/uploads/server/metadata.ts
+66 -1
View File
@@ -2,7 +2,7 @@ import { db } from '@sim/db'
import { workspaceFiles } from '@sim/db/schema'
import { createLogger } from '@sim/logger'
import { generateId } from '@sim/utils/id'
import { and, eq, isNull } from 'drizzle-orm'
import { and, eq, inArray, isNull, sql } from 'drizzle-orm'
import type { StorageContext } from '../shared/types'
const logger = createLogger('FileMetadata')
@@ -118,6 +118,44 @@ export async function insertFileMetadata(
}
}
/**
* Bulk-insert file metadata rows in a single statement.
*
* Intended for batch upload flows that create many fresh keys at once (e.g. the
* presigned batch route), replacing a fan-out of individual `insertFileMetadata`
* calls. Uses `ON CONFLICT DO NOTHING` on the active-key unique index, so it is
* safe against a concurrent single insert and idempotent for already-present
* active keys. Unlike {@link insertFileMetadata} it does NOT restore
* soft-deleted rows — callers use this only for newly generated keys.
*/
export async function insertFileMetadataMany(
rows: Array<Omit<FileMetadataInsertOptions, 'id'> & { id?: string }>
): Promise<void> {
if (rows.length === 0) {
return
}
await db
.insert(workspaceFiles)
.values(
rows.map((row) => ({
id: row.id || generateId(),
key: row.key,
userId: row.userId,
workspaceId: row.workspaceId || null,
folderId: row.folderId ?? null,
context: row.context,
originalName: row.originalName,
displayName: row.originalName,
contentType: row.contentType,
size: row.size,
deletedAt: null,
uploadedAt: new Date(),
}))
)
.onConflictDoNothing()
}
/**
* Get file metadata by key with optional context filter
*/
@@ -141,11 +179,38 @@ export async function getFileMetadataByKey(
.select()
.from(workspaceFiles)
.where(conditions.length > 1 ? and(...conditions) : conditions[0])
// Prefer the active row when includeDeleted lets both an active and a
// soft-deleted row for the same key match.
.orderBy(sql`${workspaceFiles.deletedAt} IS NULL DESC`)
.limit(1)
return record ?? null
}
/**
* Get active (non-deleted) file metadata for multiple keys in a single query.
* Batches what would otherwise be N `getFileMetadataByKey` calls.
*/
export async function getFileMetadataByKeys(
keys: string[],
context: StorageContext,
executor: Pick<typeof db, 'select'> = db
): Promise<FileMetadataRecord[]> {
if (keys.length === 0) {
return []
}
return executor
.select()
.from(workspaceFiles)
.where(
and(
inArray(workspaceFiles.key, keys),
eq(workspaceFiles.context, context),
isNull(workspaceFiles.deletedAt)
)
)
}
/**
* Get file metadata by ID
*/
packages/db/migrations/0222_kb_document_storage_key.sql
+192
View File
@@ -0,0 +1,192 @@
-- ============================================================================
-- Knowledge-base file-access authorization: document.storage_key + backfill
-- ============================================================================
-- Adds document.storage_key (the canonical kb/ key KB file authorization matches
-- on at read time), then two data steps: (1) populate storage_key from file_url
-- for existing rows; (2) backfill trusted ownership bindings into workspace_files
-- (context 'knowledge-base') for pre-existing kb/ objects, derived from the
-- populated storage_key.
--
-- Lock profile (mirrors 0220_early_hellion.sql): the ADD COLUMN runs in the
-- migration transaction, then COMMIT releases the ACCESS EXCLUSIVE lock so the
-- row-by-row backfill runs in bounded batches (committing every 5000 rows) and
-- the index is built CONCURRENTLY — no full read/write lockout on `document`.
--
-- Ownership is derived conservatively (anti-poisoning):
-- * Only active documents (not excluded/archived/deleted, in a live KB) count.
-- * A key referenced by documents in more than one workspace is ambiguous and
-- skipped (deny-by-default).
-- * Within a single workspace the earliest uploader wins.
-- * Keys with an existing active binding are skipped (idempotent; ON CONFLICT
-- DO NOTHING guards the active-key unique index against a concurrent insert).
--
-- IMPORTANT — run these read-only pre-flight checks on the DB console and
-- remediate (exclude/delete) any rows BEFORE applying this migration. The
-- multi-workspace guard catches contested keys, but a pre-existing plant that is
-- the sole referencer of an orphan key can only be caught by check (2).
--
-- (1) Multi-workspace keys — same kb/ object referenced by documents in >1 workspace:
-- WITH kb_docs AS (
-- SELECT d.id AS document_id, kb.workspace_id,
-- regexp_replace(replace(replace(
-- substring(split_part(d.file_url, '?', 1) FROM '/api/files/serve/(.*)$'),
-- '%2F','/'),'%2f','/'),'^(s3|blob)/','') AS storage_key
-- FROM document d JOIN knowledge_base kb ON kb.id = d.knowledge_base_id
-- WHERE d.user_excluded = false AND d.archived_at IS NULL
-- AND d.deleted_at IS NULL AND kb.deleted_at IS NULL
-- )
-- SELECT * FROM kb_docs WHERE storage_key LIKE 'kb/%' AND storage_key IN (
-- SELECT storage_key FROM kb_docs
-- WHERE storage_key LIKE 'kb/%' AND workspace_id IS NOT NULL
-- GROUP BY storage_key HAVING COUNT(DISTINCT workspace_id) > 1)
-- ORDER BY storage_key;
--
-- (2) Cross-origin serve URLs — foreign host mimicking the serve path. Replace
-- 'your-app-host' with your deployment host(s):
-- SELECT d.id, kb.workspace_id, d.file_url,
-- substring(split_part(d.file_url, '?', 1) FROM '^https?://([^/]+)') AS url_host
-- FROM document d JOIN knowledge_base kb ON kb.id = d.knowledge_base_id
-- WHERE d.user_excluded = false AND d.archived_at IS NULL
-- AND d.deleted_at IS NULL AND kb.deleted_at IS NULL
-- AND split_part(d.file_url, '?', 1) ~ '/api/files/serve/(s3/|blob/)?kb(/|%2F)'
-- AND substring(split_part(d.file_url, '?', 1) FROM '^https?://([^/]+)')
-- NOT IN ('your-app-host')
-- ORDER BY url_host;
ALTER TABLE "document" ADD COLUMN IF NOT EXISTS "storage_key" text;--> statement-breakpoint
COMMIT;--> statement-breakpoint
-- (1) Populate storage_key from file_url for existing documents, in bounded
-- batches. Only internal serve URLs are touched; external/data: URLs keep NULL.
CREATE OR REPLACE PROCEDURE backfill_document_storage_key_0222() LANGUAGE plpgsql AS $$
DECLARE
updated integer;
BEGIN
LOOP
WITH candidates AS (
SELECT id FROM document
WHERE storage_key IS NULL
AND file_url LIKE '%/api/files/serve/%'
LIMIT 5000
)
UPDATE document d
SET storage_key = regexp_replace(
replace(
replace(
substring(split_part(d.file_url, '?', 1) FROM '/api/files/serve/(.*)$'),
'%2F', '/'
),
'%2f', '/'
),
'^(s3|blob)/',
''
)
FROM candidates c
WHERE d.id = c.id;
GET DIAGNOSTICS updated = ROW_COUNT;
EXIT WHEN updated = 0;
COMMIT;
END LOOP;
END;
$$;--> statement-breakpoint
CALL backfill_document_storage_key_0222();--> statement-breakpoint
DROP PROCEDURE backfill_document_storage_key_0222();--> statement-breakpoint
-- (1b) Fully URL-decode keys that carry encodings beyond the separator (e.g.
-- '%2C' -> ',', '%26' -> '&', multi-byte UTF-8 like '%C3%ADa' -> 'ía'). The app
-- derives the canonical key with decodeURIComponent (extractStorageKey), so a
-- storage_key that is only %2F-decoded would never match the runtime key for
-- these documents — breaking their liveness lookups and stranding their
-- ownership bindings. The bulk pass above already handled '%2F'; only the
-- remaining residual-'%' rows are re-derived here (a small subset), so the heavy
-- decode never runs over the whole table. This single UPDATE evaluates its target
-- set once, so a key that legitimately contains a literal '%' is not reprocessed.
CREATE OR REPLACE FUNCTION url_decode_0222(input text) RETURNS text LANGUAGE sql IMMUTABLE AS $$
SELECT CASE
WHEN input IS NULL THEN NULL
WHEN position('%' IN input) = 0 THEN input
ELSE convert_from(
(
SELECT string_agg(
CASE
WHEN r.m[1] ~ '^%[0-9A-Fa-f]{2}$' THEN decode(substring(r.m[1] FROM 2), 'hex')
ELSE convert_to(r.m[1], 'UTF8')
END,
''::bytea ORDER BY r.ord
)
FROM regexp_matches(input, '%[0-9A-Fa-f]{2}|.', 'g') WITH ORDINALITY AS r(m, ord)
),
'UTF8'
)
END
$$;--> statement-breakpoint
UPDATE "document"
SET "storage_key" = regexp_replace(
url_decode_0222(substring(split_part("file_url", '?', 1) FROM '/api/files/serve/(.*)$')),
'^(s3|blob)/',
''
)
WHERE "storage_key" LIKE 'kb/%'
AND position('%' IN "storage_key") > 0;--> statement-breakpoint
DROP FUNCTION url_decode_0222(text);--> statement-breakpoint
CREATE INDEX CONCURRENTLY IF NOT EXISTS "doc_storage_key_idx" ON "document" USING btree ("storage_key") WHERE "storage_key" IS NOT NULL;--> statement-breakpoint
-- (2) Backfill workspace_files ownership bindings from the populated storage_key.
WITH kb_docs AS (
SELECT
d."uploaded_at" AS uploaded_at,
kb."workspace_id" AS workspace_id,
kb."user_id" AS user_id,
d."storage_key" AS storage_key
FROM "document" d
INNER JOIN "knowledge_base" kb ON kb."id" = d."knowledge_base_id"
WHERE d."user_excluded" = false
AND d."archived_at" IS NULL
AND d."deleted_at" IS NULL
AND kb."deleted_at" IS NULL
AND kb."workspace_id" IS NOT NULL
AND d."storage_key" LIKE 'kb/%'
),
kb_keys AS (
SELECT storage_key
FROM kb_docs
GROUP BY storage_key
HAVING COUNT(DISTINCT workspace_id) = 1
),
owners AS (
SELECT DISTINCT ON (kb_docs.storage_key)
kb_docs.storage_key AS storage_key,
kb_docs.workspace_id AS workspace_id,
kb_docs.user_id AS user_id
FROM kb_docs
INNER JOIN kb_keys ON kb_keys.storage_key = kb_docs.storage_key
ORDER BY kb_docs.storage_key, kb_docs.uploaded_at ASC
)
INSERT INTO "workspace_files" (
"id",
"key",
"user_id",
"workspace_id",
"context",
"original_name",
"content_type",
"size"
)
SELECT
gen_random_uuid()::text,
owners.storage_key,
owners.user_id,
owners.workspace_id,
'knowledge-base',
regexp_replace(owners.storage_key, '^.*/', ''),
'application/octet-stream',
0
FROM owners
WHERE NOT EXISTS (
SELECT 1
FROM "workspace_files" wf
WHERE wf."key" = owners.storage_key
AND wf."deleted_at" IS NULL
)
ON CONFLICT DO NOTHING;
packages/db/migrations/meta/0222_snapshot.json
+17583
View File
File diff suppressed because it is too large Load Diff
packages/db/migrations/meta/_journal.json
+7
View File
@@ -1548,6 +1548,13 @@
"when": 1780111474238,
"tag": "0221_secret_hannibal_king",
"breakpoints": true
},
{
"idx": 222,
"version": "7",
"when": 1780338253662,
"tag": "0222_kb_document_storage_key",
"breakpoints": true
}
]
}
packages/db/schema.ts
+8
View File
@@ -1607,6 +1607,10 @@ export const document = pgTable(
// File information
filename: text('filename').notNull(),
fileUrl: text('file_url').notNull(),
// Canonical storage key derived from fileUrl at write time (e.g. 'kb/<...>'),
// or null for external/data: ingestion URLs. KB file authorization matches on
// this exact key rather than re-parsing the URL at read time.
storageKey: text('storage_key'),
fileSize: integer('file_size').notNull(), // Size in bytes
mimeType: text('mime_type').notNull(), // e.g., 'application/pdf', 'text/plain'
@@ -1677,6 +1681,10 @@ export const document = pgTable(
.where(sql`${table.deletedAt} IS NULL`),
// Sync engine: load all active docs for a connector
connectorIdIdx: index('doc_connector_id_idx').on(table.connectorId),
// KB file-access liveness: exact lookup by canonical storage key
storageKeyIdx: index('doc_storage_key_idx')
.on(table.storageKey)
.where(sql`${table.storageKey} IS NOT NULL`),
archivedAtPartialIdx: index('doc_archived_at_partial_idx')
.on(table.archivedAt)
.where(sql`${table.archivedAt} IS NOT NULL`),