mirror of
https://github.com/simstudioai/sim.git
synced 2026-10-31 03:47:12 +08:00
fix(uploads): manual file uploads must ignore provenance stamping (#6314)
* fix(attachments): model egress attachments * fix(uploads): manual uploads provenance ignore * further checks on kb * tags fixes * fix more stuff * fix tests * address comments * fix * fix * fix * address timestamp concern
This commit is contained in:
@@ -11,6 +11,7 @@ import {
|
||||
createOrUpdateTagDefinitionsBulk,
|
||||
deleteAllTagDefinitions,
|
||||
getDocumentTagDefinitions,
|
||||
KnowledgeTagProvenanceConflictError,
|
||||
} from '@/lib/knowledge/tags/service'
|
||||
import type { BulkTagDefinitionsData } from '@/lib/knowledge/tags/types'
|
||||
import { checkDocumentAccess, checkDocumentWriteAccess } from '@/app/api/knowledge/utils'
|
||||
@@ -198,6 +199,9 @@ export const DELETE = withRouteHandler(
|
||||
data: { deleted: deletedCount },
|
||||
})
|
||||
} catch (error) {
|
||||
if (error instanceof KnowledgeTagProvenanceConflictError) {
|
||||
return NextResponse.json({ error: error.message }, { status: 409 })
|
||||
}
|
||||
logger.error(`[${requestId}] Error with tag definitions operation`, error)
|
||||
return NextResponse.json({ error: 'Failed to process tag definitions' }, { status: 500 })
|
||||
}
|
||||
|
||||
@@ -5,7 +5,10 @@ import { deleteTagDefinitionContract } from '@/lib/api/contracts/knowledge'
|
||||
import { parseRequest } from '@/lib/api/server'
|
||||
import { checkSessionOrInternalAuth } from '@/lib/auth/hybrid'
|
||||
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
|
||||
import { deleteTagDefinition } from '@/lib/knowledge/tags/service'
|
||||
import {
|
||||
deleteTagDefinition,
|
||||
KnowledgeTagProvenanceConflictError,
|
||||
} from '@/lib/knowledge/tags/service'
|
||||
import { checkKnowledgeBaseWriteAccess } from '@/app/api/knowledge/utils'
|
||||
|
||||
export const dynamic = 'force-dynamic'
|
||||
@@ -45,6 +48,9 @@ export const DELETE = withRouteHandler(
|
||||
message: `Tag definition "${deletedTag.displayName}" deleted successfully`,
|
||||
})
|
||||
} catch (error) {
|
||||
if (error instanceof KnowledgeTagProvenanceConflictError) {
|
||||
return NextResponse.json({ error: error.message }, { status: 409 })
|
||||
}
|
||||
logger.error(`[${requestId}] Error deleting tag definition`, error)
|
||||
return NextResponse.json({ error: 'Failed to delete tag definition' }, { status: 500 })
|
||||
}
|
||||
|
||||
@@ -10,17 +10,158 @@ import {
|
||||
PRIVATE_SECRET_PROVENANCE_FIELD,
|
||||
PRIVATE_SECRET_PROVENANCE_HEADER,
|
||||
} from '@/lib/execution/private-tool-metadata'
|
||||
import { resolveKnowledgeWriteSecretProvenance } from '@/app/api/knowledge/secret-provenance'
|
||||
import {
|
||||
resolveKnowledgeDocumentWriteSecretProvenance,
|
||||
resolveKnowledgeWriteSecretProvenance,
|
||||
} from '@/app/api/knowledge/secret-provenance'
|
||||
|
||||
function createRequest(payload: Record<string, unknown>): NextRequest {
|
||||
const PRIVATE_PROVENANCE_SCOPE = {
|
||||
userId: 'user-1',
|
||||
workspaceId: 'workspace-1',
|
||||
} as const
|
||||
|
||||
function createRequest(
|
||||
payload: Record<string, unknown>,
|
||||
provenanceHeader = PRIVATE_SECRET_PROVENANCE_BUNDLE_V1
|
||||
): NextRequest {
|
||||
return new NextRequest('http://localhost/api/knowledge/kb/documents', {
|
||||
method: 'POST',
|
||||
headers: { [PRIVATE_SECRET_PROVENANCE_HEADER]: provenanceHeader },
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
}
|
||||
|
||||
function createHeaderlessRequest(payload: Record<string, unknown>): NextRequest {
|
||||
return new NextRequest('http://localhost/api/knowledge/kb/documents', {
|
||||
method: 'POST',
|
||||
headers: { [PRIVATE_SECRET_PROVENANCE_HEADER]: PRIVATE_SECRET_PROVENANCE_BUNDLE_V1 },
|
||||
body: JSON.stringify(payload),
|
||||
})
|
||||
}
|
||||
|
||||
describe('knowledge write secret provenance', () => {
|
||||
it('classifies a headerless external chunk write as exact-empty', () => {
|
||||
const payload = { content: 'manual content' }
|
||||
|
||||
const result = resolveKnowledgeWriteSecretProvenance({
|
||||
request: createHeaderlessRequest(payload),
|
||||
payload,
|
||||
authType: AuthType.API_KEY,
|
||||
userId: 'user-1',
|
||||
workspaceId: 'workspace-1',
|
||||
selectionKeys: ['chunk-content'],
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
success: true,
|
||||
provenances: [{ status: 'exact', entries: [] }],
|
||||
})
|
||||
})
|
||||
|
||||
it('classifies a headerless external document write as exact-empty', () => {
|
||||
const payload = {
|
||||
filename: 'manual.txt',
|
||||
}
|
||||
|
||||
const result = resolveKnowledgeDocumentWriteSecretProvenance({
|
||||
request: createHeaderlessRequest(payload),
|
||||
payload,
|
||||
authType: AuthType.SESSION,
|
||||
userId: 'user-1',
|
||||
workspaceId: 'workspace-1',
|
||||
documents: [payload],
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
success: true,
|
||||
provenances: [
|
||||
{
|
||||
filename: { status: 'exact', entries: [] },
|
||||
content: { status: 'exact', entries: [] },
|
||||
tags: [],
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it('does not track durable provenance for a legacy headerless internal write', () => {
|
||||
const payload = { content: 'legacy workflow content' }
|
||||
|
||||
const result = resolveKnowledgeWriteSecretProvenance({
|
||||
request: createHeaderlessRequest(payload),
|
||||
payload,
|
||||
authType: AuthType.INTERNAL_JWT,
|
||||
userId: 'user-1',
|
||||
workspaceId: 'workspace-1',
|
||||
selectionKeys: ['chunk-content'],
|
||||
})
|
||||
|
||||
expect(result).toEqual({ success: true })
|
||||
})
|
||||
|
||||
it('tracks exact-empty provenance only when an internal write supplies a verified envelope', () => {
|
||||
const bundle = {
|
||||
version: 1 as const,
|
||||
complete: true,
|
||||
selections: [
|
||||
{
|
||||
key: 'chunk-content',
|
||||
provenance: {
|
||||
version: 1 as const,
|
||||
complete: true,
|
||||
entries: [],
|
||||
scope: PRIVATE_PROVENANCE_SCOPE,
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
const payload = { content: 'workflow content', [PRIVATE_SECRET_PROVENANCE_FIELD]: bundle }
|
||||
|
||||
const result = resolveKnowledgeWriteSecretProvenance({
|
||||
request: createRequest(payload),
|
||||
payload,
|
||||
authType: AuthType.INTERNAL_JWT,
|
||||
userId: 'user-1',
|
||||
workspaceId: 'workspace-1',
|
||||
selectionKeys: ['chunk-content'],
|
||||
})
|
||||
|
||||
expect(result).toEqual({
|
||||
success: true,
|
||||
provenances: [{ status: 'exact', entries: [] }],
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects a private provenance envelope from an external caller', () => {
|
||||
const bundle = {
|
||||
version: 1 as const,
|
||||
complete: true,
|
||||
selections: [
|
||||
{
|
||||
key: 'chunk-content',
|
||||
provenance: {
|
||||
version: 1 as const,
|
||||
complete: true,
|
||||
entries: [],
|
||||
scope: PRIVATE_PROVENANCE_SCOPE,
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
const payload = { content: 'external content', [PRIVATE_SECRET_PROVENANCE_FIELD]: bundle }
|
||||
|
||||
const result = resolveKnowledgeWriteSecretProvenance({
|
||||
request: createRequest(payload),
|
||||
payload,
|
||||
authType: AuthType.API_KEY,
|
||||
userId: 'user-1',
|
||||
workspaceId: 'workspace-1',
|
||||
selectionKeys: ['chunk-content'],
|
||||
})
|
||||
|
||||
expect(result.success).toBe(false)
|
||||
if (!result.success) expect(result.response.status).toBe(400)
|
||||
})
|
||||
|
||||
it('rejects an unavailable verified selection before a write can start', () => {
|
||||
const bundle = {
|
||||
version: 1 as const,
|
||||
@@ -28,7 +169,12 @@ describe('knowledge write secret provenance', () => {
|
||||
selections: [
|
||||
{
|
||||
key: 'document-source:0',
|
||||
provenance: { version: 1 as const, complete: false, entries: [] },
|
||||
provenance: {
|
||||
version: 1 as const,
|
||||
complete: false,
|
||||
entries: [],
|
||||
scope: PRIVATE_PROVENANCE_SCOPE,
|
||||
},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
vi,
|
||||
} from 'vitest'
|
||||
import type { AutoRoutingSignals } from '@/lib/model-router/resolve'
|
||||
import * as userFileBase64 from '@/lib/uploads/utils/user-file-base64.server'
|
||||
import { getAllBlocks } from '@/blocks'
|
||||
import { AGENT, BlockType, isMcpTool } from '@/executor/constants'
|
||||
import { AgentBlockHandler } from '@/executor/handlers/agent/agent-handler'
|
||||
@@ -488,6 +489,47 @@ describe('AgentBlockHandler', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('normalizes the persisted workspace-picker shape before provider execution', async () => {
|
||||
const key = 'workspace/ws-1/example.png'
|
||||
const hydrationSpy = vi
|
||||
.spyOn(userFileBase64, 'hydrateUserFilesWithBase64')
|
||||
.mockImplementationOnce(async (files) =>
|
||||
files.map((file) => ({ ...file, base64: 'aW1hZ2U=' }))
|
||||
)
|
||||
|
||||
try {
|
||||
mockGetProviderFromModel.mockReturnValue('openai')
|
||||
|
||||
await handler.execute(mockContext, mockBlock, {
|
||||
model: 'gpt-4o',
|
||||
userPrompt: 'Analyze this file',
|
||||
files: [
|
||||
{
|
||||
name: 'example.png',
|
||||
path: `/api/files/serve/${encodeURIComponent(key)}?context=workspace`,
|
||||
key,
|
||||
size: 128,
|
||||
type: 'image/png',
|
||||
},
|
||||
],
|
||||
apiKey: 'test-api-key',
|
||||
})
|
||||
|
||||
const normalizedFile = hydrationSpy.mock.calls[0][0][0]
|
||||
expect(normalizedFile).toMatchObject({
|
||||
id: expect.stringMatching(/^file-\d+$/),
|
||||
key,
|
||||
name: 'example.png',
|
||||
type: 'image/png',
|
||||
})
|
||||
expect(mockExecuteProviderRequest.mock.calls[0][1].messages.at(-1)?.files).toEqual([
|
||||
expect.objectContaining({ key, name: 'example.png', base64: 'aW1hZ2U=' }),
|
||||
])
|
||||
} finally {
|
||||
hydrationSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
it('should reject files for providers without attachment support', async () => {
|
||||
const inputs = {
|
||||
model: 'deepseek-chat',
|
||||
|
||||
@@ -496,14 +496,10 @@ describe('runCopilotLifecycle', () => {
|
||||
expect(JSON.parse(capturedRequestBody).fileAttachments).toEqual([safe])
|
||||
})
|
||||
|
||||
it('continues without attachments when durable provenance cannot be verified', async () => {
|
||||
it('rejects when durable attachment provenance cannot be verified', async () => {
|
||||
mockFilterModelSafeWorkspaceFileAttachments.mockRejectedValueOnce(new Error('db unavailable'))
|
||||
let capturedRequestBody = ''
|
||||
mockRunStreamLoop.mockImplementationOnce(async (_url: string, request: RequestInit) => {
|
||||
capturedRequestBody = String(request.body)
|
||||
})
|
||||
|
||||
await runCopilotLifecycle(
|
||||
const result = await runCopilotLifecycle(
|
||||
{
|
||||
message: 'Continue safely',
|
||||
fileAttachments: [{ id: 'wf-file', name: 'file.txt', key: 'workspace/ws-1/file.txt' }],
|
||||
@@ -517,7 +513,11 @@ describe('runCopilotLifecycle', () => {
|
||||
}
|
||||
)
|
||||
|
||||
expect(JSON.parse(capturedRequestBody)).not.toHaveProperty('fileAttachments')
|
||||
expect(result).toMatchObject({
|
||||
success: false,
|
||||
error: 'Copilot model input could not be safely projected',
|
||||
})
|
||||
expect(mockRunStreamLoop).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each(['123', 'true'])(
|
||||
|
||||
@@ -607,11 +607,11 @@ async function omitUnsafeInitialCopilotAttachments(
|
||||
try {
|
||||
safeAttachments = await filterModelSafeWorkspaceFileAttachments(attachments, { workspaceId })
|
||||
} catch (error) {
|
||||
logger.warn('Workspace file secret provenance could not be verified; omitting attachments', {
|
||||
logger.error('Workspace file secret provenance could not be verified', {
|
||||
attachmentCount: attachments.length,
|
||||
error: toError(error).message,
|
||||
})
|
||||
safeAttachments = []
|
||||
throw new CopilotModelContentProjectionError()
|
||||
}
|
||||
|
||||
if (safeAttachments.length === attachments.length) continue
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,231 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { dbChainMockFns, resetDbChainMock } from '@sim/testing'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const {
|
||||
mockCheckActorUsageLimits,
|
||||
mockGenerateEmbeddings,
|
||||
mockGetBoundWorkspaceFileSecretProvenanceByMetadata,
|
||||
mockGetFileMetadataByKeys,
|
||||
mockProcessDocument,
|
||||
} = vi.hoisted(() => ({
|
||||
mockCheckActorUsageLimits: vi.fn(),
|
||||
mockGenerateEmbeddings: vi.fn(),
|
||||
mockGetBoundWorkspaceFileSecretProvenanceByMetadata: vi.fn(),
|
||||
mockGetFileMetadataByKeys: vi.fn(),
|
||||
mockProcessDocument: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/billing/calculations/usage-monitor', () => ({
|
||||
checkActorUsageLimits: mockCheckActorUsageLimits,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/knowledge/documents/document-processor', () => ({
|
||||
processDocument: mockProcessDocument,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/knowledge/embedding-models', () => ({
|
||||
getEmbeddingModelInfo: vi.fn(() => ({ tokenizerProvider: 'openai' })),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/knowledge/embeddings', () => ({
|
||||
generateEmbeddings: mockGenerateEmbeddings,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({
|
||||
getBoundWorkspaceFileSecretProvenanceByMetadata:
|
||||
mockGetBoundWorkspaceFileSecretProvenanceByMetadata,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/core/storage-service', () => ({
|
||||
deleteFile: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/server/metadata', () => ({
|
||||
deleteFileMetadataByIdentity: vi.fn(),
|
||||
getFileMetadataByKeys: mockGetFileMetadataByKeys,
|
||||
}))
|
||||
|
||||
import { processDocumentAsync } from '@/lib/knowledge/documents/service'
|
||||
|
||||
const PERSISTED_KEY = 'workspace/workspace-1/persisted.pdf'
|
||||
const PERSISTED_URL = `/api/files/serve/${encodeURIComponent(PERSISTED_KEY)}?context=workspace`
|
||||
const CONTENT_UPDATED_AT = new Date('2026-08-05T12:00:00.000Z')
|
||||
|
||||
const PERSISTED_CONTEXT = {
|
||||
workspaceId: null,
|
||||
knowledgeBaseUserId: 'knowledge-owner',
|
||||
chunkingConfig: null,
|
||||
embeddingModel: 'text-embedding-3-small',
|
||||
billedAccountUserId: null,
|
||||
uploadedBy: 'uploader-1',
|
||||
filename: 'persisted.pdf',
|
||||
fileUrl: PERSISTED_URL,
|
||||
fileSize: 512,
|
||||
mimeType: 'application/pdf',
|
||||
tag1: null,
|
||||
tag2: null,
|
||||
tag3: null,
|
||||
tag4: null,
|
||||
tag5: null,
|
||||
tag6: null,
|
||||
tag7: null,
|
||||
number1: null,
|
||||
number2: null,
|
||||
number3: null,
|
||||
number4: null,
|
||||
number5: null,
|
||||
date1: null,
|
||||
date2: null,
|
||||
boolean1: null,
|
||||
boolean2: null,
|
||||
boolean3: null,
|
||||
}
|
||||
|
||||
const PERSISTED_PROVENANCE_ROW = {
|
||||
id: 'document-1',
|
||||
secretProvenanceVersion: null,
|
||||
filename: PERSISTED_CONTEXT.filename,
|
||||
fileUrl: PERSISTED_CONTEXT.fileUrl,
|
||||
contentHash: null,
|
||||
sourceUrl: null,
|
||||
tag1: null,
|
||||
tag2: null,
|
||||
tag3: null,
|
||||
tag4: null,
|
||||
tag5: null,
|
||||
tag6: null,
|
||||
tag7: null,
|
||||
number1: null,
|
||||
number2: null,
|
||||
number3: null,
|
||||
number4: null,
|
||||
number5: null,
|
||||
date1: null,
|
||||
date2: null,
|
||||
boolean1: null,
|
||||
boolean2: null,
|
||||
boolean3: null,
|
||||
provenanceSourceHash: null,
|
||||
status: null,
|
||||
entries: null,
|
||||
}
|
||||
|
||||
const SOURCE_BINDING = {
|
||||
id: 'source-file-1',
|
||||
key: PERSISTED_KEY,
|
||||
userId: 'uploader-1',
|
||||
workspaceId: 'workspace-1',
|
||||
context: 'workspace',
|
||||
originalName: PERSISTED_CONTEXT.filename,
|
||||
displayName: PERSISTED_CONTEXT.filename,
|
||||
contentType: PERSISTED_CONTEXT.mimeType,
|
||||
size: PERSISTED_CONTEXT.fileSize,
|
||||
folderId: null,
|
||||
uploadedAt: CONTENT_UPDATED_AT,
|
||||
contentUpdatedAt: CONTENT_UPDATED_AT,
|
||||
deletedAt: null,
|
||||
secretProvenanceVersion: null,
|
||||
}
|
||||
|
||||
describe('knowledge document processing source', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
resetDbChainMock()
|
||||
dbChainMockFns.limit
|
||||
.mockResolvedValueOnce([PERSISTED_CONTEXT])
|
||||
.mockResolvedValueOnce([PERSISTED_PROVENANCE_ROW])
|
||||
.mockResolvedValueOnce([{ id: 'document-1' }])
|
||||
mockCheckActorUsageLimits.mockResolvedValue({ isExceeded: false })
|
||||
mockGetFileMetadataByKeys.mockImplementation(async (_keys: string[], context: string) =>
|
||||
context === 'workspace' ? [SOURCE_BINDING] : []
|
||||
)
|
||||
mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(
|
||||
new Map([[SOURCE_BINDING.id, { status: 'exact', entries: [] }]])
|
||||
)
|
||||
mockProcessDocument.mockResolvedValue({
|
||||
chunks: [],
|
||||
metadata: { chunkCount: 0, tokenCount: 0, characterCount: 0 },
|
||||
})
|
||||
})
|
||||
|
||||
it('uses the persisted document source instead of stale queued source fields', async () => {
|
||||
await processDocumentAsync('knowledge-base-1', 'document-1', {
|
||||
filename: 'stale.pdf',
|
||||
fileUrl: 'https://example.com/stale.pdf',
|
||||
fileSize: 1,
|
||||
mimeType: 'text/plain',
|
||||
})
|
||||
|
||||
expect(mockGetFileMetadataByKeys).toHaveBeenCalledWith(
|
||||
[PERSISTED_KEY],
|
||||
'workspace',
|
||||
expect.anything()
|
||||
)
|
||||
expect(mockGetBoundWorkspaceFileSecretProvenanceByMetadata).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
[SOURCE_BINDING]
|
||||
)
|
||||
expect(mockProcessDocument).toHaveBeenCalledWith(
|
||||
PERSISTED_CONTEXT.fileUrl,
|
||||
PERSISTED_CONTEXT.filename,
|
||||
PERSISTED_CONTEXT.mimeType,
|
||||
1024,
|
||||
200,
|
||||
100,
|
||||
PERSISTED_CONTEXT.uploadedBy,
|
||||
null,
|
||||
undefined,
|
||||
undefined
|
||||
)
|
||||
expect(mockGenerateEmbeddings).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('processes a legacy document when its workspace metadata row no longer exists', async () => {
|
||||
mockGetFileMetadataByKeys.mockResolvedValue([])
|
||||
mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(new Map())
|
||||
|
||||
await processDocumentAsync('knowledge-base-1', 'document-1', {
|
||||
filename: 'stale.pdf',
|
||||
fileUrl: 'https://example.com/stale.pdf',
|
||||
fileSize: 1,
|
||||
mimeType: 'text/plain',
|
||||
})
|
||||
|
||||
expect(mockProcessDocument).toHaveBeenCalledWith(
|
||||
PERSISTED_CONTEXT.fileUrl,
|
||||
PERSISTED_CONTEXT.filename,
|
||||
PERSISTED_CONTEXT.mimeType,
|
||||
1024,
|
||||
200,
|
||||
100,
|
||||
PERSISTED_CONTEXT.uploadedBy,
|
||||
null,
|
||||
undefined,
|
||||
undefined
|
||||
)
|
||||
})
|
||||
|
||||
it('fails before parsing an existing document when its current source is tracked unknown', async () => {
|
||||
mockGetFileMetadataByKeys.mockImplementation(async (_keys: string[], context: string) =>
|
||||
context === 'workspace' ? [{ ...SOURCE_BINDING, secretProvenanceVersion: 1 }] : []
|
||||
)
|
||||
mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(
|
||||
new Map([[SOURCE_BINDING.id, { status: 'unknown' }]])
|
||||
)
|
||||
|
||||
await expect(
|
||||
processDocumentAsync('knowledge-base-1', 'document-1', {
|
||||
filename: 'stale.pdf',
|
||||
fileUrl: 'https://example.com/stale.pdf',
|
||||
fileSize: 1,
|
||||
mimeType: 'text/plain',
|
||||
})
|
||||
).rejects.toThrow('Knowledge document secret provenance is unavailable')
|
||||
|
||||
expect(mockProcessDocument).not.toHaveBeenCalled()
|
||||
expect(mockGenerateEmbeddings).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -7,12 +7,12 @@ const {
|
||||
mockDownloadFileFromUrl,
|
||||
mockGenerateInternalToken,
|
||||
mockGetInternalApiBaseUrl,
|
||||
mockIsModelSafeWorkspaceFileKey,
|
||||
mockParseBuffer,
|
||||
} = vi.hoisted(() => ({
|
||||
mockDownloadFileFromUrl: vi.fn(),
|
||||
mockGenerateInternalToken: vi.fn(),
|
||||
mockGetInternalApiBaseUrl: vi.fn(),
|
||||
mockIsModelSafeWorkspaceFileKey: vi.fn(),
|
||||
mockParseBuffer: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/auth/internal', () => ({
|
||||
@@ -24,10 +24,8 @@ vi.mock('@/lib/core/utils/urls', async (importOriginal) => ({
|
||||
getInternalApiBaseUrl: mockGetInternalApiBaseUrl,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({
|
||||
isModelSafeWorkspaceFileKey: mockIsModelSafeWorkspaceFileKey,
|
||||
MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE:
|
||||
'File cannot be sent to a model because its secret provenance is unavailable',
|
||||
vi.mock('@/lib/file-parsers', () => ({
|
||||
parseBuffer: mockParseBuffer,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/utils/file-utils.server', () => ({
|
||||
@@ -38,6 +36,7 @@ import { env } from '@/lib/core/config/env'
|
||||
import { RESOLVED_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata'
|
||||
import { processDocument } from '@/lib/knowledge/documents/document-processor'
|
||||
import { runWithKnowledgeModelInputProvenance } from '@/lib/knowledge/model-input-provenance'
|
||||
import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
|
||||
|
||||
describe('knowledge document model-input provenance', () => {
|
||||
beforeEach(() => {
|
||||
@@ -57,27 +56,38 @@ describe('knowledge document model-input provenance', () => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
it('rejects an unsafe workspace file before downloading or parsing its bytes', async () => {
|
||||
mockIsModelSafeWorkspaceFileKey.mockResolvedValue(false)
|
||||
it('parses tracked workspace-file bytes locally without treating parsing as model egress', async () => {
|
||||
const registry = new ResolvedSecretTraceRegistry([
|
||||
{ name: 'TOKEN', plaintext: 'tracked-secret', encryptedValue: 'encrypted-token' },
|
||||
])
|
||||
registry.recordResolved('TOKEN', 'tracked-secret')
|
||||
mockDownloadFileFromUrl.mockResolvedValue(
|
||||
Buffer.from('Locally parsed content containing tracked-secret.')
|
||||
)
|
||||
mockParseBuffer.mockResolvedValue({
|
||||
content: 'Locally parsed content containing tracked-secret.',
|
||||
metadata: {},
|
||||
})
|
||||
const fetchMock = vi.fn()
|
||||
vi.stubGlobal('fetch', fetchMock)
|
||||
|
||||
await expect(
|
||||
const processed = await runWithKnowledgeModelInputProvenance(registry, () =>
|
||||
processDocument(
|
||||
'/api/files/serve/workspace/workspace-1/opaque.txt?context=workspace',
|
||||
'opaque.txt',
|
||||
'/api/files/serve/workspace/workspace-1/tracked.txt?context=workspace',
|
||||
'tracked.txt',
|
||||
'text/plain',
|
||||
1024,
|
||||
200,
|
||||
100,
|
||||
1,
|
||||
'user-1',
|
||||
'workspace-1'
|
||||
)
|
||||
).rejects.toThrow('secret provenance is unavailable')
|
||||
|
||||
expect(mockIsModelSafeWorkspaceFileKey).toHaveBeenCalledWith(
|
||||
'workspace/workspace-1/opaque.txt',
|
||||
{ workspaceId: 'workspace-1' }
|
||||
)
|
||||
expect(mockDownloadFileFromUrl).not.toHaveBeenCalled()
|
||||
|
||||
expect(processed.metadata.processingMethod).toBe('file-parser')
|
||||
expect(processed.chunks.map((chunk) => chunk.text).join('\n')).toContain('tracked-secret')
|
||||
expect(mockDownloadFileFromUrl).toHaveBeenCalledOnce()
|
||||
expect(fetchMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects secret-bearing opaque document bytes before external OCR', async () => {
|
||||
|
||||
@@ -25,11 +25,7 @@ import {
|
||||
getKnowledgeOpaqueModelInputRegistry,
|
||||
} from '@/lib/knowledge/model-input-provenance'
|
||||
import { StorageService } from '@/lib/uploads'
|
||||
import {
|
||||
isModelSafeWorkspaceFileKey,
|
||||
MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE,
|
||||
} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
|
||||
import { extractStorageKey, isInternalFileUrl } from '@/lib/uploads/utils/file-utils'
|
||||
import { isInternalFileUrl } from '@/lib/uploads/utils/file-utils'
|
||||
import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server'
|
||||
import { MAX_FILE_SIZE } from '@/lib/uploads/utils/validation'
|
||||
import { mistralParserTool } from '@/tools/mistral/parser'
|
||||
@@ -58,20 +54,6 @@ type OCRPage = {
|
||||
|
||||
const MISTRAL_MAX_PAGES = 1000
|
||||
|
||||
async function assertDocumentFileModelSafe(
|
||||
fileUrl: string,
|
||||
workspaceId?: string | null
|
||||
): Promise<void> {
|
||||
if (!isInternalFileUrl(fileUrl)) return
|
||||
|
||||
const path = new URL(fileUrl, 'http://localhost').pathname
|
||||
const key = extractStorageKey(path)
|
||||
const safe = await isModelSafeWorkspaceFileKey(key, workspaceId ? { workspaceId } : {})
|
||||
if (!safe) {
|
||||
throw new Error(MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE)
|
||||
}
|
||||
}
|
||||
|
||||
async function getPdfPageCount(buffer: Buffer): Promise<number> {
|
||||
try {
|
||||
const { getDocumentProxy } = await import('unpdf')
|
||||
@@ -209,7 +191,6 @@ export async function processDocument(
|
||||
logger.info('Processing document', { mimeType })
|
||||
|
||||
try {
|
||||
await assertDocumentFileModelSafe(fileUrl, workspaceId)
|
||||
const parseResult = await parseDocument(fileUrl, filename, mimeType, userId, workspaceId)
|
||||
const { content, processingMethod } = parseResult
|
||||
const cloudUrl = 'cloudUrl' in parseResult ? parseResult.cloudUrl : undefined
|
||||
|
||||
@@ -93,7 +93,7 @@ import {
|
||||
} from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance'
|
||||
import { deleteFile } from '@/lib/uploads/core/storage-service'
|
||||
import {
|
||||
deleteFileMetadata,
|
||||
deleteFileMetadataByIdentity,
|
||||
type FileMetadataRecord,
|
||||
getFileMetadataByKeys,
|
||||
} from '@/lib/uploads/server/metadata'
|
||||
@@ -104,8 +104,8 @@ import { calculateCost } from '@/providers/utils'
|
||||
const logger = createLogger('DocumentService')
|
||||
|
||||
/**
|
||||
* Thrown when a knowledge-base document's `fileUrl` references an internal `kb/`
|
||||
* storage object that is not owned by the target knowledge base's workspace.
|
||||
* Thrown when a knowledge-base document's `fileUrl` references an internal
|
||||
* knowledge-base storage object not owned by the target knowledge base's workspace.
|
||||
* Routes map this to a 403.
|
||||
*/
|
||||
export class KnowledgeBaseFileOwnershipError extends Error {
|
||||
@@ -117,7 +117,7 @@ export class KnowledgeBaseFileOwnershipError extends Error {
|
||||
|
||||
/**
|
||||
* Guard document `fileUrl`s at creation time. When a URL points at an internal
|
||||
* `kb/` storage object, require that the target knowledge base owns the object,
|
||||
* knowledge-base storage object, require that the target knowledge base owns the object,
|
||||
* resolved from the trusted `workspace_files` binding:
|
||||
*
|
||||
* - Workspace KB (`kbWorkspaceId` set): the binding's `workspaceId` must match.
|
||||
@@ -125,13 +125,65 @@ export class KnowledgeBaseFileOwnershipError extends Error {
|
||||
* owner. A key bound to another tenant is rejected; an unbound key (legacy /
|
||||
* never reserved) passes since it carries no cross-tenant ownership.
|
||||
*
|
||||
* External `http(s)`/`data:` URLs (ingestion sources) and non-`kb/` internal keys
|
||||
* External `http(s)`/`data:` URLs (ingestion sources) and other internal keys
|
||||
* pass through unchanged. This blocks a user from asserting ownership of another
|
||||
* tenant's `kb/` key via a planted `fileUrl` — including in a personal KB, which
|
||||
* tenant's object via a planted `fileUrl` — including in a personal KB, which
|
||||
* otherwise could be moved into a workspace to launder the binding. All
|
||||
* referenced bindings are resolved in one query (no N+1 inside the `FOR UPDATE`
|
||||
* window). Single-document callers pass a one-element array.
|
||||
*/
|
||||
function isKnowledgeBaseOwnedStorageKey(key: string): boolean {
|
||||
return key.startsWith('kb/') || key.startsWith('knowledge-base/')
|
||||
}
|
||||
|
||||
function getKnowledgeBaseStorageKeys(fileUrls: readonly string[]): string[] {
|
||||
return [
|
||||
...new Set(
|
||||
fileUrls
|
||||
.map((url) => getKnowledgeBaseStorageKey(url))
|
||||
.filter(
|
||||
(key): key is string => typeof key === 'string' && isKnowledgeBaseOwnedStorageKey(key)
|
||||
)
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
function getWorkspaceSourceStorageKeys(fileUrls: readonly string[]): string[] {
|
||||
return [
|
||||
...new Set(
|
||||
fileUrls
|
||||
.map((url) => getKnowledgeBaseStorageKey(url))
|
||||
.filter((key): key is string => typeof key === 'string' && key.startsWith('workspace/'))
|
||||
),
|
||||
]
|
||||
}
|
||||
|
||||
async function loadKnowledgeBaseFileBindings(
|
||||
fileUrls: readonly string[],
|
||||
executor: DbExecutor = db
|
||||
): Promise<Map<string, FileMetadataRecord>> {
|
||||
const keys = getKnowledgeBaseStorageKeys(fileUrls)
|
||||
const bindings =
|
||||
keys.length > 0 ? await getFileMetadataByKeys(keys, 'knowledge-base', executor) : []
|
||||
|
||||
return new Map(bindings.map((binding) => [binding.key, binding]))
|
||||
}
|
||||
|
||||
async function loadWorkspaceSourceFileBindings(
|
||||
fileUrls: readonly string[],
|
||||
executor: DbExecutor = db
|
||||
): Promise<Map<string, FileMetadataRecord>> {
|
||||
const keys = getWorkspaceSourceStorageKeys(fileUrls)
|
||||
if (keys.length === 0) return new Map()
|
||||
|
||||
const workspaceBindings = await getFileMetadataByKeys(keys, 'workspace', executor)
|
||||
const mothershipBindings = await getFileMetadataByKeys(keys, 'mothership', executor)
|
||||
|
||||
return new Map(
|
||||
[...workspaceBindings, ...mothershipBindings].map((binding) => [binding.key, binding])
|
||||
)
|
||||
}
|
||||
|
||||
async function assertKnowledgeBaseFileUrlsOwnership(
|
||||
fileUrls: string[],
|
||||
kbWorkspaceId: string | null,
|
||||
@@ -139,21 +191,12 @@ async function assertKnowledgeBaseFileUrlsOwnership(
|
||||
requestId: string,
|
||||
executor: DbExecutor = db
|
||||
): Promise<Map<string, FileMetadataRecord>> {
|
||||
const keys = [
|
||||
...new Set(
|
||||
fileUrls
|
||||
.map((url) => getKnowledgeBaseStorageKey(url))
|
||||
.filter((key): key is string => typeof key === 'string' && key.startsWith('kb/'))
|
||||
),
|
||||
]
|
||||
const keys = getKnowledgeBaseStorageKeys(fileUrls)
|
||||
if (keys.length === 0) {
|
||||
return new Map()
|
||||
}
|
||||
|
||||
// Read bindings on the caller's transaction so the security check shares the
|
||||
// same connection/lock context as the FOR UPDATE'd insert that follows.
|
||||
const bindings = await getFileMetadataByKeys(keys, 'knowledge-base', executor)
|
||||
const bindingByKey = new Map(bindings.map((binding) => [binding.key, binding]))
|
||||
const bindingByKey = await loadKnowledgeBaseFileBindings(fileUrls, executor)
|
||||
|
||||
for (const key of keys) {
|
||||
const binding = bindingByKey.get(key)
|
||||
@@ -189,6 +232,21 @@ async function assertKnowledgeBaseFileUrlsOwnership(
|
||||
return bindingByKey
|
||||
}
|
||||
|
||||
async function loadCurrentWorkspaceSourceFileSecretProvenance(options: {
|
||||
fileUrl: string
|
||||
}): Promise<DurableSecretProvenance | undefined> {
|
||||
const storageKey = getKnowledgeBaseStorageKey(options.fileUrl)
|
||||
if (!storageKey?.startsWith('workspace/')) return undefined
|
||||
|
||||
const bindingByKey = await loadWorkspaceSourceFileBindings([options.fileUrl])
|
||||
const binding = bindingByKey.get(storageKey)
|
||||
if (!binding) return undefined
|
||||
|
||||
const provenanceById = await getBoundWorkspaceFileSecretProvenanceByMetadata(db, [binding])
|
||||
const provenance = provenanceById.get(binding.id) ?? { status: 'unknown' as const }
|
||||
return durableSecretProvenanceFromWorkspaceFile(provenance, binding)
|
||||
}
|
||||
|
||||
const TIMEOUTS = {
|
||||
OVERALL_PROCESSING: envNumber(env.KB_CONFIG_MAX_DURATION, 600) * 1000,
|
||||
} as const
|
||||
@@ -722,6 +780,10 @@ export async function processDocumentAsync(
|
||||
embeddingModel: knowledgeBase.embeddingModel,
|
||||
billedAccountUserId: workspaceTable.billedAccountUserId,
|
||||
uploadedBy: document.uploadedBy,
|
||||
filename: document.filename,
|
||||
fileUrl: document.fileUrl,
|
||||
fileSize: document.fileSize,
|
||||
mimeType: document.mimeType,
|
||||
tag1: document.tag1,
|
||||
tag2: document.tag2,
|
||||
tag3: document.tag3,
|
||||
@@ -773,6 +835,12 @@ export async function processDocumentAsync(
|
||||
}
|
||||
|
||||
const ctx = contextRows[0]
|
||||
const persistedDocData = {
|
||||
filename: ctx.filename,
|
||||
fileUrl: ctx.fileUrl,
|
||||
fileSize: ctx.fileSize,
|
||||
mimeType: ctx.mimeType,
|
||||
}
|
||||
|
||||
await db
|
||||
.update(document)
|
||||
@@ -861,19 +929,26 @@ export async function processDocumentAsync(
|
||||
let embeddingModelName = kbEmbeddingModel
|
||||
let embeddingPricingId = kbEmbeddingModel
|
||||
|
||||
const documentSecretContext = await loadKnowledgeDocumentSecretRegistry(documentId, {
|
||||
userId: documentActorUserId,
|
||||
...(ctx.workspaceId ? { workspaceId: ctx.workspaceId } : {}),
|
||||
const currentSourceFileProvenance = await loadCurrentWorkspaceSourceFileSecretProvenance({
|
||||
fileUrl: persistedDocData.fileUrl,
|
||||
})
|
||||
const documentSecretContext = await loadKnowledgeDocumentSecretRegistry(
|
||||
documentId,
|
||||
{
|
||||
userId: documentActorUserId,
|
||||
...(ctx.workspaceId ? { workspaceId: ctx.workspaceId } : {}),
|
||||
},
|
||||
currentSourceFileProvenance
|
||||
)
|
||||
|
||||
await withTimeout(
|
||||
runWithKnowledgeModelInputProvenance(
|
||||
documentSecretContext.registry,
|
||||
async () => {
|
||||
const processed = await processDocument(
|
||||
docData.fileUrl,
|
||||
docData.filename,
|
||||
docData.mimeType,
|
||||
persistedDocData.fileUrl,
|
||||
persistedDocData.filename,
|
||||
persistedDocData.mimeType,
|
||||
kbConfig.maxSize,
|
||||
kbConfig.overlap,
|
||||
kbConfig.minSize,
|
||||
@@ -1171,15 +1246,9 @@ function getServerKnownDocumentSize(
|
||||
async function resolveServerKnownDocumentSizes<
|
||||
T extends { readonly fileUrl: string; readonly fileSize: number },
|
||||
>(documents: readonly T[]): Promise<Array<T & { fileSize: number }>> {
|
||||
const keys = [
|
||||
...new Set(
|
||||
documents
|
||||
.map((docData) => getKnowledgeBaseStorageKey(docData.fileUrl))
|
||||
.filter((key): key is string => typeof key === 'string' && key.startsWith('kb/'))
|
||||
),
|
||||
]
|
||||
const bindings = keys.length > 0 ? await getFileMetadataByKeys(keys, 'knowledge-base') : []
|
||||
const bindingByKey = new Map(bindings.map((binding) => [binding.key, binding]))
|
||||
const bindingByKey = await loadKnowledgeBaseFileBindings(
|
||||
documents.map((document) => document.fileUrl)
|
||||
)
|
||||
return documents.map((docData) => ({
|
||||
...docData,
|
||||
fileSize: getServerKnownDocumentSize(docData.fileUrl, docData.fileSize, bindingByKey),
|
||||
@@ -1302,9 +1371,14 @@ export async function createDocumentRecords(
|
||||
requestId,
|
||||
tx
|
||||
)
|
||||
const trackedBindings = [...bindingByKey.values()].filter(
|
||||
(binding) => binding.secretProvenanceVersion !== null
|
||||
const sourceBindingByKey = await loadWorkspaceSourceFileBindings(
|
||||
resolvedDocuments.map((docData) => docData.fileUrl),
|
||||
tx
|
||||
)
|
||||
const trackedBindings = [
|
||||
...[...bindingByKey.values()].filter((binding) => binding.secretProvenanceVersion !== null),
|
||||
...sourceBindingByKey.values(),
|
||||
]
|
||||
const boundFileProvenanceById = await getBoundWorkspaceFileSecretProvenanceByMetadata(
|
||||
tx,
|
||||
trackedBindings
|
||||
@@ -1411,17 +1485,22 @@ export async function createDocumentRecords(
|
||||
boolean3: processedTags.boolean3 ?? null,
|
||||
}
|
||||
const source = createKnowledgeDocumentSourceValue(baseDocument)
|
||||
const binding = storageKey ? bindingByKey.get(storageKey) : undefined
|
||||
const boundFileProvenance =
|
||||
binding && binding.secretProvenanceVersion !== null
|
||||
? (boundFileProvenanceById.get(binding.id) ?? { status: 'unknown' as const })
|
||||
const binding = storageKey
|
||||
? (bindingByKey.get(storageKey) ?? sourceBindingByKey.get(storageKey))
|
||||
: undefined
|
||||
const provenanceBinding =
|
||||
binding && (binding.secretProvenanceVersion !== null || sourceBindingByKey.has(binding.key))
|
||||
? binding
|
||||
: undefined
|
||||
const boundFileProvenance = provenanceBinding
|
||||
? (boundFileProvenanceById.get(provenanceBinding.id) ?? { status: 'unknown' as const })
|
||||
: undefined
|
||||
const provenance = bindKnowledgeDocumentWriteSecretProvenance({
|
||||
source,
|
||||
provenance: secretProvenances?.[documentIndex],
|
||||
tagDefinitions,
|
||||
...(binding && boundFileProvenance
|
||||
? { boundFile: { binding, provenance: boundFileProvenance } }
|
||||
...(provenanceBinding && boundFileProvenance
|
||||
? { boundFile: { binding: provenanceBinding, provenance: boundFileProvenance } }
|
||||
: {}),
|
||||
})
|
||||
const newDocument = {
|
||||
@@ -1833,11 +1912,21 @@ export async function createSingleDocument(
|
||||
requestId,
|
||||
tx
|
||||
)
|
||||
const sourceBindingByKey = await loadWorkspaceSourceFileBindings(
|
||||
[resolvedDocumentData.fileUrl],
|
||||
tx
|
||||
)
|
||||
const storageKey = getKnowledgeBaseStorageKey(resolvedDocumentData.fileUrl)
|
||||
const binding = storageKey ? bindingByKey.get(storageKey) : undefined
|
||||
const binding = storageKey
|
||||
? (bindingByKey.get(storageKey) ?? sourceBindingByKey.get(storageKey))
|
||||
: undefined
|
||||
const provenanceBinding =
|
||||
binding && (binding.secretProvenanceVersion !== null || sourceBindingByKey.has(binding.key))
|
||||
? binding
|
||||
: undefined
|
||||
const boundFileProvenanceById = await getBoundWorkspaceFileSecretProvenanceByMetadata(
|
||||
tx,
|
||||
binding && binding.secretProvenanceVersion !== null ? [binding] : []
|
||||
provenanceBinding ? [provenanceBinding] : []
|
||||
)
|
||||
const currentSize = getServerKnownDocumentSize(
|
||||
resolvedDocumentData.fileUrl,
|
||||
@@ -1873,16 +1962,15 @@ export async function createSingleDocument(
|
||||
}
|
||||
|
||||
const source = createKnowledgeDocumentSourceValue(newDocument)
|
||||
const boundFileProvenance =
|
||||
binding && binding.secretProvenanceVersion !== null
|
||||
? (boundFileProvenanceById.get(binding.id) ?? { status: 'unknown' as const })
|
||||
: undefined
|
||||
const boundFileProvenance = provenanceBinding
|
||||
? (boundFileProvenanceById.get(provenanceBinding.id) ?? { status: 'unknown' as const })
|
||||
: undefined
|
||||
const documentProvenance = bindKnowledgeDocumentWriteSecretProvenance({
|
||||
source,
|
||||
provenance: secretProvenance,
|
||||
tagDefinitions,
|
||||
...(binding && boundFileProvenance
|
||||
? { boundFile: { binding, provenance: boundFileProvenance } }
|
||||
...(provenanceBinding && boundFileProvenance
|
||||
? { boundFile: { binding: provenanceBinding, provenance: boundFileProvenance } }
|
||||
: {}),
|
||||
})
|
||||
await tx.insert(document).values({
|
||||
@@ -2464,20 +2552,20 @@ export async function deleteDocumentStorageFiles(
|
||||
storageKey: getKnowledgeBaseStorageKey(doc.fileUrl),
|
||||
}))
|
||||
|
||||
// Resolve all kb/ ownership bindings in one query (avoids an N+1 across the
|
||||
// delete fan-out below).
|
||||
const kbKeys = [
|
||||
const storageKeys = [
|
||||
...new Set(
|
||||
entries
|
||||
.map((entry) => entry.storageKey)
|
||||
.filter((key): key is string => typeof key === 'string' && key.startsWith('kb/'))
|
||||
.filter(
|
||||
(key): key is string => typeof key === 'string' && isKnowledgeBaseOwnedStorageKey(key)
|
||||
)
|
||||
),
|
||||
]
|
||||
const ownerByKey = new Map<string, string | null>()
|
||||
if (kbKeys.length > 0) {
|
||||
const bindings = await getFileMetadataByKeys(kbKeys, 'knowledge-base')
|
||||
const bindingByKey = new Map<string, FileMetadataRecord>()
|
||||
if (storageKeys.length > 0) {
|
||||
const bindings = await getFileMetadataByKeys(storageKeys, 'knowledge-base')
|
||||
for (const binding of bindings) {
|
||||
ownerByKey.set(binding.key, binding.workspaceId)
|
||||
bindingByKey.set(binding.key, binding)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2486,32 +2574,43 @@ export async function deleteDocumentStorageFiles(
|
||||
return
|
||||
}
|
||||
|
||||
// Only delete a kb/ object when its trusted ownership binding confirms the
|
||||
// deleting document's workspace owns it. Prevents deleting another tenant's
|
||||
// object via a document with a planted fileUrl.
|
||||
if (storageKey.startsWith('kb/')) {
|
||||
const bindingWorkspaceId = ownerByKey.get(storageKey)
|
||||
if (!bindingWorkspaceId) {
|
||||
logger.warn(`[${requestId}] Skipping storage delete: no ownership binding for key`, {
|
||||
documentId: doc.id,
|
||||
storageKey,
|
||||
})
|
||||
return
|
||||
}
|
||||
if (!doc.workspaceId || bindingWorkspaceId !== doc.workspaceId) {
|
||||
logger.warn(`[${requestId}] Skipping storage delete: ownership binding mismatch`, {
|
||||
documentId: doc.id,
|
||||
storageKey,
|
||||
bindingWorkspaceId,
|
||||
documentWorkspaceId: doc.workspaceId ?? null,
|
||||
})
|
||||
return
|
||||
}
|
||||
if (!isKnowledgeBaseOwnedStorageKey(storageKey)) {
|
||||
return
|
||||
}
|
||||
|
||||
const binding = bindingByKey.get(storageKey)
|
||||
if (!binding?.workspaceId || binding.context !== 'knowledge-base') {
|
||||
logger.warn(`[${requestId}] Skipping storage delete: no ownership binding for key`, {
|
||||
documentId: doc.id,
|
||||
storageKey,
|
||||
})
|
||||
return
|
||||
}
|
||||
if (!doc.workspaceId || binding.workspaceId !== doc.workspaceId) {
|
||||
logger.warn(`[${requestId}] Skipping storage delete: ownership binding mismatch`, {
|
||||
documentId: doc.id,
|
||||
storageKey,
|
||||
bindingWorkspaceId: binding.workspaceId,
|
||||
documentWorkspaceId: doc.workspaceId ?? null,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const metadataDeleted = await deleteFileMetadataByIdentity({
|
||||
id: binding.id,
|
||||
key: binding.key,
|
||||
context: binding.context,
|
||||
contentUpdatedAt: binding.contentUpdatedAt,
|
||||
})
|
||||
if (!metadataDeleted) {
|
||||
logger.warn(`[${requestId}] Skipping storage delete: ownership binding changed`, {
|
||||
documentId: doc.id,
|
||||
storageKey,
|
||||
})
|
||||
return
|
||||
}
|
||||
await deleteFile({ key: storageKey, context: 'knowledge-base' })
|
||||
await deleteFileMetadata(storageKey)
|
||||
} catch (error) {
|
||||
logger.warn(`[${requestId}] Failed to delete document storage file`, {
|
||||
documentId: doc.id,
|
||||
|
||||
@@ -167,39 +167,43 @@ describe('knowledge document storage attribution', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('uses server-known file metadata size for quota, ledger, and document row', async () => {
|
||||
const fileUrl = '/api/files/serve/kb%2Fverified-file?context=knowledge-base'
|
||||
mockGetFileMetadataByKeys.mockResolvedValue([
|
||||
{
|
||||
key: 'kb/verified-file',
|
||||
workspaceId: 'workspace-1',
|
||||
userId: 'external-collaborator',
|
||||
size: 8,
|
||||
},
|
||||
])
|
||||
mockIncrementStorageUsageForBillingContextInTx.mockResolvedValue(13)
|
||||
it.each(['kb', 'knowledge-base'])(
|
||||
'uses server-known %s file metadata size for quota, ledger, and document row',
|
||||
async (keyPrefix) => {
|
||||
const storageKey = `${keyPrefix}/verified-file`
|
||||
const fileUrl = `/api/files/serve/${encodeURIComponent(storageKey)}?context=knowledge-base`
|
||||
mockGetFileMetadataByKeys.mockResolvedValue([
|
||||
{
|
||||
key: storageKey,
|
||||
workspaceId: 'workspace-1',
|
||||
userId: 'external-collaborator',
|
||||
size: 8,
|
||||
},
|
||||
])
|
||||
mockIncrementStorageUsageForBillingContextInTx.mockResolvedValue(13)
|
||||
|
||||
const result = await createSingleDocument(
|
||||
{
|
||||
filename: 'note.txt',
|
||||
fileUrl,
|
||||
fileSize: 5,
|
||||
mimeType: 'text/plain',
|
||||
},
|
||||
'knowledge-base-1',
|
||||
'request-1',
|
||||
'external-collaborator'
|
||||
)
|
||||
const result = await createSingleDocument(
|
||||
{
|
||||
filename: 'note.txt',
|
||||
fileUrl,
|
||||
fileSize: 5,
|
||||
mimeType: 'text/plain',
|
||||
},
|
||||
'knowledge-base-1',
|
||||
'request-1',
|
||||
'external-collaborator'
|
||||
)
|
||||
|
||||
expect(mockCheckStorageQuotaForBillingContext).toHaveBeenCalledWith(STORAGE_CONTEXT, 8)
|
||||
expect(mockIncrementStorageUsageForBillingContextInTx).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
STORAGE_CONTEXT,
|
||||
8
|
||||
)
|
||||
expect(result.fileSize).toBe(8)
|
||||
expect(dbChainMockFns.values).toHaveBeenCalledWith(expect.objectContaining({ fileSize: 8 }))
|
||||
})
|
||||
expect(mockCheckStorageQuotaForBillingContext).toHaveBeenCalledWith(STORAGE_CONTEXT, 8)
|
||||
expect(mockIncrementStorageUsageForBillingContextInTx).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
STORAGE_CONTEXT,
|
||||
8
|
||||
)
|
||||
expect(result.fileSize).toBe(8)
|
||||
expect(dbChainMockFns.values).toHaveBeenCalledWith(expect.objectContaining({ fileSize: 8 }))
|
||||
}
|
||||
)
|
||||
|
||||
it('decrements only exact bytes for document rows actually deleted', async () => {
|
||||
dbChainMockFns.where.mockResolvedValueOnce([
|
||||
|
||||
@@ -0,0 +1,355 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { dbChainMockFns, resetDbChainMock } from '@sim/testing'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const {
|
||||
mockCheckStorageQuotaForBillingContext,
|
||||
mockDeleteFile,
|
||||
mockDeleteFileMetadataByIdentity,
|
||||
mockGetBoundWorkspaceFileSecretProvenanceByMetadata,
|
||||
mockGetFileMetadataByKeys,
|
||||
mockIncrementStorageUsageForBillingContextInTx,
|
||||
mockMaybeNotifyStorageLimitForBillingContext,
|
||||
mockResolveStorageBillingContext,
|
||||
} = vi.hoisted(() => ({
|
||||
mockCheckStorageQuotaForBillingContext: vi.fn(),
|
||||
mockDeleteFile: vi.fn(),
|
||||
mockDeleteFileMetadataByIdentity: vi.fn(),
|
||||
mockGetBoundWorkspaceFileSecretProvenanceByMetadata: vi.fn(),
|
||||
mockGetFileMetadataByKeys: vi.fn(),
|
||||
mockIncrementStorageUsageForBillingContextInTx: vi.fn(),
|
||||
mockMaybeNotifyStorageLimitForBillingContext: vi.fn(),
|
||||
mockResolveStorageBillingContext: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/billing/storage', () => ({
|
||||
applyStorageUsageDeltasInTx: vi.fn(),
|
||||
checkStorageQuota: vi.fn(),
|
||||
checkStorageQuotaForBillingContext: mockCheckStorageQuotaForBillingContext,
|
||||
incrementStorageUsageForBillingContextInTx: mockIncrementStorageUsageForBillingContextInTx,
|
||||
maybeNotifyStorageLimitForBillingContext: mockMaybeNotifyStorageLimitForBillingContext,
|
||||
resolveStorageBillingContext: mockResolveStorageBillingContext,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/contexts/workspace/workspace-file-secret-provenance', () => ({
|
||||
getBoundWorkspaceFileSecretProvenanceByMetadata:
|
||||
mockGetBoundWorkspaceFileSecretProvenanceByMetadata,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/core/storage-service', () => ({
|
||||
deleteFile: mockDeleteFile,
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/uploads/server/metadata', () => ({
|
||||
deleteFileMetadataByIdentity: mockDeleteFileMetadataByIdentity,
|
||||
getFileMetadataByKeys: mockGetFileMetadataByKeys,
|
||||
}))
|
||||
|
||||
import {
|
||||
createDocumentRecords,
|
||||
createSingleDocument,
|
||||
deleteDocumentStorageFiles,
|
||||
} from '@/lib/knowledge/documents/service'
|
||||
|
||||
const WORKSPACE_ID = 'workspace-1'
|
||||
const KNOWLEDGE_BASE_ID = 'knowledge-base-1'
|
||||
const KNOWLEDGE_BASE_OWNER_ID = 'knowledge-owner'
|
||||
const SOURCE_USER_ID = 'source-user'
|
||||
const SOURCE_KEY = `workspace/${WORKSPACE_ID}/source.pdf`
|
||||
const SOURCE_URL = `/api/files/serve/${encodeURIComponent(SOURCE_KEY)}?context=workspace`
|
||||
const CONTENT_UPDATED_AT = new Date('2026-08-05T12:00:00.000Z')
|
||||
|
||||
const SOURCE_BINDING = {
|
||||
id: 'source-file-1',
|
||||
key: SOURCE_KEY,
|
||||
userId: SOURCE_USER_ID,
|
||||
workspaceId: WORKSPACE_ID,
|
||||
context: 'workspace',
|
||||
originalName: 'source.pdf',
|
||||
displayName: 'source.pdf',
|
||||
contentType: 'application/pdf',
|
||||
size: 512,
|
||||
folderId: null,
|
||||
uploadedAt: CONTENT_UPDATED_AT,
|
||||
contentUpdatedAt: CONTENT_UPDATED_AT,
|
||||
deletedAt: null,
|
||||
secretProvenanceVersion: 1,
|
||||
}
|
||||
|
||||
const STORAGE_CONTEXT = {
|
||||
workspaceId: WORKSPACE_ID,
|
||||
billedAccountUserId: KNOWLEDGE_BASE_OWNER_ID,
|
||||
billingEntity: { type: 'organization' as const, id: 'workspace-org' },
|
||||
plan: 'team_25000',
|
||||
customStorageLimitGB: null,
|
||||
}
|
||||
|
||||
function mockWorkspaceSourceBinding(context: string) {
|
||||
return context === 'workspace' ? [SOURCE_BINDING] : []
|
||||
}
|
||||
|
||||
function findDocumentProvenanceWrite() {
|
||||
return dbChainMockFns.values.mock.calls
|
||||
.map(([value]) => value)
|
||||
.find((value) => !Array.isArray(value) && value?.documentId)
|
||||
}
|
||||
|
||||
describe('knowledge workspace source provenance', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
resetDbChainMock()
|
||||
dbChainMockFns.limit.mockResolvedValue([
|
||||
{
|
||||
id: KNOWLEDGE_BASE_ID,
|
||||
workspaceId: WORKSPACE_ID,
|
||||
userId: KNOWLEDGE_BASE_OWNER_ID,
|
||||
},
|
||||
])
|
||||
mockResolveStorageBillingContext.mockResolvedValue(STORAGE_CONTEXT)
|
||||
mockCheckStorageQuotaForBillingContext.mockResolvedValue({ allowed: true })
|
||||
mockIncrementStorageUsageForBillingContextInTx.mockResolvedValue(512)
|
||||
mockMaybeNotifyStorageLimitForBillingContext.mockResolvedValue(undefined)
|
||||
mockGetFileMetadataByKeys.mockImplementation(async (_keys: string[], context: string) =>
|
||||
mockWorkspaceSourceBinding(context)
|
||||
)
|
||||
mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(
|
||||
new Map([
|
||||
[
|
||||
SOURCE_BINDING.id,
|
||||
{
|
||||
status: 'exact',
|
||||
entries: [{ name: 'OCR_SECRET', encryptedValue: 'encrypted-secret' }],
|
||||
},
|
||||
],
|
||||
])
|
||||
)
|
||||
})
|
||||
|
||||
it('merges a tracked workspace source into a single document sidecar', async () => {
|
||||
await createSingleDocument(
|
||||
{
|
||||
filename: 'source.pdf',
|
||||
fileUrl: SOURCE_URL,
|
||||
fileSize: 512,
|
||||
mimeType: 'application/pdf',
|
||||
},
|
||||
KNOWLEDGE_BASE_ID,
|
||||
'request-1',
|
||||
SOURCE_USER_ID
|
||||
)
|
||||
|
||||
expect(mockGetFileMetadataByKeys).toHaveBeenCalledWith(
|
||||
[SOURCE_KEY],
|
||||
'workspace',
|
||||
expect.anything()
|
||||
)
|
||||
expect(mockGetFileMetadataByKeys).toHaveBeenCalledWith(
|
||||
[SOURCE_KEY],
|
||||
'mothership',
|
||||
expect.anything()
|
||||
)
|
||||
expect(mockGetBoundWorkspaceFileSecretProvenanceByMetadata).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
[SOURCE_BINDING]
|
||||
)
|
||||
expect(findDocumentProvenanceWrite()).toEqual(
|
||||
expect.objectContaining({
|
||||
status: 'exact',
|
||||
entries: [
|
||||
expect.objectContaining({
|
||||
name: 'OCR_SECRET',
|
||||
encryptedValue: 'encrypted-secret',
|
||||
sourceUserId: SOURCE_USER_ID,
|
||||
sourceWorkspaceId: WORKSPACE_ID,
|
||||
sourceValueHash: expect.any(String),
|
||||
}),
|
||||
],
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('uses the same workspace source provenance path for bulk document creation', async () => {
|
||||
await createDocumentRecords(
|
||||
[
|
||||
{
|
||||
filename: 'source.pdf',
|
||||
fileUrl: SOURCE_URL,
|
||||
fileSize: 512,
|
||||
mimeType: 'application/pdf',
|
||||
},
|
||||
],
|
||||
KNOWLEDGE_BASE_ID,
|
||||
'request-1',
|
||||
SOURCE_USER_ID
|
||||
)
|
||||
|
||||
expect(findDocumentProvenanceWrite()).toEqual(
|
||||
expect.objectContaining({
|
||||
status: 'exact',
|
||||
entries: [expect.objectContaining({ name: 'OCR_SECRET' })],
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('preserves a tracked exact-empty manual upload as model-safe', async () => {
|
||||
mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(
|
||||
new Map([[SOURCE_BINDING.id, { status: 'exact', entries: [] }]])
|
||||
)
|
||||
|
||||
await createSingleDocument(
|
||||
{
|
||||
filename: 'source.pdf',
|
||||
fileUrl: SOURCE_URL,
|
||||
fileSize: 512,
|
||||
mimeType: 'application/pdf',
|
||||
},
|
||||
KNOWLEDGE_BASE_ID,
|
||||
'request-1',
|
||||
SOURCE_USER_ID
|
||||
)
|
||||
|
||||
expect(findDocumentProvenanceWrite()).toEqual(
|
||||
expect.objectContaining({ status: 'exact', entries: [] })
|
||||
)
|
||||
})
|
||||
|
||||
it('uses the trusted mothership row context instead of the URL context label', async () => {
|
||||
const mothershipBinding = {
|
||||
...SOURCE_BINDING,
|
||||
id: 'mothership-file-1',
|
||||
context: 'mothership',
|
||||
}
|
||||
mockGetFileMetadataByKeys.mockImplementation(async (_keys: string[], context: string) =>
|
||||
context === 'mothership' ? [mothershipBinding] : []
|
||||
)
|
||||
mockGetBoundWorkspaceFileSecretProvenanceByMetadata.mockResolvedValue(
|
||||
new Map([
|
||||
[
|
||||
mothershipBinding.id,
|
||||
{
|
||||
status: 'exact',
|
||||
entries: [{ name: 'CHAT_SECRET', encryptedValue: 'encrypted-chat-secret' }],
|
||||
},
|
||||
],
|
||||
])
|
||||
)
|
||||
|
||||
await createSingleDocument(
|
||||
{
|
||||
filename: 'source.pdf',
|
||||
fileUrl: SOURCE_URL,
|
||||
fileSize: 512,
|
||||
mimeType: 'application/pdf',
|
||||
},
|
||||
KNOWLEDGE_BASE_ID,
|
||||
'request-1',
|
||||
SOURCE_USER_ID
|
||||
)
|
||||
|
||||
expect(mockGetBoundWorkspaceFileSecretProvenanceByMetadata).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
[mothershipBinding]
|
||||
)
|
||||
expect(findDocumentProvenanceWrite()).toEqual(
|
||||
expect.objectContaining({
|
||||
status: 'exact',
|
||||
entries: [expect.objectContaining({ name: 'CHAT_SECRET' })],
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('preserves legacy behavior when a workspace source has no metadata binding', async () => {
|
||||
mockGetFileMetadataByKeys.mockResolvedValue([])
|
||||
|
||||
await createSingleDocument(
|
||||
{
|
||||
filename: 'legacy.pdf',
|
||||
fileUrl: SOURCE_URL,
|
||||
fileSize: 512,
|
||||
mimeType: 'application/pdf',
|
||||
},
|
||||
KNOWLEDGE_BASE_ID,
|
||||
'request-1',
|
||||
SOURCE_USER_ID
|
||||
)
|
||||
|
||||
expect(mockGetBoundWorkspaceFileSecretProvenanceByMetadata).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
[]
|
||||
)
|
||||
expect(findDocumentProvenanceWrite()).toBeUndefined()
|
||||
})
|
||||
|
||||
it('never deletes a referenced workspace source as knowledge-base storage', async () => {
|
||||
await deleteDocumentStorageFiles(
|
||||
[{ id: 'document-1', fileUrl: SOURCE_URL, workspaceId: WORKSPACE_ID }],
|
||||
'request-1'
|
||||
)
|
||||
|
||||
expect(mockGetFileMetadataByKeys).not.toHaveBeenCalled()
|
||||
expect(mockDeleteFile).not.toHaveBeenCalled()
|
||||
expect(mockDeleteFileMetadataByIdentity).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each(['kb', 'knowledge-base'])(
|
||||
'deletes a trusted %s object only after its metadata identity is claimed',
|
||||
async (keyPrefix) => {
|
||||
const storageKey = `${keyPrefix}/owned.pdf`
|
||||
const fileUrl = `/api/files/serve/${encodeURIComponent(storageKey)}?context=knowledge-base`
|
||||
const binding = {
|
||||
...SOURCE_BINDING,
|
||||
id: `binding-${keyPrefix}`,
|
||||
key: storageKey,
|
||||
context: 'knowledge-base',
|
||||
}
|
||||
mockGetFileMetadataByKeys.mockImplementation(async (_keys: string[], context: string) =>
|
||||
context === 'knowledge-base' ? [binding] : []
|
||||
)
|
||||
mockDeleteFileMetadataByIdentity.mockResolvedValue(true)
|
||||
|
||||
await deleteDocumentStorageFiles(
|
||||
[{ id: 'document-1', fileUrl, workspaceId: WORKSPACE_ID }],
|
||||
'request-1'
|
||||
)
|
||||
|
||||
expect(mockDeleteFileMetadataByIdentity).toHaveBeenCalledWith({
|
||||
id: binding.id,
|
||||
key: storageKey,
|
||||
context: 'knowledge-base',
|
||||
contentUpdatedAt: binding.contentUpdatedAt,
|
||||
})
|
||||
expect(mockDeleteFile).toHaveBeenCalledWith({
|
||||
key: storageKey,
|
||||
context: 'knowledge-base',
|
||||
})
|
||||
expect(mockDeleteFileMetadataByIdentity.mock.invocationCallOrder[0]).toBeLessThan(
|
||||
mockDeleteFile.mock.invocationCallOrder[0]
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
it('keeps the object when its metadata identity changed before deletion', async () => {
|
||||
const storageKey = 'kb/changed.pdf'
|
||||
const fileUrl = `/api/files/serve/${encodeURIComponent(storageKey)}?context=knowledge-base`
|
||||
const binding = {
|
||||
...SOURCE_BINDING,
|
||||
id: 'changed-binding',
|
||||
key: storageKey,
|
||||
context: 'knowledge-base',
|
||||
}
|
||||
mockGetFileMetadataByKeys.mockImplementation(async (_keys: string[], context: string) =>
|
||||
context === 'knowledge-base' ? [binding] : []
|
||||
)
|
||||
mockDeleteFileMetadataByIdentity.mockResolvedValue(false)
|
||||
|
||||
await deleteDocumentStorageFiles(
|
||||
[{ id: 'document-1', fileUrl, workspaceId: WORKSPACE_ID }],
|
||||
'request-1'
|
||||
)
|
||||
|
||||
expect(mockDeleteFileMetadataByIdentity).toHaveBeenCalledOnce()
|
||||
expect(mockDeleteFile).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -1,14 +1,46 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { document } from '@sim/db/schema'
|
||||
import { queueTableRows, resetDbChainMock } from '@sim/testing'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { hashDurableSecretProvenanceValue } from '@/lib/execution/durable-secret-provenance'
|
||||
import {
|
||||
createKnowledgeDocumentSourceValue,
|
||||
loadKnowledgeDocumentSecretRegistry,
|
||||
readBoundKnowledgeDocumentSecretProvenance,
|
||||
} from '@/lib/knowledge/secret-provenance'
|
||||
|
||||
const { mockDecryptSecret } = vi.hoisted(() => ({
|
||||
mockDecryptSecret: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@/lib/core/security/encryption', () => ({
|
||||
decryptSecret: mockDecryptSecret,
|
||||
}))
|
||||
|
||||
const DOCUMENT_SOURCE = createKnowledgeDocumentSourceValue({
|
||||
filename: 'source.pdf',
|
||||
fileUrl: '/api/files/serve/workspace%2Fworkspace-1%2Fsource.pdf?context=workspace',
|
||||
})
|
||||
|
||||
const DOCUMENT_ROW = {
|
||||
id: 'document-1',
|
||||
...DOCUMENT_SOURCE,
|
||||
secretProvenanceVersion: null,
|
||||
provenanceSourceHash: null,
|
||||
status: null,
|
||||
entries: null,
|
||||
}
|
||||
|
||||
describe('knowledge durable secret provenance', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
resetDbChainMock()
|
||||
queueTableRows(document, [DOCUMENT_ROW])
|
||||
mockDecryptSecret.mockResolvedValue({ decrypted: 'tracked-secret' })
|
||||
})
|
||||
|
||||
it('uses the same explicit source shape for joined rows and persisted writes', () => {
|
||||
const source = createKnowledgeDocumentSourceValue({
|
||||
filename: 'file.txt',
|
||||
@@ -51,4 +83,50 @@ describe('knowledge durable secret provenance', () => {
|
||||
})
|
||||
).toEqual({ status: 'exact', entries: [] })
|
||||
})
|
||||
|
||||
it('merges fresh source provenance into the pre-processing registry', async () => {
|
||||
const result = await loadKnowledgeDocumentSecretRegistry(
|
||||
DOCUMENT_ROW.id,
|
||||
{ userId: 'source-user', workspaceId: 'workspace-1' },
|
||||
{
|
||||
status: 'exact',
|
||||
entries: [
|
||||
{
|
||||
name: 'OCR_SECRET',
|
||||
encryptedValue: 'encrypted-secret',
|
||||
sourceUserId: 'source-user',
|
||||
sourceWorkspaceId: 'workspace-1',
|
||||
},
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
expect(result.tracked).toBe(true)
|
||||
expect(result.registry?.getActiveMatches()).toContainEqual(
|
||||
expect.objectContaining({ plaintext: 'tracked-secret' })
|
||||
)
|
||||
})
|
||||
|
||||
it('fails closed when the fresh source classification is unknown', async () => {
|
||||
await expect(
|
||||
loadKnowledgeDocumentSecretRegistry(
|
||||
DOCUMENT_ROW.id,
|
||||
{ userId: 'source-user', workspaceId: 'workspace-1' },
|
||||
{ status: 'unknown' }
|
||||
)
|
||||
).rejects.toThrow('Knowledge document secret provenance is unavailable')
|
||||
})
|
||||
|
||||
it('marks a fresh exact-empty source as tracked without creating a registry', async () => {
|
||||
const result = await loadKnowledgeDocumentSecretRegistry(
|
||||
DOCUMENT_ROW.id,
|
||||
{ userId: 'source-user', workspaceId: 'workspace-1' },
|
||||
{ status: 'exact', entries: [] }
|
||||
)
|
||||
|
||||
expect(result).toEqual({
|
||||
provenance: { status: 'exact', entries: [] },
|
||||
tracked: true,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -404,7 +404,8 @@ export async function replaceKnowledgeEmbeddingSecretProvenanceInTx(
|
||||
/** Loads a document source registry before parsing, OCR, chunking, or embedding. */
|
||||
export async function loadKnowledgeDocumentSecretRegistry(
|
||||
documentId: string,
|
||||
scope: ResolvedSecretTraceScopeV1
|
||||
scope: ResolvedSecretTraceScopeV1,
|
||||
currentSourceFileProvenance?: DurableSecretProvenance
|
||||
): Promise<{
|
||||
registry?: ResolvedSecretTraceRegistry
|
||||
provenance: DurableSecretProvenance
|
||||
@@ -413,15 +414,27 @@ export async function loadKnowledgeDocumentSecretRegistry(
|
||||
const [row] = await selectKnowledgeDocumentProvenanceRows(eq(document.id, documentId)).limit(1)
|
||||
if (!row) throw new Error('Document not found')
|
||||
const source = createKnowledgeDocumentSourceValue(row)
|
||||
const provenance = filterKnowledgeDocumentContentSecretProvenance(
|
||||
const persistedProvenance = filterKnowledgeDocumentContentSecretProvenance(
|
||||
readBoundKnowledgeDocumentSecretProvenance({ ...row, source }),
|
||||
source
|
||||
)
|
||||
const provenance = currentSourceFileProvenance
|
||||
? mergeDurableSecretProvenance(
|
||||
persistedProvenance,
|
||||
bindKnowledgeDocumentFieldSecretProvenance(currentSourceFileProvenance, 'content', {
|
||||
fileUrl: source.fileUrl,
|
||||
contentHash: source.contentHash,
|
||||
})
|
||||
)
|
||||
: persistedProvenance
|
||||
if (provenance.status === 'unknown') {
|
||||
throw new Error('Knowledge document secret provenance is unavailable')
|
||||
}
|
||||
if (provenance.entries.length === 0)
|
||||
return { provenance, tracked: row.secretProvenanceVersion === 1 }
|
||||
return {
|
||||
provenance,
|
||||
tracked: row.secretProvenanceVersion === 1 || currentSourceFileProvenance !== undefined,
|
||||
}
|
||||
const registry = new ResolvedSecretTraceRegistry([], scope)
|
||||
if (!(await importDurableSecretProvenance(registry, provenance))) {
|
||||
throw new Error('Knowledge document secret provenance is unavailable')
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { document, embedding, knowledgeBase, knowledgeBaseTagDefinitions } from '@sim/db/schema'
|
||||
import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
deleteAllTagDefinitions,
|
||||
deleteTagDefinition,
|
||||
KnowledgeTagProvenanceConflictError,
|
||||
} from '@/lib/knowledge/tags/service'
|
||||
|
||||
const KNOWLEDGE_BASE_ID = 'knowledge-base-1'
|
||||
const TAG_DEFINITION = {
|
||||
id: 'tag-definition-1',
|
||||
knowledgeBaseId: KNOWLEDGE_BASE_ID,
|
||||
tagSlot: 'tag1',
|
||||
displayName: 'Classification',
|
||||
}
|
||||
|
||||
function queueSingleTagDeletion(conflict: boolean): void {
|
||||
queueTableRows(knowledgeBase, [{ id: KNOWLEDGE_BASE_ID }])
|
||||
queueTableRows(knowledgeBaseTagDefinitions, [TAG_DEFINITION])
|
||||
queueTableRows(document, conflict ? [{ id: 'document-1' }] : [])
|
||||
}
|
||||
|
||||
describe('knowledge tag deletion provenance', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
resetDbChainMock()
|
||||
})
|
||||
|
||||
it('uses one bounded transaction and preserves the established bulk cleanup path', async () => {
|
||||
queueSingleTagDeletion(false)
|
||||
|
||||
await expect(
|
||||
deleteTagDefinition(KNOWLEDGE_BASE_ID, TAG_DEFINITION.id, 'request-1')
|
||||
).resolves.toEqual({ tagSlot: 'tag1', displayName: 'Classification' })
|
||||
|
||||
expect(dbChainMockFns.transaction).toHaveBeenCalledTimes(1)
|
||||
expect(dbChainMockFns.execute).toHaveBeenCalledTimes(1)
|
||||
expect(JSON.stringify(dbChainMockFns.execute.mock.calls[0]?.[0])).toContain('statement_timeout')
|
||||
|
||||
const selectedTables = dbChainMockFns.from.mock.calls.map(([table]) => table)
|
||||
expect(selectedTables).toEqual([knowledgeBase, knowledgeBaseTagDefinitions, document])
|
||||
expect(dbChainMockFns.for).toHaveBeenCalledWith('update', { of: document })
|
||||
expect(dbChainMockFns.update.mock.calls.map(([table]) => table)).toEqual([embedding, document])
|
||||
expect(dbChainMockFns.set).toHaveBeenCalledWith({ tag1: null })
|
||||
expect(dbChainMockFns.delete).toHaveBeenCalledWith(knowledgeBaseTagDefinitions)
|
||||
})
|
||||
|
||||
it('rejects uncertain or nonempty tracked provenance before mutating data', async () => {
|
||||
queueSingleTagDeletion(true)
|
||||
|
||||
await expect(
|
||||
deleteTagDefinition(KNOWLEDGE_BASE_ID, TAG_DEFINITION.id, 'request-1')
|
||||
).rejects.toBeInstanceOf(KnowledgeTagProvenanceConflictError)
|
||||
|
||||
expect(dbChainMockFns.update).not.toHaveBeenCalled()
|
||||
expect(dbChainMockFns.delete).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('clears every bounded tag slot through the same guarded mutation path', async () => {
|
||||
queueTableRows(knowledgeBase, [{ id: KNOWLEDGE_BASE_ID }])
|
||||
queueTableRows(knowledgeBaseTagDefinitions, [
|
||||
{ id: 'tag-definition-1', tagSlot: 'tag1' },
|
||||
{ id: 'tag-definition-2', tagSlot: 'number1' },
|
||||
])
|
||||
queueTableRows(document, [])
|
||||
|
||||
await expect(deleteAllTagDefinitions(KNOWLEDGE_BASE_ID, 'request-1')).resolves.toBe(2)
|
||||
|
||||
expect(dbChainMockFns.transaction).toHaveBeenCalledTimes(1)
|
||||
expect(dbChainMockFns.execute).toHaveBeenCalledTimes(1)
|
||||
expect(dbChainMockFns.set).toHaveBeenCalledWith({ tag1: null, number1: null })
|
||||
})
|
||||
})
|
||||
@@ -1,9 +1,15 @@
|
||||
import { db } from '@sim/db'
|
||||
import { document, embedding, knowledgeBaseTagDefinitions } from '@sim/db/schema'
|
||||
import {
|
||||
document,
|
||||
documentSecretProvenance,
|
||||
embedding,
|
||||
knowledgeBase,
|
||||
knowledgeBaseTagDefinitions,
|
||||
} from '@sim/db/schema'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { generateId } from '@sim/utils/id'
|
||||
import { and, eq, isNotNull, isNull, sql } from 'drizzle-orm'
|
||||
import type { DbOrTx } from '@/lib/db/types'
|
||||
import { and, eq, isNotNull, isNull, or, sql } from 'drizzle-orm'
|
||||
import type { DbOrTx, DbTransaction } from '@/lib/db/types'
|
||||
import { getSlotsForFieldType, SUPPORTED_FIELD_TYPES } from '@/lib/knowledge/constants'
|
||||
import type { BulkTagDefinitionsData, DocumentTagDefinition } from '@/lib/knowledge/tags/types'
|
||||
import type {
|
||||
@@ -32,6 +38,17 @@ const VALID_TAG_SLOTS = [
|
||||
] as const
|
||||
|
||||
type ValidTagSlot = (typeof VALID_TAG_SLOTS)[number]
|
||||
type ClearedTagValues = Partial<Record<ValidTagSlot, null>>
|
||||
const TAG_MUTATION_STATEMENT_TIMEOUT_MS = 120_000
|
||||
const TAG_MUTATION_LOCK_TIMEOUT_MS = 5_000
|
||||
const TAG_MUTATION_IDLE_TIMEOUT_MS = 30_000
|
||||
|
||||
export class KnowledgeTagProvenanceConflictError extends Error {
|
||||
constructor() {
|
||||
super('Tag definitions cannot be deleted while resolved-secret document provenance is present')
|
||||
this.name = 'KnowledgeTagProvenanceConflictError'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that a tag slot is a valid slot name
|
||||
@@ -42,6 +59,87 @@ function validateTagSlot(tagSlot: string): asserts tagSlot is ValidTagSlot {
|
||||
}
|
||||
}
|
||||
|
||||
/** Applies transaction-scoped bounds that are safe with pooled Postgres connections. */
|
||||
async function setTagMutationTransactionTimeouts(tx: DbTransaction): Promise<void> {
|
||||
await tx.execute(
|
||||
sql`SELECT set_config('statement_timeout', ${`${TAG_MUTATION_STATEMENT_TIMEOUT_MS}ms`}, true),
|
||||
set_config('lock_timeout', ${`${TAG_MUTATION_LOCK_TIMEOUT_MS}ms`}, true),
|
||||
set_config('idle_in_transaction_session_timeout', ${`${TAG_MUTATION_IDLE_TIMEOUT_MS}ms`}, true)`
|
||||
)
|
||||
}
|
||||
|
||||
/** Serializes tag deletion with document creation, which locks the same KB row first. */
|
||||
async function lockKnowledgeBaseForTagMutation(
|
||||
tx: DbTransaction,
|
||||
knowledgeBaseId: string
|
||||
): Promise<void> {
|
||||
await tx
|
||||
.select({ id: knowledgeBase.id })
|
||||
.from(knowledgeBase)
|
||||
.where(eq(knowledgeBase.id, knowledgeBaseId))
|
||||
.limit(1)
|
||||
.for('update')
|
||||
}
|
||||
|
||||
/** Prevents tag cleanup from demoting nonempty or unverifiable provenance to legacy state. */
|
||||
async function assertKnowledgeBaseTagsCanBeClearedInTx(
|
||||
tx: DbTransaction,
|
||||
knowledgeBaseId: string
|
||||
): Promise<void> {
|
||||
const [conflict] = await tx
|
||||
.select({ id: document.id })
|
||||
.from(document)
|
||||
.leftJoin(documentSecretProvenance, eq(documentSecretProvenance.documentId, document.id))
|
||||
.where(
|
||||
and(
|
||||
eq(document.knowledgeBaseId, knowledgeBaseId),
|
||||
eq(document.secretProvenanceVersion, 1),
|
||||
or(
|
||||
isNull(documentSecretProvenance.documentId),
|
||||
sql`${documentSecretProvenance.status} IS DISTINCT FROM 'exact'`,
|
||||
sql`${documentSecretProvenance.entries} <> '[]'::jsonb`
|
||||
)
|
||||
)
|
||||
)
|
||||
.limit(1)
|
||||
.for('update', { of: document })
|
||||
|
||||
if (conflict) throw new KnowledgeTagProvenanceConflictError()
|
||||
}
|
||||
|
||||
async function clearTagSlotsInTx(
|
||||
tx: DbTransaction,
|
||||
knowledgeBaseId: string,
|
||||
tagSlots: readonly ValidTagSlot[]
|
||||
): Promise<void> {
|
||||
if (tagSlots.length === 0) return
|
||||
|
||||
const clearedTagValues: ClearedTagValues = {}
|
||||
for (const tagSlot of tagSlots) clearedTagValues[tagSlot] = null
|
||||
|
||||
await assertKnowledgeBaseTagsCanBeClearedInTx(tx, knowledgeBaseId)
|
||||
|
||||
await tx
|
||||
.update(embedding)
|
||||
.set(clearedTagValues)
|
||||
.where(
|
||||
and(
|
||||
eq(embedding.knowledgeBaseId, knowledgeBaseId),
|
||||
or(...tagSlots.map((tagSlot) => isNotNull(embedding[tagSlot])))
|
||||
)
|
||||
)
|
||||
|
||||
await tx
|
||||
.update(document)
|
||||
.set(clearedTagValues)
|
||||
.where(
|
||||
and(
|
||||
eq(document.knowledgeBaseId, knowledgeBaseId),
|
||||
or(...tagSlots.map((tagSlot) => isNotNull(document[tagSlot])))
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the field type for a tag slot
|
||||
*/
|
||||
@@ -418,37 +516,30 @@ export async function deleteAllTagDefinitions(
|
||||
knowledgeBaseId: string,
|
||||
requestId: string
|
||||
): Promise<number> {
|
||||
const definitions = await db
|
||||
.select({ id: knowledgeBaseTagDefinitions.id, tagSlot: knowledgeBaseTagDefinitions.tagSlot })
|
||||
.from(knowledgeBaseTagDefinitions)
|
||||
.where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))
|
||||
const deletedCount = await db.transaction(async (tx) => {
|
||||
await setTagMutationTransactionTimeouts(tx)
|
||||
await lockKnowledgeBaseForTagMutation(tx, knowledgeBaseId)
|
||||
|
||||
await db.transaction(async (tx) => {
|
||||
for (const definition of definitions) {
|
||||
const definitions = await tx
|
||||
.select({ id: knowledgeBaseTagDefinitions.id, tagSlot: knowledgeBaseTagDefinitions.tagSlot })
|
||||
.from(knowledgeBaseTagDefinitions)
|
||||
.where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))
|
||||
.for('update')
|
||||
const tagSlots = definitions.map((definition) => {
|
||||
const tagSlot = definition.tagSlot as string
|
||||
validateTagSlot(tagSlot)
|
||||
return tagSlot
|
||||
})
|
||||
|
||||
await tx
|
||||
.update(document)
|
||||
.set({ [tagSlot]: null })
|
||||
.where(
|
||||
and(eq(document.knowledgeBaseId, knowledgeBaseId), isNotNull(sql`${sql.raw(tagSlot)}`))
|
||||
)
|
||||
|
||||
await tx
|
||||
.update(embedding)
|
||||
.set({ [tagSlot]: null })
|
||||
.where(
|
||||
and(eq(embedding.knowledgeBaseId, knowledgeBaseId), isNotNull(sql`${sql.raw(tagSlot)}`))
|
||||
)
|
||||
}
|
||||
await clearTagSlotsInTx(tx, knowledgeBaseId, tagSlots)
|
||||
|
||||
await tx
|
||||
.delete(knowledgeBaseTagDefinitions)
|
||||
.where(eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId))
|
||||
|
||||
return definitions.length
|
||||
})
|
||||
|
||||
const deletedCount = definitions.length
|
||||
logger.info(`[${requestId}] Deleted ${deletedCount} tag definitions for KB: ${knowledgeBaseId}`)
|
||||
|
||||
return deletedCount
|
||||
@@ -463,58 +554,41 @@ export async function deleteTagDefinition(
|
||||
tagDefinitionId: string,
|
||||
requestId: string
|
||||
): Promise<{ tagSlot: string; displayName: string }> {
|
||||
const tagDef = await db
|
||||
.select({
|
||||
id: knowledgeBaseTagDefinitions.id,
|
||||
knowledgeBaseId: knowledgeBaseTagDefinitions.knowledgeBaseId,
|
||||
tagSlot: knowledgeBaseTagDefinitions.tagSlot,
|
||||
displayName: knowledgeBaseTagDefinitions.displayName,
|
||||
})
|
||||
.from(knowledgeBaseTagDefinitions)
|
||||
.where(
|
||||
and(
|
||||
eq(knowledgeBaseTagDefinitions.id, tagDefinitionId),
|
||||
eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId)
|
||||
)
|
||||
)
|
||||
.limit(1)
|
||||
const definition = await db.transaction(async (tx) => {
|
||||
await setTagMutationTransactionTimeouts(tx)
|
||||
await lockKnowledgeBaseForTagMutation(tx, knowledgeBaseId)
|
||||
|
||||
if (tagDef.length === 0) {
|
||||
throw new Error(`Tag definition ${tagDefinitionId} not found`)
|
||||
}
|
||||
|
||||
const definition = tagDef[0]
|
||||
const definitionKnowledgeBaseId = definition.knowledgeBaseId
|
||||
const tagSlot = definition.tagSlot as string
|
||||
|
||||
validateTagSlot(tagSlot)
|
||||
|
||||
await db.transaction(async (tx) => {
|
||||
await tx
|
||||
.update(document)
|
||||
.set({ [tagSlot]: null })
|
||||
const [tagDef] = await tx
|
||||
.select({
|
||||
id: knowledgeBaseTagDefinitions.id,
|
||||
knowledgeBaseId: knowledgeBaseTagDefinitions.knowledgeBaseId,
|
||||
tagSlot: knowledgeBaseTagDefinitions.tagSlot,
|
||||
displayName: knowledgeBaseTagDefinitions.displayName,
|
||||
})
|
||||
.from(knowledgeBaseTagDefinitions)
|
||||
.where(
|
||||
and(
|
||||
eq(document.knowledgeBaseId, definitionKnowledgeBaseId),
|
||||
isNotNull(sql`${sql.raw(tagSlot)}`)
|
||||
eq(knowledgeBaseTagDefinitions.id, tagDefinitionId),
|
||||
eq(knowledgeBaseTagDefinitions.knowledgeBaseId, knowledgeBaseId)
|
||||
)
|
||||
)
|
||||
.limit(1)
|
||||
.for('update')
|
||||
if (!tagDef) throw new Error(`Tag definition ${tagDefinitionId} not found`)
|
||||
|
||||
await tx
|
||||
.update(embedding)
|
||||
.set({ [tagSlot]: null })
|
||||
.where(
|
||||
and(
|
||||
eq(embedding.knowledgeBaseId, definitionKnowledgeBaseId),
|
||||
isNotNull(sql`${sql.raw(tagSlot)}`)
|
||||
)
|
||||
)
|
||||
const tagSlot = tagDef.tagSlot as string
|
||||
validateTagSlot(tagSlot)
|
||||
await clearTagSlotsInTx(tx, tagDef.knowledgeBaseId, [tagSlot])
|
||||
|
||||
await tx
|
||||
.delete(knowledgeBaseTagDefinitions)
|
||||
.where(eq(knowledgeBaseTagDefinitions.id, tagDefinitionId))
|
||||
|
||||
return tagDef
|
||||
})
|
||||
|
||||
const tagSlot = definition.tagSlot as string
|
||||
|
||||
logger.info(
|
||||
`[${requestId}] Deleted tag definition with cleanup: ${definition.displayName} (${tagSlot})`
|
||||
)
|
||||
|
||||
@@ -141,12 +141,7 @@ describe('trackChatUpload', () => {
|
||||
displayName: 'image.png',
|
||||
})
|
||||
)
|
||||
expect(mockReplaceWorkspaceFileSecretProvenanceInTx).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
'wf_existing',
|
||||
CONTENT_UPDATED_AT,
|
||||
{ status: 'exact', entries: [] }
|
||||
)
|
||||
expect(mockReplaceWorkspaceFileSecretProvenanceInTx).not.toHaveBeenCalled()
|
||||
expectNoWorkspaceStorageAccounting()
|
||||
})
|
||||
|
||||
@@ -173,16 +168,11 @@ describe('trackChatUpload', () => {
|
||||
displayName: 'image.png',
|
||||
})
|
||||
)
|
||||
expect(mockReplaceWorkspaceFileSecretProvenanceInTx).toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
'wf_inserted',
|
||||
CONTENT_UPDATED_AT,
|
||||
{ status: 'exact', entries: [] }
|
||||
)
|
||||
expect(mockReplaceWorkspaceFileSecretProvenanceInTx).not.toHaveBeenCalled()
|
||||
expectNoWorkspaceStorageAccounting()
|
||||
})
|
||||
|
||||
it('fails the chat binding transaction when provenance classification fails', async () => {
|
||||
it('does not reclassify uploaded bytes while linking chat metadata', async () => {
|
||||
queueOwnershipLookup([existingRow()])
|
||||
dbChainMockFns.returning.mockResolvedValueOnce([
|
||||
{ id: 'wf_existing', contentUpdatedAt: CONTENT_UPDATED_AT },
|
||||
@@ -193,9 +183,10 @@ describe('trackChatUpload', () => {
|
||||
|
||||
await expect(
|
||||
trackChatUpload(WORKSPACE_ID, USER_ID, CHAT_ID, S3_KEY, 'image.png', 'image/png', 1024)
|
||||
).rejects.toThrow('provenance write failed')
|
||||
).resolves.toEqual({ displayName: 'image.png' })
|
||||
|
||||
expect(dbChainMockFns.transaction).toHaveBeenCalledTimes(1)
|
||||
expect(mockReplaceWorkspaceFileSecretProvenanceInTx).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('stamps message_id on the UPDATE arm when the birth message is known', async () => {
|
||||
|
||||
@@ -727,6 +727,9 @@ async function resolveClaimableChatUploadRow(
|
||||
* Allocates a collision-free `displayName` (the partial unique index on
|
||||
* (chat_id, display_name) WHERE context='mothership' enforces this) and returns it
|
||||
* so callers can surface the same name to the model in the VFS read hint.
|
||||
* This is a metadata-only operation: it preserves any content provenance already
|
||||
* attached to the uploaded bytes. Direct user uploads use the established
|
||||
* exact-empty/legacy classification and do not need a chat-time reclassification.
|
||||
*/
|
||||
export async function trackChatUpload(
|
||||
workspaceId: string,
|
||||
@@ -793,10 +796,7 @@ export async function trackChatUpload(
|
||||
or(isNull(workspaceFiles.chatId), eq(workspaceFiles.chatId, chatId))
|
||||
)
|
||||
)
|
||||
.returning({
|
||||
id: workspaceFiles.id,
|
||||
contentUpdatedAt: workspaceFiles.contentUpdatedAt,
|
||||
})
|
||||
.returning({ id: workspaceFiles.id })
|
||||
|
||||
if (updated.length === 0) {
|
||||
// The ownership lookup is a separate statement, so re-assert every
|
||||
@@ -807,13 +807,6 @@ export async function trackChatUpload(
|
||||
// to the chat-delete cascade.
|
||||
throw new WorkspaceFileKeyOwnershipError(s3Key)
|
||||
}
|
||||
|
||||
await replaceWorkspaceFileSecretProvenanceInTx(
|
||||
tx,
|
||||
updated[0].id,
|
||||
updated[0].contentUpdatedAt,
|
||||
EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE
|
||||
)
|
||||
})
|
||||
|
||||
logger.info(
|
||||
@@ -840,20 +833,11 @@ export async function trackChatUpload(
|
||||
contentType,
|
||||
size,
|
||||
})
|
||||
.returning({
|
||||
id: workspaceFiles.id,
|
||||
contentUpdatedAt: workspaceFiles.contentUpdatedAt,
|
||||
})
|
||||
.returning({ id: workspaceFiles.id })
|
||||
|
||||
if (!inserted) {
|
||||
throw new Error(`Failed to track chat upload for key: ${s3Key}`)
|
||||
}
|
||||
await replaceWorkspaceFileSecretProvenanceInTx(
|
||||
tx,
|
||||
inserted.id,
|
||||
inserted.contentUpdatedAt,
|
||||
EXACT_EMPTY_WORKSPACE_FILE_SECRET_PROVENANCE
|
||||
)
|
||||
})
|
||||
|
||||
logger.info(`Tracked chat upload: ${fileName} (display: ${candidate}) for chat ${chatId}`)
|
||||
|
||||
@@ -93,6 +93,36 @@ describe('workspace file secret provenance', () => {
|
||||
).rejects.toThrow('could not bind the tracked content version')
|
||||
})
|
||||
|
||||
it('binds the database content version within its JavaScript-visible millisecond', async () => {
|
||||
await replaceWorkspaceFileSecretProvenanceInTx(
|
||||
dbChainMock.db as unknown as DbTransaction,
|
||||
'file-1',
|
||||
CONTENT_UPDATED_AT,
|
||||
{ status: 'exact', entries: [] }
|
||||
)
|
||||
|
||||
expect(dbChainMockFns.where.mock.calls.at(-1)?.[0]).toEqual({
|
||||
type: 'and',
|
||||
conditions: [
|
||||
{ type: 'eq', left: 'id', right: 'file-1' },
|
||||
{ type: 'gte', left: 'contentUpdatedAt', right: CONTENT_UPDATED_AT },
|
||||
{
|
||||
type: 'lt',
|
||||
left: 'contentUpdatedAt',
|
||||
right: new Date(CONTENT_UPDATED_AT.getTime() + 1),
|
||||
},
|
||||
{ type: 'inArray', column: 'context', values: ['workspace', 'mothership'] },
|
||||
{
|
||||
type: 'or',
|
||||
conditions: [
|
||||
{ type: 'isNull', column: 'secretProvenanceVersion' },
|
||||
{ type: 'eq', left: 'secretProvenanceVersion', right: 1 },
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it('preserves provenance only from the exact preceding content version', async () => {
|
||||
const nextContentUpdatedAt = new Date('2026-08-04T00:00:01.000Z')
|
||||
queueTableRows(workspaceFileSecretProvenance, [{ contentUpdatedAt: CONTENT_UPDATED_AT }])
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
workspaceFileSecretProvenance,
|
||||
workspaceFiles,
|
||||
} from '@sim/db/schema'
|
||||
import { and, eq, inArray, isNull, or } from 'drizzle-orm'
|
||||
import { and, eq, gte, inArray, isNull, lt, or } from 'drizzle-orm'
|
||||
import type { DbTransaction } from '@/lib/db/types'
|
||||
import { importDurableSecretProvenance } from '@/lib/execution/durable-secret-provenance'
|
||||
import type { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
|
||||
@@ -167,13 +167,15 @@ async function markWorkspaceFileSecretProvenanceTrackedInTx(
|
||||
fileId: string,
|
||||
contentUpdatedAt: Date
|
||||
): Promise<void> {
|
||||
const nextContentMillisecond = new Date(contentUpdatedAt.getTime() + 1)
|
||||
const [tracked] = await tx
|
||||
.update(workspaceFiles)
|
||||
.set({ secretProvenanceVersion: 1 })
|
||||
.where(
|
||||
and(
|
||||
eq(workspaceFiles.id, fileId),
|
||||
eq(workspaceFiles.contentUpdatedAt, contentUpdatedAt),
|
||||
gte(workspaceFiles.contentUpdatedAt, contentUpdatedAt),
|
||||
lt(workspaceFiles.contentUpdatedAt, nextContentMillisecond),
|
||||
inArray(workspaceFiles.context, ['workspace', 'mothership']),
|
||||
or(
|
||||
isNull(workspaceFiles.secretProvenanceVersion),
|
||||
|
||||
@@ -3,14 +3,49 @@
|
||||
*/
|
||||
import { workspaceFiles } from '@sim/db/schema'
|
||||
import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing'
|
||||
import type { SQL } from 'drizzle-orm'
|
||||
import { PgDialect } from 'drizzle-orm/pg-core'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
vi.unmock('@sim/db/schema')
|
||||
vi.unmock('drizzle-orm')
|
||||
|
||||
import {
|
||||
ActiveFileMetadataKeyConflictError,
|
||||
deleteFileMetadataByIdentity,
|
||||
insertFileMetadata,
|
||||
insertFileMetadataMany,
|
||||
insertImmutableFileMetadata,
|
||||
} from '@/lib/uploads/server/metadata'
|
||||
|
||||
describe('deleteFileMetadataByIdentity', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
resetDbChainMock()
|
||||
})
|
||||
|
||||
it('reports whether the exact active file version was soft-deleted', async () => {
|
||||
const identity = {
|
||||
id: 'file-1',
|
||||
key: 'kb/workspace-1/file.pdf',
|
||||
context: 'knowledge-base' as const,
|
||||
contentUpdatedAt: new Date('2026-08-05T00:00:00.000Z'),
|
||||
}
|
||||
dbChainMockFns.returning.mockResolvedValueOnce([{ id: identity.id }])
|
||||
|
||||
await expect(deleteFileMetadataByIdentity(identity)).resolves.toBe(true)
|
||||
const predicate = dbChainMockFns.where.mock.calls[0]?.[0] as SQL
|
||||
const query = new PgDialect().sqlToQuery(predicate)
|
||||
expect(query.sql).toContain(
|
||||
`date_trunc('milliseconds', "workspace_files"."content_updated_at")`
|
||||
)
|
||||
expect(query.params).toContain(identity.contentUpdatedAt)
|
||||
|
||||
dbChainMockFns.returning.mockResolvedValueOnce([])
|
||||
await expect(deleteFileMetadataByIdentity(identity)).resolves.toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('insertFileMetadata content versions', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
|
||||
@@ -353,6 +353,36 @@ export async function deleteFileMetadata(key: string): Promise<boolean> {
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* Soft-deletes only the active metadata version previously authorized by a caller.
|
||||
* Postgres timestamps are compared at JavaScript `Date` precision because a selected
|
||||
* microsecond timestamp has already been rounded to milliseconds at this boundary.
|
||||
*/
|
||||
export async function deleteFileMetadataByIdentity(identity: {
|
||||
id: string
|
||||
key: string
|
||||
context: StorageContext
|
||||
contentUpdatedAt: Date
|
||||
}): Promise<boolean> {
|
||||
const deleted = await db
|
||||
.update(workspaceFiles)
|
||||
.set({ deletedAt: new Date() })
|
||||
.where(
|
||||
and(
|
||||
eq(workspaceFiles.id, identity.id),
|
||||
eq(workspaceFiles.key, identity.key),
|
||||
eq(workspaceFiles.context, identity.context),
|
||||
eq(
|
||||
sql<Date>`date_trunc('milliseconds', ${workspaceFiles.contentUpdatedAt})`,
|
||||
identity.contentUpdatedAt
|
||||
),
|
||||
isNull(workspaceFiles.deletedAt)
|
||||
)
|
||||
)
|
||||
.returning({ id: workspaceFiles.id })
|
||||
return deleted.length === 1
|
||||
}
|
||||
|
||||
/**
|
||||
* Fields needed to record a trusted storage-key -> workspace ownership binding
|
||||
* for a knowledge-base file. The `context` is always `'knowledge-base'`, so it is
|
||||
|
||||
@@ -832,37 +832,35 @@ describe('executeProviderRequest — model secret projection', () => {
|
||||
expect(mockExecuteRequest.mock.calls[0][0].messages[0].files).toEqual([safe])
|
||||
})
|
||||
|
||||
it('continues text-only when file provenance lookup is unavailable', async () => {
|
||||
it('fails explicitly when file provenance lookup is unavailable', async () => {
|
||||
mockFilterModelSafeWorkspaceFileAttachments.mockRejectedValueOnce(new Error('db unavailable'))
|
||||
|
||||
await executeProviderRequest('openai', {
|
||||
model: 'test-model',
|
||||
workspaceId: 'ws-1',
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: 'Continue without the file',
|
||||
files: [
|
||||
{
|
||||
id: 'wf-file',
|
||||
name: 'file.txt',
|
||||
url: '/file',
|
||||
size: 10,
|
||||
type: 'text/plain',
|
||||
key: 'workspace/ws-1/file.txt',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
expect(mockExecuteRequest).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
await expect(
|
||||
executeProviderRequest('openai', {
|
||||
model: 'test-model',
|
||||
workspaceId: 'ws-1',
|
||||
messages: [
|
||||
expect.objectContaining({ content: 'Continue without the file', files: undefined }),
|
||||
{
|
||||
role: 'user',
|
||||
content: 'Review the file',
|
||||
files: [
|
||||
{
|
||||
id: 'wf-file',
|
||||
name: 'file.txt',
|
||||
url: '/file',
|
||||
size: 10,
|
||||
type: 'text/plain',
|
||||
key: 'workspace/ws-1/file.txt',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
})
|
||||
)
|
||||
).rejects.toThrow('File attachments could not be verified for model use')
|
||||
|
||||
expect(mockAttachLargeFileRemoteUrls).not.toHaveBeenCalled()
|
||||
expect(mockUploadLargeFilesToProvider).not.toHaveBeenCalled()
|
||||
expect(mockExecuteRequest).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('projects JSON arguments without mutating attachment metadata before serialization', async () => {
|
||||
|
||||
@@ -298,11 +298,11 @@ async function omitUnsafeProviderFileAttachments(
|
||||
workspaceId: request.workspaceId,
|
||||
})
|
||||
} catch (error) {
|
||||
logger.warn('Workspace file secret provenance could not be verified; omitting attachments', {
|
||||
logger.error('Workspace file secret provenance could not be verified', {
|
||||
attachmentCount: attachments.length,
|
||||
error: toError(error).message,
|
||||
})
|
||||
safeAttachments = []
|
||||
throw new Error('File attachments could not be verified for model use')
|
||||
}
|
||||
|
||||
if (safeAttachments.length === attachments.length) return request
|
||||
|
||||
Reference in New Issue
Block a user