mirror of
https://github.com/simstudioai/sim.git
synced 2026-09-24 15:45:35 +08:00
improvement(agent, file-block): files in agent block, file block v4 (#4610)
* File block v4 * Support files in agent block * Clean up attachments * Fix start.files * Fix * Fix
This commit is contained in:
@@ -258,6 +258,7 @@ export const blockTypeToIconMap: Record<string, IconComponent> = {
|
||||
extend_v2: ExtendIcon,
|
||||
fathom: FathomIcon,
|
||||
file_v3: DocumentIcon,
|
||||
file_v4: DocumentIcon,
|
||||
firecrawl: FirecrawlIcon,
|
||||
fireflies_v2: FirefliesIcon,
|
||||
gamma: GammaIcon,
|
||||
|
||||
@@ -4032,18 +4032,22 @@
|
||||
"tags": ["meeting", "note-taking"]
|
||||
},
|
||||
{
|
||||
"type": "file_v3",
|
||||
"type": "file_v4",
|
||||
"slug": "file",
|
||||
"name": "File",
|
||||
"description": "Read and write workspace files",
|
||||
"longDescription": "Read and parse files from uploads or URLs, write new workspace files, or append content to existing files.",
|
||||
"description": "Read, fetch, write, and append files",
|
||||
"longDescription": "Read workspace files by picker or canonical ID, fetch and parse files from URLs with optional headers, write new workspace files, or append content to existing files.",
|
||||
"bgColor": "#40916C",
|
||||
"iconName": "DocumentIcon",
|
||||
"docsUrl": "https://docs.sim.ai/tools/file",
|
||||
"operations": [
|
||||
{
|
||||
"name": "Read",
|
||||
"description": "Parse one or more uploaded files or files from URLs (text, PDF, CSV, images, etc.)"
|
||||
"description": "Get a workspace file object from a selected file or canonical workspace file ID."
|
||||
},
|
||||
{
|
||||
"name": "Fetch",
|
||||
"description": "Parse a file from a URL with optional custom headers for authenticated downloads."
|
||||
},
|
||||
{
|
||||
"name": "Write",
|
||||
@@ -4054,7 +4058,7 @@
|
||||
"description": "Append content to an existing workspace file. The file must already exist. Content is added to the end of the file."
|
||||
}
|
||||
],
|
||||
"operationCount": 3,
|
||||
"operationCount": 4,
|
||||
"triggers": [],
|
||||
"triggerCount": 0,
|
||||
"authType": "none",
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
createMockRequest,
|
||||
hybridAuthMockFns,
|
||||
inputValidationMock,
|
||||
inputValidationMockFns,
|
||||
permissionsMock,
|
||||
permissionsMockFns,
|
||||
storageServiceMock,
|
||||
@@ -310,6 +311,39 @@ describe('File Parse API Route', () => {
|
||||
expect(data.results).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('should pass custom headers when fetching external URLs', async () => {
|
||||
inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({
|
||||
isValid: true,
|
||||
resolvedIP: '203.0.113.10',
|
||||
})
|
||||
inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue(
|
||||
new Response('private file content', {
|
||||
status: 200,
|
||||
headers: { 'content-type': 'text/plain' },
|
||||
})
|
||||
)
|
||||
|
||||
const headers = { Authorization: 'Bearer xoxb-test-token' }
|
||||
const req = createMockRequest('POST', {
|
||||
filePath: 'https://files.slack.com/files-pri/T000-F000/download/report.txt',
|
||||
headers,
|
||||
})
|
||||
|
||||
const response = await POST(req)
|
||||
const data = await response.json()
|
||||
|
||||
expect(response.status).toBe(200)
|
||||
expect(data.success).toBe(true)
|
||||
expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledWith(
|
||||
'https://files.slack.com/files-pri/T000-F000/download/report.txt',
|
||||
'203.0.113.10',
|
||||
expect.objectContaining({
|
||||
timeout: 30000,
|
||||
headers,
|
||||
})
|
||||
)
|
||||
})
|
||||
|
||||
it('should process execution file URLs with context query param', async () => {
|
||||
setupFileApiMocks({
|
||||
cloudEnabled: true,
|
||||
|
||||
@@ -110,7 +110,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
)
|
||||
if (!parsed.success) return parsed.response
|
||||
|
||||
const { filePath, fileType, workspaceId, workflowId, executionId } = parsed.data.body
|
||||
const { filePath, fileType, headers, workspaceId, workflowId, executionId } = parsed.data.body
|
||||
|
||||
if (!filePath || (typeof filePath === 'string' && filePath.trim() === '')) {
|
||||
return NextResponse.json({ success: false, error: 'No file path provided' }, { status: 400 })
|
||||
@@ -128,6 +128,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
workspaceId,
|
||||
userId,
|
||||
hasExecutionContext: !!executionContext,
|
||||
hasHeaders: Boolean(headers && Object.keys(headers).length > 0),
|
||||
})
|
||||
|
||||
if (Array.isArray(filePath)) {
|
||||
@@ -146,7 +147,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
fileType,
|
||||
workspaceId,
|
||||
userId,
|
||||
executionContext
|
||||
executionContext,
|
||||
headers
|
||||
)
|
||||
if (result.metadata) {
|
||||
result.metadata.processingTime = Date.now() - startTime
|
||||
@@ -180,7 +182,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
})
|
||||
}
|
||||
|
||||
const result = await parseFileSingle(filePath, fileType, workspaceId, userId, executionContext)
|
||||
const result = await parseFileSingle(
|
||||
filePath,
|
||||
fileType,
|
||||
workspaceId,
|
||||
userId,
|
||||
executionContext,
|
||||
headers
|
||||
)
|
||||
|
||||
if (result.metadata) {
|
||||
result.metadata.processingTime = Date.now() - startTime
|
||||
@@ -225,7 +234,8 @@ async function parseFileSingle(
|
||||
fileType: string,
|
||||
workspaceId: string,
|
||||
userId: string,
|
||||
executionContext?: ExecutionContext
|
||||
executionContext?: ExecutionContext,
|
||||
headers?: Record<string, string>
|
||||
): Promise<ParseResult> {
|
||||
logger.info('Parsing file:', filePath)
|
||||
|
||||
@@ -251,7 +261,7 @@ async function parseFileSingle(
|
||||
}
|
||||
|
||||
if (filePath.startsWith('http://') || filePath.startsWith('https://')) {
|
||||
return handleExternalUrl(filePath, fileType, workspaceId, userId, executionContext)
|
||||
return handleExternalUrl(filePath, fileType, workspaceId, userId, executionContext, headers)
|
||||
}
|
||||
|
||||
if (isUsingCloudStorage()) {
|
||||
@@ -298,7 +308,8 @@ async function handleExternalUrl(
|
||||
fileType: string,
|
||||
workspaceId: string,
|
||||
userId: string,
|
||||
executionContext?: ExecutionContext
|
||||
executionContext?: ExecutionContext,
|
||||
headers?: Record<string, string>
|
||||
): Promise<ParseResult> {
|
||||
try {
|
||||
logger.info('Fetching external URL:', url)
|
||||
@@ -382,6 +393,7 @@ async function handleExternalUrl(
|
||||
|
||||
const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, {
|
||||
timeout: DOWNLOAD_TIMEOUT_MS,
|
||||
...(headers && Object.keys(headers).length > 0 && { headers }),
|
||||
})
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch URL: ${response.status} ${response.statusText}`)
|
||||
|
||||
@@ -20,6 +20,100 @@ export const dynamic = 'force-dynamic'
|
||||
|
||||
const logger = createLogger('FileManageAPI')
|
||||
|
||||
const workspaceFileToUserFile = (file: Awaited<ReturnType<typeof getWorkspaceFile>>) => {
|
||||
if (!file) return null
|
||||
|
||||
return {
|
||||
id: file.id,
|
||||
name: file.name,
|
||||
url: ensureAbsoluteUrl(file.path),
|
||||
size: file.size,
|
||||
type: file.type,
|
||||
key: file.key,
|
||||
context: 'workspace',
|
||||
}
|
||||
}
|
||||
|
||||
const fileInputToUserFile = (fileInput: unknown) => {
|
||||
if (!fileInput || typeof fileInput !== 'object' || Array.isArray(fileInput)) return null
|
||||
|
||||
const record = fileInput as Record<string, unknown>
|
||||
const id =
|
||||
typeof record.id === 'string'
|
||||
? record.id.trim()
|
||||
: typeof record.fileId === 'string'
|
||||
? record.fileId.trim()
|
||||
: ''
|
||||
|
||||
// Objects with ids are resolved through workspace metadata. This fallback is for
|
||||
// picker/upload values that only carry storage fields.
|
||||
if (id) return null
|
||||
|
||||
const key = typeof record.key === 'string' ? record.key.trim() : ''
|
||||
const path = typeof record.path === 'string' ? record.path.trim() : ''
|
||||
const url = typeof record.url === 'string' ? record.url.trim() : ''
|
||||
const fileUrl =
|
||||
url || path || (key ? `/api/files/serve/${encodeURIComponent(key)}?context=workspace` : '')
|
||||
|
||||
if (!fileUrl && !key) return null
|
||||
|
||||
return {
|
||||
id: key || fileUrl,
|
||||
name:
|
||||
typeof record.name === 'string' && record.name.trim() ? record.name.trim() : 'workspace-file',
|
||||
url: fileUrl ? ensureAbsoluteUrl(fileUrl) : '',
|
||||
size: typeof record.size === 'number' ? record.size : 0,
|
||||
type:
|
||||
typeof record.type === 'string' && record.type.trim()
|
||||
? record.type.trim()
|
||||
: 'application/octet-stream',
|
||||
key,
|
||||
context: 'workspace',
|
||||
}
|
||||
}
|
||||
|
||||
const normalizeFileIdList = (value: unknown): string[] => {
|
||||
if (typeof value === 'string') {
|
||||
const trimmed = value.trim()
|
||||
if (!trimmed) return []
|
||||
|
||||
try {
|
||||
return normalizeFileIdList(JSON.parse(trimmed))
|
||||
} catch {
|
||||
return [trimmed]
|
||||
}
|
||||
}
|
||||
|
||||
if (!Array.isArray(value)) return []
|
||||
|
||||
return value
|
||||
.map((item) => (typeof item === 'string' ? item.trim() : ''))
|
||||
.filter((id) => id.length > 0)
|
||||
}
|
||||
|
||||
const extractUserFilesFromInput = (fileInput: unknown) => {
|
||||
const inputs = Array.isArray(fileInput) ? fileInput : fileInput ? [fileInput] : []
|
||||
return inputs
|
||||
.map((input) => fileInputToUserFile(input))
|
||||
.filter((file): file is NonNullable<ReturnType<typeof fileInputToUserFile>> => Boolean(file))
|
||||
}
|
||||
|
||||
const extractFileIdsFromInput = (fileInput: unknown): string[] => {
|
||||
const inputs = Array.isArray(fileInput) ? fileInput : fileInput ? [fileInput] : []
|
||||
|
||||
return inputs
|
||||
.flatMap((input) => {
|
||||
if (typeof input === 'string') return normalizeFileIdList(input)
|
||||
if (input && typeof input === 'object') {
|
||||
const record = input as Record<string, unknown>
|
||||
if (typeof record.id === 'string') return normalizeFileIdList(record.id)
|
||||
if (typeof record.fileId === 'string') return normalizeFileIdList(record.fileId)
|
||||
}
|
||||
return []
|
||||
})
|
||||
.filter((id) => id.length > 0)
|
||||
}
|
||||
|
||||
export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
const auth = await checkInternalAuth(request, { requireWorkflowId: false })
|
||||
if (!auth.success) {
|
||||
@@ -76,15 +170,52 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
file: {
|
||||
id: file.id,
|
||||
name: file.name,
|
||||
url: ensureAbsoluteUrl(file.path),
|
||||
size: file.size,
|
||||
type: file.type,
|
||||
key: file.key,
|
||||
context: 'workspace',
|
||||
},
|
||||
file: workspaceFileToUserFile(file),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
case 'read': {
|
||||
const { fileId, fileInput } = body
|
||||
const selectedFileIds = Array.isArray(fileId)
|
||||
? fileId.map((id) => id.trim()).filter(Boolean)
|
||||
: fileId
|
||||
? normalizeFileIdList(fileId)
|
||||
: extractFileIdsFromInput(fileInput)
|
||||
const selectedInputFiles = fileId ? [] : extractUserFilesFromInput(fileInput)
|
||||
|
||||
if (selectedFileIds.length === 0 && selectedInputFiles.length === 0) {
|
||||
return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 })
|
||||
}
|
||||
|
||||
const files = await Promise.all(
|
||||
selectedFileIds.map((id) => getWorkspaceFile(workspaceId, id))
|
||||
)
|
||||
const missingFileId = selectedFileIds.find((_, index) => !files[index])
|
||||
if (missingFileId) {
|
||||
return NextResponse.json(
|
||||
{ success: false, error: `File not found: "${missingFileId}"` },
|
||||
{ status: 404 }
|
||||
)
|
||||
}
|
||||
|
||||
const userFiles = files
|
||||
.map((file) => workspaceFileToUserFile(file))
|
||||
.filter((file): file is NonNullable<ReturnType<typeof workspaceFileToUserFile>> =>
|
||||
Boolean(file)
|
||||
)
|
||||
.concat(selectedInputFiles)
|
||||
|
||||
logger.info('Files retrieved', {
|
||||
count: userFiles.length,
|
||||
fileIds: userFiles.map((file) => file.id),
|
||||
})
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
data: {
|
||||
file: userFiles[0],
|
||||
files: userFiles,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -359,6 +359,7 @@ async function handleExecutePost(
|
||||
includeFileBase64,
|
||||
base64MaxBytes,
|
||||
workflowStateOverride,
|
||||
executionId: requestedExecutionId,
|
||||
triggerBlockId: parsedTriggerBlockId,
|
||||
startBlockId,
|
||||
stopAfterBlockId,
|
||||
@@ -508,7 +509,8 @@ async function handleExecutePost(
|
||||
)
|
||||
}
|
||||
|
||||
const executionId = generateId()
|
||||
const executionId =
|
||||
isClientSession && requestedExecutionId ? requestedExecutionId : generateId()
|
||||
reqLogger = reqLogger.withMetadata({ userId, executionId })
|
||||
|
||||
reqLogger.info('Starting server-side execution', {
|
||||
|
||||
@@ -524,6 +524,7 @@ export function useWorkflowExecution() {
|
||||
size: fileData.file.size,
|
||||
type: fileData.file.type,
|
||||
key: result.key,
|
||||
context: 'execution',
|
||||
})
|
||||
} catch (uploadError) {
|
||||
if (
|
||||
@@ -565,6 +566,7 @@ export function useWorkflowExecution() {
|
||||
size: r.size,
|
||||
type: r.type,
|
||||
key: r.key,
|
||||
context: r.context || 'execution',
|
||||
uploadedAt: r.uploadedAt,
|
||||
expiresAt: r.expiresAt,
|
||||
})
|
||||
@@ -1126,6 +1128,7 @@ export function useWorkflowExecution() {
|
||||
await executionStream.execute({
|
||||
workflowId: activeWorkflowId,
|
||||
input: finalWorkflowInput,
|
||||
executionId,
|
||||
startBlockId,
|
||||
selectedOutputs,
|
||||
triggerType: overrideTriggerType || 'manual',
|
||||
|
||||
@@ -903,6 +903,7 @@ export async function executeWorkflowWithFullLogging(
|
||||
triggerType: options.overrideTriggerType || 'manual',
|
||||
useDraftState: options.useDraftState ?? true,
|
||||
isClientSession: true,
|
||||
...(options.executionId ? { executionId: options.executionId } : {}),
|
||||
...(options.triggerBlockId ? { triggerBlockId: options.triggerBlockId } : {}),
|
||||
...(options.stopAfterBlockId ? { stopAfterBlockId: options.stopAfterBlockId } : {}),
|
||||
...(options.runFromBlock
|
||||
|
||||
@@ -94,6 +94,111 @@ describe.concurrent('Blocks Module', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('File block', () => {
|
||||
it('should keep v3 read and get routed to the legacy tools', () => {
|
||||
const block = getBlock('file_v3')
|
||||
|
||||
expect(block).toBeDefined()
|
||||
expect(block?.hideFromToolbar).toBe(true)
|
||||
expect(block?.subBlocks[0].options?.map((option) => option.id)).toEqual([
|
||||
'file_parser_v3',
|
||||
'file_get',
|
||||
'file_write',
|
||||
'file_append',
|
||||
])
|
||||
expect(block?.tools.config?.tool({ operation: 'file_parser_v3' })).toBe('file_parser_v3')
|
||||
expect(block?.tools.config?.tool({ operation: 'file_get' })).toBe('file_get')
|
||||
})
|
||||
|
||||
it('should expose v4 with read and fetch routed to the expected tools', () => {
|
||||
const block = getBlock('file_v4')
|
||||
|
||||
expect(block).toBeDefined()
|
||||
expect(block?.hideFromToolbar).toBe(false)
|
||||
expect(block?.subBlocks[0].options?.map((option) => option.id)).toEqual([
|
||||
'file_read',
|
||||
'file_fetch',
|
||||
'file_write',
|
||||
'file_append',
|
||||
])
|
||||
expect(block?.subBlocks.find((subBlock) => subBlock.id === 'readFile')?.multiple).toBe(true)
|
||||
expect(block?.tools.config?.tool({ operation: 'file_read' })).toBe('file_read')
|
||||
expect(block?.tools.config?.tool({ operation: 'file_fetch' })).toBe('file_fetch')
|
||||
expect(
|
||||
block?.tools.config?.params?.({
|
||||
operation: 'file_read',
|
||||
readFileInput: '["file-1","file-2"]',
|
||||
_context: { workspaceId: 'workspace-1' },
|
||||
})
|
||||
).toEqual({
|
||||
fileId: ['file-1', 'file-2'],
|
||||
workspaceId: 'workspace-1',
|
||||
})
|
||||
expect(
|
||||
block?.tools.config?.params?.({
|
||||
operation: 'file_read',
|
||||
readFileInput: [
|
||||
{
|
||||
key: 'workspace/workspace-1/example.md',
|
||||
name: 'example.md',
|
||||
path: '/api/files/serve/workspace%2Fworkspace-1%2Fexample.md?context=workspace',
|
||||
size: 123,
|
||||
type: 'text/markdown',
|
||||
},
|
||||
],
|
||||
_context: { workspaceId: 'workspace-1' },
|
||||
})
|
||||
).toEqual({
|
||||
fileInput: [
|
||||
{
|
||||
key: 'workspace/workspace-1/example.md',
|
||||
name: 'example.md',
|
||||
path: '/api/files/serve/workspace%2Fworkspace-1%2Fexample.md?context=workspace',
|
||||
size: 123,
|
||||
type: 'text/markdown',
|
||||
},
|
||||
],
|
||||
workspaceId: 'workspace-1',
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('Agent block', () => {
|
||||
it('should expose canonical file attachments and normalize file params', () => {
|
||||
const block = getBlock('agent')
|
||||
|
||||
expect(block).toBeDefined()
|
||||
const uploadSubBlock = block?.subBlocks.find((subBlock) => subBlock.id === 'attachmentFiles')
|
||||
const advancedSubBlock = block?.subBlocks.find((subBlock) => subBlock.id === 'files')
|
||||
|
||||
expect(uploadSubBlock?.type).toBe('file-upload')
|
||||
expect(uploadSubBlock?.canonicalParamId).toBe('files')
|
||||
expect(uploadSubBlock?.multiple).toBe(true)
|
||||
expect(advancedSubBlock?.canonicalParamId).toBe('files')
|
||||
expect(block?.inputs.files).toEqual({
|
||||
type: 'array',
|
||||
description: 'Files to include with the latest user message',
|
||||
})
|
||||
|
||||
expect(
|
||||
block?.tools.config?.params?.({
|
||||
model: 'gpt-4o',
|
||||
files:
|
||||
'[{"id":"file-1","key":"workspace/ws-1/example.png","name":"example.png","url":"/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace","size":123,"type":"image/png"}]',
|
||||
})
|
||||
).toMatchObject({
|
||||
files: [
|
||||
{
|
||||
id: 'file-1',
|
||||
key: 'workspace/ws-1/example.png',
|
||||
name: 'example.png',
|
||||
type: 'image/png',
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('getBlocksByCategory', () => {
|
||||
it('should return blocks in the "blocks" category', () => {
|
||||
const blocks = getBlocksByCategory('blocks')
|
||||
|
||||
@@ -5,6 +5,7 @@ import { AuthMode, IntegrationType } from '@/blocks/types'
|
||||
import {
|
||||
getModelOptions,
|
||||
getProviderCredentialSubBlocks,
|
||||
normalizeFileInput,
|
||||
RESPONSE_FORMAT_WAND_CONFIG,
|
||||
} from '@/blocks/utils'
|
||||
import {
|
||||
@@ -133,6 +134,25 @@ Return ONLY the JSON array.`,
|
||||
defaultValue: 'claude-sonnet-4-6',
|
||||
options: getModelOptions,
|
||||
},
|
||||
{
|
||||
id: 'attachmentFiles',
|
||||
title: 'Files',
|
||||
type: 'file-upload',
|
||||
canonicalParamId: 'files',
|
||||
placeholder: 'Upload files for the agent',
|
||||
multiple: true,
|
||||
mode: 'basic',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'files',
|
||||
title: 'Files',
|
||||
type: 'short-input',
|
||||
canonicalParamId: 'files',
|
||||
placeholder: 'Reference files from previous blocks',
|
||||
mode: 'advanced',
|
||||
required: false,
|
||||
},
|
||||
{
|
||||
id: 'reasoningEffort',
|
||||
title: 'Reasoning Effort',
|
||||
@@ -472,6 +492,9 @@ Return ONLY the JSON array.`,
|
||||
return tool
|
||||
},
|
||||
params: (params: Record<string, any>) => {
|
||||
const normalizedFiles = normalizeFileInput(params.files)
|
||||
const baseParams = normalizedFiles ? { ...params, files: normalizedFiles } : params
|
||||
|
||||
// If tools array is provided, handle tool usage control
|
||||
if (params.tools && Array.isArray(params.tools)) {
|
||||
// Transform tools to include usageControl
|
||||
@@ -506,9 +529,9 @@ Return ONLY the JSON array.`,
|
||||
logger.info('Filtered out tools set to none', { tools: filteredOutTools.join(', ') })
|
||||
}
|
||||
|
||||
return { ...params, tools: transformedTools }
|
||||
return { ...baseParams, tools: transformedTools }
|
||||
}
|
||||
return params
|
||||
return baseParams
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -518,6 +541,7 @@ Return ONLY the JSON array.`,
|
||||
description:
|
||||
'Array of message objects with role and content: [{ role: "system", content: "..." }, { role: "user", content: "..." }]',
|
||||
},
|
||||
files: { type: 'array', description: 'Files to include with the latest user message' },
|
||||
memoryType: {
|
||||
type: 'string',
|
||||
description:
|
||||
|
||||
@@ -259,6 +259,7 @@ export const FileV3Block: BlockConfig<FileParserV3Output> = {
|
||||
tags: ['document-processing'],
|
||||
bgColor: '#40916C',
|
||||
icon: DocumentIcon,
|
||||
hideFromToolbar: true,
|
||||
subBlocks: [
|
||||
{
|
||||
id: 'operation',
|
||||
@@ -514,3 +515,291 @@ export const FileV3Block: BlockConfig<FileParserV3Output> = {
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
const parseReadFileIds = (input: unknown): string | string[] | null => {
|
||||
let value = input
|
||||
|
||||
if (typeof value === 'string') {
|
||||
const trimmed = value.trim()
|
||||
if (!trimmed) return null
|
||||
|
||||
try {
|
||||
value = JSON.parse(trimmed)
|
||||
} catch {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
const fileIds = value
|
||||
.map((item) => (typeof item === 'string' ? item.trim() : ''))
|
||||
.filter((item) => item.length > 0)
|
||||
|
||||
if (fileIds.length === 0) return null
|
||||
return fileIds.length === 1 ? fileIds[0] : fileIds
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
export const FileV4Block: BlockConfig<FileParserV3Output> = {
|
||||
...FileV3Block,
|
||||
type: 'file_v4',
|
||||
name: 'File',
|
||||
description: 'Read, fetch, write, and append files',
|
||||
longDescription:
|
||||
'Read workspace files by picker or canonical ID, fetch and parse files from URLs with optional headers, write new workspace files, or append content to existing files.',
|
||||
hideFromToolbar: false,
|
||||
bestPractices: `
|
||||
- Use Read when you need an existing workspace file object by picker selection or canonical file ID.
|
||||
- Use Fetch for external file URLs. Add headers for authenticated downloads, for example Slack private file URLs require an Authorization Bearer token.
|
||||
`,
|
||||
subBlocks: [
|
||||
{
|
||||
id: 'operation',
|
||||
title: 'Operation',
|
||||
type: 'dropdown' as SubBlockType,
|
||||
options: [
|
||||
{ label: 'Read', id: 'file_read' },
|
||||
{ label: 'Fetch', id: 'file_fetch' },
|
||||
{ label: 'Write', id: 'file_write' },
|
||||
{ label: 'Append', id: 'file_append' },
|
||||
],
|
||||
value: () => 'file_read',
|
||||
},
|
||||
{
|
||||
id: 'readFile',
|
||||
title: 'File',
|
||||
type: 'file-upload' as SubBlockType,
|
||||
canonicalParamId: 'readFileInput',
|
||||
acceptedTypes: '*',
|
||||
placeholder: 'Select workspace files',
|
||||
multiple: true,
|
||||
mode: 'basic',
|
||||
condition: { field: 'operation', value: 'file_read' },
|
||||
required: { field: 'operation', value: 'file_read' },
|
||||
},
|
||||
{
|
||||
id: 'readFileId',
|
||||
title: 'File ID',
|
||||
type: 'short-input' as SubBlockType,
|
||||
canonicalParamId: 'readFileInput',
|
||||
placeholder: 'Workspace file ID or JSON array of IDs',
|
||||
mode: 'advanced',
|
||||
condition: { field: 'operation', value: 'file_read' },
|
||||
required: { field: 'operation', value: 'file_read' },
|
||||
},
|
||||
{
|
||||
id: 'fileUrl',
|
||||
title: 'File URL',
|
||||
type: 'short-input' as SubBlockType,
|
||||
placeholder: 'https://example.com/document.pdf',
|
||||
condition: { field: 'operation', value: 'file_fetch' },
|
||||
required: { field: 'operation', value: 'file_fetch' },
|
||||
},
|
||||
{
|
||||
id: 'headers',
|
||||
title: 'Headers',
|
||||
type: 'table' as SubBlockType,
|
||||
columns: ['Key', 'Value'],
|
||||
description:
|
||||
'Custom headers for fetching the file URL, such as Authorization: Bearer <token>.',
|
||||
condition: { field: 'operation', value: 'file_fetch' },
|
||||
},
|
||||
{
|
||||
id: 'fileName',
|
||||
title: 'File Name',
|
||||
type: 'short-input' as SubBlockType,
|
||||
placeholder: 'File name (e.g., data.csv)',
|
||||
condition: { field: 'operation', value: 'file_write' },
|
||||
required: { field: 'operation', value: 'file_write' },
|
||||
},
|
||||
{
|
||||
id: 'content',
|
||||
title: 'Content',
|
||||
type: 'long-input' as SubBlockType,
|
||||
placeholder: 'File content to write...',
|
||||
condition: { field: 'operation', value: 'file_write' },
|
||||
required: { field: 'operation', value: 'file_write' },
|
||||
},
|
||||
{
|
||||
id: 'contentType',
|
||||
title: 'Content Type',
|
||||
type: 'short-input' as SubBlockType,
|
||||
placeholder: 'text/plain (auto-detected from extension)',
|
||||
condition: { field: 'operation', value: 'file_write' },
|
||||
mode: 'advanced',
|
||||
},
|
||||
{
|
||||
id: 'appendFile',
|
||||
title: 'File',
|
||||
type: 'file-upload' as SubBlockType,
|
||||
canonicalParamId: 'appendFileInput',
|
||||
acceptedTypes: '.txt,.md,.json,.csv,.xml,.html,.htm,.yaml,.yml,.log,.rtf',
|
||||
placeholder: 'Select or upload a workspace file',
|
||||
mode: 'basic',
|
||||
condition: { field: 'operation', value: 'file_append' },
|
||||
required: { field: 'operation', value: 'file_append' },
|
||||
},
|
||||
{
|
||||
id: 'appendFileName',
|
||||
title: 'File',
|
||||
type: 'short-input' as SubBlockType,
|
||||
canonicalParamId: 'appendFileInput',
|
||||
placeholder: 'File name (e.g., notes.md)',
|
||||
mode: 'advanced',
|
||||
condition: { field: 'operation', value: 'file_append' },
|
||||
required: { field: 'operation', value: 'file_append' },
|
||||
},
|
||||
{
|
||||
id: 'appendContent',
|
||||
title: 'Content',
|
||||
type: 'long-input' as SubBlockType,
|
||||
placeholder: 'Content to append...',
|
||||
condition: { field: 'operation', value: 'file_append' },
|
||||
required: { field: 'operation', value: 'file_append' },
|
||||
},
|
||||
],
|
||||
tools: {
|
||||
access: ['file_fetch', 'file_read', 'file_write', 'file_append'],
|
||||
config: {
|
||||
tool: (params) => {
|
||||
const operation = params.operation || 'file_read'
|
||||
if (operation === 'file_read') return 'file_read'
|
||||
if (operation === 'file_fetch') return 'file_fetch'
|
||||
return operation
|
||||
},
|
||||
params: (params) => {
|
||||
const operation = params.operation || 'file_read'
|
||||
|
||||
if (operation === 'file_write') {
|
||||
return {
|
||||
fileName: params.fileName,
|
||||
content: params.content,
|
||||
contentType: params.contentType,
|
||||
workspaceId: params._context?.workspaceId,
|
||||
}
|
||||
}
|
||||
|
||||
if (operation === 'file_append') {
|
||||
const appendInput = params.appendFileInput
|
||||
if (!appendInput) {
|
||||
throw new Error('File is required for append')
|
||||
}
|
||||
|
||||
let fileName: string
|
||||
if (typeof appendInput === 'string') {
|
||||
fileName = appendInput.trim()
|
||||
} else {
|
||||
const normalized = normalizeFileInput(appendInput, { single: true })
|
||||
const file = normalized as Record<string, unknown> | null
|
||||
fileName = (file?.name as string) ?? ''
|
||||
}
|
||||
|
||||
if (!fileName) {
|
||||
throw new Error('Could not determine file name')
|
||||
}
|
||||
|
||||
return {
|
||||
fileName,
|
||||
content: params.appendContent,
|
||||
workspaceId: params._context?.workspaceId,
|
||||
}
|
||||
}
|
||||
|
||||
if (operation === 'file_read') {
|
||||
const readInput = params.readFileInput
|
||||
if (!readInput) {
|
||||
throw new Error('File is required for read')
|
||||
}
|
||||
|
||||
const fileIds = parseReadFileIds(readInput)
|
||||
if (fileIds) {
|
||||
return {
|
||||
fileId: fileIds,
|
||||
workspaceId: params._context?.workspaceId,
|
||||
}
|
||||
}
|
||||
|
||||
const normalized = normalizeFileInput(readInput)
|
||||
if (!normalized || normalized.length === 0) {
|
||||
throw new Error('File is required for read')
|
||||
}
|
||||
|
||||
return {
|
||||
fileInput: normalized,
|
||||
workspaceId: params._context?.workspaceId,
|
||||
}
|
||||
}
|
||||
|
||||
if (operation === 'file_fetch') {
|
||||
const fileUrl = typeof params.fileUrl === 'string' ? params.fileUrl.trim() : ''
|
||||
if (!fileUrl) {
|
||||
logger.error('No file URL provided')
|
||||
throw new Error('File URL is required')
|
||||
}
|
||||
|
||||
return {
|
||||
filePath: fileUrl,
|
||||
fileType: params.fileType || 'auto',
|
||||
headers: params.headers,
|
||||
workspaceId: params._context?.workspaceId,
|
||||
workflowId: params._context?.workflowId,
|
||||
executionId: params._context?.executionId,
|
||||
}
|
||||
}
|
||||
|
||||
logger.error(`Invalid file operation: ${operation}`)
|
||||
throw new Error('Invalid file operation')
|
||||
},
|
||||
},
|
||||
},
|
||||
inputs: {
|
||||
operation: {
|
||||
type: 'string',
|
||||
description: 'Operation to perform (read, fetch, write, or append)',
|
||||
},
|
||||
readFileInput: {
|
||||
type: 'json',
|
||||
description: 'Selected workspace file or canonical file ID for read',
|
||||
},
|
||||
fileUrl: { type: 'string', description: 'External file URL for fetch' },
|
||||
headers: { type: 'json', description: 'Request headers for fetch' },
|
||||
fileType: { type: 'string', description: 'File type for fetch' },
|
||||
fileName: { type: 'string', description: 'Name for a new file (write)' },
|
||||
content: { type: 'string', description: 'File content to write' },
|
||||
contentType: { type: 'string', description: 'MIME content type for write' },
|
||||
appendFileInput: { type: 'json', description: 'File to append to' },
|
||||
appendContent: { type: 'string', description: 'Content to append to file' },
|
||||
},
|
||||
outputs: {
|
||||
file: {
|
||||
type: 'file',
|
||||
description: 'First workspace file object (read)',
|
||||
},
|
||||
files: {
|
||||
type: 'file[]',
|
||||
description: 'Workspace file objects (read) or fetched file objects (fetch)',
|
||||
},
|
||||
combinedContent: {
|
||||
type: 'string',
|
||||
description: 'All fetched file contents merged into a single text string (fetch)',
|
||||
},
|
||||
id: {
|
||||
type: 'string',
|
||||
description: 'File ID (write)',
|
||||
},
|
||||
name: {
|
||||
type: 'string',
|
||||
description: 'File name (write)',
|
||||
},
|
||||
size: {
|
||||
type: 'number',
|
||||
description: 'File size in bytes (write)',
|
||||
},
|
||||
url: {
|
||||
type: 'string',
|
||||
description: 'URL to access the file (write)',
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -55,7 +55,7 @@ import { EvernoteBlock } from '@/blocks/blocks/evernote'
|
||||
import { ExaBlock } from '@/blocks/blocks/exa'
|
||||
import { ExtendBlock, ExtendV2Block } from '@/blocks/blocks/extend'
|
||||
import { FathomBlock } from '@/blocks/blocks/fathom'
|
||||
import { FileBlock, FileV2Block, FileV3Block } from '@/blocks/blocks/file'
|
||||
import { FileBlock, FileV2Block, FileV3Block, FileV4Block } from '@/blocks/blocks/file'
|
||||
import { FirecrawlBlock } from '@/blocks/blocks/firecrawl'
|
||||
import { FirefliesBlock, FirefliesV2Block } from '@/blocks/blocks/fireflies'
|
||||
import { FunctionBlock } from '@/blocks/blocks/function'
|
||||
@@ -299,6 +299,7 @@ export const registry: Record<string, BlockConfig> = {
|
||||
file: FileBlock,
|
||||
file_v2: FileV2Block,
|
||||
file_v3: FileV3Block,
|
||||
file_v4: FileV4Block,
|
||||
firecrawl: FirecrawlBlock,
|
||||
fireflies: FirefliesBlock,
|
||||
fireflies_v2: FirefliesV2Block,
|
||||
|
||||
@@ -449,19 +449,6 @@ export class ExecutionEngine {
|
||||
|
||||
const readyNodes = this.edgeManager.processOutgoingEdges(node, output, false)
|
||||
|
||||
this.execLogger.info('Processing outgoing edges', {
|
||||
nodeId,
|
||||
outgoingEdgesCount: node.outgoingEdges.size,
|
||||
outgoingEdges: Array.from(node.outgoingEdges.entries()).map(([id, e]) => ({
|
||||
id,
|
||||
target: e.target,
|
||||
sourceHandle: e.sourceHandle,
|
||||
})),
|
||||
output,
|
||||
readyNodesCount: readyNodes.length,
|
||||
readyNodes,
|
||||
})
|
||||
|
||||
this.addMultipleToQueue(readyNodes)
|
||||
|
||||
if (this.context.pendingDynamicNodes && this.context.pendingDynamicNodes.length > 0) {
|
||||
|
||||
@@ -264,6 +264,78 @@ describe('AgentBlockHandler', () => {
|
||||
expect(result).toEqual(expectedOutput)
|
||||
})
|
||||
|
||||
it('should attach files to the last user message only', async () => {
|
||||
const inputs = {
|
||||
model: 'gpt-4o',
|
||||
messages: [
|
||||
{ role: 'system' as const, content: 'You are helpful.' },
|
||||
{ role: 'user' as const, content: 'Earlier question' },
|
||||
{ role: 'assistant' as const, content: 'Earlier answer' },
|
||||
{ role: 'user' as const, content: 'Analyze this file' },
|
||||
],
|
||||
files: [
|
||||
{
|
||||
id: 'file-1',
|
||||
key: 'workspace/ws-1/example.png',
|
||||
name: 'example.png',
|
||||
url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace',
|
||||
size: 128,
|
||||
type: 'image/png',
|
||||
base64: 'aW1hZ2U=',
|
||||
},
|
||||
],
|
||||
apiKey: 'test-api-key',
|
||||
}
|
||||
|
||||
mockGetProviderFromModel.mockReturnValue('openai')
|
||||
|
||||
await handler.execute(mockContext, mockBlock, inputs)
|
||||
|
||||
const requestBody = mockExecuteProviderRequest.mock.calls[0][1]
|
||||
expect(requestBody.messages[1]).toMatchObject({
|
||||
role: 'user',
|
||||
content: 'Earlier question',
|
||||
})
|
||||
expect(requestBody.messages[1].files).toBeUndefined()
|
||||
expect(requestBody.messages[3]).toMatchObject({
|
||||
role: 'user',
|
||||
content: 'Analyze this file',
|
||||
files: [
|
||||
{
|
||||
id: 'file-1',
|
||||
name: 'example.png',
|
||||
type: 'image/png',
|
||||
base64: 'aW1hZ2U=',
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
it('should reject files for providers without attachment support', async () => {
|
||||
const inputs = {
|
||||
model: 'deepseek-chat',
|
||||
messages: [{ role: 'user' as const, content: 'Analyze this file' }],
|
||||
files: [
|
||||
{
|
||||
id: 'file-1',
|
||||
key: 'workspace/ws-1/example.png',
|
||||
name: 'example.png',
|
||||
url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace',
|
||||
size: 128,
|
||||
type: 'image/png',
|
||||
base64: 'aW1hZ2U=',
|
||||
},
|
||||
],
|
||||
apiKey: 'test-api-key',
|
||||
}
|
||||
|
||||
mockGetProviderFromModel.mockReturnValue('deepseek')
|
||||
|
||||
await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow(
|
||||
'File attachments are not supported for provider "deepseek"'
|
||||
)
|
||||
})
|
||||
|
||||
it('should preserve usageControl for custom tools and filter out "none"', async () => {
|
||||
const inputs = {
|
||||
model: 'gpt-4o',
|
||||
|
||||
@@ -6,9 +6,12 @@ import { sleep } from '@sim/utils/helpers'
|
||||
import { and, eq, inArray, isNull } from 'drizzle-orm'
|
||||
import { normalizeStringRecord, normalizeWorkflowVariables } from '@/lib/core/utils/records'
|
||||
import { createMcpToolId } from '@/lib/mcp/utils'
|
||||
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
|
||||
import { hydrateUserFilesWithBase64 } from '@/lib/uploads/utils/user-file-base64.server'
|
||||
import { getCustomToolById } from '@/lib/workflows/custom-tools/operations'
|
||||
import { getAllBlocks } from '@/blocks'
|
||||
import type { BlockOutput } from '@/blocks/types'
|
||||
import { normalizeFileInput } from '@/blocks/utils'
|
||||
import {
|
||||
validateBlockType,
|
||||
validateCustomToolsAllowed,
|
||||
@@ -36,6 +39,7 @@ import { buildAPIUrl, buildAuthHeaders } from '@/executor/utils/http'
|
||||
import { stringifyJSON } from '@/executor/utils/json'
|
||||
import { resolveVertexCredential } from '@/executor/utils/vertex-credential'
|
||||
import { executeProviderRequest } from '@/providers'
|
||||
import { getProviderAttachmentMaxBytes, supportsFileAttachments } from '@/providers/attachments'
|
||||
import { getProviderFromModel, transformBlockTool } from '@/providers/utils'
|
||||
import type { SerializedBlock } from '@/serializer/types'
|
||||
import { filterSchemaForLLM, type ToolSchema } from '@/tools/params'
|
||||
@@ -87,12 +91,22 @@ export class AgentBlockHandler implements BlockHandler {
|
||||
|
||||
const streamingConfig = this.getStreamingConfig(ctx, block)
|
||||
const messages = await this.buildMessages(ctx, filteredInputs, skillMetadata)
|
||||
const messagesWithInputFiles = this.attachFilesToLastUserMessage(
|
||||
ctx,
|
||||
messages,
|
||||
filteredInputs.files
|
||||
)
|
||||
const messagesWithFiles = await this.hydrateMessageFilesForProvider(
|
||||
ctx,
|
||||
messagesWithInputFiles,
|
||||
providerId
|
||||
)
|
||||
|
||||
const providerRequest = this.buildProviderRequest({
|
||||
ctx,
|
||||
providerId,
|
||||
model,
|
||||
messages,
|
||||
messages: messagesWithFiles,
|
||||
inputs: filteredInputs,
|
||||
formattedTools,
|
||||
responseFormat,
|
||||
@@ -672,6 +686,107 @@ export class AgentBlockHandler implements BlockHandler {
|
||||
return messages.length > 0 ? messages : undefined
|
||||
}
|
||||
|
||||
private attachFilesToLastUserMessage(
|
||||
ctx: ExecutionContext,
|
||||
messages: Message[] | undefined,
|
||||
filesInput: unknown
|
||||
): Message[] | undefined {
|
||||
const normalizedFiles = normalizeFileInput(filesInput)
|
||||
if (!normalizedFiles || normalizedFiles.length === 0) {
|
||||
return messages
|
||||
}
|
||||
|
||||
if (!messages || messages.length === 0) {
|
||||
throw new Error('Files require at least one user message in the agent prompt')
|
||||
}
|
||||
|
||||
let lastUserMessageIndex = -1
|
||||
for (let index = messages.length - 1; index >= 0; index--) {
|
||||
if (messages[index].role === 'user') {
|
||||
lastUserMessageIndex = index
|
||||
break
|
||||
}
|
||||
}
|
||||
if (lastUserMessageIndex === -1) {
|
||||
throw new Error('Files require at least one user message in the agent prompt')
|
||||
}
|
||||
|
||||
const requestId = ctx.executionId || ctx.workflowId || 'agent-files'
|
||||
const userFiles = processFilesToUserFiles(normalizedFiles as RawFileInput[], requestId, logger)
|
||||
if (userFiles.length === 0) {
|
||||
throw new Error('Files must include at least one valid file object')
|
||||
}
|
||||
|
||||
const lastUserMessage = messages[lastUserMessageIndex]
|
||||
const nextMessages = [...messages]
|
||||
nextMessages[lastUserMessageIndex] = {
|
||||
...lastUserMessage,
|
||||
files: [...(lastUserMessage.files ?? []), ...userFiles],
|
||||
}
|
||||
|
||||
return nextMessages
|
||||
}
|
||||
|
||||
private async hydrateMessageFilesForProvider(
|
||||
ctx: ExecutionContext,
|
||||
messages: Message[] | undefined,
|
||||
providerId: string
|
||||
): Promise<Message[] | undefined> {
|
||||
if (!messages?.some((message) => message.files?.length)) {
|
||||
return messages
|
||||
}
|
||||
|
||||
if (!supportsFileAttachments(providerId)) {
|
||||
throw new Error(`File attachments are not supported for provider "${providerId}"`)
|
||||
}
|
||||
|
||||
const requestId = ctx.executionId || ctx.workflowId || 'agent-files'
|
||||
const nextMessages = [...messages]
|
||||
|
||||
for (let messageIndex = 0; messageIndex < messages.length; messageIndex++) {
|
||||
const message = messages[messageIndex]
|
||||
const normalizedFiles = normalizeFileInput(message.files)
|
||||
if (!normalizedFiles || normalizedFiles.length === 0) {
|
||||
continue
|
||||
}
|
||||
|
||||
const userFiles = processFilesToUserFiles(
|
||||
normalizedFiles as RawFileInput[],
|
||||
requestId,
|
||||
logger
|
||||
)
|
||||
if (userFiles.length === 0) {
|
||||
throw new Error('Files must include at least one valid file object')
|
||||
}
|
||||
|
||||
const hydratedFiles = await hydrateUserFilesWithBase64(userFiles, {
|
||||
requestId,
|
||||
workspaceId: ctx.workspaceId,
|
||||
workflowId: ctx.workflowId,
|
||||
executionId: ctx.executionId,
|
||||
largeValueExecutionIds: ctx.largeValueExecutionIds,
|
||||
allowLargeValueWorkflowScope: ctx.allowLargeValueWorkflowScope,
|
||||
userId: ctx.userId,
|
||||
logger,
|
||||
maxBytes: getProviderAttachmentMaxBytes(providerId),
|
||||
})
|
||||
|
||||
const missingFile = hydratedFiles.find((file) => !file.base64)
|
||||
if (missingFile) {
|
||||
throw new Error(
|
||||
`File "${missingFile.name}" could not be read for provider "${providerId}". Make sure the file is still accessible and under the provider attachment size limit.`
|
||||
)
|
||||
}
|
||||
|
||||
nextMessages[messageIndex] = {
|
||||
...message,
|
||||
files: hydratedFiles,
|
||||
}
|
||||
}
|
||||
|
||||
return nextMessages
|
||||
}
|
||||
|
||||
private extractValidMessages(messages?: Message[]): Message[] {
|
||||
if (!messages || !Array.isArray(messages)) return []
|
||||
|
||||
|
||||
@@ -178,6 +178,34 @@ describe('Memory', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('sanitizeMessageForStorage', () => {
|
||||
it('should strip file payloads and provider-only fields before memory persistence', () => {
|
||||
const message: Message = {
|
||||
role: 'user',
|
||||
content: 'Analyze this file',
|
||||
executionId: 'exec-1',
|
||||
files: [
|
||||
{
|
||||
id: 'file-1',
|
||||
key: 'workspace/ws-1/example.png',
|
||||
name: 'example.png',
|
||||
url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace',
|
||||
size: 128,
|
||||
type: 'image/png',
|
||||
base64: 'iVBORw0KGgo=',
|
||||
},
|
||||
],
|
||||
tool_calls: [{ id: 'call-1' }],
|
||||
}
|
||||
|
||||
expect((memoryService as any).sanitizeMessageForStorage(message)).toEqual({
|
||||
role: 'user',
|
||||
content: 'Analyze this file',
|
||||
executionId: 'exec-1',
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('Token-based vs Message-based comparison', () => {
|
||||
it('should produce different results for same limit concept', () => {
|
||||
const messages: Message[] = [
|
||||
|
||||
@@ -122,6 +122,14 @@ export class Memory {
|
||||
return messages.slice(-limit)
|
||||
}
|
||||
|
||||
private sanitizeMessageForStorage(message: Message): Message {
|
||||
return {
|
||||
role: message.role,
|
||||
content: message.content,
|
||||
...(message.executionId && { executionId: message.executionId }),
|
||||
}
|
||||
}
|
||||
|
||||
private applyTokenWindow(messages: Message[], maxTokens: number, model?: string): Message[] {
|
||||
const result: Message[] = []
|
||||
let tokenCount = 0
|
||||
@@ -177,9 +185,17 @@ export class Memory {
|
||||
const data = result[0].data
|
||||
if (!Array.isArray(data)) return []
|
||||
|
||||
return data.filter(
|
||||
(msg): msg is Message => msg && typeof msg === 'object' && 'role' in msg && 'content' in msg
|
||||
)
|
||||
return data
|
||||
.filter(
|
||||
(msg): msg is Message =>
|
||||
msg &&
|
||||
typeof msg === 'object' &&
|
||||
'role' in msg &&
|
||||
'content' in msg &&
|
||||
['system', 'user', 'assistant'].includes(msg.role) &&
|
||||
typeof msg.content === 'string'
|
||||
)
|
||||
.map((msg) => this.sanitizeMessageForStorage(msg))
|
||||
}
|
||||
|
||||
private async seedMemoryRecord(
|
||||
@@ -189,13 +205,15 @@ export class Memory {
|
||||
): Promise<void> {
|
||||
const now = new Date()
|
||||
|
||||
const sanitizedMessages = messages.map((message) => this.sanitizeMessageForStorage(message))
|
||||
|
||||
await db
|
||||
.insert(memory)
|
||||
.values({
|
||||
id: generateId(),
|
||||
workspaceId,
|
||||
key,
|
||||
data: messages,
|
||||
data: sanitizedMessages,
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
@@ -205,20 +223,22 @@ export class Memory {
|
||||
private async appendMessage(workspaceId: string, key: string, message: Message): Promise<void> {
|
||||
const now = new Date()
|
||||
|
||||
const sanitizedMessage = this.sanitizeMessageForStorage(message)
|
||||
|
||||
await db
|
||||
.insert(memory)
|
||||
.values({
|
||||
id: generateId(),
|
||||
workspaceId,
|
||||
key,
|
||||
data: [message],
|
||||
data: [sanitizedMessage],
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
})
|
||||
.onConflictDoUpdate({
|
||||
target: [memory.workspaceId, memory.key],
|
||||
set: {
|
||||
data: sql`${memory.data} || ${JSON.stringify([message])}::jsonb`,
|
||||
data: sql`${memory.data} || ${JSON.stringify([sanitizedMessage])}::jsonb`,
|
||||
updatedAt: now,
|
||||
},
|
||||
})
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { UserFile } from '@/executor/types'
|
||||
|
||||
export interface SkillInput {
|
||||
skillId: string
|
||||
name?: string
|
||||
@@ -37,6 +39,7 @@ export interface AgentInputs {
|
||||
reasoningEffort?: string
|
||||
verbosity?: string
|
||||
thinkingLevel?: string
|
||||
files?: unknown
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -66,6 +69,7 @@ export interface ToolInput {
|
||||
export interface Message {
|
||||
role: 'system' | 'user' | 'assistant'
|
||||
content: string
|
||||
files?: UserFile[]
|
||||
executionId?: string
|
||||
function_call?: any
|
||||
tool_calls?: any[]
|
||||
|
||||
@@ -119,6 +119,42 @@ describe('start-block utilities', () => {
|
||||
expect(output.files).toEqual(files)
|
||||
})
|
||||
|
||||
it.concurrent('buildStartBlockOutput normalizes Start files from internal serve URLs', () => {
|
||||
const block = createBlock('start_trigger', 'start')
|
||||
const resolution = {
|
||||
blockId: 'start',
|
||||
block,
|
||||
path: StartBlockPath.UNIFIED,
|
||||
} as const
|
||||
|
||||
const output = buildStartBlockOutput({
|
||||
resolution,
|
||||
workflowInput: {
|
||||
files: [
|
||||
{
|
||||
id: 'file_1',
|
||||
name: 'screenshot.png',
|
||||
url: '/api/files/serve/s3/execution%2Fworkspace-id%2Fworkflow-id%2Fexecution-id%2Fscreenshot.png?context=execution',
|
||||
size: 243289,
|
||||
type: 'image/png',
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
|
||||
expect(output.files).toEqual([
|
||||
{
|
||||
id: 'file_1',
|
||||
name: 'screenshot.png',
|
||||
url: '/api/files/serve/s3/execution%2Fworkspace-id%2Fworkflow-id%2Fexecution-id%2Fscreenshot.png?context=execution',
|
||||
size: 243289,
|
||||
type: 'image/png',
|
||||
key: 'execution/workspace-id/workflow-id/execution-id/screenshot.png',
|
||||
context: 'execution',
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it.concurrent('rejects inputFormat fields that collide with executor routing keys', () => {
|
||||
const block = createBlock('start_trigger', 'start', {
|
||||
subBlocks: {
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
import { isUserFileWithMetadata } from '@/lib/core/utils/user-file'
|
||||
import {
|
||||
inferContextFromKey,
|
||||
isInternalFileUrl,
|
||||
parseInternalFileUrl,
|
||||
} from '@/lib/uploads/utils/file-utils'
|
||||
import {
|
||||
classifyStartBlockType,
|
||||
resolveStartCandidates,
|
||||
@@ -323,13 +327,68 @@ function getRawInputCandidate(workflowInput: unknown): unknown {
|
||||
return workflowInput
|
||||
}
|
||||
|
||||
function normalizeStartFile(file: unknown): UserFile | null {
|
||||
if (!isPlainObject(file)) {
|
||||
return null
|
||||
}
|
||||
|
||||
const id = typeof file.id === 'string' ? file.id : ''
|
||||
const name = typeof file.name === 'string' ? file.name : ''
|
||||
const url =
|
||||
typeof file.url === 'string' ? file.url : typeof file.path === 'string' ? file.path : ''
|
||||
const size = typeof file.size === 'number' ? file.size : Number.NaN
|
||||
const type = typeof file.type === 'string' ? file.type : ''
|
||||
const explicitKey = typeof file.key === 'string' ? file.key : ''
|
||||
|
||||
let key = explicitKey
|
||||
let context = typeof file.context === 'string' ? file.context : undefined
|
||||
|
||||
if (!key && url && isInternalFileUrl(url)) {
|
||||
try {
|
||||
const parsed = parseInternalFileUrl(url)
|
||||
key = parsed.key
|
||||
context = context || parsed.context
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
if (!context && key) {
|
||||
try {
|
||||
context = inferContextFromKey(key)
|
||||
} catch {
|
||||
// Older file outputs may have opaque keys; keep the file shape intact.
|
||||
}
|
||||
}
|
||||
|
||||
if (!id || !name || !url || !Number.isFinite(size) || !type || !key) {
|
||||
return null
|
||||
}
|
||||
|
||||
return {
|
||||
id,
|
||||
name,
|
||||
url,
|
||||
size,
|
||||
type,
|
||||
key,
|
||||
...(context && { context }),
|
||||
...(typeof file.base64 === 'string' && { base64: file.base64 }),
|
||||
}
|
||||
}
|
||||
|
||||
function getFilesFromWorkflowInput(workflowInput: unknown): UserFile[] | undefined {
|
||||
if (!isPlainObject(workflowInput)) {
|
||||
return undefined
|
||||
}
|
||||
const files = workflowInput.files
|
||||
if (Array.isArray(files) && files.every(isUserFileWithMetadata)) {
|
||||
return files
|
||||
if (!Array.isArray(files)) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const normalizedFiles = files.map(normalizeStartFile)
|
||||
if (normalizedFiles.every((file): file is UserFile => Boolean(file))) {
|
||||
return normalizedFiles
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
@@ -341,6 +400,8 @@ function mergeFilesIntoOutput(
|
||||
const files = getFilesFromWorkflowInput(workflowInput)
|
||||
if (files) {
|
||||
output.files = files
|
||||
} else if (isPlainObject(workflowInput) && Object.hasOwn(workflowInput, 'files')) {
|
||||
output.files = undefined
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
@@ -197,6 +197,7 @@ export interface ExecuteStreamOptions {
|
||||
envVarValues?: Record<string, string>
|
||||
workflowVariables?: Record<string, any>
|
||||
selectedOutputs?: string[]
|
||||
executionId?: string
|
||||
startBlockId?: string
|
||||
triggerType?: string
|
||||
useDraftState?: boolean
|
||||
|
||||
@@ -302,6 +302,7 @@ export const fileParseBodySchema = z
|
||||
.object({
|
||||
filePath: z.union([z.string(), z.array(z.string())]).optional(),
|
||||
fileType: z.string().optional().default(''),
|
||||
headers: z.record(z.string(), z.string()).optional(),
|
||||
workspaceId: z.string().optional().default(''),
|
||||
workflowId: z.string().optional(),
|
||||
executionId: z.string().optional(),
|
||||
|
||||
@@ -33,10 +33,22 @@ export const fileManageGetBodySchema = z
|
||||
message: 'Either fileId or fileInput is required for get operation',
|
||||
})
|
||||
|
||||
export const fileManageReadBodySchema = z
|
||||
.object({
|
||||
operation: z.literal('read'),
|
||||
workspaceId: z.string().min(1).optional(),
|
||||
fileId: z.union([z.string().min(1), z.array(z.string().min(1)).min(1)]).optional(),
|
||||
fileInput: z.any().optional(),
|
||||
})
|
||||
.refine((data) => data.fileId !== undefined || data.fileInput !== undefined, {
|
||||
message: 'Either fileId or fileInput is required for read operation',
|
||||
})
|
||||
|
||||
export const fileManageBodySchema = z.union([
|
||||
fileManageWriteBodySchema,
|
||||
fileManageAppendBodySchema,
|
||||
fileManageGetBodySchema,
|
||||
fileManageReadBodySchema,
|
||||
])
|
||||
|
||||
export const fileManageContract = defineRouteContract({
|
||||
|
||||
@@ -336,6 +336,7 @@ export const executeWorkflowBodySchema = z.object({
|
||||
includeFileBase64: z.boolean().optional().default(true),
|
||||
base64MaxBytes: z.number().int().positive().optional(),
|
||||
workflowStateOverride: workflowStateSchema.optional(),
|
||||
executionId: z.string().optional(),
|
||||
triggerBlockId: z.string().optional(),
|
||||
startBlockId: z.string().optional(),
|
||||
stopAfterBlockId: z.string().optional(),
|
||||
|
||||
@@ -172,7 +172,7 @@ export function createFileContentFromBase64(base64: string, mimeType: string): M
|
||||
return null
|
||||
}
|
||||
|
||||
if (contentType === 'image' && !CLAUDE_SUPPORTED_IMAGE_MIME_TYPES.has(mimeType.toLowerCase())) {
|
||||
if (contentType === 'image' && !MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mimeType.toLowerCase())) {
|
||||
return null
|
||||
}
|
||||
|
||||
@@ -186,7 +186,7 @@ export function createFileContentFromBase64(base64: string, mimeType: string): M
|
||||
}
|
||||
}
|
||||
|
||||
const CLAUDE_SUPPORTED_IMAGE_MIME_TYPES = new Set([
|
||||
export const MODEL_SUPPORTED_IMAGE_MIME_TYPES = new Set([
|
||||
'image/jpeg',
|
||||
'image/jpg',
|
||||
'image/png',
|
||||
|
||||
@@ -361,8 +361,7 @@ async function resolveBase64(
|
||||
options: Base64HydrationOptions,
|
||||
logger: Logger
|
||||
): Promise<string | null> {
|
||||
const requestedMaxBytes = options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES
|
||||
const maxBytes = Math.min(requestedMaxBytes, DEFAULT_MAX_BASE64_BYTES)
|
||||
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES
|
||||
|
||||
if (file.base64) {
|
||||
const base64Bytes = Buffer.byteLength(file.base64, 'base64')
|
||||
@@ -440,10 +439,7 @@ async function hydrateUserFile(
|
||||
|
||||
const cached = await state.cache.get(file)
|
||||
if (cached) {
|
||||
const maxBytes = Math.min(
|
||||
options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES,
|
||||
DEFAULT_MAX_BASE64_BYTES
|
||||
)
|
||||
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES
|
||||
if (Buffer.byteLength(cached, 'base64') > maxBytes) {
|
||||
return stripBase64(file)
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
checkForForcedToolUsage,
|
||||
createReadableStreamFromAnthropicStream,
|
||||
} from '@/providers/anthropic/utils'
|
||||
import { buildAnthropicMessageContent } from '@/providers/attachments'
|
||||
import {
|
||||
getMaxOutputTokensForModel,
|
||||
getThinkingCapability,
|
||||
@@ -229,9 +230,11 @@ export async function executeAnthropicProviderRequest(
|
||||
],
|
||||
})
|
||||
} else {
|
||||
const content = buildAnthropicMessageContent(msg.content, msg.files, config.providerId)
|
||||
messages.push({
|
||||
role: msg.role === 'assistant' ? 'assistant' : 'user',
|
||||
content: msg.content ? [{ type: 'text', text: msg.content }] : [],
|
||||
// double-cast-allowed: shared attachment builder returns Anthropic-compatible content blocks but avoids importing SDK-only union types
|
||||
content: content as unknown as Anthropic.Messages.ContentBlockParam[],
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { UserFile } from '@/executor/types'
|
||||
import {
|
||||
buildAnthropicMessageContent,
|
||||
buildBedrockMessageContent,
|
||||
buildGeminiMessageParts,
|
||||
buildOpenAIMessageContent,
|
||||
buildOpenRouterMessageContent,
|
||||
formatMessagesForProvider,
|
||||
inferAttachmentMimeType,
|
||||
prepareProviderAttachments,
|
||||
} from '@/providers/attachments'
|
||||
|
||||
const imageFile: UserFile = {
|
||||
id: 'file-1',
|
||||
name: 'example.png',
|
||||
url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace',
|
||||
size: 128,
|
||||
type: 'image/png',
|
||||
key: 'workspace/ws-1/example.png',
|
||||
base64: 'iVBORw0KGgo=',
|
||||
}
|
||||
|
||||
const pdfFile: UserFile = {
|
||||
id: 'file-2',
|
||||
name: 'example.pdf',
|
||||
url: '/api/files/serve/workspace%2Fws-1%2Fexample.pdf?context=workspace',
|
||||
size: 256,
|
||||
type: 'application/pdf',
|
||||
key: 'workspace/ws-1/example.pdf',
|
||||
base64: 'cGRm',
|
||||
}
|
||||
|
||||
const markdownFile: UserFile = {
|
||||
id: 'file-3',
|
||||
name: 'notes.md',
|
||||
url: '/api/files/serve/workspace%2Fws-1%2Fnotes.md?context=workspace',
|
||||
size: 17,
|
||||
type: 'text/markdown',
|
||||
key: 'workspace/ws-1/notes.md',
|
||||
base64: Buffer.from('# Notes\n\nHello').toString('base64'),
|
||||
}
|
||||
|
||||
describe('provider attachments', () => {
|
||||
it('infers MIME type from filename when file type is generic', () => {
|
||||
expect(
|
||||
inferAttachmentMimeType({
|
||||
...imageFile,
|
||||
type: 'application/octet-stream',
|
||||
})
|
||||
).toBe('image/png')
|
||||
})
|
||||
|
||||
it('formats OpenAI Responses content with text, image, and file parts', () => {
|
||||
const content = buildOpenAIMessageContent(
|
||||
'Analyze these files',
|
||||
[imageFile, pdfFile, markdownFile],
|
||||
'openai'
|
||||
)
|
||||
|
||||
expect(content).toEqual([
|
||||
{ type: 'input_text', text: 'Analyze these files' },
|
||||
{
|
||||
type: 'input_image',
|
||||
image_url: 'data:image/png;base64,iVBORw0KGgo=',
|
||||
},
|
||||
{
|
||||
type: 'input_file',
|
||||
filename: 'example.pdf',
|
||||
file_data: 'data:application/pdf;base64,cGRm',
|
||||
},
|
||||
{
|
||||
type: 'input_file',
|
||||
filename: 'notes.md',
|
||||
file_data: `data:text/markdown;base64,${markdownFile.base64}`,
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('formats Anthropic content with image, PDF document, and text document blocks', () => {
|
||||
const content = buildAnthropicMessageContent(
|
||||
'Analyze these files',
|
||||
[imageFile, pdfFile, markdownFile],
|
||||
'anthropic'
|
||||
)
|
||||
|
||||
expect(content).toEqual([
|
||||
{ type: 'text', text: 'Analyze these files' },
|
||||
{
|
||||
type: 'image',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: 'image/png',
|
||||
data: 'iVBORw0KGgo=',
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'document',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: 'application/pdf',
|
||||
data: 'cGRm',
|
||||
},
|
||||
title: 'example.pdf',
|
||||
},
|
||||
{
|
||||
type: 'document',
|
||||
source: {
|
||||
type: 'text',
|
||||
media_type: 'text/plain',
|
||||
data: '# Notes\n\nHello',
|
||||
},
|
||||
title: 'notes.md',
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('formats Gemini content with text and inline data parts', () => {
|
||||
const parts = buildGeminiMessageParts('Analyze this file', [imageFile, markdownFile], 'google')
|
||||
|
||||
expect(parts).toEqual([
|
||||
{ text: 'Analyze this file' },
|
||||
{
|
||||
inlineData: {
|
||||
mimeType: 'image/png',
|
||||
data: 'iVBORw0KGgo=',
|
||||
},
|
||||
},
|
||||
{
|
||||
inlineData: {
|
||||
mimeType: 'text/plain',
|
||||
data: markdownFile.base64,
|
||||
},
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('formats Bedrock content with native document blocks', () => {
|
||||
const parts = buildBedrockMessageContent('Analyze this file', [markdownFile], 'bedrock')
|
||||
|
||||
expect(parts).toEqual([
|
||||
{ text: 'Analyze this file' },
|
||||
{
|
||||
document: {
|
||||
format: 'md',
|
||||
name: 'notes',
|
||||
source: {
|
||||
bytes: Buffer.from(markdownFile.base64, 'base64'),
|
||||
},
|
||||
},
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('formats OpenRouter images and PDFs with native multimodal message parts', () => {
|
||||
const content = buildOpenRouterMessageContent(
|
||||
'Analyze these files',
|
||||
[imageFile, pdfFile],
|
||||
'openrouter'
|
||||
)
|
||||
|
||||
expect(content).toEqual([
|
||||
{ type: 'text', text: 'Analyze these files' },
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' },
|
||||
},
|
||||
{
|
||||
type: 'file',
|
||||
file: {
|
||||
filename: 'example.pdf',
|
||||
file_data: 'data:application/pdf;base64,cGRm',
|
||||
},
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('formats image-only provider messages and strips file fields', () => {
|
||||
const messages = formatMessagesForProvider(
|
||||
[{ role: 'user', content: 'Analyze this image', files: [imageFile] }],
|
||||
'groq'
|
||||
)
|
||||
|
||||
expect(messages).toEqual([
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'Analyze this image' },
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' },
|
||||
},
|
||||
],
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it('fails fast for unsupported MIME types', () => {
|
||||
expect(() =>
|
||||
prepareProviderAttachments(
|
||||
[
|
||||
{
|
||||
...imageFile,
|
||||
name: 'archive.zip',
|
||||
type: 'application/zip',
|
||||
},
|
||||
],
|
||||
'openai'
|
||||
)
|
||||
).toThrow('application/zip')
|
||||
})
|
||||
|
||||
it('sniffs image bytes and corrects a wrong declared image MIME type', () => {
|
||||
const content = buildAnthropicMessageContent(
|
||||
'Analyze this image',
|
||||
[
|
||||
{
|
||||
...imageFile,
|
||||
name: 'wrong.ico',
|
||||
type: 'image/x-icon',
|
||||
},
|
||||
],
|
||||
'anthropic'
|
||||
)
|
||||
|
||||
expect(content[1]).toEqual({
|
||||
type: 'image',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: 'image/png',
|
||||
data: 'iVBORw0KGgo=',
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects image attachments when the bytes are not a supported image format', () => {
|
||||
expect(() =>
|
||||
prepareProviderAttachments(
|
||||
[
|
||||
{
|
||||
...imageFile,
|
||||
name: 'not-an-image.png',
|
||||
base64: Buffer.from('not an image').toString('base64'),
|
||||
},
|
||||
],
|
||||
'anthropic'
|
||||
)
|
||||
).toThrow('not a supported model image format')
|
||||
})
|
||||
|
||||
it('rejects documents for image-only providers', () => {
|
||||
expect(() =>
|
||||
formatMessagesForProvider(
|
||||
[{ role: 'user', content: 'Analyze this file', files: [pdfFile] }],
|
||||
'groq'
|
||||
)
|
||||
).toThrow('Supported attachments: images')
|
||||
})
|
||||
|
||||
it('rejects providers without file attachment support', () => {
|
||||
expect(() =>
|
||||
formatMessagesForProvider(
|
||||
[{ role: 'user', content: 'Analyze this file', files: [imageFile] }],
|
||||
'deepseek'
|
||||
)
|
||||
).toThrow('not supported')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,571 @@
|
||||
import {
|
||||
getContentType,
|
||||
getExtensionFromMimeType,
|
||||
getFileExtension,
|
||||
getMimeTypeFromExtension,
|
||||
MIME_TYPE_MAPPING,
|
||||
MODEL_SUPPORTED_IMAGE_MIME_TYPES,
|
||||
} from '@/lib/uploads/utils/file-utils'
|
||||
import type { UserFile } from '@/executor/types'
|
||||
import type { ProviderId } from '@/providers/types'
|
||||
|
||||
export type AttachmentProvider =
|
||||
| 'openai'
|
||||
| 'anthropic'
|
||||
| 'google'
|
||||
| 'bedrock'
|
||||
| 'openrouter'
|
||||
| 'mistral'
|
||||
| 'groq'
|
||||
| 'fireworks'
|
||||
| 'ollama'
|
||||
| 'vllm'
|
||||
| 'xai'
|
||||
| 'deepseek'
|
||||
| 'cerebras'
|
||||
|
||||
export interface PreparedProviderAttachment {
|
||||
file: UserFile
|
||||
filename: string
|
||||
mimeType: string
|
||||
providerMimeType: string
|
||||
base64: string
|
||||
dataUrl: string
|
||||
text?: string
|
||||
extension: string
|
||||
contentType: 'image' | 'document' | 'audio' | 'video'
|
||||
}
|
||||
|
||||
type ProviderMessageInput = {
|
||||
role: string
|
||||
content?: string | null
|
||||
files?: UserFile[]
|
||||
}
|
||||
|
||||
type ProviderFormattedMessage = {
|
||||
role: string
|
||||
content?: string | null | Array<Record<string, unknown>>
|
||||
files?: UserFile[]
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
const AGENT_ATTACHMENT_MAX_BYTES = 10 * 1024 * 1024
|
||||
const PDF_MIME_TYPE = 'application/pdf'
|
||||
|
||||
const DOCUMENT_MIME_TYPES = new Set(
|
||||
Object.entries(MIME_TYPE_MAPPING)
|
||||
.filter(([, contentType]) => contentType === 'document')
|
||||
.map(([mimeType]) => mimeType)
|
||||
)
|
||||
|
||||
const OPENAI_DOCUMENT_MIME_TYPES = new Set([...DOCUMENT_MIME_TYPES, 'application/x-yaml'])
|
||||
|
||||
const GEMINI_INLINE_MIME_TYPES = new Set([...Object.keys(MIME_TYPE_MAPPING), 'application/x-yaml'])
|
||||
|
||||
const BEDROCK_DOCUMENT_FORMATS = new Set([
|
||||
'pdf',
|
||||
'csv',
|
||||
'doc',
|
||||
'docx',
|
||||
'xls',
|
||||
'xlsx',
|
||||
'html',
|
||||
'txt',
|
||||
'md',
|
||||
])
|
||||
const BEDROCK_IMAGE_FORMATS = new Set(['png', 'jpeg', 'jpg', 'gif', 'webp'])
|
||||
const BEDROCK_VIDEO_FORMATS = new Set(['mp4', 'mov', 'mkv', 'webm'])
|
||||
|
||||
const IMAGE_ONLY_PROVIDERS = new Set<AttachmentProvider>([
|
||||
'mistral',
|
||||
'groq',
|
||||
'fireworks',
|
||||
'ollama',
|
||||
'vllm',
|
||||
])
|
||||
|
||||
const UNSUPPORTED_FILE_PROVIDERS = new Set<AttachmentProvider>(['xai', 'deepseek', 'cerebras'])
|
||||
|
||||
const PROVIDER_SUPPORTED_LABELS: Record<AttachmentProvider, string> = {
|
||||
openai: 'images and documents through the Responses API input_image/input_file parts',
|
||||
anthropic: 'images, PDFs, and text documents through Claude content blocks',
|
||||
google: 'images, audio, video, PDFs, and text documents through Gemini inlineData',
|
||||
bedrock: 'Bedrock Converse image, document, and video content blocks',
|
||||
openrouter: 'images and PDFs through OpenRouter multimodal message parts',
|
||||
mistral: 'images through image_url message parts',
|
||||
groq: 'images through image_url message parts on multimodal models',
|
||||
fireworks: 'images through image_url message parts on vision models',
|
||||
ollama: 'images through image_url message parts on vision models',
|
||||
vllm: 'images through image_url message parts on multimodal models',
|
||||
xai: 'no file attachments in the current chat-completions adapter',
|
||||
deepseek: 'no file attachments in the current API adapter',
|
||||
cerebras: 'no file attachments in the current API adapter',
|
||||
}
|
||||
|
||||
export function getAttachmentProvider(providerId: ProviderId | string): AttachmentProvider | null {
|
||||
if (providerId === 'openai' || providerId === 'azure-openai') return 'openai'
|
||||
if (providerId === 'anthropic' || providerId === 'azure-anthropic') return 'anthropic'
|
||||
if (providerId === 'google' || providerId === 'vertex') return 'google'
|
||||
if (providerId === 'bedrock') return 'bedrock'
|
||||
if (providerId === 'openrouter') return 'openrouter'
|
||||
if (providerId === 'mistral') return 'mistral'
|
||||
if (providerId === 'groq') return 'groq'
|
||||
if (providerId === 'fireworks') return 'fireworks'
|
||||
if (providerId === 'ollama') return 'ollama'
|
||||
if (providerId === 'vllm') return 'vllm'
|
||||
if (providerId === 'xai') return 'xai'
|
||||
if (providerId === 'deepseek') return 'deepseek'
|
||||
if (providerId === 'cerebras') return 'cerebras'
|
||||
return null
|
||||
}
|
||||
|
||||
export function getProviderAttachmentMaxBytes(_providerId: ProviderId | string): number {
|
||||
return AGENT_ATTACHMENT_MAX_BYTES
|
||||
}
|
||||
|
||||
export function supportsFileAttachments(providerId: ProviderId | string): boolean {
|
||||
const provider = getAttachmentProvider(providerId)
|
||||
return Boolean(provider && !UNSUPPORTED_FILE_PROVIDERS.has(provider))
|
||||
}
|
||||
|
||||
export function inferAttachmentMimeType(file: UserFile): string {
|
||||
const explicitType = file.type?.trim().toLowerCase()
|
||||
if (explicitType && explicitType !== 'application/octet-stream') {
|
||||
return explicitType
|
||||
}
|
||||
|
||||
const inferred = getMimeTypeFromExtension(getFileExtension(file.name))
|
||||
return inferred.toLowerCase()
|
||||
}
|
||||
|
||||
function isTextDocumentMimeType(mimeType: string): boolean {
|
||||
return (
|
||||
mimeType.startsWith('text/') ||
|
||||
mimeType === 'application/json' ||
|
||||
mimeType === 'application/xml' ||
|
||||
mimeType === 'application/x-yaml'
|
||||
)
|
||||
}
|
||||
|
||||
function isImageMimeType(mimeType: string): boolean {
|
||||
return MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mimeType)
|
||||
}
|
||||
|
||||
function isOpenAIDocumentMimeType(mimeType: string): boolean {
|
||||
return OPENAI_DOCUMENT_MIME_TYPES.has(mimeType) || isTextDocumentMimeType(mimeType)
|
||||
}
|
||||
|
||||
function getAttachmentContentType(
|
||||
mimeType: string
|
||||
): PreparedProviderAttachment['contentType'] | null {
|
||||
return getContentType(mimeType) || (isTextDocumentMimeType(mimeType) ? 'document' : null)
|
||||
}
|
||||
|
||||
function sniffImageMimeType(base64: string): string {
|
||||
let bytes: Buffer
|
||||
try {
|
||||
bytes = Buffer.from(base64, 'base64')
|
||||
} catch {
|
||||
return ''
|
||||
}
|
||||
|
||||
if (
|
||||
bytes.length >= 8 &&
|
||||
bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))
|
||||
) {
|
||||
return 'image/png'
|
||||
}
|
||||
|
||||
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
|
||||
return 'image/jpeg'
|
||||
}
|
||||
|
||||
if (
|
||||
bytes.length >= 6 &&
|
||||
(bytes.subarray(0, 6).equals(Buffer.from('GIF87a')) ||
|
||||
bytes.subarray(0, 6).equals(Buffer.from('GIF89a')))
|
||||
) {
|
||||
return 'image/gif'
|
||||
}
|
||||
|
||||
if (
|
||||
bytes.length >= 12 &&
|
||||
bytes.subarray(0, 4).equals(Buffer.from('RIFF')) &&
|
||||
bytes.subarray(8, 12).equals(Buffer.from('WEBP'))
|
||||
) {
|
||||
return 'image/webp'
|
||||
}
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
function getAttachmentExtension(file: UserFile, mimeType: string): string {
|
||||
if (mimeType === 'text/markdown') return 'md'
|
||||
return getExtensionFromMimeType(mimeType) || getFileExtension(file.name)
|
||||
}
|
||||
|
||||
function normalizeProviderMimeType(mimeType: string, provider: AttachmentProvider): string {
|
||||
if ((provider === 'anthropic' || provider === 'google') && isTextDocumentMimeType(mimeType)) {
|
||||
return 'text/plain'
|
||||
}
|
||||
return mimeType
|
||||
}
|
||||
|
||||
function decodeBase64Text(base64: string, filename: string): string {
|
||||
try {
|
||||
return Buffer.from(base64, 'base64').toString('utf8')
|
||||
} catch {
|
||||
throw new Error(`File "${filename}" could not be decoded as UTF-8 text`)
|
||||
}
|
||||
}
|
||||
|
||||
function toDataUrl(mimeType: string, base64: string): string {
|
||||
return `data:${mimeType};base64,${base64}`
|
||||
}
|
||||
|
||||
function getProviderSupportedLabel(provider: AttachmentProvider): string {
|
||||
return PROVIDER_SUPPORTED_LABELS[provider]
|
||||
}
|
||||
|
||||
function validateProviderSupport(
|
||||
attachment: Omit<PreparedProviderAttachment, 'providerMimeType' | 'dataUrl' | 'text'>,
|
||||
provider: AttachmentProvider,
|
||||
providerId: ProviderId | string
|
||||
) {
|
||||
const { filename, mimeType, contentType, extension } = attachment
|
||||
const supportedLabel = getProviderSupportedLabel(provider)
|
||||
|
||||
const supported =
|
||||
provider === 'openai'
|
||||
? isImageMimeType(mimeType) || isOpenAIDocumentMimeType(mimeType)
|
||||
: provider === 'anthropic'
|
||||
? isImageMimeType(mimeType) ||
|
||||
mimeType === PDF_MIME_TYPE ||
|
||||
isTextDocumentMimeType(mimeType)
|
||||
: provider === 'google'
|
||||
? GEMINI_INLINE_MIME_TYPES.has(mimeType) || isTextDocumentMimeType(mimeType)
|
||||
: provider === 'bedrock'
|
||||
? (contentType === 'image' && BEDROCK_IMAGE_FORMATS.has(extension)) ||
|
||||
(contentType === 'document' && BEDROCK_DOCUMENT_FORMATS.has(extension)) ||
|
||||
(contentType === 'video' && BEDROCK_VIDEO_FORMATS.has(extension))
|
||||
: provider === 'openrouter'
|
||||
? isImageMimeType(mimeType) || mimeType === PDF_MIME_TYPE
|
||||
: IMAGE_ONLY_PROVIDERS.has(provider)
|
||||
? isImageMimeType(mimeType)
|
||||
: !UNSUPPORTED_FILE_PROVIDERS.has(provider)
|
||||
|
||||
if (!supported) {
|
||||
throw new Error(
|
||||
`File "${filename}" has MIME type "${mimeType}", which is not supported by provider "${providerId}". Supported attachments: ${supportedLabel}.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export function prepareProviderAttachments(
|
||||
files: UserFile[] | undefined,
|
||||
providerId: ProviderId | string
|
||||
): PreparedProviderAttachment[] {
|
||||
if (!files || files.length === 0) return []
|
||||
|
||||
const provider = getAttachmentProvider(providerId)
|
||||
if (!provider) {
|
||||
throw new Error(`File attachments are not supported for provider "${providerId}"`)
|
||||
}
|
||||
|
||||
if (UNSUPPORTED_FILE_PROVIDERS.has(provider)) {
|
||||
throw new Error(
|
||||
`File attachments are not supported for provider "${providerId}" in the current adapter. Supported attachments: ${getProviderSupportedLabel(provider)}.`
|
||||
)
|
||||
}
|
||||
|
||||
const maxBytes = getProviderAttachmentMaxBytes(providerId)
|
||||
|
||||
return files.map((file) => {
|
||||
const declaredMimeType = inferAttachmentMimeType(file)
|
||||
const contentType = getAttachmentContentType(declaredMimeType)
|
||||
|
||||
if (!contentType) {
|
||||
throw new Error(
|
||||
`File "${file.name}" has MIME type "${declaredMimeType}", which is not supported by provider "${providerId}". Supported attachments: ${getProviderSupportedLabel(provider)}.`
|
||||
)
|
||||
}
|
||||
|
||||
if (Number.isFinite(file.size) && file.size > maxBytes) {
|
||||
const sizeMB = (file.size / (1024 * 1024)).toFixed(2)
|
||||
const maxMB = (maxBytes / (1024 * 1024)).toFixed(0)
|
||||
throw new Error(
|
||||
`File "${file.name}" (${sizeMB}MB) exceeds the ${maxMB}MB agent attachment limit for provider "${providerId}"`
|
||||
)
|
||||
}
|
||||
|
||||
if (!file.base64) {
|
||||
throw new Error(`File "${file.name}" could not be read for provider "${providerId}"`)
|
||||
}
|
||||
|
||||
const sniffedImageMimeType = contentType === 'image' ? sniffImageMimeType(file.base64) : ''
|
||||
if (contentType === 'image' && !sniffedImageMimeType) {
|
||||
throw new Error(
|
||||
`Image bytes in "${file.name}" are not a supported model image format (declared MIME type "${declaredMimeType}"). Supported image formats: image/jpeg, image/png, image/gif, image/webp.`
|
||||
)
|
||||
}
|
||||
|
||||
const mimeType = sniffedImageMimeType || declaredMimeType
|
||||
const extension = getAttachmentExtension(file, mimeType)
|
||||
const attachment = {
|
||||
file,
|
||||
filename: file.name,
|
||||
mimeType,
|
||||
base64: file.base64,
|
||||
extension,
|
||||
contentType,
|
||||
}
|
||||
|
||||
validateProviderSupport(attachment, provider, providerId)
|
||||
|
||||
const providerMimeType = normalizeProviderMimeType(mimeType, provider)
|
||||
return {
|
||||
...attachment,
|
||||
providerMimeType,
|
||||
dataUrl: toDataUrl(providerMimeType, file.base64),
|
||||
...(isTextDocumentMimeType(mimeType) && {
|
||||
text: decodeBase64Text(file.base64, file.name),
|
||||
}),
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export function buildOpenAIMessageContent(
|
||||
content: string | null | undefined,
|
||||
files: UserFile[] | undefined,
|
||||
providerId: ProviderId | string
|
||||
): string | Array<Record<string, unknown>> {
|
||||
const attachments = prepareProviderAttachments(files, providerId)
|
||||
if (attachments.length === 0) return content ?? ''
|
||||
|
||||
const parts: Array<Record<string, unknown>> = []
|
||||
if (content) {
|
||||
parts.push({ type: 'input_text', text: content })
|
||||
}
|
||||
|
||||
for (const attachment of attachments) {
|
||||
if (attachment.contentType === 'image') {
|
||||
parts.push({ type: 'input_image', image_url: attachment.dataUrl })
|
||||
} else {
|
||||
parts.push({
|
||||
type: 'input_file',
|
||||
filename: attachment.filename,
|
||||
file_data: attachment.dataUrl,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return parts
|
||||
}
|
||||
|
||||
export function buildAnthropicMessageContent(
|
||||
content: string | null | undefined,
|
||||
files: UserFile[] | undefined,
|
||||
providerId: ProviderId | string
|
||||
): Array<Record<string, unknown>> {
|
||||
const parts: Array<Record<string, unknown>> = []
|
||||
if (content) {
|
||||
parts.push({ type: 'text', text: content })
|
||||
}
|
||||
|
||||
for (const attachment of prepareProviderAttachments(files, providerId)) {
|
||||
if (attachment.contentType === 'image') {
|
||||
parts.push({
|
||||
type: 'image',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: attachment.providerMimeType,
|
||||
data: attachment.base64,
|
||||
},
|
||||
})
|
||||
} else if (attachment.text) {
|
||||
parts.push({
|
||||
type: 'document',
|
||||
source: {
|
||||
type: 'text',
|
||||
media_type: 'text/plain',
|
||||
data: attachment.text,
|
||||
},
|
||||
title: attachment.filename,
|
||||
})
|
||||
} else {
|
||||
parts.push({
|
||||
type: 'document',
|
||||
source: {
|
||||
type: 'base64',
|
||||
media_type: attachment.providerMimeType,
|
||||
data: attachment.base64,
|
||||
},
|
||||
title: attachment.filename,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return parts
|
||||
}
|
||||
|
||||
export function buildGeminiMessageParts(
|
||||
content: string | null | undefined,
|
||||
files: UserFile[] | undefined,
|
||||
providerId: ProviderId | string
|
||||
): Array<Record<string, unknown>> {
|
||||
const parts: Array<Record<string, unknown>> = []
|
||||
if (content) {
|
||||
parts.push({ text: content })
|
||||
}
|
||||
|
||||
for (const attachment of prepareProviderAttachments(files, providerId)) {
|
||||
parts.push({
|
||||
inlineData: {
|
||||
mimeType: attachment.providerMimeType,
|
||||
data: attachment.base64,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return parts
|
||||
}
|
||||
|
||||
export function buildOpenAICompatibleChatContent(
|
||||
content: string | null | undefined,
|
||||
files: UserFile[] | undefined,
|
||||
providerId: ProviderId | string
|
||||
): string | Array<Record<string, unknown>> {
|
||||
const attachments = prepareProviderAttachments(files, providerId)
|
||||
if (attachments.length === 0) return content ?? ''
|
||||
|
||||
const parts: Array<Record<string, unknown>> = []
|
||||
if (content) {
|
||||
parts.push({ type: 'text', text: content })
|
||||
}
|
||||
|
||||
for (const attachment of attachments) {
|
||||
parts.push({
|
||||
type: 'image_url',
|
||||
image_url: {
|
||||
url: attachment.dataUrl,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return parts
|
||||
}
|
||||
|
||||
export function buildOpenRouterMessageContent(
|
||||
content: string | null | undefined,
|
||||
files: UserFile[] | undefined,
|
||||
providerId: ProviderId | string
|
||||
): string | Array<Record<string, unknown>> {
|
||||
const attachments = prepareProviderAttachments(files, providerId)
|
||||
if (attachments.length === 0) return content ?? ''
|
||||
|
||||
const parts: Array<Record<string, unknown>> = []
|
||||
if (content) {
|
||||
parts.push({ type: 'text', text: content })
|
||||
}
|
||||
|
||||
for (const attachment of attachments) {
|
||||
if (attachment.contentType === 'image') {
|
||||
parts.push({
|
||||
type: 'image_url',
|
||||
image_url: { url: attachment.dataUrl },
|
||||
})
|
||||
} else {
|
||||
parts.push({
|
||||
type: 'file',
|
||||
file: {
|
||||
filename: attachment.filename,
|
||||
file_data: attachment.dataUrl,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return parts
|
||||
}
|
||||
|
||||
function sanitizeBedrockName(filename: string): string {
|
||||
const baseName = filename.replace(/\.[^/.]+$/, '').replace(/[^a-zA-Z0-9\s()[\]-]/g, ' ')
|
||||
const compacted = baseName.replace(/\s+/g, ' ').trim()
|
||||
return compacted || 'Document'
|
||||
}
|
||||
|
||||
function getBedrockDocumentFormat(attachment: PreparedProviderAttachment): string {
|
||||
if (attachment.extension === 'md' || attachment.mimeType === 'text/markdown') return 'md'
|
||||
if (attachment.extension === 'txt' || attachment.mimeType === 'text/plain') return 'txt'
|
||||
return attachment.extension || 'txt'
|
||||
}
|
||||
|
||||
function getBedrockImageFormat(attachment: PreparedProviderAttachment): string {
|
||||
return attachment.extension === 'jpg' ? 'jpeg' : attachment.extension
|
||||
}
|
||||
|
||||
export function buildBedrockMessageContent(
|
||||
content: string | null | undefined,
|
||||
files: UserFile[] | undefined,
|
||||
providerId: ProviderId | string
|
||||
): Array<Record<string, unknown>> {
|
||||
const parts: Array<Record<string, unknown>> = []
|
||||
if (content) {
|
||||
parts.push({ text: content })
|
||||
}
|
||||
|
||||
for (const attachment of prepareProviderAttachments(files, providerId)) {
|
||||
const bytes = Buffer.from(attachment.base64, 'base64')
|
||||
if (attachment.contentType === 'image') {
|
||||
parts.push({
|
||||
image: {
|
||||
format: getBedrockImageFormat(attachment),
|
||||
source: { bytes },
|
||||
},
|
||||
})
|
||||
} else if (attachment.contentType === 'video') {
|
||||
parts.push({
|
||||
video: {
|
||||
format: attachment.extension,
|
||||
source: { bytes },
|
||||
},
|
||||
})
|
||||
} else {
|
||||
parts.push({
|
||||
document: {
|
||||
format: getBedrockDocumentFormat(attachment),
|
||||
name: sanitizeBedrockName(attachment.filename),
|
||||
source: { bytes },
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return parts
|
||||
}
|
||||
|
||||
export function formatMessagesForProvider(
|
||||
messages: ProviderMessageInput[],
|
||||
providerId: ProviderId | string
|
||||
): ProviderFormattedMessage[] {
|
||||
return messages.map((message) => {
|
||||
if (!message.files?.length || (message.role !== 'user' && message.role !== 'assistant')) {
|
||||
return message as ProviderFormattedMessage
|
||||
}
|
||||
|
||||
const provider = getAttachmentProvider(providerId)
|
||||
if (provider === 'openrouter') {
|
||||
const { files: _files, ...rest } = message
|
||||
return {
|
||||
...rest,
|
||||
content: buildOpenRouterMessageContent(message.content, message.files, providerId),
|
||||
}
|
||||
}
|
||||
|
||||
const { files: _files, ...rest } = message
|
||||
return {
|
||||
...rest,
|
||||
content: buildOpenAICompatibleChatContent(message.content, message.files, providerId),
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -90,6 +90,9 @@ async function executeChatCompletionsRequest(
|
||||
}
|
||||
|
||||
if (request.messages) {
|
||||
if (request.messages.some((message) => message.files?.length)) {
|
||||
throw new Error('File attachments require an Azure OpenAI Responses API endpoint')
|
||||
}
|
||||
allMessages.push(...(request.messages as ChatCompletionMessageParam[]))
|
||||
}
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import type { IterationToolCall, StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { buildBedrockMessageContent } from '@/providers/attachments'
|
||||
import {
|
||||
checkForForcedToolUsage,
|
||||
createReadableStreamFromBedrockStream,
|
||||
@@ -178,9 +179,11 @@ export const bedrockProvider: ProviderConfig = {
|
||||
}
|
||||
} else {
|
||||
const role: ConversationRole = msg.role === 'assistant' ? 'assistant' : 'user'
|
||||
const content = buildBedrockMessageContent(msg.content, msg.files, 'bedrock')
|
||||
messages.push({
|
||||
role,
|
||||
content: [{ text: msg.content || '' }],
|
||||
// double-cast-allowed: shared attachment builder emits Bedrock Converse content blocks while keeping provider-neutral attachment types
|
||||
content: content as unknown as ContentBlock[],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import type { StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import type { CerebrasResponse } from '@/providers/cerebras/types'
|
||||
import { createReadableStreamFromCerebrasStream } from '@/providers/cerebras/utils'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
@@ -64,6 +65,7 @@ export const cerebrasProvider: ProviderConfig = {
|
||||
if (request.messages) {
|
||||
allMessages.push(...request.messages)
|
||||
}
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'cerebras')
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
@@ -78,7 +80,7 @@ export const cerebrasProvider: ProviderConfig = {
|
||||
|
||||
const payload: any = {
|
||||
model: request.model.replace('cerebras/', ''),
|
||||
messages: allMessages,
|
||||
messages: formattedMessages,
|
||||
}
|
||||
if (request.temperature !== undefined) payload.temperature = request.temperature
|
||||
if (request.maxTokens != null) payload.max_completion_tokens = request.maxTokens
|
||||
@@ -199,7 +201,7 @@ export const cerebrasProvider: ProviderConfig = {
|
||||
}
|
||||
const toolCalls = []
|
||||
const toolResults: Record<string, unknown>[] = []
|
||||
const currentMessages = [...allMessages]
|
||||
const currentMessages = [...formattedMessages]
|
||||
let iterationCount = 0
|
||||
|
||||
let modelTime = firstResponseTime
|
||||
|
||||
@@ -3,6 +3,7 @@ import { toError } from '@sim/utils/errors'
|
||||
import OpenAI from 'openai'
|
||||
import type { StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { createReadableStreamFromDeepseekStream } from '@/providers/deepseek/utils'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
@@ -67,6 +68,7 @@ export const deepseekProvider: ProviderConfig = {
|
||||
if (request.messages) {
|
||||
allMessages.push(...request.messages)
|
||||
}
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'deepseek')
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
@@ -81,7 +83,7 @@ export const deepseekProvider: ProviderConfig = {
|
||||
|
||||
const payload: any = {
|
||||
model: request.model,
|
||||
messages: allMessages,
|
||||
messages: formattedMessages,
|
||||
}
|
||||
|
||||
if (request.temperature !== undefined) payload.temperature = request.temperature
|
||||
@@ -209,7 +211,7 @@ export const deepseekProvider: ProviderConfig = {
|
||||
}
|
||||
const toolCalls = []
|
||||
const toolResults: Record<string, unknown>[] = []
|
||||
const currentMessages = [...allMessages]
|
||||
const currentMessages = [...formattedMessages]
|
||||
let iterationCount = 0
|
||||
let hasUsedForcedTool = false
|
||||
let modelTime = firstResponseTime
|
||||
|
||||
@@ -4,6 +4,7 @@ import OpenAI from 'openai'
|
||||
import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions'
|
||||
import type { StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import {
|
||||
checkForForcedToolUsage,
|
||||
createReadableStreamFromOpenAIStream,
|
||||
@@ -108,6 +109,7 @@ export const fireworksProvider: ProviderConfig = {
|
||||
if (request.messages) {
|
||||
allMessages.push(...request.messages)
|
||||
}
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'fireworks') as Message[]
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
@@ -122,7 +124,7 @@ export const fireworksProvider: ProviderConfig = {
|
||||
|
||||
const payload: any = {
|
||||
model: requestedModel,
|
||||
messages: allMessages,
|
||||
messages: formattedMessages,
|
||||
}
|
||||
|
||||
if (request.temperature !== undefined) payload.temperature = request.temperature
|
||||
@@ -250,7 +252,7 @@ export const fireworksProvider: ProviderConfig = {
|
||||
}
|
||||
const toolCalls: FunctionCallResponse[] = []
|
||||
const toolResults: Record<string, unknown>[] = []
|
||||
const currentMessages = [...allMessages]
|
||||
const currentMessages = [...formattedMessages]
|
||||
let iterationCount = 0
|
||||
let modelTime = firstResponseTime
|
||||
let toolsTime = 0
|
||||
|
||||
@@ -917,7 +917,7 @@ export async function executeGeminiRequest(
|
||||
const providerStartTimeISO = new Date(providerStartTime).toISOString()
|
||||
|
||||
try {
|
||||
const { contents, tools, systemInstruction } = convertToGeminiFormat(request)
|
||||
const { contents, tools, systemInstruction } = convertToGeminiFormat(request, providerType)
|
||||
|
||||
// Build configuration
|
||||
const geminiConfig: GenerateContentConfig = {}
|
||||
|
||||
@@ -111,6 +111,44 @@ describe('ensureStructResponse', () => {
|
||||
})
|
||||
|
||||
describe('convertToGeminiFormat', () => {
|
||||
it('should convert user message files to inline data parts', () => {
|
||||
const request: ProviderRequest = {
|
||||
model: 'gemini-2.5-flash',
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: 'Analyze this image',
|
||||
files: [
|
||||
{
|
||||
id: 'file-1',
|
||||
key: 'workspace/ws-1/example.png',
|
||||
name: 'example.png',
|
||||
url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace',
|
||||
size: 128,
|
||||
type: 'image/png',
|
||||
base64: 'iVBORw0KGgo=',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
const result = convertToGeminiFormat(request)
|
||||
|
||||
expect(result.contents[0]).toEqual({
|
||||
role: 'user',
|
||||
parts: [
|
||||
{ text: 'Analyze this image' },
|
||||
{
|
||||
inlineData: {
|
||||
mimeType: 'image/png',
|
||||
data: 'iVBORw0KGgo=',
|
||||
},
|
||||
},
|
||||
],
|
||||
})
|
||||
})
|
||||
|
||||
describe('tool message handling', () => {
|
||||
it('should convert tool message with object response correctly', () => {
|
||||
const request: ProviderRequest = {
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
} from '@google/genai'
|
||||
import { createLogger } from '@sim/logger'
|
||||
import { toError } from '@sim/utils/errors'
|
||||
import { buildGeminiMessageParts } from '@/providers/attachments'
|
||||
import type { ProviderRequest } from '@/providers/types'
|
||||
import { trackForcedToolUsage } from '@/providers/utils'
|
||||
|
||||
@@ -166,7 +167,10 @@ export interface GeminiToolDef {
|
||||
/**
|
||||
* Converts OpenAI-style request format to Gemini format
|
||||
*/
|
||||
export function convertToGeminiFormat(request: ProviderRequest): {
|
||||
export function convertToGeminiFormat(
|
||||
request: ProviderRequest,
|
||||
providerId = 'google'
|
||||
): {
|
||||
contents: Content[]
|
||||
tools: GeminiToolDef[] | undefined
|
||||
systemInstruction: Content | undefined
|
||||
@@ -192,9 +196,10 @@ export function convertToGeminiFormat(request: ProviderRequest): {
|
||||
}
|
||||
} else if (message.role === 'user' || message.role === 'assistant') {
|
||||
const geminiRole = message.role === 'user' ? 'user' : 'model'
|
||||
const parts = buildGeminiMessageParts(message.content, message.files, providerId) as Part[]
|
||||
|
||||
if (message.content) {
|
||||
contents.push({ role: geminiRole, parts: [{ text: message.content }] })
|
||||
if (parts.length > 0) {
|
||||
contents.push({ role: geminiRole, parts })
|
||||
}
|
||||
|
||||
if (message.role === 'assistant' && message.tool_calls?.length) {
|
||||
|
||||
@@ -3,6 +3,7 @@ import { toError } from '@sim/utils/errors'
|
||||
import { Groq } from 'groq-sdk'
|
||||
import type { StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { createReadableStreamFromGroqStream } from '@/providers/groq/utils'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
@@ -60,6 +61,7 @@ export const groqProvider: ProviderConfig = {
|
||||
if (request.messages) {
|
||||
allMessages.push(...request.messages)
|
||||
}
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'groq')
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
@@ -74,7 +76,7 @@ export const groqProvider: ProviderConfig = {
|
||||
|
||||
const payload: any = {
|
||||
model: request.model.replace('groq/', ''),
|
||||
messages: allMessages,
|
||||
messages: formattedMessages,
|
||||
}
|
||||
|
||||
if (request.temperature !== undefined) payload.temperature = request.temperature
|
||||
@@ -205,7 +207,7 @@ export const groqProvider: ProviderConfig = {
|
||||
}
|
||||
const toolCalls = []
|
||||
const toolResults: Record<string, unknown>[] = []
|
||||
const currentMessages = [...allMessages]
|
||||
const currentMessages = [...formattedMessages]
|
||||
let iterationCount = 0
|
||||
let modelTime = firstResponseTime
|
||||
let toolsTime = 0
|
||||
|
||||
@@ -4,6 +4,7 @@ import OpenAI from 'openai'
|
||||
import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions'
|
||||
import type { StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { createReadableStreamFromMistralStream } from '@/providers/mistral/utils'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
@@ -77,6 +78,7 @@ export const mistralProvider: ProviderConfig = {
|
||||
if (request.messages) {
|
||||
allMessages.push(...request.messages)
|
||||
}
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'mistral')
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
@@ -91,7 +93,7 @@ export const mistralProvider: ProviderConfig = {
|
||||
|
||||
const payload: any = {
|
||||
model: request.model,
|
||||
messages: allMessages,
|
||||
messages: formattedMessages,
|
||||
}
|
||||
|
||||
if (request.temperature !== undefined) payload.temperature = request.temperature
|
||||
@@ -262,7 +264,7 @@ export const mistralProvider: ProviderConfig = {
|
||||
}
|
||||
const toolCalls = []
|
||||
const toolResults: Record<string, unknown>[] = []
|
||||
const currentMessages = [...allMessages]
|
||||
const currentMessages = [...formattedMessages]
|
||||
let iterationCount = 0
|
||||
|
||||
let modelTime = firstResponseTime
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/
|
||||
import { getOllamaUrl } from '@/lib/core/utils/urls'
|
||||
import type { StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import type { ModelsObject } from '@/providers/ollama/types'
|
||||
import { createReadableStreamFromOllamaStream } from '@/providers/ollama/utils'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
@@ -91,6 +92,7 @@ export const ollamaProvider: ProviderConfig = {
|
||||
if (request.messages) {
|
||||
allMessages.push(...request.messages)
|
||||
}
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'ollama')
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
@@ -105,7 +107,7 @@ export const ollamaProvider: ProviderConfig = {
|
||||
|
||||
const payload: any = {
|
||||
model: request.model,
|
||||
messages: allMessages,
|
||||
messages: formattedMessages,
|
||||
}
|
||||
|
||||
if (request.temperature !== undefined) payload.temperature = request.temperature
|
||||
@@ -274,7 +276,7 @@ export const ollamaProvider: ProviderConfig = {
|
||||
}
|
||||
const toolCalls = []
|
||||
const toolResults: Record<string, unknown>[] = []
|
||||
const currentMessages = [...allMessages]
|
||||
const currentMessages = [...formattedMessages]
|
||||
let iterationCount = 0
|
||||
|
||||
let modelTime = firstResponseTime
|
||||
|
||||
@@ -133,7 +133,7 @@ export async function executeResponsesProviderRequest(
|
||||
allMessages.push(...request.messages)
|
||||
}
|
||||
|
||||
const initialInput = buildResponsesInputFromMessages(allMessages)
|
||||
const initialInput = buildResponsesInputFromMessages(allMessages, config.providerId)
|
||||
|
||||
const basePayload: Record<string, unknown> = {
|
||||
model: config.modelName,
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
/**
|
||||
* @vitest-environment node
|
||||
*/
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { buildResponsesInputFromMessages } from '@/providers/openai/utils'
|
||||
|
||||
describe('buildResponsesInputFromMessages', () => {
|
||||
it('should convert user message files to Responses multipart content', () => {
|
||||
const input = buildResponsesInputFromMessages([
|
||||
{
|
||||
role: 'user',
|
||||
content: 'Analyze this image',
|
||||
files: [
|
||||
{
|
||||
id: 'file-1',
|
||||
key: 'workspace/ws-1/example.png',
|
||||
name: 'example.png',
|
||||
url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace',
|
||||
size: 128,
|
||||
type: 'image/png',
|
||||
base64: 'iVBORw0KGgo=',
|
||||
},
|
||||
],
|
||||
},
|
||||
])
|
||||
|
||||
expect(input).toEqual([
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'input_text', text: 'Analyze this image' },
|
||||
{
|
||||
type: 'input_image',
|
||||
image_url: 'data:image/png;base64,iVBORw0KGgo=',
|
||||
},
|
||||
],
|
||||
},
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -1,5 +1,6 @@
|
||||
import { createLogger } from '@sim/logger'
|
||||
import type OpenAI from 'openai'
|
||||
import { buildOpenAIMessageContent } from '@/providers/attachments'
|
||||
import type { Message } from '@/providers/types'
|
||||
|
||||
const logger = createLogger('ResponsesUtils')
|
||||
@@ -21,7 +22,7 @@ export interface ResponsesToolCall {
|
||||
export type ResponsesInputItem =
|
||||
| {
|
||||
role: 'system' | 'user' | 'assistant'
|
||||
content: string
|
||||
content: string | Array<Record<string, unknown>>
|
||||
}
|
||||
| {
|
||||
type: 'function_call'
|
||||
@@ -45,7 +46,10 @@ export interface ResponsesToolDefinition {
|
||||
/**
|
||||
* Converts chat-style messages into Responses API input items.
|
||||
*/
|
||||
export function buildResponsesInputFromMessages(messages: Message[]): ResponsesInputItem[] {
|
||||
export function buildResponsesInputFromMessages(
|
||||
messages: Message[],
|
||||
providerId = 'openai'
|
||||
): ResponsesInputItem[] {
|
||||
const input: ResponsesInputItem[] = []
|
||||
|
||||
for (const message of messages) {
|
||||
@@ -58,13 +62,21 @@ export function buildResponsesInputFromMessages(messages: Message[]): ResponsesI
|
||||
continue
|
||||
}
|
||||
|
||||
if (
|
||||
message.content &&
|
||||
(message.role === 'system' || message.role === 'user' || message.role === 'assistant')
|
||||
) {
|
||||
if (message.role === 'system' || message.role === 'user' || message.role === 'assistant') {
|
||||
const content =
|
||||
message.role === 'user'
|
||||
? buildOpenAIMessageContent(message.content, message.files, providerId)
|
||||
: (message.content ?? '')
|
||||
if (
|
||||
(typeof content === 'string' && !content) ||
|
||||
(Array.isArray(content) && content.length === 0)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
|
||||
input.push({
|
||||
role: message.role,
|
||||
content: message.content,
|
||||
content,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import OpenAI from 'openai'
|
||||
import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions'
|
||||
import type { StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import {
|
||||
checkForForcedToolUsage,
|
||||
@@ -109,6 +110,7 @@ export const openRouterProvider: ProviderConfig = {
|
||||
if (request.messages) {
|
||||
allMessages.push(...request.messages)
|
||||
}
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'openrouter') as Message[]
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
@@ -123,7 +125,7 @@ export const openRouterProvider: ProviderConfig = {
|
||||
|
||||
const payload: any = {
|
||||
model: requestedModel,
|
||||
messages: allMessages,
|
||||
messages: formattedMessages,
|
||||
}
|
||||
|
||||
if (request.temperature !== undefined) payload.temperature = request.temperature
|
||||
@@ -251,7 +253,7 @@ export const openRouterProvider: ProviderConfig = {
|
||||
}
|
||||
const toolCalls: FunctionCallResponse[] = []
|
||||
const toolResults: Record<string, unknown>[] = []
|
||||
const currentMessages = [...allMessages]
|
||||
const currentMessages = [...formattedMessages]
|
||||
let iterationCount = 0
|
||||
let modelTime = firstResponseTime
|
||||
let toolsTime = 0
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { ProviderTimingSegment, StreamingExecution } from '@/executor/types'
|
||||
import type { ProviderTimingSegment, StreamingExecution, UserFile } from '@/executor/types'
|
||||
|
||||
export type ProviderId =
|
||||
| 'openai'
|
||||
@@ -121,6 +121,7 @@ export interface ProviderToolConfig {
|
||||
export interface Message {
|
||||
role: 'system' | 'user' | 'assistant' | 'function' | 'tool'
|
||||
content: string | null
|
||||
files?: UserFile[]
|
||||
name?: string
|
||||
function_call?: {
|
||||
name: string
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/
|
||||
import { env } from '@/lib/core/config/env'
|
||||
import type { StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
@@ -123,6 +124,7 @@ export const vllmProvider: ProviderConfig = {
|
||||
if (request.messages) {
|
||||
allMessages.push(...request.messages)
|
||||
}
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'vllm') as Message[]
|
||||
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
@@ -137,7 +139,7 @@ export const vllmProvider: ProviderConfig = {
|
||||
|
||||
const payload: any = {
|
||||
model: request.model.replace(/^vllm\//, ''),
|
||||
messages: allMessages,
|
||||
messages: formattedMessages,
|
||||
}
|
||||
|
||||
if (request.temperature !== undefined) payload.temperature = request.temperature
|
||||
@@ -319,7 +321,7 @@ export const vllmProvider: ProviderConfig = {
|
||||
}
|
||||
const toolCalls = []
|
||||
const toolResults: Record<string, unknown>[] = []
|
||||
const currentMessages = [...allMessages]
|
||||
const currentMessages = [...formattedMessages]
|
||||
let iterationCount = 0
|
||||
|
||||
let modelTime = firstResponseTime
|
||||
|
||||
@@ -4,6 +4,7 @@ import OpenAI from 'openai'
|
||||
import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions'
|
||||
import type { StreamingExecution } from '@/executor/types'
|
||||
import { MAX_TOOL_ITERATIONS } from '@/providers'
|
||||
import { formatMessagesForProvider } from '@/providers/attachments'
|
||||
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
|
||||
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
|
||||
import type {
|
||||
@@ -76,6 +77,7 @@ export const xAIProvider: ProviderConfig = {
|
||||
if (request.messages) {
|
||||
allMessages.push(...request.messages)
|
||||
}
|
||||
const formattedMessages = formatMessagesForProvider(allMessages, 'xai') as Message[]
|
||||
const tools = request.tools?.length
|
||||
? request.tools.map((tool) => ({
|
||||
type: 'function',
|
||||
@@ -93,7 +95,7 @@ export const xAIProvider: ProviderConfig = {
|
||||
}
|
||||
const basePayload: any = {
|
||||
model: request.model,
|
||||
messages: allMessages,
|
||||
messages: formattedMessages,
|
||||
}
|
||||
|
||||
if (request.temperature !== undefined) basePayload.temperature = request.temperature
|
||||
@@ -219,7 +221,7 @@ export const xAIProvider: ProviderConfig = {
|
||||
}
|
||||
const toolCalls = []
|
||||
const toolResults: Record<string, unknown>[] = []
|
||||
const currentMessages = [...allMessages]
|
||||
const currentMessages = [...formattedMessages]
|
||||
let iterationCount = 0
|
||||
|
||||
let hasUsedForcedTool = false
|
||||
|
||||
@@ -7,6 +7,64 @@ interface FileGetParams {
|
||||
_context?: WorkflowToolExecutionContext
|
||||
}
|
||||
|
||||
interface FileReadParams {
|
||||
fileId?: string | string[]
|
||||
fileInput?: unknown
|
||||
workspaceId?: string
|
||||
_context?: WorkflowToolExecutionContext
|
||||
}
|
||||
|
||||
const createFileReadTool = (config: {
|
||||
id: 'file_read'
|
||||
name: string
|
||||
description: string
|
||||
}): ToolConfig<FileReadParams, ToolResponse> => ({
|
||||
id: config.id,
|
||||
name: config.name,
|
||||
description: config.description,
|
||||
version: '1.0.0',
|
||||
|
||||
params: {
|
||||
fileId: {
|
||||
type: 'string',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'Canonical workspace file ID, or an array of canonical workspace file IDs.',
|
||||
},
|
||||
fileInput: {
|
||||
type: 'file',
|
||||
required: false,
|
||||
visibility: 'user-only',
|
||||
description: 'Selected workspace file object.',
|
||||
},
|
||||
},
|
||||
|
||||
request: {
|
||||
url: '/api/tools/file/manage',
|
||||
method: 'POST',
|
||||
headers: () => ({ 'Content-Type': 'application/json' }),
|
||||
body: (params) => ({
|
||||
operation: 'read',
|
||||
fileId: params.fileId,
|
||||
fileInput: params.fileInput,
|
||||
workspaceId: params.workspaceId || params._context?.workspaceId,
|
||||
}),
|
||||
},
|
||||
|
||||
transformResponse: async (response) => {
|
||||
const data = await response.json()
|
||||
if (!response.ok || !data.success) {
|
||||
return { success: false, output: {}, error: data.error || 'Failed to get file' }
|
||||
}
|
||||
return { success: true, output: data.data }
|
||||
},
|
||||
|
||||
outputs: {
|
||||
file: { type: 'file', description: 'Workspace file object' },
|
||||
files: { type: 'file[]', description: 'Workspace file objects' },
|
||||
},
|
||||
})
|
||||
|
||||
export const fileGetTool: ToolConfig<FileGetParams, ToolResponse> = {
|
||||
id: 'file_get',
|
||||
name: 'File Get',
|
||||
@@ -52,3 +110,9 @@ export const fileGetTool: ToolConfig<FileGetParams, ToolResponse> = {
|
||||
file: { type: 'file', description: 'Workspace file object' },
|
||||
},
|
||||
}
|
||||
|
||||
export const fileReadTool = createFileReadTool({
|
||||
id: 'file_read',
|
||||
name: 'File Read',
|
||||
description: 'Read workspace file objects from selected files or canonical workspace file IDs.',
|
||||
})
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
import { fileParserTool, fileParserV2Tool, fileParserV3Tool } from '@/tools/file/parser'
|
||||
import {
|
||||
fileFetchTool,
|
||||
fileParserTool,
|
||||
fileParserV2Tool,
|
||||
fileParserV3Tool,
|
||||
} from '@/tools/file/parser'
|
||||
|
||||
export { fileAppendTool } from '@/tools/file/append'
|
||||
export { fileGetTool } from '@/tools/file/get'
|
||||
export { fileGetTool, fileReadTool } from '@/tools/file/get'
|
||||
export { fileWriteTool } from '@/tools/file/write'
|
||||
|
||||
export const fileParseTool = fileParserTool
|
||||
export { fileFetchTool }
|
||||
export { fileParserV2Tool }
|
||||
export { fileParserV3Tool }
|
||||
|
||||
@@ -10,6 +10,7 @@ import type {
|
||||
FileParserV3OutputData,
|
||||
FileUploadInput,
|
||||
} from '@/tools/file/types'
|
||||
import { transformTable } from '@/tools/shared/table'
|
||||
import type { ToolConfig } from '@/tools/types'
|
||||
|
||||
const logger = createLogger('FileParserTool')
|
||||
@@ -34,6 +35,20 @@ const isFileParseResult = (value: unknown): value is FileParseResult =>
|
||||
typeof value.name === 'string' &&
|
||||
typeof value.binary === 'boolean'
|
||||
|
||||
const normalizeHeaders = (headers: FileParserInput['headers']): Record<string, string> => {
|
||||
const transformed = transformTable(headers ?? null)
|
||||
return Object.entries(transformed).reduce(
|
||||
(acc, [key, value]) => {
|
||||
const headerName = key.trim()
|
||||
if (headerName && value !== undefined && value !== null) {
|
||||
acc[headerName] = String(value)
|
||||
}
|
||||
return acc
|
||||
},
|
||||
{} as Record<string, string>
|
||||
)
|
||||
}
|
||||
|
||||
const normalizeFileParseResult = (value: unknown): FileParseResult => {
|
||||
if (isRecord(value) && isFileParseResult(value.output)) {
|
||||
return value.output
|
||||
@@ -245,9 +260,11 @@ export const fileParserTool: ToolConfig<FileParserInput, FileParserOutput> = {
|
||||
}
|
||||
|
||||
logger.info('Tool body determined filePath:', determinedFilePath)
|
||||
const headers = normalizeHeaders(params.headers)
|
||||
return {
|
||||
filePath: determinedFilePath,
|
||||
fileType: determinedFileType,
|
||||
...(Object.keys(headers).length > 0 && { headers }),
|
||||
workspaceId: params.workspaceId || params._context?.workspaceId,
|
||||
workflowId: params._context?.workflowId,
|
||||
executionId: params._context?.executionId,
|
||||
@@ -286,6 +303,25 @@ export const fileParserV2Tool: ToolConfig<FileParserInput, FileParserOutput> = {
|
||||
},
|
||||
}
|
||||
|
||||
const parseFileParserV3Response = async (response: Response): Promise<FileParserV3Output> => {
|
||||
const parsed = await parseFileParserResponse(response)
|
||||
const output = parsed.output as FileParserOutputData
|
||||
const files =
|
||||
Array.isArray(output.processedFiles) && output.processedFiles.length > 0
|
||||
? output.processedFiles
|
||||
: []
|
||||
|
||||
const cleanedOutput: FileParserV3OutputData = {
|
||||
files,
|
||||
combinedContent: output.combinedContent,
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: cleanedOutput,
|
||||
}
|
||||
}
|
||||
|
||||
export const fileParserV3Tool: ToolConfig<FileParserInput, FileParserV3Output> = {
|
||||
id: 'file_parser_v3',
|
||||
name: 'File Parser',
|
||||
@@ -293,26 +329,31 @@ export const fileParserV3Tool: ToolConfig<FileParserInput, FileParserV3Output> =
|
||||
version: '3.0.0',
|
||||
params: fileParserTool.params,
|
||||
request: fileParserTool.request,
|
||||
transformResponse: async (response: Response): Promise<FileParserV3Output> => {
|
||||
const parsed = await parseFileParserResponse(response)
|
||||
const output = parsed.output as FileParserOutputData
|
||||
const files =
|
||||
Array.isArray(output.processedFiles) && output.processedFiles.length > 0
|
||||
? output.processedFiles
|
||||
: []
|
||||
|
||||
const cleanedOutput: FileParserV3OutputData = {
|
||||
files,
|
||||
combinedContent: output.combinedContent,
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
output: cleanedOutput,
|
||||
}
|
||||
},
|
||||
transformResponse: parseFileParserV3Response,
|
||||
outputs: {
|
||||
files: { type: 'file[]', description: 'Parsed files as UserFile objects' },
|
||||
combinedContent: { type: 'string', description: 'Combined content of all parsed files' },
|
||||
},
|
||||
}
|
||||
|
||||
export const fileFetchTool: ToolConfig<FileParserInput, FileParserV3Output> = {
|
||||
id: 'file_fetch',
|
||||
name: 'File Fetch',
|
||||
description: 'Fetch and parse a file from a URL with optional custom headers.',
|
||||
version: '1.0.0',
|
||||
params: {
|
||||
...fileParserTool.params,
|
||||
headers: {
|
||||
type: 'object',
|
||||
required: false,
|
||||
visibility: 'user-or-llm',
|
||||
description: 'HTTP headers to include when fetching URL-based files.',
|
||||
},
|
||||
},
|
||||
request: fileParserTool.request,
|
||||
transformResponse: parseFileParserV3Response,
|
||||
outputs: {
|
||||
files: { type: 'file[]', description: 'Fetched files as UserFile objects' },
|
||||
combinedContent: { type: 'string', description: 'Combined content of all fetched files' },
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import type { UserFile } from '@/executor/types'
|
||||
import type { ToolResponse } from '@/tools/types'
|
||||
import type { TableRow, ToolResponse } from '@/tools/types'
|
||||
|
||||
export interface FileParserInput {
|
||||
filePath?: string | string[]
|
||||
file?: UserFile | UserFile[] | FileUploadInput | FileUploadInput[]
|
||||
fileType?: string
|
||||
headers?: TableRow[] | Record<string, unknown> | string | null
|
||||
workspaceId?: string
|
||||
workflowId?: string
|
||||
executionId?: string
|
||||
|
||||
@@ -645,10 +645,12 @@ import {
|
||||
} from '@/tools/fathom'
|
||||
import {
|
||||
fileAppendTool,
|
||||
fileFetchTool,
|
||||
fileGetTool,
|
||||
fileParserV2Tool,
|
||||
fileParserV3Tool,
|
||||
fileParseTool,
|
||||
fileReadTool,
|
||||
fileWriteTool,
|
||||
} from '@/tools/file'
|
||||
import {
|
||||
@@ -3216,7 +3218,9 @@ export const tools: Record<string, ToolConfig> = {
|
||||
file_parser_v2: fileParserV2Tool,
|
||||
file_parser_v3: fileParserV3Tool,
|
||||
file_append: fileAppendTool,
|
||||
file_fetch: fileFetchTool,
|
||||
file_get: fileGetTool,
|
||||
file_read: fileReadTool,
|
||||
file_write: fileWriteTool,
|
||||
firecrawl_scrape: firecrawlScrapeTool,
|
||||
firecrawl_search: firecrawlSearchTool,
|
||||
|
||||
Reference in New Issue
Block a user