improvement(agent, file-block): files in agent block, file block v4 (#4610)

* File block v4

* Support files in agent block

* Clean up attachments

* Fix start.files

* Fix

* Fix
This commit is contained in:
Siddharth Ganesan
2026-05-15 10:48:21 -07:00
committed by GitHub
parent 9bbbe0adcb
commit bad21cb23d
52 changed files with 2131 additions and 109 deletions
@@ -258,6 +258,7 @@ export const blockTypeToIconMap: Record<string, IconComponent> = {
extend_v2: ExtendIcon,
fathom: FathomIcon,
file_v3: DocumentIcon,
file_v4: DocumentIcon,
firecrawl: FirecrawlIcon,
fireflies_v2: FirefliesIcon,
gamma: GammaIcon,
@@ -4032,18 +4032,22 @@
"tags": ["meeting", "note-taking"]
},
{
"type": "file_v3",
"type": "file_v4",
"slug": "file",
"name": "File",
"description": "Read and write workspace files",
"longDescription": "Read and parse files from uploads or URLs, write new workspace files, or append content to existing files.",
"description": "Read, fetch, write, and append files",
"longDescription": "Read workspace files by picker or canonical ID, fetch and parse files from URLs with optional headers, write new workspace files, or append content to existing files.",
"bgColor": "#40916C",
"iconName": "DocumentIcon",
"docsUrl": "https://docs.sim.ai/tools/file",
"operations": [
{
"name": "Read",
"description": "Parse one or more uploaded files or files from URLs (text, PDF, CSV, images, etc.)"
"description": "Get a workspace file object from a selected file or canonical workspace file ID."
},
{
"name": "Fetch",
"description": "Parse a file from a URL with optional custom headers for authenticated downloads."
},
{
"name": "Write",
@@ -4054,7 +4058,7 @@
"description": "Append content to an existing workspace file. The file must already exist. Content is added to the end of the file."
}
],
"operationCount": 3,
"operationCount": 4,
"triggers": [],
"triggerCount": 0,
"authType": "none",
@@ -8,6 +8,7 @@ import {
createMockRequest,
hybridAuthMockFns,
inputValidationMock,
inputValidationMockFns,
permissionsMock,
permissionsMockFns,
storageServiceMock,
@@ -310,6 +311,39 @@ describe('File Parse API Route', () => {
expect(data.results).toHaveLength(2)
})
it('should pass custom headers when fetching external URLs', async () => {
inputValidationMockFns.mockValidateUrlWithDNS.mockResolvedValue({
isValid: true,
resolvedIP: '203.0.113.10',
})
inputValidationMockFns.mockSecureFetchWithPinnedIP.mockResolvedValue(
new Response('private file content', {
status: 200,
headers: { 'content-type': 'text/plain' },
})
)
const headers = { Authorization: 'Bearer xoxb-test-token' }
const req = createMockRequest('POST', {
filePath: 'https://files.slack.com/files-pri/T000-F000/download/report.txt',
headers,
})
const response = await POST(req)
const data = await response.json()
expect(response.status).toBe(200)
expect(data.success).toBe(true)
expect(inputValidationMockFns.mockSecureFetchWithPinnedIP).toHaveBeenCalledWith(
'https://files.slack.com/files-pri/T000-F000/download/report.txt',
'203.0.113.10',
expect.objectContaining({
timeout: 30000,
headers,
})
)
})
it('should process execution file URLs with context query param', async () => {
setupFileApiMocks({
cloudEnabled: true,
+18 -6
View File
@@ -110,7 +110,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
)
if (!parsed.success) return parsed.response
const { filePath, fileType, workspaceId, workflowId, executionId } = parsed.data.body
const { filePath, fileType, headers, workspaceId, workflowId, executionId } = parsed.data.body
if (!filePath || (typeof filePath === 'string' && filePath.trim() === '')) {
return NextResponse.json({ success: false, error: 'No file path provided' }, { status: 400 })
@@ -128,6 +128,7 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
workspaceId,
userId,
hasExecutionContext: !!executionContext,
hasHeaders: Boolean(headers && Object.keys(headers).length > 0),
})
if (Array.isArray(filePath)) {
@@ -146,7 +147,8 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
fileType,
workspaceId,
userId,
executionContext
executionContext,
headers
)
if (result.metadata) {
result.metadata.processingTime = Date.now() - startTime
@@ -180,7 +182,14 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
})
}
const result = await parseFileSingle(filePath, fileType, workspaceId, userId, executionContext)
const result = await parseFileSingle(
filePath,
fileType,
workspaceId,
userId,
executionContext,
headers
)
if (result.metadata) {
result.metadata.processingTime = Date.now() - startTime
@@ -225,7 +234,8 @@ async function parseFileSingle(
fileType: string,
workspaceId: string,
userId: string,
executionContext?: ExecutionContext
executionContext?: ExecutionContext,
headers?: Record<string, string>
): Promise<ParseResult> {
logger.info('Parsing file:', filePath)
@@ -251,7 +261,7 @@ async function parseFileSingle(
}
if (filePath.startsWith('http://') || filePath.startsWith('https://')) {
return handleExternalUrl(filePath, fileType, workspaceId, userId, executionContext)
return handleExternalUrl(filePath, fileType, workspaceId, userId, executionContext, headers)
}
if (isUsingCloudStorage()) {
@@ -298,7 +308,8 @@ async function handleExternalUrl(
fileType: string,
workspaceId: string,
userId: string,
executionContext?: ExecutionContext
executionContext?: ExecutionContext,
headers?: Record<string, string>
): Promise<ParseResult> {
try {
logger.info('Fetching external URL:', url)
@@ -382,6 +393,7 @@ async function handleExternalUrl(
const response = await secureFetchWithPinnedIP(url, urlValidation.resolvedIP!, {
timeout: DOWNLOAD_TIMEOUT_MS,
...(headers && Object.keys(headers).length > 0 && { headers }),
})
if (!response.ok) {
throw new Error(`Failed to fetch URL: ${response.status} ${response.statusText}`)
+140 -9
View File
@@ -20,6 +20,100 @@ export const dynamic = 'force-dynamic'
const logger = createLogger('FileManageAPI')
const workspaceFileToUserFile = (file: Awaited<ReturnType<typeof getWorkspaceFile>>) => {
if (!file) return null
return {
id: file.id,
name: file.name,
url: ensureAbsoluteUrl(file.path),
size: file.size,
type: file.type,
key: file.key,
context: 'workspace',
}
}
const fileInputToUserFile = (fileInput: unknown) => {
if (!fileInput || typeof fileInput !== 'object' || Array.isArray(fileInput)) return null
const record = fileInput as Record<string, unknown>
const id =
typeof record.id === 'string'
? record.id.trim()
: typeof record.fileId === 'string'
? record.fileId.trim()
: ''
// Objects with ids are resolved through workspace metadata. This fallback is for
// picker/upload values that only carry storage fields.
if (id) return null
const key = typeof record.key === 'string' ? record.key.trim() : ''
const path = typeof record.path === 'string' ? record.path.trim() : ''
const url = typeof record.url === 'string' ? record.url.trim() : ''
const fileUrl =
url || path || (key ? `/api/files/serve/${encodeURIComponent(key)}?context=workspace` : '')
if (!fileUrl && !key) return null
return {
id: key || fileUrl,
name:
typeof record.name === 'string' && record.name.trim() ? record.name.trim() : 'workspace-file',
url: fileUrl ? ensureAbsoluteUrl(fileUrl) : '',
size: typeof record.size === 'number' ? record.size : 0,
type:
typeof record.type === 'string' && record.type.trim()
? record.type.trim()
: 'application/octet-stream',
key,
context: 'workspace',
}
}
const normalizeFileIdList = (value: unknown): string[] => {
if (typeof value === 'string') {
const trimmed = value.trim()
if (!trimmed) return []
try {
return normalizeFileIdList(JSON.parse(trimmed))
} catch {
return [trimmed]
}
}
if (!Array.isArray(value)) return []
return value
.map((item) => (typeof item === 'string' ? item.trim() : ''))
.filter((id) => id.length > 0)
}
const extractUserFilesFromInput = (fileInput: unknown) => {
const inputs = Array.isArray(fileInput) ? fileInput : fileInput ? [fileInput] : []
return inputs
.map((input) => fileInputToUserFile(input))
.filter((file): file is NonNullable<ReturnType<typeof fileInputToUserFile>> => Boolean(file))
}
const extractFileIdsFromInput = (fileInput: unknown): string[] => {
const inputs = Array.isArray(fileInput) ? fileInput : fileInput ? [fileInput] : []
return inputs
.flatMap((input) => {
if (typeof input === 'string') return normalizeFileIdList(input)
if (input && typeof input === 'object') {
const record = input as Record<string, unknown>
if (typeof record.id === 'string') return normalizeFileIdList(record.id)
if (typeof record.fileId === 'string') return normalizeFileIdList(record.fileId)
}
return []
})
.filter((id) => id.length > 0)
}
export const POST = withRouteHandler(async (request: NextRequest) => {
const auth = await checkInternalAuth(request, { requireWorkflowId: false })
if (!auth.success) {
@@ -76,15 +170,52 @@ export const POST = withRouteHandler(async (request: NextRequest) => {
return NextResponse.json({
success: true,
data: {
file: {
id: file.id,
name: file.name,
url: ensureAbsoluteUrl(file.path),
size: file.size,
type: file.type,
key: file.key,
context: 'workspace',
},
file: workspaceFileToUserFile(file),
},
})
}
case 'read': {
const { fileId, fileInput } = body
const selectedFileIds = Array.isArray(fileId)
? fileId.map((id) => id.trim()).filter(Boolean)
: fileId
? normalizeFileIdList(fileId)
: extractFileIdsFromInput(fileInput)
const selectedInputFiles = fileId ? [] : extractUserFilesFromInput(fileInput)
if (selectedFileIds.length === 0 && selectedInputFiles.length === 0) {
return NextResponse.json({ success: false, error: 'File is required' }, { status: 400 })
}
const files = await Promise.all(
selectedFileIds.map((id) => getWorkspaceFile(workspaceId, id))
)
const missingFileId = selectedFileIds.find((_, index) => !files[index])
if (missingFileId) {
return NextResponse.json(
{ success: false, error: `File not found: "${missingFileId}"` },
{ status: 404 }
)
}
const userFiles = files
.map((file) => workspaceFileToUserFile(file))
.filter((file): file is NonNullable<ReturnType<typeof workspaceFileToUserFile>> =>
Boolean(file)
)
.concat(selectedInputFiles)
logger.info('Files retrieved', {
count: userFiles.length,
fileIds: userFiles.map((file) => file.id),
})
return NextResponse.json({
success: true,
data: {
file: userFiles[0],
files: userFiles,
},
})
}
@@ -359,6 +359,7 @@ async function handleExecutePost(
includeFileBase64,
base64MaxBytes,
workflowStateOverride,
executionId: requestedExecutionId,
triggerBlockId: parsedTriggerBlockId,
startBlockId,
stopAfterBlockId,
@@ -508,7 +509,8 @@ async function handleExecutePost(
)
}
const executionId = generateId()
const executionId =
isClientSession && requestedExecutionId ? requestedExecutionId : generateId()
reqLogger = reqLogger.withMetadata({ userId, executionId })
reqLogger.info('Starting server-side execution', {
@@ -524,6 +524,7 @@ export function useWorkflowExecution() {
size: fileData.file.size,
type: fileData.file.type,
key: result.key,
context: 'execution',
})
} catch (uploadError) {
if (
@@ -565,6 +566,7 @@ export function useWorkflowExecution() {
size: r.size,
type: r.type,
key: r.key,
context: r.context || 'execution',
uploadedAt: r.uploadedAt,
expiresAt: r.expiresAt,
})
@@ -1126,6 +1128,7 @@ export function useWorkflowExecution() {
await executionStream.execute({
workflowId: activeWorkflowId,
input: finalWorkflowInput,
executionId,
startBlockId,
selectedOutputs,
triggerType: overrideTriggerType || 'manual',
@@ -903,6 +903,7 @@ export async function executeWorkflowWithFullLogging(
triggerType: options.overrideTriggerType || 'manual',
useDraftState: options.useDraftState ?? true,
isClientSession: true,
...(options.executionId ? { executionId: options.executionId } : {}),
...(options.triggerBlockId ? { triggerBlockId: options.triggerBlockId } : {}),
...(options.stopAfterBlockId ? { stopAfterBlockId: options.stopAfterBlockId } : {}),
...(options.runFromBlock
+105
View File
@@ -94,6 +94,111 @@ describe.concurrent('Blocks Module', () => {
})
})
describe('File block', () => {
it('should keep v3 read and get routed to the legacy tools', () => {
const block = getBlock('file_v3')
expect(block).toBeDefined()
expect(block?.hideFromToolbar).toBe(true)
expect(block?.subBlocks[0].options?.map((option) => option.id)).toEqual([
'file_parser_v3',
'file_get',
'file_write',
'file_append',
])
expect(block?.tools.config?.tool({ operation: 'file_parser_v3' })).toBe('file_parser_v3')
expect(block?.tools.config?.tool({ operation: 'file_get' })).toBe('file_get')
})
it('should expose v4 with read and fetch routed to the expected tools', () => {
const block = getBlock('file_v4')
expect(block).toBeDefined()
expect(block?.hideFromToolbar).toBe(false)
expect(block?.subBlocks[0].options?.map((option) => option.id)).toEqual([
'file_read',
'file_fetch',
'file_write',
'file_append',
])
expect(block?.subBlocks.find((subBlock) => subBlock.id === 'readFile')?.multiple).toBe(true)
expect(block?.tools.config?.tool({ operation: 'file_read' })).toBe('file_read')
expect(block?.tools.config?.tool({ operation: 'file_fetch' })).toBe('file_fetch')
expect(
block?.tools.config?.params?.({
operation: 'file_read',
readFileInput: '["file-1","file-2"]',
_context: { workspaceId: 'workspace-1' },
})
).toEqual({
fileId: ['file-1', 'file-2'],
workspaceId: 'workspace-1',
})
expect(
block?.tools.config?.params?.({
operation: 'file_read',
readFileInput: [
{
key: 'workspace/workspace-1/example.md',
name: 'example.md',
path: '/api/files/serve/workspace%2Fworkspace-1%2Fexample.md?context=workspace',
size: 123,
type: 'text/markdown',
},
],
_context: { workspaceId: 'workspace-1' },
})
).toEqual({
fileInput: [
{
key: 'workspace/workspace-1/example.md',
name: 'example.md',
path: '/api/files/serve/workspace%2Fworkspace-1%2Fexample.md?context=workspace',
size: 123,
type: 'text/markdown',
},
],
workspaceId: 'workspace-1',
})
})
})
describe('Agent block', () => {
it('should expose canonical file attachments and normalize file params', () => {
const block = getBlock('agent')
expect(block).toBeDefined()
const uploadSubBlock = block?.subBlocks.find((subBlock) => subBlock.id === 'attachmentFiles')
const advancedSubBlock = block?.subBlocks.find((subBlock) => subBlock.id === 'files')
expect(uploadSubBlock?.type).toBe('file-upload')
expect(uploadSubBlock?.canonicalParamId).toBe('files')
expect(uploadSubBlock?.multiple).toBe(true)
expect(advancedSubBlock?.canonicalParamId).toBe('files')
expect(block?.inputs.files).toEqual({
type: 'array',
description: 'Files to include with the latest user message',
})
expect(
block?.tools.config?.params?.({
model: 'gpt-4o',
files:
'[{"id":"file-1","key":"workspace/ws-1/example.png","name":"example.png","url":"/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace","size":123,"type":"image/png"}]',
})
).toMatchObject({
files: [
{
id: 'file-1',
key: 'workspace/ws-1/example.png',
name: 'example.png',
type: 'image/png',
},
],
})
})
})
describe('getBlocksByCategory', () => {
it('should return blocks in the "blocks" category', () => {
const blocks = getBlocksByCategory('blocks')
+26 -2
View File
@@ -5,6 +5,7 @@ import { AuthMode, IntegrationType } from '@/blocks/types'
import {
getModelOptions,
getProviderCredentialSubBlocks,
normalizeFileInput,
RESPONSE_FORMAT_WAND_CONFIG,
} from '@/blocks/utils'
import {
@@ -133,6 +134,25 @@ Return ONLY the JSON array.`,
defaultValue: 'claude-sonnet-4-6',
options: getModelOptions,
},
{
id: 'attachmentFiles',
title: 'Files',
type: 'file-upload',
canonicalParamId: 'files',
placeholder: 'Upload files for the agent',
multiple: true,
mode: 'basic',
required: false,
},
{
id: 'files',
title: 'Files',
type: 'short-input',
canonicalParamId: 'files',
placeholder: 'Reference files from previous blocks',
mode: 'advanced',
required: false,
},
{
id: 'reasoningEffort',
title: 'Reasoning Effort',
@@ -472,6 +492,9 @@ Return ONLY the JSON array.`,
return tool
},
params: (params: Record<string, any>) => {
const normalizedFiles = normalizeFileInput(params.files)
const baseParams = normalizedFiles ? { ...params, files: normalizedFiles } : params
// If tools array is provided, handle tool usage control
if (params.tools && Array.isArray(params.tools)) {
// Transform tools to include usageControl
@@ -506,9 +529,9 @@ Return ONLY the JSON array.`,
logger.info('Filtered out tools set to none', { tools: filteredOutTools.join(', ') })
}
return { ...params, tools: transformedTools }
return { ...baseParams, tools: transformedTools }
}
return params
return baseParams
},
},
},
@@ -518,6 +541,7 @@ Return ONLY the JSON array.`,
description:
'Array of message objects with role and content: [{ role: "system", content: "..." }, { role: "user", content: "..." }]',
},
files: { type: 'array', description: 'Files to include with the latest user message' },
memoryType: {
type: 'string',
description:
+289
View File
@@ -259,6 +259,7 @@ export const FileV3Block: BlockConfig<FileParserV3Output> = {
tags: ['document-processing'],
bgColor: '#40916C',
icon: DocumentIcon,
hideFromToolbar: true,
subBlocks: [
{
id: 'operation',
@@ -514,3 +515,291 @@ export const FileV3Block: BlockConfig<FileParserV3Output> = {
},
},
}
const parseReadFileIds = (input: unknown): string | string[] | null => {
let value = input
if (typeof value === 'string') {
const trimmed = value.trim()
if (!trimmed) return null
try {
value = JSON.parse(trimmed)
} catch {
return trimmed
}
}
if (Array.isArray(value)) {
const fileIds = value
.map((item) => (typeof item === 'string' ? item.trim() : ''))
.filter((item) => item.length > 0)
if (fileIds.length === 0) return null
return fileIds.length === 1 ? fileIds[0] : fileIds
}
return null
}
export const FileV4Block: BlockConfig<FileParserV3Output> = {
...FileV3Block,
type: 'file_v4',
name: 'File',
description: 'Read, fetch, write, and append files',
longDescription:
'Read workspace files by picker or canonical ID, fetch and parse files from URLs with optional headers, write new workspace files, or append content to existing files.',
hideFromToolbar: false,
bestPractices: `
- Use Read when you need an existing workspace file object by picker selection or canonical file ID.
- Use Fetch for external file URLs. Add headers for authenticated downloads, for example Slack private file URLs require an Authorization Bearer token.
`,
subBlocks: [
{
id: 'operation',
title: 'Operation',
type: 'dropdown' as SubBlockType,
options: [
{ label: 'Read', id: 'file_read' },
{ label: 'Fetch', id: 'file_fetch' },
{ label: 'Write', id: 'file_write' },
{ label: 'Append', id: 'file_append' },
],
value: () => 'file_read',
},
{
id: 'readFile',
title: 'File',
type: 'file-upload' as SubBlockType,
canonicalParamId: 'readFileInput',
acceptedTypes: '*',
placeholder: 'Select workspace files',
multiple: true,
mode: 'basic',
condition: { field: 'operation', value: 'file_read' },
required: { field: 'operation', value: 'file_read' },
},
{
id: 'readFileId',
title: 'File ID',
type: 'short-input' as SubBlockType,
canonicalParamId: 'readFileInput',
placeholder: 'Workspace file ID or JSON array of IDs',
mode: 'advanced',
condition: { field: 'operation', value: 'file_read' },
required: { field: 'operation', value: 'file_read' },
},
{
id: 'fileUrl',
title: 'File URL',
type: 'short-input' as SubBlockType,
placeholder: 'https://example.com/document.pdf',
condition: { field: 'operation', value: 'file_fetch' },
required: { field: 'operation', value: 'file_fetch' },
},
{
id: 'headers',
title: 'Headers',
type: 'table' as SubBlockType,
columns: ['Key', 'Value'],
description:
'Custom headers for fetching the file URL, such as Authorization: Bearer <token>.',
condition: { field: 'operation', value: 'file_fetch' },
},
{
id: 'fileName',
title: 'File Name',
type: 'short-input' as SubBlockType,
placeholder: 'File name (e.g., data.csv)',
condition: { field: 'operation', value: 'file_write' },
required: { field: 'operation', value: 'file_write' },
},
{
id: 'content',
title: 'Content',
type: 'long-input' as SubBlockType,
placeholder: 'File content to write...',
condition: { field: 'operation', value: 'file_write' },
required: { field: 'operation', value: 'file_write' },
},
{
id: 'contentType',
title: 'Content Type',
type: 'short-input' as SubBlockType,
placeholder: 'text/plain (auto-detected from extension)',
condition: { field: 'operation', value: 'file_write' },
mode: 'advanced',
},
{
id: 'appendFile',
title: 'File',
type: 'file-upload' as SubBlockType,
canonicalParamId: 'appendFileInput',
acceptedTypes: '.txt,.md,.json,.csv,.xml,.html,.htm,.yaml,.yml,.log,.rtf',
placeholder: 'Select or upload a workspace file',
mode: 'basic',
condition: { field: 'operation', value: 'file_append' },
required: { field: 'operation', value: 'file_append' },
},
{
id: 'appendFileName',
title: 'File',
type: 'short-input' as SubBlockType,
canonicalParamId: 'appendFileInput',
placeholder: 'File name (e.g., notes.md)',
mode: 'advanced',
condition: { field: 'operation', value: 'file_append' },
required: { field: 'operation', value: 'file_append' },
},
{
id: 'appendContent',
title: 'Content',
type: 'long-input' as SubBlockType,
placeholder: 'Content to append...',
condition: { field: 'operation', value: 'file_append' },
required: { field: 'operation', value: 'file_append' },
},
],
tools: {
access: ['file_fetch', 'file_read', 'file_write', 'file_append'],
config: {
tool: (params) => {
const operation = params.operation || 'file_read'
if (operation === 'file_read') return 'file_read'
if (operation === 'file_fetch') return 'file_fetch'
return operation
},
params: (params) => {
const operation = params.operation || 'file_read'
if (operation === 'file_write') {
return {
fileName: params.fileName,
content: params.content,
contentType: params.contentType,
workspaceId: params._context?.workspaceId,
}
}
if (operation === 'file_append') {
const appendInput = params.appendFileInput
if (!appendInput) {
throw new Error('File is required for append')
}
let fileName: string
if (typeof appendInput === 'string') {
fileName = appendInput.trim()
} else {
const normalized = normalizeFileInput(appendInput, { single: true })
const file = normalized as Record<string, unknown> | null
fileName = (file?.name as string) ?? ''
}
if (!fileName) {
throw new Error('Could not determine file name')
}
return {
fileName,
content: params.appendContent,
workspaceId: params._context?.workspaceId,
}
}
if (operation === 'file_read') {
const readInput = params.readFileInput
if (!readInput) {
throw new Error('File is required for read')
}
const fileIds = parseReadFileIds(readInput)
if (fileIds) {
return {
fileId: fileIds,
workspaceId: params._context?.workspaceId,
}
}
const normalized = normalizeFileInput(readInput)
if (!normalized || normalized.length === 0) {
throw new Error('File is required for read')
}
return {
fileInput: normalized,
workspaceId: params._context?.workspaceId,
}
}
if (operation === 'file_fetch') {
const fileUrl = typeof params.fileUrl === 'string' ? params.fileUrl.trim() : ''
if (!fileUrl) {
logger.error('No file URL provided')
throw new Error('File URL is required')
}
return {
filePath: fileUrl,
fileType: params.fileType || 'auto',
headers: params.headers,
workspaceId: params._context?.workspaceId,
workflowId: params._context?.workflowId,
executionId: params._context?.executionId,
}
}
logger.error(`Invalid file operation: ${operation}`)
throw new Error('Invalid file operation')
},
},
},
inputs: {
operation: {
type: 'string',
description: 'Operation to perform (read, fetch, write, or append)',
},
readFileInput: {
type: 'json',
description: 'Selected workspace file or canonical file ID for read',
},
fileUrl: { type: 'string', description: 'External file URL for fetch' },
headers: { type: 'json', description: 'Request headers for fetch' },
fileType: { type: 'string', description: 'File type for fetch' },
fileName: { type: 'string', description: 'Name for a new file (write)' },
content: { type: 'string', description: 'File content to write' },
contentType: { type: 'string', description: 'MIME content type for write' },
appendFileInput: { type: 'json', description: 'File to append to' },
appendContent: { type: 'string', description: 'Content to append to file' },
},
outputs: {
file: {
type: 'file',
description: 'First workspace file object (read)',
},
files: {
type: 'file[]',
description: 'Workspace file objects (read) or fetched file objects (fetch)',
},
combinedContent: {
type: 'string',
description: 'All fetched file contents merged into a single text string (fetch)',
},
id: {
type: 'string',
description: 'File ID (write)',
},
name: {
type: 'string',
description: 'File name (write)',
},
size: {
type: 'number',
description: 'File size in bytes (write)',
},
url: {
type: 'string',
description: 'URL to access the file (write)',
},
},
}
+2 -1
View File
@@ -55,7 +55,7 @@ import { EvernoteBlock } from '@/blocks/blocks/evernote'
import { ExaBlock } from '@/blocks/blocks/exa'
import { ExtendBlock, ExtendV2Block } from '@/blocks/blocks/extend'
import { FathomBlock } from '@/blocks/blocks/fathom'
import { FileBlock, FileV2Block, FileV3Block } from '@/blocks/blocks/file'
import { FileBlock, FileV2Block, FileV3Block, FileV4Block } from '@/blocks/blocks/file'
import { FirecrawlBlock } from '@/blocks/blocks/firecrawl'
import { FirefliesBlock, FirefliesV2Block } from '@/blocks/blocks/fireflies'
import { FunctionBlock } from '@/blocks/blocks/function'
@@ -299,6 +299,7 @@ export const registry: Record<string, BlockConfig> = {
file: FileBlock,
file_v2: FileV2Block,
file_v3: FileV3Block,
file_v4: FileV4Block,
firecrawl: FirecrawlBlock,
fireflies: FirefliesBlock,
fireflies_v2: FirefliesV2Block,
-13
View File
@@ -449,19 +449,6 @@ export class ExecutionEngine {
const readyNodes = this.edgeManager.processOutgoingEdges(node, output, false)
this.execLogger.info('Processing outgoing edges', {
nodeId,
outgoingEdgesCount: node.outgoingEdges.size,
outgoingEdges: Array.from(node.outgoingEdges.entries()).map(([id, e]) => ({
id,
target: e.target,
sourceHandle: e.sourceHandle,
})),
output,
readyNodesCount: readyNodes.length,
readyNodes,
})
this.addMultipleToQueue(readyNodes)
if (this.context.pendingDynamicNodes && this.context.pendingDynamicNodes.length > 0) {
@@ -264,6 +264,78 @@ describe('AgentBlockHandler', () => {
expect(result).toEqual(expectedOutput)
})
it('should attach files to the last user message only', async () => {
const inputs = {
model: 'gpt-4o',
messages: [
{ role: 'system' as const, content: 'You are helpful.' },
{ role: 'user' as const, content: 'Earlier question' },
{ role: 'assistant' as const, content: 'Earlier answer' },
{ role: 'user' as const, content: 'Analyze this file' },
],
files: [
{
id: 'file-1',
key: 'workspace/ws-1/example.png',
name: 'example.png',
url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace',
size: 128,
type: 'image/png',
base64: 'aW1hZ2U=',
},
],
apiKey: 'test-api-key',
}
mockGetProviderFromModel.mockReturnValue('openai')
await handler.execute(mockContext, mockBlock, inputs)
const requestBody = mockExecuteProviderRequest.mock.calls[0][1]
expect(requestBody.messages[1]).toMatchObject({
role: 'user',
content: 'Earlier question',
})
expect(requestBody.messages[1].files).toBeUndefined()
expect(requestBody.messages[3]).toMatchObject({
role: 'user',
content: 'Analyze this file',
files: [
{
id: 'file-1',
name: 'example.png',
type: 'image/png',
base64: 'aW1hZ2U=',
},
],
})
})
it('should reject files for providers without attachment support', async () => {
const inputs = {
model: 'deepseek-chat',
messages: [{ role: 'user' as const, content: 'Analyze this file' }],
files: [
{
id: 'file-1',
key: 'workspace/ws-1/example.png',
name: 'example.png',
url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace',
size: 128,
type: 'image/png',
base64: 'aW1hZ2U=',
},
],
apiKey: 'test-api-key',
}
mockGetProviderFromModel.mockReturnValue('deepseek')
await expect(handler.execute(mockContext, mockBlock, inputs)).rejects.toThrow(
'File attachments are not supported for provider "deepseek"'
)
})
it('should preserve usageControl for custom tools and filter out "none"', async () => {
const inputs = {
model: 'gpt-4o',
@@ -6,9 +6,12 @@ import { sleep } from '@sim/utils/helpers'
import { and, eq, inArray, isNull } from 'drizzle-orm'
import { normalizeStringRecord, normalizeWorkflowVariables } from '@/lib/core/utils/records'
import { createMcpToolId } from '@/lib/mcp/utils'
import { processFilesToUserFiles, type RawFileInput } from '@/lib/uploads/utils/file-utils'
import { hydrateUserFilesWithBase64 } from '@/lib/uploads/utils/user-file-base64.server'
import { getCustomToolById } from '@/lib/workflows/custom-tools/operations'
import { getAllBlocks } from '@/blocks'
import type { BlockOutput } from '@/blocks/types'
import { normalizeFileInput } from '@/blocks/utils'
import {
validateBlockType,
validateCustomToolsAllowed,
@@ -36,6 +39,7 @@ import { buildAPIUrl, buildAuthHeaders } from '@/executor/utils/http'
import { stringifyJSON } from '@/executor/utils/json'
import { resolveVertexCredential } from '@/executor/utils/vertex-credential'
import { executeProviderRequest } from '@/providers'
import { getProviderAttachmentMaxBytes, supportsFileAttachments } from '@/providers/attachments'
import { getProviderFromModel, transformBlockTool } from '@/providers/utils'
import type { SerializedBlock } from '@/serializer/types'
import { filterSchemaForLLM, type ToolSchema } from '@/tools/params'
@@ -87,12 +91,22 @@ export class AgentBlockHandler implements BlockHandler {
const streamingConfig = this.getStreamingConfig(ctx, block)
const messages = await this.buildMessages(ctx, filteredInputs, skillMetadata)
const messagesWithInputFiles = this.attachFilesToLastUserMessage(
ctx,
messages,
filteredInputs.files
)
const messagesWithFiles = await this.hydrateMessageFilesForProvider(
ctx,
messagesWithInputFiles,
providerId
)
const providerRequest = this.buildProviderRequest({
ctx,
providerId,
model,
messages,
messages: messagesWithFiles,
inputs: filteredInputs,
formattedTools,
responseFormat,
@@ -672,6 +686,107 @@ export class AgentBlockHandler implements BlockHandler {
return messages.length > 0 ? messages : undefined
}
private attachFilesToLastUserMessage(
ctx: ExecutionContext,
messages: Message[] | undefined,
filesInput: unknown
): Message[] | undefined {
const normalizedFiles = normalizeFileInput(filesInput)
if (!normalizedFiles || normalizedFiles.length === 0) {
return messages
}
if (!messages || messages.length === 0) {
throw new Error('Files require at least one user message in the agent prompt')
}
let lastUserMessageIndex = -1
for (let index = messages.length - 1; index >= 0; index--) {
if (messages[index].role === 'user') {
lastUserMessageIndex = index
break
}
}
if (lastUserMessageIndex === -1) {
throw new Error('Files require at least one user message in the agent prompt')
}
const requestId = ctx.executionId || ctx.workflowId || 'agent-files'
const userFiles = processFilesToUserFiles(normalizedFiles as RawFileInput[], requestId, logger)
if (userFiles.length === 0) {
throw new Error('Files must include at least one valid file object')
}
const lastUserMessage = messages[lastUserMessageIndex]
const nextMessages = [...messages]
nextMessages[lastUserMessageIndex] = {
...lastUserMessage,
files: [...(lastUserMessage.files ?? []), ...userFiles],
}
return nextMessages
}
private async hydrateMessageFilesForProvider(
ctx: ExecutionContext,
messages: Message[] | undefined,
providerId: string
): Promise<Message[] | undefined> {
if (!messages?.some((message) => message.files?.length)) {
return messages
}
if (!supportsFileAttachments(providerId)) {
throw new Error(`File attachments are not supported for provider "${providerId}"`)
}
const requestId = ctx.executionId || ctx.workflowId || 'agent-files'
const nextMessages = [...messages]
for (let messageIndex = 0; messageIndex < messages.length; messageIndex++) {
const message = messages[messageIndex]
const normalizedFiles = normalizeFileInput(message.files)
if (!normalizedFiles || normalizedFiles.length === 0) {
continue
}
const userFiles = processFilesToUserFiles(
normalizedFiles as RawFileInput[],
requestId,
logger
)
if (userFiles.length === 0) {
throw new Error('Files must include at least one valid file object')
}
const hydratedFiles = await hydrateUserFilesWithBase64(userFiles, {
requestId,
workspaceId: ctx.workspaceId,
workflowId: ctx.workflowId,
executionId: ctx.executionId,
largeValueExecutionIds: ctx.largeValueExecutionIds,
allowLargeValueWorkflowScope: ctx.allowLargeValueWorkflowScope,
userId: ctx.userId,
logger,
maxBytes: getProviderAttachmentMaxBytes(providerId),
})
const missingFile = hydratedFiles.find((file) => !file.base64)
if (missingFile) {
throw new Error(
`File "${missingFile.name}" could not be read for provider "${providerId}". Make sure the file is still accessible and under the provider attachment size limit.`
)
}
nextMessages[messageIndex] = {
...message,
files: hydratedFiles,
}
}
return nextMessages
}
private extractValidMessages(messages?: Message[]): Message[] {
if (!messages || !Array.isArray(messages)) return []
@@ -178,6 +178,34 @@ describe('Memory', () => {
})
})
describe('sanitizeMessageForStorage', () => {
it('should strip file payloads and provider-only fields before memory persistence', () => {
const message: Message = {
role: 'user',
content: 'Analyze this file',
executionId: 'exec-1',
files: [
{
id: 'file-1',
key: 'workspace/ws-1/example.png',
name: 'example.png',
url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace',
size: 128,
type: 'image/png',
base64: 'iVBORw0KGgo=',
},
],
tool_calls: [{ id: 'call-1' }],
}
expect((memoryService as any).sanitizeMessageForStorage(message)).toEqual({
role: 'user',
content: 'Analyze this file',
executionId: 'exec-1',
})
})
})
describe('Token-based vs Message-based comparison', () => {
it('should produce different results for same limit concept', () => {
const messages: Message[] = [
+26 -6
View File
@@ -122,6 +122,14 @@ export class Memory {
return messages.slice(-limit)
}
private sanitizeMessageForStorage(message: Message): Message {
return {
role: message.role,
content: message.content,
...(message.executionId && { executionId: message.executionId }),
}
}
private applyTokenWindow(messages: Message[], maxTokens: number, model?: string): Message[] {
const result: Message[] = []
let tokenCount = 0
@@ -177,9 +185,17 @@ export class Memory {
const data = result[0].data
if (!Array.isArray(data)) return []
return data.filter(
(msg): msg is Message => msg && typeof msg === 'object' && 'role' in msg && 'content' in msg
)
return data
.filter(
(msg): msg is Message =>
msg &&
typeof msg === 'object' &&
'role' in msg &&
'content' in msg &&
['system', 'user', 'assistant'].includes(msg.role) &&
typeof msg.content === 'string'
)
.map((msg) => this.sanitizeMessageForStorage(msg))
}
private async seedMemoryRecord(
@@ -189,13 +205,15 @@ export class Memory {
): Promise<void> {
const now = new Date()
const sanitizedMessages = messages.map((message) => this.sanitizeMessageForStorage(message))
await db
.insert(memory)
.values({
id: generateId(),
workspaceId,
key,
data: messages,
data: sanitizedMessages,
createdAt: now,
updatedAt: now,
})
@@ -205,20 +223,22 @@ export class Memory {
private async appendMessage(workspaceId: string, key: string, message: Message): Promise<void> {
const now = new Date()
const sanitizedMessage = this.sanitizeMessageForStorage(message)
await db
.insert(memory)
.values({
id: generateId(),
workspaceId,
key,
data: [message],
data: [sanitizedMessage],
createdAt: now,
updatedAt: now,
})
.onConflictDoUpdate({
target: [memory.workspaceId, memory.key],
set: {
data: sql`${memory.data} || ${JSON.stringify([message])}::jsonb`,
data: sql`${memory.data} || ${JSON.stringify([sanitizedMessage])}::jsonb`,
updatedAt: now,
},
})
@@ -1,3 +1,5 @@
import type { UserFile } from '@/executor/types'
export interface SkillInput {
skillId: string
name?: string
@@ -37,6 +39,7 @@ export interface AgentInputs {
reasoningEffort?: string
verbosity?: string
thinkingLevel?: string
files?: unknown
}
/**
@@ -66,6 +69,7 @@ export interface ToolInput {
export interface Message {
role: 'system' | 'user' | 'assistant'
content: string
files?: UserFile[]
executionId?: string
function_call?: any
tool_calls?: any[]
@@ -119,6 +119,42 @@ describe('start-block utilities', () => {
expect(output.files).toEqual(files)
})
it.concurrent('buildStartBlockOutput normalizes Start files from internal serve URLs', () => {
const block = createBlock('start_trigger', 'start')
const resolution = {
blockId: 'start',
block,
path: StartBlockPath.UNIFIED,
} as const
const output = buildStartBlockOutput({
resolution,
workflowInput: {
files: [
{
id: 'file_1',
name: 'screenshot.png',
url: '/api/files/serve/s3/execution%2Fworkspace-id%2Fworkflow-id%2Fexecution-id%2Fscreenshot.png?context=execution',
size: 243289,
type: 'image/png',
},
],
},
})
expect(output.files).toEqual([
{
id: 'file_1',
name: 'screenshot.png',
url: '/api/files/serve/s3/execution%2Fworkspace-id%2Fworkflow-id%2Fexecution-id%2Fscreenshot.png?context=execution',
size: 243289,
type: 'image/png',
key: 'execution/workspace-id/workflow-id/execution-id/screenshot.png',
context: 'execution',
},
])
})
it.concurrent('rejects inputFormat fields that collide with executor routing keys', () => {
const block = createBlock('start_trigger', 'start', {
subBlocks: {
+64 -3
View File
@@ -1,4 +1,8 @@
import { isUserFileWithMetadata } from '@/lib/core/utils/user-file'
import {
inferContextFromKey,
isInternalFileUrl,
parseInternalFileUrl,
} from '@/lib/uploads/utils/file-utils'
import {
classifyStartBlockType,
resolveStartCandidates,
@@ -323,13 +327,68 @@ function getRawInputCandidate(workflowInput: unknown): unknown {
return workflowInput
}
function normalizeStartFile(file: unknown): UserFile | null {
if (!isPlainObject(file)) {
return null
}
const id = typeof file.id === 'string' ? file.id : ''
const name = typeof file.name === 'string' ? file.name : ''
const url =
typeof file.url === 'string' ? file.url : typeof file.path === 'string' ? file.path : ''
const size = typeof file.size === 'number' ? file.size : Number.NaN
const type = typeof file.type === 'string' ? file.type : ''
const explicitKey = typeof file.key === 'string' ? file.key : ''
let key = explicitKey
let context = typeof file.context === 'string' ? file.context : undefined
if (!key && url && isInternalFileUrl(url)) {
try {
const parsed = parseInternalFileUrl(url)
key = parsed.key
context = context || parsed.context
} catch {
return null
}
}
if (!context && key) {
try {
context = inferContextFromKey(key)
} catch {
// Older file outputs may have opaque keys; keep the file shape intact.
}
}
if (!id || !name || !url || !Number.isFinite(size) || !type || !key) {
return null
}
return {
id,
name,
url,
size,
type,
key,
...(context && { context }),
...(typeof file.base64 === 'string' && { base64: file.base64 }),
}
}
function getFilesFromWorkflowInput(workflowInput: unknown): UserFile[] | undefined {
if (!isPlainObject(workflowInput)) {
return undefined
}
const files = workflowInput.files
if (Array.isArray(files) && files.every(isUserFileWithMetadata)) {
return files
if (!Array.isArray(files)) {
return undefined
}
const normalizedFiles = files.map(normalizeStartFile)
if (normalizedFiles.every((file): file is UserFile => Boolean(file))) {
return normalizedFiles
}
return undefined
}
@@ -341,6 +400,8 @@ function mergeFilesIntoOutput(
const files = getFilesFromWorkflowInput(workflowInput)
if (files) {
output.files = files
} else if (isPlainObject(workflowInput) && Object.hasOwn(workflowInput, 'files')) {
output.files = undefined
}
return output
}
+1
View File
@@ -197,6 +197,7 @@ export interface ExecuteStreamOptions {
envVarValues?: Record<string, string>
workflowVariables?: Record<string, any>
selectedOutputs?: string[]
executionId?: string
startBlockId?: string
triggerType?: string
useDraftState?: boolean
@@ -302,6 +302,7 @@ export const fileParseBodySchema = z
.object({
filePath: z.union([z.string(), z.array(z.string())]).optional(),
fileType: z.string().optional().default(''),
headers: z.record(z.string(), z.string()).optional(),
workspaceId: z.string().optional().default(''),
workflowId: z.string().optional(),
executionId: z.string().optional(),
+12
View File
@@ -33,10 +33,22 @@ export const fileManageGetBodySchema = z
message: 'Either fileId or fileInput is required for get operation',
})
export const fileManageReadBodySchema = z
.object({
operation: z.literal('read'),
workspaceId: z.string().min(1).optional(),
fileId: z.union([z.string().min(1), z.array(z.string().min(1)).min(1)]).optional(),
fileInput: z.any().optional(),
})
.refine((data) => data.fileId !== undefined || data.fileInput !== undefined, {
message: 'Either fileId or fileInput is required for read operation',
})
export const fileManageBodySchema = z.union([
fileManageWriteBodySchema,
fileManageAppendBodySchema,
fileManageGetBodySchema,
fileManageReadBodySchema,
])
export const fileManageContract = defineRouteContract({
+1
View File
@@ -336,6 +336,7 @@ export const executeWorkflowBodySchema = z.object({
includeFileBase64: z.boolean().optional().default(true),
base64MaxBytes: z.number().int().positive().optional(),
workflowStateOverride: workflowStateSchema.optional(),
executionId: z.string().optional(),
triggerBlockId: z.string().optional(),
startBlockId: z.string().optional(),
stopAfterBlockId: z.string().optional(),
+2 -2
View File
@@ -172,7 +172,7 @@ export function createFileContentFromBase64(base64: string, mimeType: string): M
return null
}
if (contentType === 'image' && !CLAUDE_SUPPORTED_IMAGE_MIME_TYPES.has(mimeType.toLowerCase())) {
if (contentType === 'image' && !MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mimeType.toLowerCase())) {
return null
}
@@ -186,7 +186,7 @@ export function createFileContentFromBase64(base64: string, mimeType: string): M
}
}
const CLAUDE_SUPPORTED_IMAGE_MIME_TYPES = new Set([
export const MODEL_SUPPORTED_IMAGE_MIME_TYPES = new Set([
'image/jpeg',
'image/jpg',
'image/png',
@@ -361,8 +361,7 @@ async function resolveBase64(
options: Base64HydrationOptions,
logger: Logger
): Promise<string | null> {
const requestedMaxBytes = options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES
const maxBytes = Math.min(requestedMaxBytes, DEFAULT_MAX_BASE64_BYTES)
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES
if (file.base64) {
const base64Bytes = Buffer.byteLength(file.base64, 'base64')
@@ -440,10 +439,7 @@ async function hydrateUserFile(
const cached = await state.cache.get(file)
if (cached) {
const maxBytes = Math.min(
options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES,
DEFAULT_MAX_BASE64_BYTES
)
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BASE64_BYTES
if (Buffer.byteLength(cached, 'base64') > maxBytes) {
return stripBase64(file)
}
+4 -1
View File
@@ -9,6 +9,7 @@ import {
checkForForcedToolUsage,
createReadableStreamFromAnthropicStream,
} from '@/providers/anthropic/utils'
import { buildAnthropicMessageContent } from '@/providers/attachments'
import {
getMaxOutputTokensForModel,
getThinkingCapability,
@@ -229,9 +230,11 @@ export async function executeAnthropicProviderRequest(
],
})
} else {
const content = buildAnthropicMessageContent(msg.content, msg.files, config.providerId)
messages.push({
role: msg.role === 'assistant' ? 'assistant' : 'user',
content: msg.content ? [{ type: 'text', text: msg.content }] : [],
// double-cast-allowed: shared attachment builder returns Anthropic-compatible content blocks but avoids importing SDK-only union types
content: content as unknown as Anthropic.Messages.ContentBlockParam[],
})
}
})
+271
View File
@@ -0,0 +1,271 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import type { UserFile } from '@/executor/types'
import {
buildAnthropicMessageContent,
buildBedrockMessageContent,
buildGeminiMessageParts,
buildOpenAIMessageContent,
buildOpenRouterMessageContent,
formatMessagesForProvider,
inferAttachmentMimeType,
prepareProviderAttachments,
} from '@/providers/attachments'
const imageFile: UserFile = {
id: 'file-1',
name: 'example.png',
url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace',
size: 128,
type: 'image/png',
key: 'workspace/ws-1/example.png',
base64: 'iVBORw0KGgo=',
}
const pdfFile: UserFile = {
id: 'file-2',
name: 'example.pdf',
url: '/api/files/serve/workspace%2Fws-1%2Fexample.pdf?context=workspace',
size: 256,
type: 'application/pdf',
key: 'workspace/ws-1/example.pdf',
base64: 'cGRm',
}
const markdownFile: UserFile = {
id: 'file-3',
name: 'notes.md',
url: '/api/files/serve/workspace%2Fws-1%2Fnotes.md?context=workspace',
size: 17,
type: 'text/markdown',
key: 'workspace/ws-1/notes.md',
base64: Buffer.from('# Notes\n\nHello').toString('base64'),
}
describe('provider attachments', () => {
it('infers MIME type from filename when file type is generic', () => {
expect(
inferAttachmentMimeType({
...imageFile,
type: 'application/octet-stream',
})
).toBe('image/png')
})
it('formats OpenAI Responses content with text, image, and file parts', () => {
const content = buildOpenAIMessageContent(
'Analyze these files',
[imageFile, pdfFile, markdownFile],
'openai'
)
expect(content).toEqual([
{ type: 'input_text', text: 'Analyze these files' },
{
type: 'input_image',
image_url: 'data:image/png;base64,iVBORw0KGgo=',
},
{
type: 'input_file',
filename: 'example.pdf',
file_data: 'data:application/pdf;base64,cGRm',
},
{
type: 'input_file',
filename: 'notes.md',
file_data: `data:text/markdown;base64,${markdownFile.base64}`,
},
])
})
it('formats Anthropic content with image, PDF document, and text document blocks', () => {
const content = buildAnthropicMessageContent(
'Analyze these files',
[imageFile, pdfFile, markdownFile],
'anthropic'
)
expect(content).toEqual([
{ type: 'text', text: 'Analyze these files' },
{
type: 'image',
source: {
type: 'base64',
media_type: 'image/png',
data: 'iVBORw0KGgo=',
},
},
{
type: 'document',
source: {
type: 'base64',
media_type: 'application/pdf',
data: 'cGRm',
},
title: 'example.pdf',
},
{
type: 'document',
source: {
type: 'text',
media_type: 'text/plain',
data: '# Notes\n\nHello',
},
title: 'notes.md',
},
])
})
it('formats Gemini content with text and inline data parts', () => {
const parts = buildGeminiMessageParts('Analyze this file', [imageFile, markdownFile], 'google')
expect(parts).toEqual([
{ text: 'Analyze this file' },
{
inlineData: {
mimeType: 'image/png',
data: 'iVBORw0KGgo=',
},
},
{
inlineData: {
mimeType: 'text/plain',
data: markdownFile.base64,
},
},
])
})
it('formats Bedrock content with native document blocks', () => {
const parts = buildBedrockMessageContent('Analyze this file', [markdownFile], 'bedrock')
expect(parts).toEqual([
{ text: 'Analyze this file' },
{
document: {
format: 'md',
name: 'notes',
source: {
bytes: Buffer.from(markdownFile.base64, 'base64'),
},
},
},
])
})
it('formats OpenRouter images and PDFs with native multimodal message parts', () => {
const content = buildOpenRouterMessageContent(
'Analyze these files',
[imageFile, pdfFile],
'openrouter'
)
expect(content).toEqual([
{ type: 'text', text: 'Analyze these files' },
{
type: 'image_url',
image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' },
},
{
type: 'file',
file: {
filename: 'example.pdf',
file_data: 'data:application/pdf;base64,cGRm',
},
},
])
})
it('formats image-only provider messages and strips file fields', () => {
const messages = formatMessagesForProvider(
[{ role: 'user', content: 'Analyze this image', files: [imageFile] }],
'groq'
)
expect(messages).toEqual([
{
role: 'user',
content: [
{ type: 'text', text: 'Analyze this image' },
{
type: 'image_url',
image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' },
},
],
},
])
})
it('fails fast for unsupported MIME types', () => {
expect(() =>
prepareProviderAttachments(
[
{
...imageFile,
name: 'archive.zip',
type: 'application/zip',
},
],
'openai'
)
).toThrow('application/zip')
})
it('sniffs image bytes and corrects a wrong declared image MIME type', () => {
const content = buildAnthropicMessageContent(
'Analyze this image',
[
{
...imageFile,
name: 'wrong.ico',
type: 'image/x-icon',
},
],
'anthropic'
)
expect(content[1]).toEqual({
type: 'image',
source: {
type: 'base64',
media_type: 'image/png',
data: 'iVBORw0KGgo=',
},
})
})
it('rejects image attachments when the bytes are not a supported image format', () => {
expect(() =>
prepareProviderAttachments(
[
{
...imageFile,
name: 'not-an-image.png',
base64: Buffer.from('not an image').toString('base64'),
},
],
'anthropic'
)
).toThrow('not a supported model image format')
})
it('rejects documents for image-only providers', () => {
expect(() =>
formatMessagesForProvider(
[{ role: 'user', content: 'Analyze this file', files: [pdfFile] }],
'groq'
)
).toThrow('Supported attachments: images')
})
it('rejects providers without file attachment support', () => {
expect(() =>
formatMessagesForProvider(
[{ role: 'user', content: 'Analyze this file', files: [imageFile] }],
'deepseek'
)
).toThrow('not supported')
})
})
+571
View File
@@ -0,0 +1,571 @@
import {
getContentType,
getExtensionFromMimeType,
getFileExtension,
getMimeTypeFromExtension,
MIME_TYPE_MAPPING,
MODEL_SUPPORTED_IMAGE_MIME_TYPES,
} from '@/lib/uploads/utils/file-utils'
import type { UserFile } from '@/executor/types'
import type { ProviderId } from '@/providers/types'
export type AttachmentProvider =
| 'openai'
| 'anthropic'
| 'google'
| 'bedrock'
| 'openrouter'
| 'mistral'
| 'groq'
| 'fireworks'
| 'ollama'
| 'vllm'
| 'xai'
| 'deepseek'
| 'cerebras'
export interface PreparedProviderAttachment {
file: UserFile
filename: string
mimeType: string
providerMimeType: string
base64: string
dataUrl: string
text?: string
extension: string
contentType: 'image' | 'document' | 'audio' | 'video'
}
type ProviderMessageInput = {
role: string
content?: string | null
files?: UserFile[]
}
type ProviderFormattedMessage = {
role: string
content?: string | null | Array<Record<string, unknown>>
files?: UserFile[]
[key: string]: unknown
}
const AGENT_ATTACHMENT_MAX_BYTES = 10 * 1024 * 1024
const PDF_MIME_TYPE = 'application/pdf'
const DOCUMENT_MIME_TYPES = new Set(
Object.entries(MIME_TYPE_MAPPING)
.filter(([, contentType]) => contentType === 'document')
.map(([mimeType]) => mimeType)
)
const OPENAI_DOCUMENT_MIME_TYPES = new Set([...DOCUMENT_MIME_TYPES, 'application/x-yaml'])
const GEMINI_INLINE_MIME_TYPES = new Set([...Object.keys(MIME_TYPE_MAPPING), 'application/x-yaml'])
const BEDROCK_DOCUMENT_FORMATS = new Set([
'pdf',
'csv',
'doc',
'docx',
'xls',
'xlsx',
'html',
'txt',
'md',
])
const BEDROCK_IMAGE_FORMATS = new Set(['png', 'jpeg', 'jpg', 'gif', 'webp'])
const BEDROCK_VIDEO_FORMATS = new Set(['mp4', 'mov', 'mkv', 'webm'])
const IMAGE_ONLY_PROVIDERS = new Set<AttachmentProvider>([
'mistral',
'groq',
'fireworks',
'ollama',
'vllm',
])
const UNSUPPORTED_FILE_PROVIDERS = new Set<AttachmentProvider>(['xai', 'deepseek', 'cerebras'])
const PROVIDER_SUPPORTED_LABELS: Record<AttachmentProvider, string> = {
openai: 'images and documents through the Responses API input_image/input_file parts',
anthropic: 'images, PDFs, and text documents through Claude content blocks',
google: 'images, audio, video, PDFs, and text documents through Gemini inlineData',
bedrock: 'Bedrock Converse image, document, and video content blocks',
openrouter: 'images and PDFs through OpenRouter multimodal message parts',
mistral: 'images through image_url message parts',
groq: 'images through image_url message parts on multimodal models',
fireworks: 'images through image_url message parts on vision models',
ollama: 'images through image_url message parts on vision models',
vllm: 'images through image_url message parts on multimodal models',
xai: 'no file attachments in the current chat-completions adapter',
deepseek: 'no file attachments in the current API adapter',
cerebras: 'no file attachments in the current API adapter',
}
export function getAttachmentProvider(providerId: ProviderId | string): AttachmentProvider | null {
if (providerId === 'openai' || providerId === 'azure-openai') return 'openai'
if (providerId === 'anthropic' || providerId === 'azure-anthropic') return 'anthropic'
if (providerId === 'google' || providerId === 'vertex') return 'google'
if (providerId === 'bedrock') return 'bedrock'
if (providerId === 'openrouter') return 'openrouter'
if (providerId === 'mistral') return 'mistral'
if (providerId === 'groq') return 'groq'
if (providerId === 'fireworks') return 'fireworks'
if (providerId === 'ollama') return 'ollama'
if (providerId === 'vllm') return 'vllm'
if (providerId === 'xai') return 'xai'
if (providerId === 'deepseek') return 'deepseek'
if (providerId === 'cerebras') return 'cerebras'
return null
}
export function getProviderAttachmentMaxBytes(_providerId: ProviderId | string): number {
return AGENT_ATTACHMENT_MAX_BYTES
}
export function supportsFileAttachments(providerId: ProviderId | string): boolean {
const provider = getAttachmentProvider(providerId)
return Boolean(provider && !UNSUPPORTED_FILE_PROVIDERS.has(provider))
}
export function inferAttachmentMimeType(file: UserFile): string {
const explicitType = file.type?.trim().toLowerCase()
if (explicitType && explicitType !== 'application/octet-stream') {
return explicitType
}
const inferred = getMimeTypeFromExtension(getFileExtension(file.name))
return inferred.toLowerCase()
}
function isTextDocumentMimeType(mimeType: string): boolean {
return (
mimeType.startsWith('text/') ||
mimeType === 'application/json' ||
mimeType === 'application/xml' ||
mimeType === 'application/x-yaml'
)
}
function isImageMimeType(mimeType: string): boolean {
return MODEL_SUPPORTED_IMAGE_MIME_TYPES.has(mimeType)
}
function isOpenAIDocumentMimeType(mimeType: string): boolean {
return OPENAI_DOCUMENT_MIME_TYPES.has(mimeType) || isTextDocumentMimeType(mimeType)
}
function getAttachmentContentType(
mimeType: string
): PreparedProviderAttachment['contentType'] | null {
return getContentType(mimeType) || (isTextDocumentMimeType(mimeType) ? 'document' : null)
}
function sniffImageMimeType(base64: string): string {
let bytes: Buffer
try {
bytes = Buffer.from(base64, 'base64')
} catch {
return ''
}
if (
bytes.length >= 8 &&
bytes.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))
) {
return 'image/png'
}
if (bytes.length >= 3 && bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff) {
return 'image/jpeg'
}
if (
bytes.length >= 6 &&
(bytes.subarray(0, 6).equals(Buffer.from('GIF87a')) ||
bytes.subarray(0, 6).equals(Buffer.from('GIF89a')))
) {
return 'image/gif'
}
if (
bytes.length >= 12 &&
bytes.subarray(0, 4).equals(Buffer.from('RIFF')) &&
bytes.subarray(8, 12).equals(Buffer.from('WEBP'))
) {
return 'image/webp'
}
return ''
}
function getAttachmentExtension(file: UserFile, mimeType: string): string {
if (mimeType === 'text/markdown') return 'md'
return getExtensionFromMimeType(mimeType) || getFileExtension(file.name)
}
function normalizeProviderMimeType(mimeType: string, provider: AttachmentProvider): string {
if ((provider === 'anthropic' || provider === 'google') && isTextDocumentMimeType(mimeType)) {
return 'text/plain'
}
return mimeType
}
function decodeBase64Text(base64: string, filename: string): string {
try {
return Buffer.from(base64, 'base64').toString('utf8')
} catch {
throw new Error(`File "${filename}" could not be decoded as UTF-8 text`)
}
}
function toDataUrl(mimeType: string, base64: string): string {
return `data:${mimeType};base64,${base64}`
}
function getProviderSupportedLabel(provider: AttachmentProvider): string {
return PROVIDER_SUPPORTED_LABELS[provider]
}
function validateProviderSupport(
attachment: Omit<PreparedProviderAttachment, 'providerMimeType' | 'dataUrl' | 'text'>,
provider: AttachmentProvider,
providerId: ProviderId | string
) {
const { filename, mimeType, contentType, extension } = attachment
const supportedLabel = getProviderSupportedLabel(provider)
const supported =
provider === 'openai'
? isImageMimeType(mimeType) || isOpenAIDocumentMimeType(mimeType)
: provider === 'anthropic'
? isImageMimeType(mimeType) ||
mimeType === PDF_MIME_TYPE ||
isTextDocumentMimeType(mimeType)
: provider === 'google'
? GEMINI_INLINE_MIME_TYPES.has(mimeType) || isTextDocumentMimeType(mimeType)
: provider === 'bedrock'
? (contentType === 'image' && BEDROCK_IMAGE_FORMATS.has(extension)) ||
(contentType === 'document' && BEDROCK_DOCUMENT_FORMATS.has(extension)) ||
(contentType === 'video' && BEDROCK_VIDEO_FORMATS.has(extension))
: provider === 'openrouter'
? isImageMimeType(mimeType) || mimeType === PDF_MIME_TYPE
: IMAGE_ONLY_PROVIDERS.has(provider)
? isImageMimeType(mimeType)
: !UNSUPPORTED_FILE_PROVIDERS.has(provider)
if (!supported) {
throw new Error(
`File "${filename}" has MIME type "${mimeType}", which is not supported by provider "${providerId}". Supported attachments: ${supportedLabel}.`
)
}
}
export function prepareProviderAttachments(
files: UserFile[] | undefined,
providerId: ProviderId | string
): PreparedProviderAttachment[] {
if (!files || files.length === 0) return []
const provider = getAttachmentProvider(providerId)
if (!provider) {
throw new Error(`File attachments are not supported for provider "${providerId}"`)
}
if (UNSUPPORTED_FILE_PROVIDERS.has(provider)) {
throw new Error(
`File attachments are not supported for provider "${providerId}" in the current adapter. Supported attachments: ${getProviderSupportedLabel(provider)}.`
)
}
const maxBytes = getProviderAttachmentMaxBytes(providerId)
return files.map((file) => {
const declaredMimeType = inferAttachmentMimeType(file)
const contentType = getAttachmentContentType(declaredMimeType)
if (!contentType) {
throw new Error(
`File "${file.name}" has MIME type "${declaredMimeType}", which is not supported by provider "${providerId}". Supported attachments: ${getProviderSupportedLabel(provider)}.`
)
}
if (Number.isFinite(file.size) && file.size > maxBytes) {
const sizeMB = (file.size / (1024 * 1024)).toFixed(2)
const maxMB = (maxBytes / (1024 * 1024)).toFixed(0)
throw new Error(
`File "${file.name}" (${sizeMB}MB) exceeds the ${maxMB}MB agent attachment limit for provider "${providerId}"`
)
}
if (!file.base64) {
throw new Error(`File "${file.name}" could not be read for provider "${providerId}"`)
}
const sniffedImageMimeType = contentType === 'image' ? sniffImageMimeType(file.base64) : ''
if (contentType === 'image' && !sniffedImageMimeType) {
throw new Error(
`Image bytes in "${file.name}" are not a supported model image format (declared MIME type "${declaredMimeType}"). Supported image formats: image/jpeg, image/png, image/gif, image/webp.`
)
}
const mimeType = sniffedImageMimeType || declaredMimeType
const extension = getAttachmentExtension(file, mimeType)
const attachment = {
file,
filename: file.name,
mimeType,
base64: file.base64,
extension,
contentType,
}
validateProviderSupport(attachment, provider, providerId)
const providerMimeType = normalizeProviderMimeType(mimeType, provider)
return {
...attachment,
providerMimeType,
dataUrl: toDataUrl(providerMimeType, file.base64),
...(isTextDocumentMimeType(mimeType) && {
text: decodeBase64Text(file.base64, file.name),
}),
}
})
}
export function buildOpenAIMessageContent(
content: string | null | undefined,
files: UserFile[] | undefined,
providerId: ProviderId | string
): string | Array<Record<string, unknown>> {
const attachments = prepareProviderAttachments(files, providerId)
if (attachments.length === 0) return content ?? ''
const parts: Array<Record<string, unknown>> = []
if (content) {
parts.push({ type: 'input_text', text: content })
}
for (const attachment of attachments) {
if (attachment.contentType === 'image') {
parts.push({ type: 'input_image', image_url: attachment.dataUrl })
} else {
parts.push({
type: 'input_file',
filename: attachment.filename,
file_data: attachment.dataUrl,
})
}
}
return parts
}
export function buildAnthropicMessageContent(
content: string | null | undefined,
files: UserFile[] | undefined,
providerId: ProviderId | string
): Array<Record<string, unknown>> {
const parts: Array<Record<string, unknown>> = []
if (content) {
parts.push({ type: 'text', text: content })
}
for (const attachment of prepareProviderAttachments(files, providerId)) {
if (attachment.contentType === 'image') {
parts.push({
type: 'image',
source: {
type: 'base64',
media_type: attachment.providerMimeType,
data: attachment.base64,
},
})
} else if (attachment.text) {
parts.push({
type: 'document',
source: {
type: 'text',
media_type: 'text/plain',
data: attachment.text,
},
title: attachment.filename,
})
} else {
parts.push({
type: 'document',
source: {
type: 'base64',
media_type: attachment.providerMimeType,
data: attachment.base64,
},
title: attachment.filename,
})
}
}
return parts
}
export function buildGeminiMessageParts(
content: string | null | undefined,
files: UserFile[] | undefined,
providerId: ProviderId | string
): Array<Record<string, unknown>> {
const parts: Array<Record<string, unknown>> = []
if (content) {
parts.push({ text: content })
}
for (const attachment of prepareProviderAttachments(files, providerId)) {
parts.push({
inlineData: {
mimeType: attachment.providerMimeType,
data: attachment.base64,
},
})
}
return parts
}
export function buildOpenAICompatibleChatContent(
content: string | null | undefined,
files: UserFile[] | undefined,
providerId: ProviderId | string
): string | Array<Record<string, unknown>> {
const attachments = prepareProviderAttachments(files, providerId)
if (attachments.length === 0) return content ?? ''
const parts: Array<Record<string, unknown>> = []
if (content) {
parts.push({ type: 'text', text: content })
}
for (const attachment of attachments) {
parts.push({
type: 'image_url',
image_url: {
url: attachment.dataUrl,
},
})
}
return parts
}
export function buildOpenRouterMessageContent(
content: string | null | undefined,
files: UserFile[] | undefined,
providerId: ProviderId | string
): string | Array<Record<string, unknown>> {
const attachments = prepareProviderAttachments(files, providerId)
if (attachments.length === 0) return content ?? ''
const parts: Array<Record<string, unknown>> = []
if (content) {
parts.push({ type: 'text', text: content })
}
for (const attachment of attachments) {
if (attachment.contentType === 'image') {
parts.push({
type: 'image_url',
image_url: { url: attachment.dataUrl },
})
} else {
parts.push({
type: 'file',
file: {
filename: attachment.filename,
file_data: attachment.dataUrl,
},
})
}
}
return parts
}
function sanitizeBedrockName(filename: string): string {
const baseName = filename.replace(/\.[^/.]+$/, '').replace(/[^a-zA-Z0-9\s()[\]-]/g, ' ')
const compacted = baseName.replace(/\s+/g, ' ').trim()
return compacted || 'Document'
}
function getBedrockDocumentFormat(attachment: PreparedProviderAttachment): string {
if (attachment.extension === 'md' || attachment.mimeType === 'text/markdown') return 'md'
if (attachment.extension === 'txt' || attachment.mimeType === 'text/plain') return 'txt'
return attachment.extension || 'txt'
}
function getBedrockImageFormat(attachment: PreparedProviderAttachment): string {
return attachment.extension === 'jpg' ? 'jpeg' : attachment.extension
}
export function buildBedrockMessageContent(
content: string | null | undefined,
files: UserFile[] | undefined,
providerId: ProviderId | string
): Array<Record<string, unknown>> {
const parts: Array<Record<string, unknown>> = []
if (content) {
parts.push({ text: content })
}
for (const attachment of prepareProviderAttachments(files, providerId)) {
const bytes = Buffer.from(attachment.base64, 'base64')
if (attachment.contentType === 'image') {
parts.push({
image: {
format: getBedrockImageFormat(attachment),
source: { bytes },
},
})
} else if (attachment.contentType === 'video') {
parts.push({
video: {
format: attachment.extension,
source: { bytes },
},
})
} else {
parts.push({
document: {
format: getBedrockDocumentFormat(attachment),
name: sanitizeBedrockName(attachment.filename),
source: { bytes },
},
})
}
}
return parts
}
export function formatMessagesForProvider(
messages: ProviderMessageInput[],
providerId: ProviderId | string
): ProviderFormattedMessage[] {
return messages.map((message) => {
if (!message.files?.length || (message.role !== 'user' && message.role !== 'assistant')) {
return message as ProviderFormattedMessage
}
const provider = getAttachmentProvider(providerId)
if (provider === 'openrouter') {
const { files: _files, ...rest } = message
return {
...rest,
content: buildOpenRouterMessageContent(message.content, message.files, providerId),
}
}
const { files: _files, ...rest } = message
return {
...rest,
content: buildOpenAICompatibleChatContent(message.content, message.files, providerId),
}
})
}
+3
View File
@@ -90,6 +90,9 @@ async function executeChatCompletionsRequest(
}
if (request.messages) {
if (request.messages.some((message) => message.files?.length)) {
throw new Error('File attachments require an Azure OpenAI Responses API endpoint')
}
allMessages.push(...(request.messages as ChatCompletionMessageParam[]))
}
+4 -1
View File
@@ -17,6 +17,7 @@ import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import type { IterationToolCall, StreamingExecution } from '@/executor/types'
import { MAX_TOOL_ITERATIONS } from '@/providers'
import { buildBedrockMessageContent } from '@/providers/attachments'
import {
checkForForcedToolUsage,
createReadableStreamFromBedrockStream,
@@ -178,9 +179,11 @@ export const bedrockProvider: ProviderConfig = {
}
} else {
const role: ConversationRole = msg.role === 'assistant' ? 'assistant' : 'user'
const content = buildBedrockMessageContent(msg.content, msg.files, 'bedrock')
messages.push({
role,
content: [{ text: msg.content || '' }],
// double-cast-allowed: shared attachment builder emits Bedrock Converse content blocks while keeping provider-neutral attachment types
content: content as unknown as ContentBlock[],
})
}
}
+4 -2
View File
@@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import type { StreamingExecution } from '@/executor/types'
import { MAX_TOOL_ITERATIONS } from '@/providers'
import { formatMessagesForProvider } from '@/providers/attachments'
import type { CerebrasResponse } from '@/providers/cerebras/types'
import { createReadableStreamFromCerebrasStream } from '@/providers/cerebras/utils'
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
@@ -64,6 +65,7 @@ export const cerebrasProvider: ProviderConfig = {
if (request.messages) {
allMessages.push(...request.messages)
}
const formattedMessages = formatMessagesForProvider(allMessages, 'cerebras')
const tools = request.tools?.length
? request.tools.map((tool) => ({
@@ -78,7 +80,7 @@ export const cerebrasProvider: ProviderConfig = {
const payload: any = {
model: request.model.replace('cerebras/', ''),
messages: allMessages,
messages: formattedMessages,
}
if (request.temperature !== undefined) payload.temperature = request.temperature
if (request.maxTokens != null) payload.max_completion_tokens = request.maxTokens
@@ -199,7 +201,7 @@ export const cerebrasProvider: ProviderConfig = {
}
const toolCalls = []
const toolResults: Record<string, unknown>[] = []
const currentMessages = [...allMessages]
const currentMessages = [...formattedMessages]
let iterationCount = 0
let modelTime = firstResponseTime
+4 -2
View File
@@ -3,6 +3,7 @@ import { toError } from '@sim/utils/errors'
import OpenAI from 'openai'
import type { StreamingExecution } from '@/executor/types'
import { MAX_TOOL_ITERATIONS } from '@/providers'
import { formatMessagesForProvider } from '@/providers/attachments'
import { createReadableStreamFromDeepseekStream } from '@/providers/deepseek/utils'
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
@@ -67,6 +68,7 @@ export const deepseekProvider: ProviderConfig = {
if (request.messages) {
allMessages.push(...request.messages)
}
const formattedMessages = formatMessagesForProvider(allMessages, 'deepseek')
const tools = request.tools?.length
? request.tools.map((tool) => ({
@@ -81,7 +83,7 @@ export const deepseekProvider: ProviderConfig = {
const payload: any = {
model: request.model,
messages: allMessages,
messages: formattedMessages,
}
if (request.temperature !== undefined) payload.temperature = request.temperature
@@ -209,7 +211,7 @@ export const deepseekProvider: ProviderConfig = {
}
const toolCalls = []
const toolResults: Record<string, unknown>[] = []
const currentMessages = [...allMessages]
const currentMessages = [...formattedMessages]
let iterationCount = 0
let hasUsedForcedTool = false
let modelTime = firstResponseTime
+4 -2
View File
@@ -4,6 +4,7 @@ import OpenAI from 'openai'
import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions'
import type { StreamingExecution } from '@/executor/types'
import { MAX_TOOL_ITERATIONS } from '@/providers'
import { formatMessagesForProvider } from '@/providers/attachments'
import {
checkForForcedToolUsage,
createReadableStreamFromOpenAIStream,
@@ -108,6 +109,7 @@ export const fireworksProvider: ProviderConfig = {
if (request.messages) {
allMessages.push(...request.messages)
}
const formattedMessages = formatMessagesForProvider(allMessages, 'fireworks') as Message[]
const tools = request.tools?.length
? request.tools.map((tool) => ({
@@ -122,7 +124,7 @@ export const fireworksProvider: ProviderConfig = {
const payload: any = {
model: requestedModel,
messages: allMessages,
messages: formattedMessages,
}
if (request.temperature !== undefined) payload.temperature = request.temperature
@@ -250,7 +252,7 @@ export const fireworksProvider: ProviderConfig = {
}
const toolCalls: FunctionCallResponse[] = []
const toolResults: Record<string, unknown>[] = []
const currentMessages = [...allMessages]
const currentMessages = [...formattedMessages]
let iterationCount = 0
let modelTime = firstResponseTime
let toolsTime = 0
+1 -1
View File
@@ -917,7 +917,7 @@ export async function executeGeminiRequest(
const providerStartTimeISO = new Date(providerStartTime).toISOString()
try {
const { contents, tools, systemInstruction } = convertToGeminiFormat(request)
const { contents, tools, systemInstruction } = convertToGeminiFormat(request, providerType)
// Build configuration
const geminiConfig: GenerateContentConfig = {}
+38
View File
@@ -111,6 +111,44 @@ describe('ensureStructResponse', () => {
})
describe('convertToGeminiFormat', () => {
it('should convert user message files to inline data parts', () => {
const request: ProviderRequest = {
model: 'gemini-2.5-flash',
messages: [
{
role: 'user',
content: 'Analyze this image',
files: [
{
id: 'file-1',
key: 'workspace/ws-1/example.png',
name: 'example.png',
url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace',
size: 128,
type: 'image/png',
base64: 'iVBORw0KGgo=',
},
],
},
],
}
const result = convertToGeminiFormat(request)
expect(result.contents[0]).toEqual({
role: 'user',
parts: [
{ text: 'Analyze this image' },
{
inlineData: {
mimeType: 'image/png',
data: 'iVBORw0KGgo=',
},
},
],
})
})
describe('tool message handling', () => {
it('should convert tool message with object response correctly', () => {
const request: ProviderRequest = {
+8 -3
View File
@@ -14,6 +14,7 @@ import {
} from '@google/genai'
import { createLogger } from '@sim/logger'
import { toError } from '@sim/utils/errors'
import { buildGeminiMessageParts } from '@/providers/attachments'
import type { ProviderRequest } from '@/providers/types'
import { trackForcedToolUsage } from '@/providers/utils'
@@ -166,7 +167,10 @@ export interface GeminiToolDef {
/**
* Converts OpenAI-style request format to Gemini format
*/
export function convertToGeminiFormat(request: ProviderRequest): {
export function convertToGeminiFormat(
request: ProviderRequest,
providerId = 'google'
): {
contents: Content[]
tools: GeminiToolDef[] | undefined
systemInstruction: Content | undefined
@@ -192,9 +196,10 @@ export function convertToGeminiFormat(request: ProviderRequest): {
}
} else if (message.role === 'user' || message.role === 'assistant') {
const geminiRole = message.role === 'user' ? 'user' : 'model'
const parts = buildGeminiMessageParts(message.content, message.files, providerId) as Part[]
if (message.content) {
contents.push({ role: geminiRole, parts: [{ text: message.content }] })
if (parts.length > 0) {
contents.push({ role: geminiRole, parts })
}
if (message.role === 'assistant' && message.tool_calls?.length) {
+4 -2
View File
@@ -3,6 +3,7 @@ import { toError } from '@sim/utils/errors'
import { Groq } from 'groq-sdk'
import type { StreamingExecution } from '@/executor/types'
import { MAX_TOOL_ITERATIONS } from '@/providers'
import { formatMessagesForProvider } from '@/providers/attachments'
import { createReadableStreamFromGroqStream } from '@/providers/groq/utils'
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
@@ -60,6 +61,7 @@ export const groqProvider: ProviderConfig = {
if (request.messages) {
allMessages.push(...request.messages)
}
const formattedMessages = formatMessagesForProvider(allMessages, 'groq')
const tools = request.tools?.length
? request.tools.map((tool) => ({
@@ -74,7 +76,7 @@ export const groqProvider: ProviderConfig = {
const payload: any = {
model: request.model.replace('groq/', ''),
messages: allMessages,
messages: formattedMessages,
}
if (request.temperature !== undefined) payload.temperature = request.temperature
@@ -205,7 +207,7 @@ export const groqProvider: ProviderConfig = {
}
const toolCalls = []
const toolResults: Record<string, unknown>[] = []
const currentMessages = [...allMessages]
const currentMessages = [...formattedMessages]
let iterationCount = 0
let modelTime = firstResponseTime
let toolsTime = 0
+4 -2
View File
@@ -4,6 +4,7 @@ import OpenAI from 'openai'
import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions'
import type { StreamingExecution } from '@/executor/types'
import { MAX_TOOL_ITERATIONS } from '@/providers'
import { formatMessagesForProvider } from '@/providers/attachments'
import { createReadableStreamFromMistralStream } from '@/providers/mistral/utils'
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
@@ -77,6 +78,7 @@ export const mistralProvider: ProviderConfig = {
if (request.messages) {
allMessages.push(...request.messages)
}
const formattedMessages = formatMessagesForProvider(allMessages, 'mistral')
const tools = request.tools?.length
? request.tools.map((tool) => ({
@@ -91,7 +93,7 @@ export const mistralProvider: ProviderConfig = {
const payload: any = {
model: request.model,
messages: allMessages,
messages: formattedMessages,
}
if (request.temperature !== undefined) payload.temperature = request.temperature
@@ -262,7 +264,7 @@ export const mistralProvider: ProviderConfig = {
}
const toolCalls = []
const toolResults: Record<string, unknown>[] = []
const currentMessages = [...allMessages]
const currentMessages = [...formattedMessages]
let iterationCount = 0
let modelTime = firstResponseTime
+4 -2
View File
@@ -5,6 +5,7 @@ import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/
import { getOllamaUrl } from '@/lib/core/utils/urls'
import type { StreamingExecution } from '@/executor/types'
import { MAX_TOOL_ITERATIONS } from '@/providers'
import { formatMessagesForProvider } from '@/providers/attachments'
import type { ModelsObject } from '@/providers/ollama/types'
import { createReadableStreamFromOllamaStream } from '@/providers/ollama/utils'
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
@@ -91,6 +92,7 @@ export const ollamaProvider: ProviderConfig = {
if (request.messages) {
allMessages.push(...request.messages)
}
const formattedMessages = formatMessagesForProvider(allMessages, 'ollama')
const tools = request.tools?.length
? request.tools.map((tool) => ({
@@ -105,7 +107,7 @@ export const ollamaProvider: ProviderConfig = {
const payload: any = {
model: request.model,
messages: allMessages,
messages: formattedMessages,
}
if (request.temperature !== undefined) payload.temperature = request.temperature
@@ -274,7 +276,7 @@ export const ollamaProvider: ProviderConfig = {
}
const toolCalls = []
const toolResults: Record<string, unknown>[] = []
const currentMessages = [...allMessages]
const currentMessages = [...formattedMessages]
let iterationCount = 0
let modelTime = firstResponseTime
+1 -1
View File
@@ -133,7 +133,7 @@ export async function executeResponsesProviderRequest(
allMessages.push(...request.messages)
}
const initialInput = buildResponsesInputFromMessages(allMessages)
const initialInput = buildResponsesInputFromMessages(allMessages, config.providerId)
const basePayload: Record<string, unknown> = {
model: config.modelName,
+40
View File
@@ -0,0 +1,40 @@
/**
* @vitest-environment node
*/
import { describe, expect, it } from 'vitest'
import { buildResponsesInputFromMessages } from '@/providers/openai/utils'
describe('buildResponsesInputFromMessages', () => {
it('should convert user message files to Responses multipart content', () => {
const input = buildResponsesInputFromMessages([
{
role: 'user',
content: 'Analyze this image',
files: [
{
id: 'file-1',
key: 'workspace/ws-1/example.png',
name: 'example.png',
url: '/api/files/serve/workspace%2Fws-1%2Fexample.png?context=workspace',
size: 128,
type: 'image/png',
base64: 'iVBORw0KGgo=',
},
],
},
])
expect(input).toEqual([
{
role: 'user',
content: [
{ type: 'input_text', text: 'Analyze this image' },
{
type: 'input_image',
image_url: 'data:image/png;base64,iVBORw0KGgo=',
},
],
},
])
})
})
+19 -7
View File
@@ -1,5 +1,6 @@
import { createLogger } from '@sim/logger'
import type OpenAI from 'openai'
import { buildOpenAIMessageContent } from '@/providers/attachments'
import type { Message } from '@/providers/types'
const logger = createLogger('ResponsesUtils')
@@ -21,7 +22,7 @@ export interface ResponsesToolCall {
export type ResponsesInputItem =
| {
role: 'system' | 'user' | 'assistant'
content: string
content: string | Array<Record<string, unknown>>
}
| {
type: 'function_call'
@@ -45,7 +46,10 @@ export interface ResponsesToolDefinition {
/**
* Converts chat-style messages into Responses API input items.
*/
export function buildResponsesInputFromMessages(messages: Message[]): ResponsesInputItem[] {
export function buildResponsesInputFromMessages(
messages: Message[],
providerId = 'openai'
): ResponsesInputItem[] {
const input: ResponsesInputItem[] = []
for (const message of messages) {
@@ -58,13 +62,21 @@ export function buildResponsesInputFromMessages(messages: Message[]): ResponsesI
continue
}
if (
message.content &&
(message.role === 'system' || message.role === 'user' || message.role === 'assistant')
) {
if (message.role === 'system' || message.role === 'user' || message.role === 'assistant') {
const content =
message.role === 'user'
? buildOpenAIMessageContent(message.content, message.files, providerId)
: (message.content ?? '')
if (
(typeof content === 'string' && !content) ||
(Array.isArray(content) && content.length === 0)
) {
continue
}
input.push({
role: message.role,
content: message.content,
content,
})
}
+4 -2
View File
@@ -4,6 +4,7 @@ import OpenAI from 'openai'
import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions'
import type { StreamingExecution } from '@/executor/types'
import { MAX_TOOL_ITERATIONS } from '@/providers'
import { formatMessagesForProvider } from '@/providers/attachments'
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
import {
checkForForcedToolUsage,
@@ -109,6 +110,7 @@ export const openRouterProvider: ProviderConfig = {
if (request.messages) {
allMessages.push(...request.messages)
}
const formattedMessages = formatMessagesForProvider(allMessages, 'openrouter') as Message[]
const tools = request.tools?.length
? request.tools.map((tool) => ({
@@ -123,7 +125,7 @@ export const openRouterProvider: ProviderConfig = {
const payload: any = {
model: requestedModel,
messages: allMessages,
messages: formattedMessages,
}
if (request.temperature !== undefined) payload.temperature = request.temperature
@@ -251,7 +253,7 @@ export const openRouterProvider: ProviderConfig = {
}
const toolCalls: FunctionCallResponse[] = []
const toolResults: Record<string, unknown>[] = []
const currentMessages = [...allMessages]
const currentMessages = [...formattedMessages]
let iterationCount = 0
let modelTime = firstResponseTime
let toolsTime = 0
+2 -1
View File
@@ -1,4 +1,4 @@
import type { ProviderTimingSegment, StreamingExecution } from '@/executor/types'
import type { ProviderTimingSegment, StreamingExecution, UserFile } from '@/executor/types'
export type ProviderId =
| 'openai'
@@ -121,6 +121,7 @@ export interface ProviderToolConfig {
export interface Message {
role: 'system' | 'user' | 'assistant' | 'function' | 'tool'
content: string | null
files?: UserFile[]
name?: string
function_call?: {
name: string
+4 -2
View File
@@ -5,6 +5,7 @@ import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/
import { env } from '@/lib/core/config/env'
import type { StreamingExecution } from '@/executor/types'
import { MAX_TOOL_ITERATIONS } from '@/providers'
import { formatMessagesForProvider } from '@/providers/attachments'
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
import type {
@@ -123,6 +124,7 @@ export const vllmProvider: ProviderConfig = {
if (request.messages) {
allMessages.push(...request.messages)
}
const formattedMessages = formatMessagesForProvider(allMessages, 'vllm') as Message[]
const tools = request.tools?.length
? request.tools.map((tool) => ({
@@ -137,7 +139,7 @@ export const vllmProvider: ProviderConfig = {
const payload: any = {
model: request.model.replace(/^vllm\//, ''),
messages: allMessages,
messages: formattedMessages,
}
if (request.temperature !== undefined) payload.temperature = request.temperature
@@ -319,7 +321,7 @@ export const vllmProvider: ProviderConfig = {
}
const toolCalls = []
const toolResults: Record<string, unknown>[] = []
const currentMessages = [...allMessages]
const currentMessages = [...formattedMessages]
let iterationCount = 0
let modelTime = firstResponseTime
+4 -2
View File
@@ -4,6 +4,7 @@ import OpenAI from 'openai'
import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions'
import type { StreamingExecution } from '@/executor/types'
import { MAX_TOOL_ITERATIONS } from '@/providers'
import { formatMessagesForProvider } from '@/providers/attachments'
import { getProviderDefaultModel, getProviderModels } from '@/providers/models'
import { enrichLastModelSegmentFromChatCompletions } from '@/providers/trace-enrichment'
import type {
@@ -76,6 +77,7 @@ export const xAIProvider: ProviderConfig = {
if (request.messages) {
allMessages.push(...request.messages)
}
const formattedMessages = formatMessagesForProvider(allMessages, 'xai') as Message[]
const tools = request.tools?.length
? request.tools.map((tool) => ({
type: 'function',
@@ -93,7 +95,7 @@ export const xAIProvider: ProviderConfig = {
}
const basePayload: any = {
model: request.model,
messages: allMessages,
messages: formattedMessages,
}
if (request.temperature !== undefined) basePayload.temperature = request.temperature
@@ -219,7 +221,7 @@ export const xAIProvider: ProviderConfig = {
}
const toolCalls = []
const toolResults: Record<string, unknown>[] = []
const currentMessages = [...allMessages]
const currentMessages = [...formattedMessages]
let iterationCount = 0
let hasUsedForcedTool = false
+64
View File
@@ -7,6 +7,64 @@ interface FileGetParams {
_context?: WorkflowToolExecutionContext
}
interface FileReadParams {
fileId?: string | string[]
fileInput?: unknown
workspaceId?: string
_context?: WorkflowToolExecutionContext
}
const createFileReadTool = (config: {
id: 'file_read'
name: string
description: string
}): ToolConfig<FileReadParams, ToolResponse> => ({
id: config.id,
name: config.name,
description: config.description,
version: '1.0.0',
params: {
fileId: {
type: 'string',
required: false,
visibility: 'user-or-llm',
description: 'Canonical workspace file ID, or an array of canonical workspace file IDs.',
},
fileInput: {
type: 'file',
required: false,
visibility: 'user-only',
description: 'Selected workspace file object.',
},
},
request: {
url: '/api/tools/file/manage',
method: 'POST',
headers: () => ({ 'Content-Type': 'application/json' }),
body: (params) => ({
operation: 'read',
fileId: params.fileId,
fileInput: params.fileInput,
workspaceId: params.workspaceId || params._context?.workspaceId,
}),
},
transformResponse: async (response) => {
const data = await response.json()
if (!response.ok || !data.success) {
return { success: false, output: {}, error: data.error || 'Failed to get file' }
}
return { success: true, output: data.data }
},
outputs: {
file: { type: 'file', description: 'Workspace file object' },
files: { type: 'file[]', description: 'Workspace file objects' },
},
})
export const fileGetTool: ToolConfig<FileGetParams, ToolResponse> = {
id: 'file_get',
name: 'File Get',
@@ -52,3 +110,9 @@ export const fileGetTool: ToolConfig<FileGetParams, ToolResponse> = {
file: { type: 'file', description: 'Workspace file object' },
},
}
export const fileReadTool = createFileReadTool({
id: 'file_read',
name: 'File Read',
description: 'Read workspace file objects from selected files or canonical workspace file IDs.',
})
+8 -2
View File
@@ -1,9 +1,15 @@
import { fileParserTool, fileParserV2Tool, fileParserV3Tool } from '@/tools/file/parser'
import {
fileFetchTool,
fileParserTool,
fileParserV2Tool,
fileParserV3Tool,
} from '@/tools/file/parser'
export { fileAppendTool } from '@/tools/file/append'
export { fileGetTool } from '@/tools/file/get'
export { fileGetTool, fileReadTool } from '@/tools/file/get'
export { fileWriteTool } from '@/tools/file/write'
export const fileParseTool = fileParserTool
export { fileFetchTool }
export { fileParserV2Tool }
export { fileParserV3Tool }
+59 -18
View File
@@ -10,6 +10,7 @@ import type {
FileParserV3OutputData,
FileUploadInput,
} from '@/tools/file/types'
import { transformTable } from '@/tools/shared/table'
import type { ToolConfig } from '@/tools/types'
const logger = createLogger('FileParserTool')
@@ -34,6 +35,20 @@ const isFileParseResult = (value: unknown): value is FileParseResult =>
typeof value.name === 'string' &&
typeof value.binary === 'boolean'
const normalizeHeaders = (headers: FileParserInput['headers']): Record<string, string> => {
const transformed = transformTable(headers ?? null)
return Object.entries(transformed).reduce(
(acc, [key, value]) => {
const headerName = key.trim()
if (headerName && value !== undefined && value !== null) {
acc[headerName] = String(value)
}
return acc
},
{} as Record<string, string>
)
}
const normalizeFileParseResult = (value: unknown): FileParseResult => {
if (isRecord(value) && isFileParseResult(value.output)) {
return value.output
@@ -245,9 +260,11 @@ export const fileParserTool: ToolConfig<FileParserInput, FileParserOutput> = {
}
logger.info('Tool body determined filePath:', determinedFilePath)
const headers = normalizeHeaders(params.headers)
return {
filePath: determinedFilePath,
fileType: determinedFileType,
...(Object.keys(headers).length > 0 && { headers }),
workspaceId: params.workspaceId || params._context?.workspaceId,
workflowId: params._context?.workflowId,
executionId: params._context?.executionId,
@@ -286,6 +303,25 @@ export const fileParserV2Tool: ToolConfig<FileParserInput, FileParserOutput> = {
},
}
const parseFileParserV3Response = async (response: Response): Promise<FileParserV3Output> => {
const parsed = await parseFileParserResponse(response)
const output = parsed.output as FileParserOutputData
const files =
Array.isArray(output.processedFiles) && output.processedFiles.length > 0
? output.processedFiles
: []
const cleanedOutput: FileParserV3OutputData = {
files,
combinedContent: output.combinedContent,
}
return {
success: true,
output: cleanedOutput,
}
}
export const fileParserV3Tool: ToolConfig<FileParserInput, FileParserV3Output> = {
id: 'file_parser_v3',
name: 'File Parser',
@@ -293,26 +329,31 @@ export const fileParserV3Tool: ToolConfig<FileParserInput, FileParserV3Output> =
version: '3.0.0',
params: fileParserTool.params,
request: fileParserTool.request,
transformResponse: async (response: Response): Promise<FileParserV3Output> => {
const parsed = await parseFileParserResponse(response)
const output = parsed.output as FileParserOutputData
const files =
Array.isArray(output.processedFiles) && output.processedFiles.length > 0
? output.processedFiles
: []
const cleanedOutput: FileParserV3OutputData = {
files,
combinedContent: output.combinedContent,
}
return {
success: true,
output: cleanedOutput,
}
},
transformResponse: parseFileParserV3Response,
outputs: {
files: { type: 'file[]', description: 'Parsed files as UserFile objects' },
combinedContent: { type: 'string', description: 'Combined content of all parsed files' },
},
}
export const fileFetchTool: ToolConfig<FileParserInput, FileParserV3Output> = {
id: 'file_fetch',
name: 'File Fetch',
description: 'Fetch and parse a file from a URL with optional custom headers.',
version: '1.0.0',
params: {
...fileParserTool.params,
headers: {
type: 'object',
required: false,
visibility: 'user-or-llm',
description: 'HTTP headers to include when fetching URL-based files.',
},
},
request: fileParserTool.request,
transformResponse: parseFileParserV3Response,
outputs: {
files: { type: 'file[]', description: 'Fetched files as UserFile objects' },
combinedContent: { type: 'string', description: 'Combined content of all fetched files' },
},
}
+2 -1
View File
@@ -1,10 +1,11 @@
import type { UserFile } from '@/executor/types'
import type { ToolResponse } from '@/tools/types'
import type { TableRow, ToolResponse } from '@/tools/types'
export interface FileParserInput {
filePath?: string | string[]
file?: UserFile | UserFile[] | FileUploadInput | FileUploadInput[]
fileType?: string
headers?: TableRow[] | Record<string, unknown> | string | null
workspaceId?: string
workflowId?: string
executionId?: string
+4
View File
@@ -645,10 +645,12 @@ import {
} from '@/tools/fathom'
import {
fileAppendTool,
fileFetchTool,
fileGetTool,
fileParserV2Tool,
fileParserV3Tool,
fileParseTool,
fileReadTool,
fileWriteTool,
} from '@/tools/file'
import {
@@ -3216,7 +3218,9 @@ export const tools: Record<string, ToolConfig> = {
file_parser_v2: fileParserV2Tool,
file_parser_v3: fileParserV3Tool,
file_append: fileAppendTool,
file_fetch: fileFetchTool,
file_get: fileGetTool,
file_read: fileReadTool,
file_write: fileWriteTool,
firecrawl_scrape: firecrawlScrapeTool,
firecrawl_search: firecrawlSearchTool,