From c7fd47d31fcc23bdcdae5cfa2aa385ccb3b031fc Mon Sep 17 00:00:00 2001 From: Suguru Inoue Date: Wed, 10 Dec 2025 08:38:29 +0100 Subject: [PATCH] fix(core): Files are not included in memory in chat (no-changelog) (#22922) --- .../chat-hub-workflow.service.test.ts | 576 ++++++++++++++++++ .../chat-hub/chat-hub-workflow.service.ts | 155 ++++- .../chat-hub/chat-hub.attachment.service.ts | 16 + .../src/modules/chat-hub/chat-hub.types.ts | 7 +- .../src/features/ai/chatHub/chat.utils.ts | 10 +- .../ai/chatHub/components/ChatPrompt.vue | 7 +- 6 files changed, 737 insertions(+), 34 deletions(-) create mode 100644 packages/cli/src/modules/chat-hub/__tests__/chat-hub-workflow.service.test.ts diff --git a/packages/cli/src/modules/chat-hub/__tests__/chat-hub-workflow.service.test.ts b/packages/cli/src/modules/chat-hub/__tests__/chat-hub-workflow.service.test.ts new file mode 100644 index 00000000000..f7f23da3823 --- /dev/null +++ b/packages/cli/src/modules/chat-hub/__tests__/chat-hub-workflow.service.test.ts @@ -0,0 +1,576 @@ +import type { WorkflowRepository, SharedWorkflowRepository } from '@n8n/db'; +import type { Logger } from '@n8n/backend-common'; +import { mock } from 'jest-mock-extended'; +import type { BinaryDataService } from 'n8n-core'; +import type { IBinaryData } from 'n8n-workflow'; + +import { ChatHubWorkflowService } from '../chat-hub-workflow.service'; +import { ChatHubMessage } from '../chat-hub-message.entity'; +import { ChatHubSession } from '../chat-hub-session.entity'; +import { ChatHubAttachmentService } from '../chat-hub.attachment.service'; +import type { ChatHubMessageRepository } from '../chat-message.repository'; + +describe('ChatHubWorkflowService', () => { + const logger = mock(); + const workflowRepository = mock(); + const sharedWorkflowRepository = mock(); + const binaryDataService = mock(); + const messageRepository = mock(); + + let chatHubAttachmentService: ChatHubAttachmentService; + let service: ChatHubWorkflowService; + + beforeEach(() => { + jest.resetAllMocks(); + + // Create real ChatHubAttachmentService with mocked dependencies + chatHubAttachmentService = new ChatHubAttachmentService(binaryDataService, messageRepository); + + service = new ChatHubWorkflowService( + logger, + workflowRepository, + sharedWorkflowRepository, + chatHubAttachmentService, + ); + + // Mock repository methods + const mockEntityManager = { + save: jest.fn().mockImplementation(async (entity) => { + // Return the entity with an ID added + return { ...entity, id: 'workflow-123' }; + }), + } as any; + + Object.defineProperty(workflowRepository, 'manager', { + value: { + transaction: jest.fn((cb) => cb(mockEntityManager)), + }, + writable: true, + }); + + (sharedWorkflowRepository.create as jest.Mock) = jest.fn().mockReturnValue({}); + }); + + describe('createChatWorkflow', () => { + describe('message history handling', () => { + it('should handle empty history', async () => { + const mockHistory: ChatHubMessage[] = []; + + const result = await service.createChatWorkflow( + 'user-123', + 'session-456', + 'project-789', + mockHistory, + 'Hello', + [], + { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, + { provider: 'openai', model: 'gpt-4' }, + undefined, + [], + 'UTC', + ); + + const restoreMemoryNode = result.workflowData.nodes.find( + (node) => node.name === 'Restore Chat Memory', + ); + expect(restoreMemoryNode?.parameters?.messages).toBeDefined(); + + const messageValues = (restoreMemoryNode?.parameters?.messages as any)?.messageValues; + expect(messageValues).toHaveLength(0); + }); + + it('should handle multiple messages', async () => { + const message1 = new ChatHubMessage(); + message1.id = 'msg-1'; + message1.content = 'First message'; + message1.type = 'human'; + message1.attachments = []; + message1.sessionId = 'session-456'; + message1.session = new ChatHubSession(); + message1.status = 'running'; + + const message2 = new ChatHubMessage(); + message2.id = 'msg-2'; + message2.content = 'Second message'; + message2.type = 'ai'; + message2.attachments = []; + message2.sessionId = 'session-456'; + message2.session = new ChatHubSession(); + message2.status = 'running'; + + const message3 = new ChatHubMessage(); + message3.id = 'msg-3'; + message3.content = 'Third message'; + message3.type = 'human'; + message3.attachments = []; + message3.sessionId = 'session-456'; + message3.session = new ChatHubSession(); + message3.status = 'running'; + + const mockHistory: ChatHubMessage[] = [message1, message2, message3]; + + const result = await service.createChatWorkflow( + 'user-123', + 'session-456', + 'project-789', + mockHistory, + 'Hello', + [], + { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, + { provider: 'openai', model: 'gpt-4' }, + undefined, + [], + 'UTC', + ); + + const restoreMemoryNode = result.workflowData.nodes.find( + (node) => node.name === 'Restore Chat Memory', + ); + expect(restoreMemoryNode?.parameters?.messages).toBeDefined(); + + const messageValues = (restoreMemoryNode?.parameters?.messages as any)?.messageValues; + expect(messageValues).toHaveLength(3); + expect(messageValues[0]).toEqual({ + type: 'user', + message: 'First message', + hideFromUI: false, + }); + expect(messageValues[1]).toEqual({ + type: 'ai', + message: 'Second message', + hideFromUI: false, + }); + expect(messageValues[2]).toEqual({ + type: 'user', + message: 'Third message', + hideFromUI: false, + }); + }); + }); + + describe('attachment files', () => { + it('should convert binary data to data URL when attachment has id', async () => { + const mockAttachment: IBinaryData = { + id: 'filesystem-v2:chat-hub/sessions/session-456/messages/msg-1/binary_data/bin-1', + data: 'filesystem-v2', + mimeType: 'image/png', + fileName: 'test.png', + fileSize: '100 kB', + fileExtension: 'png', + }; + + const mockMessage = new ChatHubMessage(); + mockMessage.id = 'msg-1'; + mockMessage.content = 'Check this image'; + mockMessage.type = 'human'; + mockMessage.attachments = [mockAttachment]; + mockMessage.sessionId = 'session-456'; + mockMessage.session = new ChatHubSession(); + mockMessage.status = 'running'; + + const mockHistory: ChatHubMessage[] = [mockMessage]; + + const mockImageBuffer = Buffer.from('fake-image-data', 'base64'); + binaryDataService.getAsBuffer.mockResolvedValue(mockImageBuffer); + + const result = await service.createChatWorkflow( + 'user-123', + 'session-456', + 'project-789', + mockHistory, + 'Hello', + [], + { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, + { provider: 'openai', model: 'gpt-4' }, + undefined, + [], + 'UTC', + ); + + expect(binaryDataService.getAsBuffer).toHaveBeenCalledWith(mockAttachment); + + const expectedDataUrl = `data:${mockAttachment.mimeType};base64,${mockImageBuffer.toString('base64')}`; + const restoreMemoryNode = result.workflowData.nodes.find( + (node) => node.name === 'Restore Chat Memory', + ); + expect(restoreMemoryNode?.parameters?.messages).toBeDefined(); + expect((restoreMemoryNode?.parameters?.messages as any)?.messageValues[0].message).toEqual([ + { type: 'text', text: 'Check this image' }, + { type: 'image_url', image_url: expectedDataUrl }, + ]); + }); + + it('should use data URL directly when attachment has data but no id', async () => { + const mockAttachment: IBinaryData = { + data: 'data:image/png;base64,iVBORw0KGgoAAAANS', + mimeType: 'image/png', + fileName: 'test.png', + }; + + const mockMessage = new ChatHubMessage(); + mockMessage.id = 'msg-1'; + mockMessage.content = 'Check this image'; + mockMessage.type = 'human'; + mockMessage.attachments = [mockAttachment]; + mockMessage.sessionId = 'session-456'; + mockMessage.session = new ChatHubSession(); + mockMessage.status = 'running'; + + const mockHistory: ChatHubMessage[] = [mockMessage]; + + const result = await service.createChatWorkflow( + 'user-123', + 'session-456', + 'project-789', + mockHistory, + 'Hello', + [], + { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, + { provider: 'openai', model: 'gpt-4' }, + undefined, + [], + 'UTC', + ); + + const restoreMemoryNode = result.workflowData.nodes.find( + (node) => node.name === 'Restore Chat Memory', + ); + expect(restoreMemoryNode?.parameters?.messages).toBeDefined(); + expect((restoreMemoryNode?.parameters?.messages as any)?.messageValues[0].message).toEqual([ + { type: 'text', text: 'Check this image' }, + { type: 'image_url', image_url: mockAttachment.data }, + ]); + }); + + it('should handle multiple attachments correctly', async () => { + const mockAttachmentWithId: IBinaryData = { + id: 'filesystem-v2:chat-hub/sessions/session-456/messages/msg-1/binary_data/bin-2', + data: 'filesystem-v2', + mimeType: 'image/png', + fileName: 'test1.png', + fileSize: '150 kB', + fileExtension: 'png', + }; + + const mockAttachmentWithData: IBinaryData = { + data: 'data:image/jpeg;base64,/9j/4AAQ', + mimeType: 'image/jpeg', + fileName: 'test2.jpg', + }; + + const mockMessage = new ChatHubMessage(); + mockMessage.id = 'msg-1'; + mockMessage.content = 'Check these images'; + mockMessage.type = 'human'; + mockMessage.attachments = [mockAttachmentWithId, mockAttachmentWithData]; + mockMessage.sessionId = 'session-456'; + mockMessage.session = new ChatHubSession(); + mockMessage.status = 'running'; + + const mockHistory: ChatHubMessage[] = [mockMessage]; + + const mockImageBuffer = Buffer.from('fake-image-data-1', 'base64'); + binaryDataService.getAsBuffer.mockResolvedValue(mockImageBuffer); + + const result = await service.createChatWorkflow( + 'user-123', + 'session-456', + 'project-789', + mockHistory, + 'Hello', + [], + { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, + { provider: 'openai', model: 'gpt-4' }, + undefined, + [], + 'UTC', + ); + + expect(binaryDataService.getAsBuffer).toHaveBeenCalledTimes(1); + expect(binaryDataService.getAsBuffer).toHaveBeenCalledWith(mockAttachmentWithId); + expect(binaryDataService.createSignedToken).not.toHaveBeenCalled(); + + const expectedDataUrl = `data:${mockAttachmentWithId.mimeType};base64,${mockImageBuffer.toString('base64')}`; + const restoreMemoryNode = result.workflowData.nodes.find( + (node) => node.name === 'Restore Chat Memory', + ); + expect(restoreMemoryNode?.parameters?.messages).toBeDefined(); + expect((restoreMemoryNode?.parameters?.messages as any)?.messageValues[0].message).toEqual([ + { type: 'text', text: 'Check these images' }, + { type: 'image_url', image_url: expectedDataUrl }, + { type: 'image_url', image_url: mockAttachmentWithData.data }, + ]); + }); + + it('should omit attachments that exceed maxTotalPayloadSize limit', async () => { + // Create a large data URL that, when added to another attachment, will exceed the 20MB limit (90% of 20MB = 18MB) + const largeDataUrl = 'data:image/png;base64,' + 'A'.repeat(10 * 1024 * 1024); + const smallDataUrl = 'data:image/png;base64,' + 'B'.repeat(9 * 1024 * 1024); + + const mockAttachment1: IBinaryData = { + data: largeDataUrl, + mimeType: 'image/png', + fileName: 'large.png', + }; + + const mockAttachment2: IBinaryData = { + data: smallDataUrl, + mimeType: 'image/png', + fileName: 'small.png', + }; + + const mockMessage = new ChatHubMessage(); + mockMessage.id = 'msg-1'; + mockMessage.content = 'Check these images'; + mockMessage.type = 'human'; + mockMessage.attachments = [mockAttachment1, mockAttachment2]; + mockMessage.sessionId = 'session-456'; + mockMessage.session = new ChatHubSession(); + mockMessage.status = 'running'; + + const mockHistory: ChatHubMessage[] = [mockMessage]; + + const result = await service.createChatWorkflow( + 'user-123', + 'session-456', + 'project-789', + mockHistory, + 'Hello', + [], + { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, + { provider: 'openai', model: 'gpt-4' }, + undefined, + [], + 'UTC', + ); + + const restoreMemoryNode = result.workflowData.nodes.find( + (node) => node.name === 'Restore Chat Memory', + ); + expect(restoreMemoryNode?.parameters?.messages).toBeDefined(); + + const messageContent = (restoreMemoryNode?.parameters?.messages as any)?.messageValues[0] + .message; + + // Should include text and only the first attachment since the second would exceed the limit + expect(messageContent).toEqual([ + { type: 'text', text: 'Check these images' }, + { type: 'image_url', image_url: largeDataUrl }, + { type: 'text', text: 'File: small.png\n(Content omitted due to size limit)' }, + ]); + }); + + it('should omit attachments across multiple messages when total size exceeds limit', async () => { + const attachment1DataUrl = 'data:image/png;base64,' + 'A'.repeat(5 * 1024 * 1024); + const mockAttachment1: IBinaryData = { + data: attachment1DataUrl, + mimeType: 'image/png', + fileName: 'first.png', + }; + + const message1 = new ChatHubMessage(); + message1.id = 'msg-1'; + message1.content = 'First message with attachment'; + message1.type = 'human'; + message1.attachments = [mockAttachment1]; + message1.sessionId = 'session-456'; + message1.session = new ChatHubSession(); + message1.status = 'running'; + + const attachment2DataUrl = 'data:image/jpeg;base64,' + 'B'.repeat(8 * 1024 * 1024); + const mockAttachment2: IBinaryData = { + data: attachment2DataUrl, + mimeType: 'image/jpeg', + fileName: 'second.jpg', + }; + + const message2 = new ChatHubMessage(); + message2.id = 'msg-2'; + message2.content = 'Second message'; + message2.type = 'ai'; + message2.attachments = [mockAttachment2]; + message2.sessionId = 'session-456'; + message2.session = new ChatHubSession(); + message2.status = 'running'; + + const attachment3DataUrl = 'data:image/png;base64,' + 'C'.repeat(6 * 1024 * 1024); + const mockAttachment3: IBinaryData = { + data: attachment3DataUrl, + mimeType: 'image/png', + fileName: 'third.png', + }; + + const message3 = new ChatHubMessage(); + message3.id = 'msg-3'; + message3.content = 'Third message'; + message3.type = 'human'; + message3.attachments = [mockAttachment3]; + message3.sessionId = 'session-456'; + message3.session = new ChatHubSession(); + message3.status = 'running'; + + const mockHistory: ChatHubMessage[] = [message1, message2, message3]; + + const result = await service.createChatWorkflow( + 'user-123', + 'session-456', + 'project-789', + mockHistory, + 'Hello', + [], + { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, + { provider: 'openai', model: 'gpt-4' }, + undefined, + [], + 'UTC', + ); + + const restoreMemoryNode = result.workflowData.nodes.find( + (node) => node.name === 'Restore Chat Memory', + ); + expect(restoreMemoryNode?.parameters?.messages).toBeDefined(); + + const messageValues = (restoreMemoryNode?.parameters?.messages as any)?.messageValues; + expect(messageValues).toHaveLength(3); + + // First message attachment is skipped due to cumulative size limit (processed last, size limit reached) + expect(messageValues[0].message).toEqual([ + { type: 'text', text: 'First message with attachment' }, + { type: 'text', text: 'File: first.png\n(Content omitted due to size limit)' }, + ]); + + // Second message includes attachment + expect(messageValues[1].message).toEqual([ + { type: 'text', text: 'Second message' }, + { type: 'image_url', image_url: attachment2DataUrl }, + ]); + + // Third message includes attachment (processed first) + expect(messageValues[2].message).toEqual([ + { type: 'text', text: 'Third message' }, + { type: 'image_url', image_url: attachment3DataUrl }, + ]); + }); + + it('should include omitted content message when attachment exceeds size limit', async () => { + const largeDataUrl = 'data:image/png;base64,' + 'A'.repeat(17 * 1024 * 1024); + + const mockAttachment: IBinaryData = { + data: largeDataUrl, + mimeType: 'image/png', + fileName: 'large-image.png', + }; + + const mockMessage = new ChatHubMessage(); + mockMessage.id = 'msg-1'; + mockMessage.content = 'Check this large image'; + mockMessage.type = 'human'; + mockMessage.attachments = [mockAttachment]; + mockMessage.sessionId = 'session-456'; + mockMessage.session = new ChatHubSession(); + mockMessage.status = 'running'; + + const anotherAttachment: IBinaryData = { + data: 'data:image/jpeg;base64,' + 'B'.repeat(2 * 1024 * 1024), + mimeType: 'image/jpeg', + fileName: 'small-image.jpg', + }; + + const mockMessage2 = new ChatHubMessage(); + mockMessage2.id = 'msg-2'; + mockMessage2.content = 'And this small one'; + mockMessage2.type = 'human'; + mockMessage2.attachments = [anotherAttachment]; + mockMessage2.sessionId = 'session-456'; + mockMessage2.session = new ChatHubSession(); + mockMessage2.status = 'running'; + + const mockHistory: ChatHubMessage[] = [mockMessage, mockMessage2]; + + const result = await service.createChatWorkflow( + 'user-123', + 'session-456', + 'project-789', + mockHistory, + 'Hello', + [], + { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, + { provider: 'openai', model: 'gpt-4' }, + undefined, + [], + 'UTC', + ); + + const restoreMemoryNode = result.workflowData.nodes.find( + (node) => node.name === 'Restore Chat Memory', + ); + expect(restoreMemoryNode?.parameters?.messages).toBeDefined(); + + const messageValues = (restoreMemoryNode?.parameters?.messages as any)?.messageValues; + + expect(messageValues[0].message).toEqual([ + { type: 'text', text: 'Check this large image' }, + { type: 'text', text: 'File: large-image.png\n(Content omitted due to size limit)' }, + ]); + + expect(messageValues[1].message).toEqual([ + { type: 'text', text: 'And this small one' }, + { type: 'image_url', image_url: anotherAttachment.data }, + ]); + }); + + it('should handle text file attachments as text blocks', async () => { + const textContent = 'This is the content of the text file.\nIt has multiple lines.'; + const mockAttachment: IBinaryData = { + id: 'filesystem-v2:chat-hub/sessions/session-456/messages/msg-1/binary_data/text-1', + data: 'filesystem-v2', + mimeType: 'text/plain', + fileName: 'document.txt', + fileSize: '60', + fileExtension: 'txt', + }; + + const mockMessage = new ChatHubMessage(); + mockMessage.id = 'msg-1'; + mockMessage.content = 'Here is a text file'; + mockMessage.type = 'human'; + mockMessage.attachments = [mockAttachment]; + mockMessage.sessionId = 'session-456'; + mockMessage.session = new ChatHubSession(); + mockMessage.status = 'running'; + + const mockHistory: ChatHubMessage[] = [mockMessage]; + + // Mock getAsBuffer to return the text content + binaryDataService.getAsBuffer.mockResolvedValue(Buffer.from(textContent, 'utf-8')); + + const result = await service.createChatWorkflow( + 'user-123', + 'session-456', + 'project-789', + mockHistory, + 'Hello', + [], + { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, + { provider: 'openai', model: 'gpt-4' }, + undefined, + [], + 'UTC', + ); + + expect(binaryDataService.getAsBuffer).toHaveBeenCalledWith(mockAttachment); + + const restoreMemoryNode = result.workflowData.nodes.find( + (node) => node.name === 'Restore Chat Memory', + ); + expect(restoreMemoryNode?.parameters?.messages).toBeDefined(); + + const messageValues = (restoreMemoryNode?.parameters?.messages as any)?.messageValues; + expect(messageValues[0].message).toEqual([ + { type: 'text', text: 'Here is a text file' }, + { type: 'text', text: `File: document.txt\nContent: \n${textContent}` }, + ]); + }); + }); + }); +}); diff --git a/packages/cli/src/modules/chat-hub/chat-hub-workflow.service.ts b/packages/cli/src/modules/chat-hub/chat-hub-workflow.service.ts index 334df173816..baf699ed316 100644 --- a/packages/cli/src/modules/chat-hub/chat-hub-workflow.service.ts +++ b/packages/cli/src/modules/chat-hub/chat-hub-workflow.service.ts @@ -26,13 +26,15 @@ import { NodeConnectionTypes, OperationalError, type IBinaryData, + type NodeParameterValueType, } from 'n8n-workflow'; import { v4 as uuidv4 } from 'uuid'; import { ChatHubMessage } from './chat-hub-message.entity'; import { NODE_NAMES, PROVIDER_NODE_TYPE_MAP } from './chat-hub.constants'; -import { MessageRecord, type ChatTriggerResponseMode } from './chat-hub.types'; +import { MessageRecord, type ContentBlock, type ChatTriggerResponseMode } from './chat-hub.types'; import { getMaxContextWindowTokens } from './context-limits'; +import { ChatHubAttachmentService } from './chat-hub.attachment.service'; @Service() export class ChatHubWorkflowService { @@ -40,6 +42,7 @@ export class ChatHubWorkflowService { private readonly logger: Logger, private readonly workflowRepository: WorkflowRepository, private readonly sharedWorkflowRepository: SharedWorkflowRepository, + private readonly chatHubAttachmentService: ChatHubAttachmentService, ) {} async createChatWorkflow( @@ -65,7 +68,7 @@ export class ChatHubWorkflowService { `Creating chat workflow for user ${userId} and session ${sessionId}, provider ${model.provider}`, ); - const { nodes, connections, executionData } = this.buildChatWorkflow({ + const { nodes, connections, executionData } = await this.buildChatWorkflow({ userId, sessionId, history, @@ -255,7 +258,7 @@ export class ChatHubWorkflowService { return uniqueName; } - private buildChatWorkflow({ + private async buildChatWorkflow({ userId, sessionId, history, @@ -280,7 +283,7 @@ export class ChatHubWorkflowService { const toolsAgentNode = this.buildToolsAgentNode(model, systemMessage); const modelNode = this.buildModelNode(credentials, model); const memoryNode = this.buildMemoryNode(20); - const restoreMemoryNode = this.buildRestoreMemoryNode(history); + const restoreMemoryNode = await this.buildRestoreMemoryNode(history); const clearMemoryNode = this.buildClearMemoryNode(); const mergeNode = this.buildMergeNode(); @@ -673,29 +676,15 @@ ${this.getSystemMessageMetadata(timeZone)}`; }; } - private buildRestoreMemoryNode(history: ChatHubMessage[]): INode { + private async buildRestoreMemoryNode(history: ChatHubMessage[]): Promise { + const messageValues = await this.buildMessageValuesWithAttachments(history); + return { parameters: { mode: 'insert', insertMode: 'override', messages: { - messageValues: history - // Empty messages can't be restored by the memory manager - .filter((message) => message.content.length > 0) - .map((message) => { - const typeMap: Record = { - human: 'user', - ai: 'ai', - system: 'system', - }; - - // TODO: Tool messages etc? - return { - type: typeMap[message.type] || 'system', - message: message.content, - hideFromUI: false, - }; - }), + messageValues: messageValues as unknown as NodeParameterValueType, }, }, type: MEMORY_MANAGER_NODE_TYPE, @@ -706,6 +695,128 @@ ${this.getSystemMessageMetadata(timeZone)}`; }; } + private async buildMessageValuesWithAttachments( + history: ChatHubMessage[], + ): Promise { + // Gemini has 20MB limit, the value should also be what n8n instance can safely handle + const maxTotalPayloadSize = 20 * 1024 * 1024 * 0.9; + + const typeMap: Record = { + human: 'user', + ai: 'ai', + system: 'system', + }; + + const messageValues: MessageRecord[] = []; + let currentTotalSize = 0; + + const messages = history.slice().reverse(); // Traversing messages from last to prioritize newer attachments + + for (const message of messages) { + // Empty messages can't be restored by the memory manager + if (message.content.length === 0) { + continue; + } + + const attachments = message.attachments ?? []; + const type = typeMap[message.type] || 'system'; + + // TODO: Tool messages etc? + + const textSize = message.content.length; + currentTotalSize += textSize; + + if (attachments.length === 0) { + messageValues.push({ + type, + message: message.content, + hideFromUI: false, + }); + continue; + } + + const blocks: ContentBlock[] = [{ type: 'text', text: message.content }]; + + // Add attachments if within size limit + for (const attachment of attachments) { + const block = await this.buildContentBlockForAttachment( + attachment, + currentTotalSize, + maxTotalPayloadSize, + ); + blocks.push(block); + currentTotalSize += block.type === 'text' ? block.text.length : block.image_url.length; + } + + messageValues.push({ + type, + message: blocks, + hideFromUI: false, + }); + } + + // Reverse to restore original order + messageValues.reverse(); + + return messageValues; + } + + private async buildContentBlockForAttachment( + attachment: IBinaryData, + currentTotalSize: number, + maxTotalPayloadSize: number, + ): Promise { + class TotalFileSizeExceededError extends Error {} + + try { + if (currentTotalSize >= maxTotalPayloadSize) { + throw new TotalFileSizeExceededError(); + } + + if (this.isTextFile(attachment.mimeType)) { + const buffer = await this.chatHubAttachmentService.getAsBuffer(attachment); + const content = buffer.toString('utf-8'); + + if (currentTotalSize + content.length > maxTotalPayloadSize) { + throw new TotalFileSizeExceededError(); + } + + return { + type: 'text', + text: `File: ${attachment.fileName ?? 'attachment'}\nContent: \n${content}`, + }; + } + + const url = await this.chatHubAttachmentService.getDataUrl(attachment); + + if (currentTotalSize + url.length > maxTotalPayloadSize) { + throw new TotalFileSizeExceededError(); + } + + return { type: 'image_url', image_url: url }; + } catch (e) { + if (e instanceof TotalFileSizeExceededError) { + return { + type: 'text', + text: `File: ${attachment.fileName ?? 'attachment'}\n(Content omitted due to size limit)`, + }; + } + + throw e; + } + } + + private isTextFile(mimeType: string): boolean { + return ( + mimeType.startsWith('text/') || + mimeType === 'application/json' || + mimeType === 'application/xml' || + mimeType === 'application/csv' || + mimeType === 'application/x-yaml' || + mimeType === 'application/yaml' + ); + } + private buildClearMemoryNode(): INode { return { parameters: { diff --git a/packages/cli/src/modules/chat-hub/chat-hub.attachment.service.ts b/packages/cli/src/modules/chat-hub/chat-hub.attachment.service.ts index 7f73c759ef9..a7aeffd4d39 100644 --- a/packages/cli/src/modules/chat-hub/chat-hub.attachment.service.ts +++ b/packages/cli/src/modules/chat-hub/chat-hub.attachment.service.ts @@ -126,6 +126,22 @@ export class ChatHubAttachmentService { ); } + async getDataUrl(binaryData: IBinaryData): Promise { + if (binaryData.data.startsWith('data:')) { + return binaryData.data; + } + + const buffer = await this.binaryDataService.getAsBuffer(binaryData); + const base64Data = buffer.toString(BINARY_ENCODING); + const mimeType = binaryData.mimeType || 'application/octet-stream'; + + return `data:${mimeType};base64,${base64Data}`; + } + + async getAsBuffer(binaryData: IBinaryData): Promise> { + return await this.binaryDataService.getAsBuffer(binaryData); + } + /** * Processes a single attachment by populating metadata and storing it. */ diff --git a/packages/cli/src/modules/chat-hub/chat-hub.types.ts b/packages/cli/src/modules/chat-hub/chat-hub.types.ts index a88711b63bb..f40b0ff9ec9 100644 --- a/packages/cli/src/modules/chat-hub/chat-hub.types.ts +++ b/packages/cli/src/modules/chat-hub/chat-hub.types.ts @@ -43,11 +43,16 @@ export interface EditMessagePayload extends BaseMessagePayload { message: string; } +// From @langchain/core +export type ContentBlock = + | { type: 'text'; text: string } + | { type: 'image_url'; image_url: string }; + // From packages/@n8n/nodes-langchain/nodes/memory/MemoryManager/MemoryManager.node.ts export type MessageRole = 'ai' | 'system' | 'user'; export interface MessageRecord { type: MessageRole; - message: string; + message: string | ContentBlock[]; hideFromUI: boolean; } diff --git a/packages/frontend/editor-ui/src/features/ai/chatHub/chat.utils.ts b/packages/frontend/editor-ui/src/features/ai/chatHub/chat.utils.ts index 3f60576b2af..457d57e1c85 100644 --- a/packages/frontend/editor-ui/src/features/ai/chatHub/chat.utils.ts +++ b/packages/frontend/editor-ui/src/features/ai/chatHub/chat.utils.ts @@ -349,17 +349,13 @@ export function createSessionFromStreamingState(streaming: ChatStreamingState): }; } -export function createMimeTypes(modalities: ChatHubInputModality[]): string | undefined { - if (modalities.length === 0) { - return undefined; - } - +export function createMimeTypes(modalities: ChatHubInputModality[]): string { // If 'file' modality is present, accept all file types if (modalities.includes('file')) { return '*/*'; } - const mimeTypes: string[] = []; + const mimeTypes: string[] = ['text/*']; for (const modality of modalities) { if (modality === 'image') { @@ -373,5 +369,5 @@ export function createMimeTypes(modalities: ChatHubInputModality[]): string | un } } - return mimeTypes.length > 0 ? mimeTypes.join(',') : undefined; + return mimeTypes.join(','); } diff --git a/packages/frontend/editor-ui/src/features/ai/chatHub/components/ChatPrompt.vue b/packages/frontend/editor-ui/src/features/ai/chatHub/components/ChatPrompt.vue index d8e93315089..6ed57061aa5 100644 --- a/packages/frontend/editor-ui/src/features/ai/chatHub/components/ChatPrompt.vue +++ b/packages/frontend/editor-ui/src/features/ai/chatHub/components/ChatPrompt.vue @@ -60,10 +60,9 @@ const llmProvider = computed(() => isLlmProviderModel(selectedModel?.model) ? selectedModel?.model.provider : undefined, ); -const acceptedMimeTypes = computed(() => { - const modalities = selectedModel?.metadata.inputModalities; - return modalities ? createMimeTypes(modalities) : undefined; -}); +const acceptedMimeTypes = computed(() => + createMimeTypes(selectedModel?.metadata.inputModalities ?? []), +); const canUploadFiles = computed(() => !!acceptedMimeTypes.value);