From 5c3b7b6186c63b50b1dcbd4fa21eecbd705899e0 Mon Sep 17 00:00:00 2001 From: Suguru Inoue Date: Tue, 16 Dec 2025 15:35:52 +0100 Subject: [PATCH] fix(core): Check mime type when restoring attachments in chat memory (#23287) --- .../chat-hub-workflow.service.test.ts | 62 ++++++++++++++--- .../chat-hub/chat-hub-workflow.service.ts | 69 ++++++++++++++----- 2 files changed, 105 insertions(+), 26 deletions(-) diff --git a/packages/cli/src/modules/chat-hub/__tests__/chat-hub-workflow.service.test.ts b/packages/cli/src/modules/chat-hub/__tests__/chat-hub-workflow.service.test.ts index f7f23da3823..bbb0aedcfcb 100644 --- a/packages/cli/src/modules/chat-hub/__tests__/chat-hub-workflow.service.test.ts +++ b/packages/cli/src/modules/chat-hub/__tests__/chat-hub-workflow.service.test.ts @@ -64,7 +64,7 @@ describe('ChatHubWorkflowService', () => { 'Hello', [], { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, - { provider: 'openai', model: 'gpt-4' }, + { provider: 'openai', model: 'gpt-4-turbo' }, undefined, [], 'UTC', @@ -117,7 +117,7 @@ describe('ChatHubWorkflowService', () => { 'Hello', [], { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, - { provider: 'openai', model: 'gpt-4' }, + { provider: 'openai', model: 'gpt-4-turbo' }, undefined, [], 'UTC', @@ -181,7 +181,7 @@ describe('ChatHubWorkflowService', () => { 'Hello', [], { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, - { provider: 'openai', model: 'gpt-4' }, + { provider: 'openai', model: 'gpt-4-turbo' }, undefined, [], 'UTC', @@ -226,7 +226,7 @@ describe('ChatHubWorkflowService', () => { 'Hello', [], { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, - { provider: 'openai', model: 'gpt-4' }, + { provider: 'openai', model: 'gpt-4-turbo' }, undefined, [], 'UTC', @@ -280,7 +280,7 @@ describe('ChatHubWorkflowService', () => { 'Hello', [], { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, - { provider: 'openai', model: 'gpt-4' }, + { provider: 'openai', model: 'gpt-4-turbo' }, undefined, [], 'UTC', @@ -338,7 +338,7 @@ describe('ChatHubWorkflowService', () => { 'Hello', [], { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, - { provider: 'openai', model: 'gpt-4' }, + { provider: 'openai', model: 'gpt-4-turbo' }, undefined, [], 'UTC', @@ -419,7 +419,7 @@ describe('ChatHubWorkflowService', () => { 'Hello', [], { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, - { provider: 'openai', model: 'gpt-4' }, + { provider: 'openai', model: 'gpt-4-turbo' }, undefined, [], 'UTC', @@ -495,7 +495,7 @@ describe('ChatHubWorkflowService', () => { 'Hello', [], { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, - { provider: 'openai', model: 'gpt-4' }, + { provider: 'openai', model: 'gpt-4-turbo' }, undefined, [], 'UTC', @@ -552,7 +552,7 @@ describe('ChatHubWorkflowService', () => { 'Hello', [], { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, - { provider: 'openai', model: 'gpt-4' }, + { provider: 'openai', model: 'gpt-4-turbo' }, undefined, [], 'UTC', @@ -571,6 +571,50 @@ describe('ChatHubWorkflowService', () => { { type: 'text', text: `File: document.txt\nContent: \n${textContent}` }, ]); }); + + it('should replace unsupported attachment with unsupported message', async () => { + const mockAudioAttachment: IBinaryData = { + data: 'data:audio/mp3;base64,SUQzBAAAAAAAI1RTU0UAAAAPAAADTGF2ZjU4Ljc2LjEwMAAAAAAAAAAAAAAA', + mimeType: 'audio/mp3', + fileName: 'audio.mp3', + }; + + const mockMessage = new ChatHubMessage(); + mockMessage.id = 'msg-1'; + mockMessage.content = 'Listen to this audio'; + mockMessage.type = 'human'; + mockMessage.attachments = [mockAudioAttachment]; + mockMessage.sessionId = 'session-456'; + mockMessage.session = new ChatHubSession(); + mockMessage.status = 'running'; + + const mockHistory: ChatHubMessage[] = [mockMessage]; + + const result = await service.createChatWorkflow( + 'user-123', + 'session-456', + 'project-789', + mockHistory, + 'Hello', + [], + { openAiApi: { id: 'cred-123', name: 'OpenAI' } }, + { provider: 'openai', model: 'gpt-4' }, + undefined, + [], + 'UTC', + ); + + const restoreMemoryNode = result.workflowData.nodes.find( + (node) => node.name === 'Restore Chat Memory', + ); + expect(restoreMemoryNode?.parameters?.messages).toBeDefined(); + + const messageValues = (restoreMemoryNode?.parameters?.messages as any)?.messageValues; + expect(messageValues[0].message).toEqual([ + { type: 'text', text: 'Listen to this audio' }, + { type: 'text', text: 'File: audio.mp3\n(Unsupported file type)' }, + ]); + }); }); }); }); diff --git a/packages/cli/src/modules/chat-hub/chat-hub-workflow.service.ts b/packages/cli/src/modules/chat-hub/chat-hub-workflow.service.ts index baf699ed316..5cd8d44243a 100644 --- a/packages/cli/src/modules/chat-hub/chat-hub-workflow.service.ts +++ b/packages/cli/src/modules/chat-hub/chat-hub-workflow.service.ts @@ -1,4 +1,10 @@ -import { ChatHubConversationModel, ChatSessionId, type ChatHubInputModality } from '@n8n/api-types'; +import { + ChatHubConversationModel, + ChatSessionId, + type ChatHubBaseLLMModel, + type ChatHubInputModality, + type ChatModelMetadataDto, +} from '@n8n/api-types'; import { Logger } from '@n8n/backend-common'; import { SharedWorkflow, @@ -31,7 +37,7 @@ import { import { v4 as uuidv4 } from 'uuid'; import { ChatHubMessage } from './chat-hub-message.entity'; -import { NODE_NAMES, PROVIDER_NODE_TYPE_MAP } from './chat-hub.constants'; +import { getModelMetadata, NODE_NAMES, PROVIDER_NODE_TYPE_MAP } from './chat-hub.constants'; import { MessageRecord, type ContentBlock, type ChatTriggerResponseMode } from './chat-hub.types'; import { getMaxContextWindowTokens } from './context-limits'; import { ChatHubAttachmentService } from './chat-hub.attachment.service'; @@ -53,7 +59,7 @@ export class ChatHubWorkflowService { humanMessage: string, attachments: IBinaryData[], credentials: INodeCredentials, - model: ChatHubConversationModel, + model: ChatHubBaseLLMModel, systemMessage: string | undefined, tools: INode[], timeZone: string, @@ -227,16 +233,7 @@ export class ChatHubWorkflowService { const modalities = new Set(['text']); for (const mimeType of mimeTypes) { - if (mimeType.startsWith('image/')) { - modalities.add('image'); - } else if (mimeType.startsWith('audio/')) { - modalities.add('audio'); - } else if (mimeType.startsWith('video/')) { - modalities.add('video'); - } else { - // Any other MIME type falls under generic 'file' - modalities.add('file'); - } + modalities.add(this.getMimeTypeModality(mimeType)); } return Array.from(modalities); @@ -275,7 +272,7 @@ export class ChatHubWorkflowService { humanMessage: string; attachments: IBinaryData[]; credentials: INodeCredentials; - model: ChatHubConversationModel; + model: ChatHubBaseLLMModel; systemMessage: string; tools: INode[]; }) { @@ -283,7 +280,7 @@ export class ChatHubWorkflowService { const toolsAgentNode = this.buildToolsAgentNode(model, systemMessage); const modelNode = this.buildModelNode(credentials, model); const memoryNode = this.buildMemoryNode(20); - const restoreMemoryNode = await this.buildRestoreMemoryNode(history); + const restoreMemoryNode = await this.buildRestoreMemoryNode(history, model); const clearMemoryNode = this.buildClearMemoryNode(); const mergeNode = this.buildMergeNode(); @@ -676,8 +673,11 @@ ${this.getSystemMessageMetadata(timeZone)}`; }; } - private async buildRestoreMemoryNode(history: ChatHubMessage[]): Promise { - const messageValues = await this.buildMessageValuesWithAttachments(history); + private async buildRestoreMemoryNode( + history: ChatHubMessage[], + model: ChatHubBaseLLMModel, + ): Promise { + const messageValues = await this.buildMessageValuesWithAttachments(history, model); return { parameters: { @@ -697,7 +697,10 @@ ${this.getSystemMessageMetadata(timeZone)}`; private async buildMessageValuesWithAttachments( history: ChatHubMessage[], + model: ChatHubBaseLLMModel, ): Promise { + const metadata = getModelMetadata(model.provider, model.model); + // Gemini has 20MB limit, the value should also be what n8n instance can safely handle const maxTotalPayloadSize = 20 * 1024 * 1024 * 0.9; @@ -743,6 +746,7 @@ ${this.getSystemMessageMetadata(timeZone)}`; attachment, currentTotalSize, maxTotalPayloadSize, + metadata, ); blocks.push(block); currentTotalSize += block.type === 'text' ? block.text.length : block.image_url.length; @@ -765,8 +769,10 @@ ${this.getSystemMessageMetadata(timeZone)}`; attachment: IBinaryData, currentTotalSize: number, maxTotalPayloadSize: number, + modelMetadata: ChatModelMetadataDto, ): Promise { class TotalFileSizeExceededError extends Error {} + class UnsupportedMimeTypeError extends Error {} try { if (currentTotalSize >= maxTotalPayloadSize) { @@ -787,6 +793,12 @@ ${this.getSystemMessageMetadata(timeZone)}`; }; } + const modality = this.getMimeTypeModality(attachment.mimeType); + + if (!modelMetadata.inputModalities.includes(modality)) { + throw new UnsupportedMimeTypeError(); + } + const url = await this.chatHubAttachmentService.getDataUrl(attachment); if (currentTotalSize + url.length > maxTotalPayloadSize) { @@ -802,6 +814,13 @@ ${this.getSystemMessageMetadata(timeZone)}`; }; } + if (e instanceof UnsupportedMimeTypeError) { + return { + type: 'text', + text: `File: ${attachment.fileName ?? 'attachment'}\n(Unsupported file type)`, + }; + } + throw e; } } @@ -876,4 +895,20 @@ Respond the title only:`, name: NODE_NAMES.TITLE_GENERATOR_AGENT, }; } + + /** + * Determines the input modality for a given MIME type + */ + private getMimeTypeModality(mimeType: string): ChatHubInputModality { + if (mimeType.startsWith('image/')) { + return 'image'; + } + if (mimeType.startsWith('audio/')) { + return 'audio'; + } + if (mimeType.startsWith('video/')) { + return 'video'; + } + return 'file'; + } }