opensource/n8n
3
0
Fork 0
mirror of https://github.com/n8n-io/n8n.git synced 2026-09-24 23:22:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(core): Check mime type when restoring attachments in chat memory (#23287)

Browse Source
This commit is contained in:
Suguru Inoue
2025-12-16 15:35:52 +01:00
committed by GitHub
parent 2f4cda5a8e
commit 5c3b7b6186
2 changed files with 105 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/cli/src/modules/chat-hub/__tests__/chat-hub-workflow.service.test.ts
+53 -9
View File
@@ -64,7 +64,7 @@ describe('ChatHubWorkflowService', () => {
'Hello',
[],
{ openAiApi: { id: 'cred-123', name: 'OpenAI' } },
{ provider: 'openai', model: 'gpt-4' },
{ provider: 'openai', model: 'gpt-4-turbo' },
undefined,
[],
'UTC',
@@ -117,7 +117,7 @@ describe('ChatHubWorkflowService', () => {
'Hello',
[],
{ openAiApi: { id: 'cred-123', name: 'OpenAI' } },
{ provider: 'openai', model: 'gpt-4' },
{ provider: 'openai', model: 'gpt-4-turbo' },
undefined,
[],
'UTC',
@@ -181,7 +181,7 @@ describe('ChatHubWorkflowService', () => {
'Hello',
[],
{ openAiApi: { id: 'cred-123', name: 'OpenAI' } },
{ provider: 'openai', model: 'gpt-4' },
{ provider: 'openai', model: 'gpt-4-turbo' },
undefined,
[],
'UTC',
@@ -226,7 +226,7 @@ describe('ChatHubWorkflowService', () => {
'Hello',
[],
{ openAiApi: { id: 'cred-123', name: 'OpenAI' } },
{ provider: 'openai', model: 'gpt-4' },
{ provider: 'openai', model: 'gpt-4-turbo' },
undefined,
[],
'UTC',
@@ -280,7 +280,7 @@ describe('ChatHubWorkflowService', () => {
'Hello',
[],
{ openAiApi: { id: 'cred-123', name: 'OpenAI' } },
{ provider: 'openai', model: 'gpt-4' },
{ provider: 'openai', model: 'gpt-4-turbo' },
undefined,
[],
'UTC',
@@ -338,7 +338,7 @@ describe('ChatHubWorkflowService', () => {
'Hello',
[],
{ openAiApi: { id: 'cred-123', name: 'OpenAI' } },
{ provider: 'openai', model: 'gpt-4' },
{ provider: 'openai', model: 'gpt-4-turbo' },
undefined,
[],
'UTC',
@@ -419,7 +419,7 @@ describe('ChatHubWorkflowService', () => {
'Hello',
[],
{ openAiApi: { id: 'cred-123', name: 'OpenAI' } },
{ provider: 'openai', model: 'gpt-4' },
{ provider: 'openai', model: 'gpt-4-turbo' },
undefined,
[],
'UTC',
@@ -495,7 +495,7 @@ describe('ChatHubWorkflowService', () => {
'Hello',
[],
{ openAiApi: { id: 'cred-123', name: 'OpenAI' } },
{ provider: 'openai', model: 'gpt-4' },
{ provider: 'openai', model: 'gpt-4-turbo' },
undefined,
[],
'UTC',
@@ -552,7 +552,7 @@ describe('ChatHubWorkflowService', () => {
'Hello',
[],
{ openAiApi: { id: 'cred-123', name: 'OpenAI' } },
{ provider: 'openai', model: 'gpt-4' },
{ provider: 'openai', model: 'gpt-4-turbo' },
undefined,
[],
'UTC',
@@ -571,6 +571,50 @@ describe('ChatHubWorkflowService', () => {
{ type: 'text', text: `File: document.txt\nContent: \n${textContent}` },
]);
});
it('should replace unsupported attachment with unsupported message', async () => {
const mockAudioAttachment: IBinaryData = {
data: 'data:audio/mp3;base64,SUQzBAAAAAAAI1RTU0UAAAAPAAADTGF2ZjU4Ljc2LjEwMAAAAAAAAAAAAAAA',
mimeType: 'audio/mp3',
fileName: 'audio.mp3',
};
const mockMessage = new ChatHubMessage();
mockMessage.id = 'msg-1';
mockMessage.content = 'Listen to this audio';
mockMessage.type = 'human';
mockMessage.attachments = [mockAudioAttachment];
mockMessage.sessionId = 'session-456';
mockMessage.session = new ChatHubSession();
mockMessage.status = 'running';
const mockHistory: ChatHubMessage[] = [mockMessage];
const result = await service.createChatWorkflow(
'user-123',
'session-456',
'project-789',
mockHistory,
'Hello',
[],
{ openAiApi: { id: 'cred-123', name: 'OpenAI' } },
{ provider: 'openai', model: 'gpt-4' },
undefined,
[],
'UTC',
);
const restoreMemoryNode = result.workflowData.nodes.find(
(node) => node.name === 'Restore Chat Memory',
);
expect(restoreMemoryNode?.parameters?.messages).toBeDefined();
const messageValues = (restoreMemoryNode?.parameters?.messages as any)?.messageValues;
expect(messageValues[0].message).toEqual([
{ type: 'text', text: 'Listen to this audio' },
{ type: 'text', text: 'File: audio.mp3\n(Unsupported file type)' },
]);
});
});
});
});
packages/cli/src/modules/chat-hub/chat-hub-workflow.service.ts
+52 -17
View File
@@ -1,4 +1,10 @@
import { ChatHubConversationModel, ChatSessionId, type ChatHubInputModality } from '@n8n/api-types';
import {
ChatHubConversationModel,
ChatSessionId,
type ChatHubBaseLLMModel,
type ChatHubInputModality,
type ChatModelMetadataDto,
} from '@n8n/api-types';
import { Logger } from '@n8n/backend-common';
import {
SharedWorkflow,
@@ -31,7 +37,7 @@ import {
import { v4 as uuidv4 } from 'uuid';
import { ChatHubMessage } from './chat-hub-message.entity';
import { NODE_NAMES, PROVIDER_NODE_TYPE_MAP } from './chat-hub.constants';
import { getModelMetadata, NODE_NAMES, PROVIDER_NODE_TYPE_MAP } from './chat-hub.constants';
import { MessageRecord, type ContentBlock, type ChatTriggerResponseMode } from './chat-hub.types';
import { getMaxContextWindowTokens } from './context-limits';
import { ChatHubAttachmentService } from './chat-hub.attachment.service';
@@ -53,7 +59,7 @@ export class ChatHubWorkflowService {
humanMessage: string,
attachments: IBinaryData[],
credentials: INodeCredentials,
model: ChatHubConversationModel,
model: ChatHubBaseLLMModel,
systemMessage: string | undefined,
tools: INode[],
timeZone: string,
@@ -227,16 +233,7 @@ export class ChatHubWorkflowService {
const modalities = new Set<ChatHubInputModality>(['text']);
for (const mimeType of mimeTypes) {
if (mimeType.startsWith('image/')) {
modalities.add('image');
} else if (mimeType.startsWith('audio/')) {
modalities.add('audio');
} else if (mimeType.startsWith('video/')) {
modalities.add('video');
} else {
// Any other MIME type falls under generic 'file'
modalities.add('file');
}
modalities.add(this.getMimeTypeModality(mimeType));
}
return Array.from(modalities);
@@ -275,7 +272,7 @@ export class ChatHubWorkflowService {
humanMessage: string;
attachments: IBinaryData[];
credentials: INodeCredentials;
model: ChatHubConversationModel;
model: ChatHubBaseLLMModel;
systemMessage: string;
tools: INode[];
}) {
@@ -283,7 +280,7 @@ export class ChatHubWorkflowService {
const toolsAgentNode = this.buildToolsAgentNode(model, systemMessage);
const modelNode = this.buildModelNode(credentials, model);
const memoryNode = this.buildMemoryNode(20);
const restoreMemoryNode = await this.buildRestoreMemoryNode(history);
const restoreMemoryNode = await this.buildRestoreMemoryNode(history, model);
const clearMemoryNode = this.buildClearMemoryNode();
const mergeNode = this.buildMergeNode();
@@ -676,8 +673,11 @@ ${this.getSystemMessageMetadata(timeZone)}`;
};
}
private async buildRestoreMemoryNode(history: ChatHubMessage[]): Promise<INode> {
const messageValues = await this.buildMessageValuesWithAttachments(history);
private async buildRestoreMemoryNode(
history: ChatHubMessage[],
model: ChatHubBaseLLMModel,
): Promise<INode> {
const messageValues = await this.buildMessageValuesWithAttachments(history, model);
return {
parameters: {
@@ -697,7 +697,10 @@ ${this.getSystemMessageMetadata(timeZone)}`;
private async buildMessageValuesWithAttachments(
history: ChatHubMessage[],
model: ChatHubBaseLLMModel,
): Promise<MessageRecord[]> {
const metadata = getModelMetadata(model.provider, model.model);
// Gemini has 20MB limit, the value should also be what n8n instance can safely handle
const maxTotalPayloadSize = 20 * 1024 * 1024 * 0.9;
@@ -743,6 +746,7 @@ ${this.getSystemMessageMetadata(timeZone)}`;
attachment,
currentTotalSize,
maxTotalPayloadSize,
metadata,
);
blocks.push(block);
currentTotalSize += block.type === 'text' ? block.text.length : block.image_url.length;
@@ -765,8 +769,10 @@ ${this.getSystemMessageMetadata(timeZone)}`;
attachment: IBinaryData,
currentTotalSize: number,
maxTotalPayloadSize: number,
modelMetadata: ChatModelMetadataDto,
): Promise<ContentBlock> {
class TotalFileSizeExceededError extends Error {}
class UnsupportedMimeTypeError extends Error {}
try {
if (currentTotalSize >= maxTotalPayloadSize) {
@@ -787,6 +793,12 @@ ${this.getSystemMessageMetadata(timeZone)}`;
};
}
const modality = this.getMimeTypeModality(attachment.mimeType);
if (!modelMetadata.inputModalities.includes(modality)) {
throw new UnsupportedMimeTypeError();
}
const url = await this.chatHubAttachmentService.getDataUrl(attachment);
if (currentTotalSize + url.length > maxTotalPayloadSize) {
@@ -802,6 +814,13 @@ ${this.getSystemMessageMetadata(timeZone)}`;
};
}
if (e instanceof UnsupportedMimeTypeError) {
return {
type: 'text',
text: `File: ${attachment.fileName ?? 'attachment'}\n(Unsupported file type)`,
};
}
throw e;
}
}
@@ -876,4 +895,20 @@ Respond the title only:`,
name: NODE_NAMES.TITLE_GENERATOR_AGENT,
};
}
/**
* Determines the input modality for a given MIME type
*/
private getMimeTypeModality(mimeType: string): ChatHubInputModality {
if (mimeType.startsWith('image/')) {
return 'image';
}
if (mimeType.startsWith('audio/')) {
return 'audio';
}
if (mimeType.startsWith('video/')) {
return 'video';
}
return 'file';
}
}