From 991bd29a14f243ff00528cecb2f416476b3c1cb2 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 11 Mar 2026 17:17:32 +0800 Subject: [PATCH] feat: implement image upload and multimodal support - Added functionality for image uploads in chat, allowing users to attach images for multimodal Q&A. - Enhanced the input field to handle image selection via drag-and-drop and paste, with validation for file types and sizes. - Updated the backend to process images alongside text queries, including support for image analysis. - Introduced new UI components for image previews and management, improving user interaction with uploaded content. - Added localization strings for image upload features and error messages, enhancing accessibility for users in multiple languages. These changes significantly improve the chat experience by enabling users to incorporate images into their queries, facilitating richer interactions and responses. --- config/config.yaml | 87 +++-- frontend/src/api/agent/index.ts | 5 + frontend/src/api/chat/streame.ts | 6 +- frontend/src/api/model/index.ts | 1 + frontend/src/components/Input-field.vue | 216 +++++++++++- frontend/src/components/ModelEditorDialog.vue | 16 +- frontend/src/i18n/locales/en-US.ts | 25 ++ frontend/src/i18n/locales/zh-CN.ts | 25 ++ frontend/src/stores/menu.ts | 5 +- frontend/src/views/agent/AgentEditorModal.vue | 128 ++++++- .../chat/components/AgentStreamDisplay.vue | 81 +++-- .../src/views/chat/components/usermsg.vue | 73 +++- frontend/src/views/chat/index.vue | 43 ++- frontend/src/views/creatChat/creatChat.vue | 12 +- frontend/src/views/settings/ModelSettings.vue | 6 +- internal/agent/engine.go | 18 +- internal/agent/prompts.go | 37 +- .../service/chat_pipline/common.go | 9 +- .../service/chat_pipline/into_chat_message.go | 6 + .../service/chat_pipline/rewrite.go | 320 ++++++++++++------ .../service/chat_pipline/search_parallel.go | 9 + internal/application/service/session.go | 46 ++- internal/handler/session/handler.go | 6 + internal/handler/session/helpers.go | 45 ++- internal/handler/session/image_upload.go | 160 +++++++++ internal/handler/session/qa.go | 78 ++++- internal/handler/session/types.go | 10 + internal/models/chat/chat.go | 1 + internal/models/chat/image_resolve.go | 82 +++++ internal/models/chat/ollama.go | 33 ++ internal/models/chat/remote_api.go | 42 ++- internal/models/vlm/remote_api.go | 6 +- internal/types/agent.go | 18 +- internal/types/chat_manage.go | 12 + internal/types/custom_agent.go | 18 +- internal/types/interfaces/agent.go | 2 + internal/types/interfaces/session.go | 2 + internal/types/message.go | 59 +++- internal/types/model.go | 1 + internal/utils/log_sanitize.go | 29 ++ .../versioned/000021_message_images.down.sql | 1 + .../versioned/000021_message_images.up.sql | 1 + 42 files changed, 1554 insertions(+), 226 deletions(-) create mode 100644 internal/handler/session/image_upload.go create mode 100644 internal/models/chat/image_resolve.go create mode 100644 internal/utils/log_sanitize.go create mode 100644 migrations/versioned/000021_message_images.down.sql create mode 100644 migrations/versioned/000021_message_images.up.sql diff --git a/config/config.yaml b/config/config.yaml index 2a14bac61..c32fbc1c6 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -29,59 +29,94 @@ conversation: enable_query_expansion: true enable_rerank: true rewrite_prompt_system: | - You are an intelligent assistant specialized in coreference resolution and ellipsis completion. Your task is to clearly identify pronouns in the user's question based on the conversation history and replace them with explicit subjects, while completing any omitted key information. + You are an intelligent assistant that performs TWO tasks on the user's question: + 1. Rewrite the question (coreference resolution and ellipsis completion) + 2. Classify whether the question requires knowledge base retrieval - ## Rewriting Goals - Based on the conversation history, rewrite the current user question with the following objectives: + ## Task 1: Rewriting Goals + Based on the conversation history, rewrite the current user question: - Perform coreference resolution: replace pronouns such as "it", "this", "that", "they", "them", etc. with explicit subjects - Complete omitted key information to ensure the question is semantically complete - Preserve the original meaning and expression style of the question - The rewritten result must also be a question - The rewritten question should be within 30 words - - Output ONLY the rewritten question without any explanation, and do NOT attempt to answer the question - IMPORTANT: The rewritten question must be in the same language as the original question + ## Task 2: Intent Classification + Determine if the question requires knowledge base retrieval. + - If retrieval is NOT needed, prefix your output with [NO_SEARCH] + - Otherwise, output the rewritten question directly (no prefix = default to search) + - IMPORTANT: This is a knowledge base Q&A system. Default to SEARCH. Only add [NO_SEARCH] when you are very confident the question has nothing to do with the knowledge base. + + When to output [NO_SEARCH] (ONLY these narrow cases): + - Pure greetings, thanks, or farewell with no question ("谢谢", "你好", "再见") + - Requests to summarize or manipulate the previous conversation itself ("总结一下我们的对话") + - When unsure, do NOT add the prefix (default to search) + + When NOT to output [NO_SEARCH] (common mistakes to avoid): + - User uploads an image and asks to find/search/look up something → SEARCH (the image content is the search query) + - User asks about image content that may relate to documents ("这个报错怎么解决") → SEARCH + - User asks a question with an image, even if it seems like a simple image description → SEARCH (the knowledge base may contain relevant information) + + ## Task 3: Image Analysis (only when images are attached) + If the user's message includes images, after the rewritten question, add a separator "---" on a new line, then provide a concise description of the image content. This description is used as a text fallback for models that cannot process images directly. + + ## Output Format + - Text only (needs search): rewritten question + - Text only (no search): [NO_SEARCH] rewritten question + - With images (needs search): rewritten question\n---\nimage description + - With images (no search): [NO_SEARCH] rewritten question\n---\nimage description + ## Few-shot Examples - Example 1: + Example 1 (text, needs search): Conversation history: User: What features does Slack have? Assistant: Slack's main features include messaging, file sharing, channel organization, and integration with various tools. - User question: Is it secure? - Rewritten: Is Slack secure? + Output: Is Slack secure? - Example 2: + Example 2 (text, needs search): Conversation history: User: My laptop battery drains too fast, what should I do? Assistant: You can extend battery life by reducing screen brightness, closing background apps, and regularly updating your system. - User question: Would that affect the user experience? - Rewritten: Would reducing screen brightness and closing background apps affect the user experience? + Output: Would reducing screen brightness and closing background apps affect the user experience? - Example 3: + Example 3 (text, no search - greeting): Conversation history: - User: How do you make pasta carbonara? - Assistant: Pasta carbonara requires cooking spaghetti, then mixing it with a sauce made from eggs, cheese, and pancetta. + User: How do I install WeKnora? + Assistant: You can install WeKnora using docker compose... + User question: Thanks! + Output: [NO_SEARCH] Thanks! - User question: How long does it take? - Rewritten: How long does it take to make pasta carbonara? - - Example 4: + Example 4 (text, no search - summarize conversation): Conversation history: - User: How much does a flight from New York to London cost? - Assistant: Flights from New York to London vary by airline and class. Economy tickets are around $400-800, and business class around $2000-5000. + User: What is RAG? + Assistant: RAG stands for Retrieval Augmented Generation... + User question: Summarize what we discussed + Output: [NO_SEARCH] Summarize our discussion about RAG - User question: What about the duration? - Rewritten: How long is the flight from New York to London? + Example 5 (image, needs search - user wants to find related info): + User question: 找一下这张图 [image attached] + Output: 找一下这张图片相关的内容 + --- + 图片中显示了一个系统架构图，包含前端、后端和数据库三层。 - Example 5: + Example 6 (image, needs search - error screenshot): Conversation history: - User: How do I create a GitHub account? - Assistant: To create a GitHub account, go to github.com, click "Sign up", enter your email, create a password, and choose a username. + User: How do I configure the search settings? + Assistant: You can configure search settings in the admin panel... + User question: I got this error, how to fix it? [image attached] + Output: How to fix this search configuration error? + --- + The image shows an error dialog with the message "Invalid rerank model ID: model not found". - User question: Can I use a company email? - Rewritten: Can I use a company email to create a GitHub account? + Example 7 (image, needs search - image as query): + User question: What is this? [image attached] + Output: What is the content shown in this image? + --- + The image shows a product specification table with model numbers and parameters. rewrite_prompt_user: | ## Conversation History {{conversation}} diff --git a/frontend/src/api/agent/index.ts b/frontend/src/api/agent/index.ts index 9aba699fe..a6515dcb3 100644 --- a/frontend/src/api/agent/index.ts +++ b/frontend/src/api/agent/index.ts @@ -35,6 +35,11 @@ export interface CustomAgentConfig { // false: 根据 kb_selection_mode 自动检索知识库 retrieve_kb_only_when_mentioned?: boolean; + // ===== 图片上传/多模态设置 ===== + image_upload_enabled?: boolean; // 是否启用图片上传（默认: false） + vlm_model_id?: string; // VLM模型ID（图片分析用） + image_storage_provider?: string; // 图片存储提供商 + // ===== 文件类型限制 ===== // 支持的文件类型（如 ["csv", "xlsx", "xls"]） // 为空表示支持所有文件类型 diff --git a/frontend/src/api/chat/streame.ts b/frontend/src/api/chat/streame.ts index 479c7ba1a..b8764f7d2 100644 --- a/frontend/src/api/chat/streame.ts +++ b/frontend/src/api/chat/streame.ts @@ -29,7 +29,7 @@ export function useStream() { let renderTimer: number | null = null // 启动流式请求 - const startStream = async (params: { session_id: any; query: any; knowledge_base_ids?: string[]; knowledge_ids?: string[]; agent_enabled?: boolean; agent_id?: string; web_search_enabled?: boolean; enable_memory?: boolean; summary_model_id?: string; mcp_service_ids?: string[]; mentioned_items?: Array<{id: string; name: string; type: string; kb_type?: string}>; method: string; url: string }) => { + const startStream = async (params: { session_id: any; query: any; knowledge_base_ids?: string[]; knowledge_ids?: string[]; agent_enabled?: boolean; agent_id?: string; web_search_enabled?: boolean; enable_memory?: boolean; summary_model_id?: string; mcp_service_ids?: string[]; mentioned_items?: Array<{id: string; name: string; type: string; kb_type?: string}>; images?: Array<{data: string}>; method: string; url: string }) => { // 重置状态 output.value = ''; error.value = null; @@ -114,6 +114,10 @@ export function useStream() { if (params.mentioned_items !== undefined && params.mentioned_items.length > 0) { postBody.mentioned_items = params.mentioned_items; } + // Include images if provided (base64 data URIs for multimodal chat) + if (params.images !== undefined && params.images.length > 0) { + postBody.images = params.images; + } await fetchEventSource(url, { method: params.method, diff --git a/frontend/src/api/model/index.ts b/frontend/src/api/model/index.ts index e683ceb8c..79448c4ef 100644 --- a/frontend/src/api/model/index.ts +++ b/frontend/src/api/model/index.ts @@ -22,6 +22,7 @@ export interface ModelConfig { interface_type?: 'ollama' | 'openai'; // VLLM专用 parameter_size?: string; // Ollama模型参数大小 (e.g., "7B", "13B", "70B") extra_config?: Record; // Provider-specific configuration + supports_vision?: boolean; // Whether the model accepts image/multimodal input }; is_default?: boolean; is_builtin?: boolean; diff --git a/frontend/src/components/Input-field.vue b/frontend/src/components/Input-field.vue index 2c3e54ff6..5103d31b9 100644 --- a/frontend/src/components/Input-field.vue +++ b/frontend/src/components/Input-field.vue @@ -29,6 +29,48 @@ const { t } = useI18n(); let query = ref(""); const showKbSelector = ref(false); + +// Image upload state +const uploadedImages = ref>([]); +const imageInputRef = ref(); +const imageUploading = ref(false); + +const handleImageSelect = (event: Event) => { + const input = event.target as HTMLInputElement; + if (!input.files) return; + addImageFiles(Array.from(input.files)); + input.value = ''; +}; + +const addImageFiles = (files: File[]) => { + if (!isImageUploadEnabledByAgent.value) return; + const allowed = ['image/jpeg', 'image/png', 'image/gif', 'image/webp']; + const maxSize = 10 * 1024 * 1024; + for (const file of files) { + if (uploadedImages.value.length >= 5) { + MessagePlugin.warning(t('chat.imageTooMany')); + break; + } + if (!allowed.includes(file.type)) { + MessagePlugin.warning(t('chat.imageTypeSizeError')); + continue; + } + if (file.size > maxSize) { + MessagePlugin.warning(t('chat.imageTypeSizeError')); + continue; + } + uploadedImages.value.push({ file, preview: URL.createObjectURL(file) }); + } +}; + +const removeImage = (index: number) => { + const removed = uploadedImages.value.splice(index, 1); + if (removed.length > 0) URL.revokeObjectURL(removed[0].preview); +}; + +const triggerImageUpload = () => { + imageInputRef.value?.click(); +}; const atButtonRef = ref(); const showAgentModeSelector = ref(false); const agentModeButtonRef = ref(); @@ -115,6 +157,11 @@ watch([selectedAgentId, agentKnowledgeBases, agentKBSelectionMode], ([newAgentId if (showMention.value) { loadMentionItems(mentionQuery.value, true); } + // Clear images when switching to an agent that doesn't support image upload + if (!isImageUploadEnabledByAgent.value && uploadedImages.value.length > 0) { + uploadedImages.value.forEach(img => URL.revokeObjectURL(img.preview)); + uploadedImages.value = []; + } } }, { immediate: true }); @@ -177,6 +224,12 @@ const agentSupportedFileTypes = computed(() => { return currentAgentConfig.value?.supported_file_types || []; }); +// 智能体是否启用了图片上传（多模态） +const isImageUploadEnabledByAgent = computed(() => { + if (!hasAgentConfig.value) return false; + return currentAgentConfig.value?.image_upload_enabled === true; +}); + // 模型选择是否被智能体锁定 - 已移除锁定逻辑，允许用户自由切换模型 const isModelLockedByAgent = computed(() => { return false; @@ -1321,7 +1374,11 @@ const createSession = async (val: string) => { type: item.type, kb_type: item.type === 'kb' ? (item.kbType || 'document') : undefined })); - emit('send-msg', val, selectedModelId.value, mentionedItems); + const imageFiles = uploadedImages.value.map(img => img.file); + emit('send-msg', val, selectedModelId.value, mentionedItems, imageFiles); + // Clean up image previews + uploadedImages.value.forEach(img => URL.revokeObjectURL(img.preview)); + uploadedImages.value = []; clearvalue(); } @@ -1554,6 +1611,36 @@ const onKeydown = (val: string, event: { e: { preventDefault(): unknown; keyCode } } +const onPaste = (e: ClipboardEvent) => { + const items = e.clipboardData?.items; + if (!items) return; + const imageFiles: File[] = []; + for (const item of items) { + if (item.type.startsWith('image/')) { + const file = item.getAsFile(); + if (file) imageFiles.push(file); + } + } + if (imageFiles.length > 0 && isImageUploadEnabledByAgent.value) { + e.preventDefault(); + addImageFiles(imageFiles); + } +}; + +const onDrop = (e: DragEvent) => { + e.preventDefault(); + const files = e.dataTransfer?.files; + if (!files) return; + const imageFiles = Array.from(files).filter(f => f.type.startsWith('image/')); + if (imageFiles.length > 0 && isImageUploadEnabledByAgent.value) { + addImageFiles(imageFiles); + } +}; + +const onDragOver = (e: DragEvent) => { + e.preventDefault(); +}; + const handleGoToWebSearchSettings = () => { uiStore.openSettings('websearch'); if (route.path !== '/platform/settings') { @@ -1732,9 +1819,25 @@ onBeforeRouteUpdate((to, from, next) => {