From 160672ea9c20d9bc1e42e567e44f3ac2af6c9598 Mon Sep 17 00:00:00 2001
From: wizardchen <wizardchen@tencent.com>
Date: Mon, 16 Mar 2026 17:33:50 +0800
Subject: [PATCH] feat: enhance message structure to support multi-content
 messages

- Introduced MessageContentPart and ImageURL types to allow for multi-content messages that can include both text and images.
- Updated the Message struct to include a MultiContent field for handling multiple message parts.
- Modified the ConvertMessages function to prioritize processing of multi-content messages, improving the handling of user messages with images.

These changes enhance the flexibility and functionality of chat messages, enabling richer content delivery.
---
 internal/models/chat/chat.go       | 26 ++++++++++++++++++++------
 internal/models/chat/remote_api.go | 24 +++++++++++++++++++++++-
 2 files changed, 43 insertions(+), 7 deletions(-)

diff --git a/internal/models/chat/chat.go b/internal/models/chat/chat.go
index a4c9f173b..c4719c02c 100644
--- a/internal/models/chat/chat.go
+++ b/internal/models/chat/chat.go
@@ -39,14 +39,28 @@ type ChatOptions struct {
 	Format              json.RawMessage `json:"format,omitempty"`      // 响应格式定义
 }
 
+// MessageContentPart represents a part of multi-content message
+type MessageContentPart struct {
+	Type     string    `json:"type"`                // "text" or "image_url"
+	Text     string    `json:"text,omitempty"`      // For type="text"
+	ImageURL *ImageURL `json:"image_url,omitempty"` // For type="image_url"
+}
+
+// ImageURL represents the image URL structure
+type ImageURL struct {
+	URL    string `json:"url"`              // URL or base64 data URI
+	Detail string `json:"detail,omitempty"` // "auto", "low", "high"
+}
+
 // Message 表示聊天消息
 type Message struct {
-	Role       string     `json:"role"`                   // 角色：system, user, assistant, tool
-	Content    string     `json:"content"`                // 消息内容
-	Name       string     `json:"name,omitempty"`         // Function/tool name (for tool role)
-	ToolCallID string     `json:"tool_call_id,omitempty"` // Tool call ID (for tool role)
-	ToolCalls  []ToolCall `json:"tool_calls,omitempty"`   // Tool calls (for assistant role)
-	Images     []string   `json:"images,omitempty"`       // Image URLs for multimodal (only for current user message)
+	Role         string               `json:"role"`                    // 角色：system, user, assistant, tool
+	Content      string               `json:"content"`                 // 消息内容
+	MultiContent []MessageContentPart `json:"multi_content,omitempty"` // 多内容消息（文本+图片）
+	Name         string               `json:"name,omitempty"`          // Function/tool name (for tool role)
+	ToolCallID   string               `json:"tool_call_id,omitempty"`  // Tool call ID (for tool role)
+	ToolCalls    []ToolCall           `json:"tool_calls,omitempty"`    // Tool calls (for assistant role)
+	Images       []string             `json:"images,omitempty"`        // Image URLs for multimodal (only for current user message)
 }
 
 // ToolCall represents a tool call in a message
diff --git a/internal/models/chat/remote_api.go b/internal/models/chat/remote_api.go
index 18cff6007..69f6d9575 100644
--- a/internal/models/chat/remote_api.go
+++ b/internal/models/chat/remote_api.go
@@ -76,7 +76,29 @@ func (c *RemoteAPIChat) ConvertMessages(messages []Message) []openai.ChatComplet
 			Role: msg.Role,
 		}
 
-		if len(msg.Images) > 0 && msg.Role == "user" {
+		// 优先处理多内容消息（包含图片等）
+		if len(msg.MultiContent) > 0 {
+			openaiMsg.MultiContent = make([]openai.ChatMessagePart, 0, len(msg.MultiContent))
+			for _, part := range msg.MultiContent {
+				switch part.Type {
+				case "text":
+					openaiMsg.MultiContent = append(openaiMsg.MultiContent, openai.ChatMessagePart{
+						Type: openai.ChatMessagePartTypeText,
+						Text: part.Text,
+					})
+				case "image_url":
+					if part.ImageURL != nil {
+						openaiMsg.MultiContent = append(openaiMsg.MultiContent, openai.ChatMessagePart{
+							Type: openai.ChatMessagePartTypeImageURL,
+							ImageURL: &openai.ChatMessageImageURL{
+								URL:    part.ImageURL.URL,
+								Detail: openai.ImageURLDetail(part.ImageURL.Detail),
+							},
+						})
+					}
+				}
+			}
+		} else if len(msg.Images) > 0 && msg.Role == "user" {
 			parts := make([]openai.ChatMessagePart, 0, len(msg.Images)+1)
 			for _, imgURL := range msg.Images {
 				resolved := resolveImageURLForLLM(imgURL)