diff --git a/src-tauri/src/models/openai.rs b/src-tauri/src/models/openai.rs index e6454a789..4ea0b8af4 100644 --- a/src-tauri/src/models/openai.rs +++ b/src-tauri/src/models/openai.rs @@ -78,6 +78,41 @@ impl ChatMessage { None => String::new(), } } + + /// 提取消息中的图片 URL 列表 + /// 返回 (format, base64_data) 元组列表 + pub fn get_images(&self) -> Vec<(String, String)> { + match &self.content { + Some(MessageContent::Parts(parts)) => parts + .iter() + .filter_map(|p| { + if let ContentPart::ImageUrl { image_url } = p { + // 解析 data URL: data:image/jpeg;base64,xxxxx + if image_url.url.starts_with("data:") { + let parts: Vec<&str> = image_url.url.splitn(2, ',').collect(); + if parts.len() == 2 { + // 提取 media_type: data:image/jpeg;base64 -> image/jpeg + let header = parts[0]; + let data = parts[1]; + let media_type = header + .strip_prefix("data:") + .and_then(|s| s.split(';').next()) + .unwrap_or("image/jpeg"); + // 提取格式: image/jpeg -> jpeg + let format = + media_type.split('/').nth(1).unwrap_or("jpeg").to_string(); + return Some((format, data.to_string())); + } + } + None + } else { + None + } + }) + .collect(), + _ => Vec::new(), + } + } } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/src-tauri/src/translator/kiro/anthropic/request.rs b/src-tauri/src/translator/kiro/anthropic/request.rs index 58296ae44..d3bdd1635 100644 --- a/src-tauri/src/translator/kiro/anthropic/request.rs +++ b/src-tauri/src/translator/kiro/anthropic/request.rs @@ -60,6 +60,7 @@ struct ProcessedMessage { content: String, tool_uses: Option>, tool_results: Option>, + images: Option>, } // ============================================================================ @@ -105,7 +106,7 @@ pub fn convert_anthropic_to_codewhisperer( content: combined, model_id: cw_model.clone(), origin: "AI_EDITOR".to_string(), - images: None, + images: messages[0].images.clone(), // 传递图片 user_input_message_context: None, }; @@ -144,7 +145,7 @@ pub fn convert_anthropic_to_codewhisperer( content, model_id: cw_model.clone(), origin: "AI_EDITOR".to_string(), - images: None, + images: msg.images.clone(), // 传递图片 user_input_message_context: None, }; @@ -181,24 +182,29 @@ pub fn convert_anthropic_to_codewhisperer( let history = fix_history_alternation(history, &cw_model); // 构建当前消息 - let (current_content, current_tool_results) = if let Some(last_msg) = messages.last() { - if last_msg.role == "assistant" { - ("Continue".to_string(), None) - } else { - let content = if last_msg.content.is_empty() { - if last_msg.tool_results.is_some() { - "Tool results provided.".to_string() - } else { - "Continue".to_string() - } + let (current_content, current_tool_results, current_images) = + if let Some(last_msg) = messages.last() { + if last_msg.role == "assistant" { + ("Continue".to_string(), None, None) } else { - last_msg.content.clone() - }; - (content, last_msg.tool_results.clone()) - } - } else { - ("Continue".to_string(), None) - }; + let content = if last_msg.content.is_empty() { + if last_msg.tool_results.is_some() { + "Tool results provided.".to_string() + } else { + "Continue".to_string() + } + } else { + last_msg.content.clone() + }; + ( + content, + last_msg.tool_results.clone(), + last_msg.images.clone(), + ) + } + } else { + ("Continue".to_string(), None, None) + }; // 构建 tools let tools = convert_anthropic_tools(&request.tools); @@ -212,6 +218,14 @@ pub fn convert_anthropic_to_codewhisperer( None }; + // 记录图片信息 + if let Some(ref imgs) = current_images { + tracing::info!( + "[KIRO_TRANSLATE] Current message contains {} image(s)", + imgs.len() + ); + } + CodeWhispererRequest { conversation_state: ConversationState { chat_trigger_type: "MANUAL".to_string(), @@ -221,7 +235,7 @@ pub fn convert_anthropic_to_codewhisperer( content: current_content, model_id: cw_model, origin: "AI_EDITOR".to_string(), - images: None, + images: current_images, // 传递当前消息的图片 user_input_message_context, }, }, @@ -292,6 +306,7 @@ fn preprocess_anthropic_messages(messages: &[AnthropicMessage]) -> Vec Vec Vec Vec { content: s.clone(), tool_uses: None, tool_results: None, + images: None, }); } serde_json::Value::Array(parts) => { let mut text_parts: Vec = Vec::new(); let mut tool_uses: Vec = Vec::new(); let mut tool_results: Vec = Vec::new(); + let mut images: Vec = Vec::new(); for part in parts { let part_type = part.get("type").and_then(|t| t.as_str()).unwrap_or(""); @@ -356,6 +375,39 @@ fn convert_anthropic_message(msg: &AnthropicMessage) -> Vec { text_parts.push(text.to_string()); } } + "image" => { + // 处理 Anthropic 格式的图片 + // { "type": "image", "source": { "type": "base64", "media_type": "image/jpeg", "data": "..." } } + if let Some(source) = part.get("source") { + let source_type = + source.get("type").and_then(|t| t.as_str()).unwrap_or(""); + if source_type == "base64" { + let media_type = source + .get("media_type") + .and_then(|m| m.as_str()) + .unwrap_or("image/jpeg"); + let data = + source.get("data").and_then(|d| d.as_str()).unwrap_or(""); + + if !data.is_empty() { + // 从 media_type 提取格式 (image/jpeg -> jpeg) + let format = + media_type.split('/').nth(1).unwrap_or("jpeg").to_string(); + + images.push(CWImage { + format, + source: CWImageSource { + bytes: data.to_string(), + }, + }); + tracing::debug!( + "[KIRO_TRANSLATE] Converted image: media_type={}", + media_type + ); + } + } + } + } "tool_use" => { let default_id = format!("toolu_{}", &Uuid::new_v4().to_string()[..8]); let id = part @@ -407,6 +459,7 @@ fn convert_anthropic_message(msg: &AnthropicMessage) -> Vec { Some(tool_uses) }, tool_results: None, + images: None, // assistant 消息不包含图片 }); } // 处理 user 消息 @@ -418,16 +471,22 @@ fn convert_anthropic_message(msg: &AnthropicMessage) -> Vec { content: String::new(), tool_uses: None, tool_results: Some(tool_results), + images: None, }); } - // 添加文本内容 - if !text_parts.is_empty() { + // 添加文本内容和图片 + if !text_parts.is_empty() || !images.is_empty() { result.push(ProcessedMessage { role: "user".to_string(), content: text_parts.join(""), tool_uses: None, tool_results: None, + images: if images.is_empty() { + None + } else { + Some(images) + }, }); } } diff --git a/src-tauri/src/translator/kiro/openai/request.rs b/src-tauri/src/translator/kiro/openai/request.rs index 60aa5ebd8..232863a20 100644 --- a/src-tauri/src/translator/kiro/openai/request.rs +++ b/src-tauri/src/translator/kiro/openai/request.rs @@ -119,6 +119,7 @@ struct ProcessedMessage { content: String, tool_calls: Option>, tool_results: Option>, + images: Option>, } // ============================================================================ @@ -181,7 +182,7 @@ pub fn convert_openai_to_codewhisperer( content: combined, model_id: cw_model.clone(), origin: "AI_EDITOR".to_string(), - images: None, + images: messages[0].images.clone(), // 传递图片 user_input_message_context: None, }; @@ -220,7 +221,7 @@ pub fn convert_openai_to_codewhisperer( content, model_id: cw_model.clone(), origin: "AI_EDITOR".to_string(), - images: None, + images: msg.images.clone(), // 传递图片 user_input_message_context: None, }; @@ -257,24 +258,29 @@ pub fn convert_openai_to_codewhisperer( let history = fix_history_alternation(history, &cw_model); // 构建当前消息 - let (current_content, current_tool_results) = if let Some(last_msg) = messages.last() { - if last_msg.role == "assistant" { - ("Continue".to_string(), None) - } else { - let content = if last_msg.content.is_empty() { - if last_msg.tool_results.is_some() { - "Tool results provided.".to_string() - } else { - "Continue".to_string() - } + let (current_content, current_tool_results, current_images) = + if let Some(last_msg) = messages.last() { + if last_msg.role == "assistant" { + ("Continue".to_string(), None, None) } else { - last_msg.content.clone() - }; - (content, last_msg.tool_results.clone()) - } - } else { - ("Continue".to_string(), None) - }; + let content = if last_msg.content.is_empty() { + if last_msg.tool_results.is_some() { + "Tool results provided.".to_string() + } else { + "Continue".to_string() + } + } else { + last_msg.content.clone() + }; + ( + content, + last_msg.tool_results.clone(), + last_msg.images.clone(), + ) + } + } else { + ("Continue".to_string(), None, None) + }; // 构建 tools let tools = convert_tools(&request.tools); @@ -288,6 +294,14 @@ pub fn convert_openai_to_codewhisperer( None }; + // 记录图片信息 + if let Some(ref imgs) = current_images { + tracing::info!( + "[KIRO_TRANSLATE] OpenAI current message contains {} image(s)", + imgs.len() + ); + } + CodeWhispererRequest { conversation_state: ConversationState { chat_trigger_type: "MANUAL".to_string(), @@ -297,7 +311,7 @@ pub fn convert_openai_to_codewhisperer( content: current_content, model_id: cw_model, origin: "AI_EDITOR".to_string(), - images: None, + images: current_images, // 传递当前消息的图片 user_input_message_context, }, }, @@ -336,6 +350,29 @@ fn preprocess_messages(messages: &[&ChatMessage]) -> Vec { let mut seen_ids = HashSet::new(); tool_results.retain(|tr| seen_ids.insert(tr.tool_use_id.clone())); + // 提取图片 + let raw_images = msg.get_images(); + let images = if raw_images.is_empty() { + None + } else { + Some( + raw_images + .into_iter() + .map(|(format, data)| CWImage { + format, + source: CWImageSource { bytes: data }, + }) + .collect(), + ) + }; + + if images.is_some() { + tracing::debug!( + "[KIRO_TRANSLATE] OpenAI user message contains {} image(s)", + images.as_ref().map(|v: &Vec| v.len()).unwrap_or(0) + ); + } + result.push(ProcessedMessage { role: "user".to_string(), content, @@ -345,6 +382,7 @@ fn preprocess_messages(messages: &[&ChatMessage]) -> Vec { } else { Some(tool_results) }, + images, }); } "assistant" => { @@ -361,6 +399,7 @@ fn preprocess_messages(messages: &[&ChatMessage]) -> Vec { content: "Tool results provided.".to_string(), tool_calls: None, tool_results: Some(tool_results), + images: None, }); } @@ -382,6 +421,7 @@ fn preprocess_messages(messages: &[&ChatMessage]) -> Vec { content, tool_calls, tool_results: None, + images: None, // assistant 消息不包含图片 }); } _ => {} @@ -399,6 +439,7 @@ fn preprocess_messages(messages: &[&ChatMessage]) -> Vec { content: "Tool results provided.".to_string(), tool_calls: None, tool_results: Some(tool_results), + images: None, }); } diff --git a/src/components/terminal/README.md b/src/components/terminal/README.md index 4d6aec369..003bf575f 100644 --- a/src/components/terminal/README.md +++ b/src/components/terminal/README.md @@ -48,13 +48,14 @@ - **VDOM 模式**: 支持在终端内嵌入 React 组件 - **贴纸系统**: 支持在终端上显示可定位的贴纸标注 - **分块布局**: 支持在主终端旁添加附加面板(对齐 Waveterm TileLayout) -- **右侧小部件栏**: 快速添加 Terminal/Files/Web/Sysinfo 面板 +- **右侧小部件栏**: 快速添加 Terminal/Files/Web/Sysinfo/AI 面板 +- **Terminal AI**: 内置 AI 助手,支持终端上下文理解 ## 文件索引 - `index.ts` - 模块导出 - `TerminalPage.tsx` - 终端页面组件(多标签页管理) -- `TerminalWorkspace.tsx` - 终端工作区组件(分块布局 + 小部件栏) +- `TerminalWorkspace.tsx` - 终端工作区组件(分块布局 + 小部件栏 + AI 面板) - `TerminalPanel.tsx` - 独立终端面板组件(用于分块布局) - `TerminalView.tsx` - 终端视图组件(使用 Jotai 原子状态) - `TerminalSearch.tsx` - 终端搜索组件 @@ -69,6 +70,7 @@ - `termwrap.ts` - 终端封装类(连接模式,WebGL/Unicode11 支持) - `fitaddon.ts` - 自定义 FitAddon - `terminal.css` - 终端样式(Tokyo Night 主题) +- `ai/` - Terminal AI 模块(AI 助手面板) - `widgets/` - 小部件系统子目录 ## widgets 子目录 diff --git a/src/components/terminal/TerminalWorkspace.tsx b/src/components/terminal/TerminalWorkspace.tsx index e7290af89..d7581e4b8 100644 --- a/src/components/terminal/TerminalWorkspace.tsx +++ b/src/components/terminal/TerminalWorkspace.tsx @@ -15,7 +15,7 @@ * - 右侧小部件栏 */ -import { useState, useCallback } from "react"; +import { useState, useCallback, useRef } from "react"; import styled from "styled-components"; import { TerminalPanel } from "./TerminalPanel"; import { @@ -26,6 +26,7 @@ import { WidgetProvider, WidgetType, } from "./widgets"; +import { TerminalAIPanel } from "./ai"; import { ConnectionSelector, type ConnectionListEntry, @@ -38,7 +39,7 @@ import { Page } from "@/types/page"; // ============================================================================ /** 附加面板类型 */ -export type SidePanelType = "terminal" | "files" | "web" | "sysinfo"; +export type SidePanelType = "terminal" | "files" | "web" | "sysinfo" | "ai"; /** 附加面板配置 */ export interface SidePanel { @@ -184,6 +185,14 @@ const SysinfoIcon = () => ( ); +const AIIcon = () => ( + + + + + +); + const CloseIcon = () => ( @@ -209,6 +218,12 @@ export function TerminalWorkspace({ onNavigate }: TerminalWorkspaceProps) { { id: "main-terminal", type: "terminal", title: "Terminal" }, ]); + // AI 面板状态 + const [showAIPanel, setShowAIPanel] = useState(false); + + // 终端输出引用(用于 AI 上下文) + const terminalOutputRef = useRef(null); + // 连接编辑器模态窗口状态 const [isConnectionsEditorOpen, setIsConnectionsEditorOpen] = useState(false); @@ -224,6 +239,7 @@ export function TerminalWorkspace({ onNavigate }: TerminalWorkspaceProps) { files: "Files", web: "Web", sysinfo: "Sysinfo", + ai: "AI", }; // 如果有连接配置,使用连接标签作为标题 @@ -281,6 +297,8 @@ export function TerminalWorkspace({ onNavigate }: TerminalWorkspaceProps) { return ; case "sysinfo": return ; + case "ai": + return ; default: return null; } @@ -294,6 +312,11 @@ export function TerminalWorkspace({ onNavigate }: TerminalWorkspaceProps) { [addPanel], ); + // 获取终端输出(用于 AI 上下文) + const getTerminalOutput = useCallback(() => { + return terminalOutputRef.current; + }, []); + // 渲染面板内容 const renderPanelContent = (panel: SidePanel) => { switch (panel.type) { @@ -305,6 +328,8 @@ export function TerminalWorkspace({ onNavigate }: TerminalWorkspaceProps) { return ; case "sysinfo": return ; + case "ai": + return ; default: return null; } @@ -329,6 +354,10 @@ export function TerminalWorkspace({ onNavigate }: TerminalWorkspaceProps) { case "sysinfo": addPanel("sysinfo"); break; + case "ai": + // 切换 AI 面板显示 + setShowAIPanel((prev) => !prev); + break; case "settings": onNavigate("settings"); break; @@ -349,6 +378,15 @@ export function TerminalWorkspace({ onNavigate }: TerminalWorkspaceProps) { return ( + {/* AI 面板(左侧,参考 Waveterm) */} + {showAIPanel && ( +
+ +
+ )} + {/* 所有面板统一渲染,都可以关闭和多开 */} {panels.map((panel) => ( diff --git a/src/components/terminal/ai/README.md b/src/components/terminal/ai/README.md new file mode 100644 index 000000000..649c3d61f --- /dev/null +++ b/src/components/terminal/ai/README.md @@ -0,0 +1,78 @@ +# Terminal AI 模块 + + + +## 架构说明 + +Terminal AI 是终端内置的 AI 助手功能,参考 Waveterm 的 AI 面板设计。 + +**核心特性:** +- 复用 AI Agent 的模型选择器和 API +- 支持终端上下文(Widget Context) +- 流式响应显示 +- 工具调用支持 + +## 文件索引 + +| 文件 | 说明 | +|------|------| +| `index.ts` | 模块导出 | +| `types.ts` | 类型定义 | +| `useTerminalAI.ts` | Terminal AI Hook | +| `TerminalAIPanel.tsx` | AI 面板主组件 | +| `TerminalAIInput.tsx` | 输入框组件 | +| `TerminalAIMessages.tsx` | 消息列表组件 | +| `TerminalAIModeSelector.tsx` | 模式/模型选择器 | +| `TerminalAIWelcome.tsx` | 欢迎页面组件 | + +## 使用方式 + +```tsx +import { TerminalAIPanel } from "@/components/terminal/ai"; + +function MyComponent() { + const getTerminalOutput = () => { + // 返回终端输出内容 + return "$ ls -la\ntotal 0\n..."; + }; + + return ( + + ); +} +``` + +## 功能说明 + +### Widget Context + +开启后,AI 可以看到终端的最近输出(默认 50 行),用于: +- 解释命令输出 +- 调试错误信息 +- 提供上下文相关的建议 + +### 模型选择 + +复用 AI Agent 的 Provider/Model 选择器,支持: +- OAuth 凭证(Kiro、Gemini、Antigravity 等) +- API Key 凭证(OpenAI、Claude 等) + +### 快捷操作 + +欢迎页面提供快捷操作按钮: +- 解释命令 +- 调试错误 +- 写脚本 +- 优化命令 + +## 依赖 + +- `@/lib/api/agent` - Agent API +- `@/hooks/useProviderPool` - Provider 凭证 +- `@/hooks/useApiKeyProvider` - API Key 凭证 +- `@/hooks/useModelRegistry` - 模型注册表 +- `@/components/ui/*` - UI 组件 + +## 更新提醒 + +任何文件变更后,请更新此文档和相关的上级文档。 diff --git a/src/components/terminal/ai/TerminalAIInput.tsx b/src/components/terminal/ai/TerminalAIInput.tsx new file mode 100644 index 000000000..5a0c4dbe5 --- /dev/null +++ b/src/components/terminal/ai/TerminalAIInput.tsx @@ -0,0 +1,145 @@ +/** + * @file TerminalAIInput.tsx + * @description Terminal AI 输入框组件 + * @module components/terminal/ai/TerminalAIInput + * + * 参考 Waveterm 的 AIPanelInput 设计 + */ + +import React, { useRef, useCallback, useEffect } from "react"; +import { Send, Square, Paperclip } from "lucide-react"; +import { cn } from "@/lib/utils"; + +interface TerminalAIInputProps { + /** 输入值 */ + value: string; + /** 输入变化回调 */ + onChange: (value: string) => void; + /** 提交回调 */ + onSubmit: () => void; + /** 停止回调 */ + onStop?: () => void; + /** 是否正在发送 */ + isSending: boolean; + /** 是否禁用 */ + disabled?: boolean; + /** 占位符 */ + placeholder?: string; +} + +export const TerminalAIInput: React.FC = ({ + value, + onChange, + onSubmit, + onStop, + isSending, + disabled = false, + placeholder = "Continue...", +}) => { + const textareaRef = useRef(null); + + /** + * 自动调整高度 + */ + const resizeTextarea = useCallback(() => { + const textarea = textareaRef.current; + if (!textarea) return; + + textarea.style.height = "auto"; + const scrollHeight = textarea.scrollHeight; + const maxHeight = 7 * 24; // 7 行 + textarea.style.height = `${Math.min(scrollHeight, maxHeight)}px`; + }, []); + + useEffect(() => { + resizeTextarea(); + }, [value, resizeTextarea]); + + /** + * 处理键盘事件 + */ + const handleKeyDown = (e: React.KeyboardEvent) => { + const isComposing = e.nativeEvent?.isComposing || e.keyCode === 229; + if (e.key === "Enter" && !e.shiftKey && !isComposing) { + e.preventDefault(); + if (!isSending && value.trim()) { + onSubmit(); + } + } + }; + + /** + * 处理提交 + */ + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + if (!isSending && value.trim()) { + onSubmit(); + } + }; + + return ( +
+
+
+