add voice clone init

.claude/skills/add-new-node/SKILL.md

@@ -0,0 +1,444 @@
+---
+name: add-new-node
+description: 在 FireRed-OpenStoryline 中添加新节点功能的完整指南。适用于用户想要扩展新的视频编辑节点（如新的 AI 处理步骤、自定义滤镜、数据转换等）时使用。触发词：添加新节点、新建节点、扩展节点、add node、new node。
+---
+
+# FireRed-OpenStoryline 添加新节点指南
+
+## 系统架构速览
+
+FireRed-OpenStoryline 是一个基于 MCP（Model Context Protocol）的 AI 视频剪辑系统。整个编辑流程由一系列**节点（Node）**组成 DAG（有向无环图），每个节点是一个独立的处理单元，通过 MCP Tool 的形式暴露给 LLM Agent 调用。
+
+### 核心文件位置
+
+```
+src/open_storyline/
+├── nodes/
+│   ├── core_nodes/          # 所有节点实现（每个节点一个文件）
+│   │   ├── base_node.py     # 基类 BaseNode + NodeMeta
+│   │   ├── load_media.py    # 示例：入口节点
+│   │   ├── split_shots.py   # 示例：处理节点
+│   │   └── ...
+│   ├── node_schema.py       # 所有节点的 Input/Output Pydantic 模型
+│   ├── node_manager.py      # 节点注册与依赖管理器
+│   ├── node_state.py        # 节点执行状态（session、artifact、llm 等）
+│   └── node_summary.py      # 节点执行摘要（日志/进度）
+├── utils/register.py        # 全局 NODE_REGISTRY 注册表
+└── config.py                # Settings 配置 Schema（Pydantic）
+config.toml                  # 主配置文件
+```
+
+---
+
+## 节点执行机制
+
+### Mode 模式
+
+每个节点调用时会传入 `mode` 参数：
+
+| mode | 调用方法 | 含义 |
+|------|----------|------|
+| `"auto"` | `process()` | 正常执行核心逻辑（LLM 调用、算法处理等） |
+| `"skip"` | `default_process()` | 跳过该节点，返回空/透传数据 |
+| `"default"` | `default_process()` | 同 skip，使用默认/简化逻辑 |
+
+### 数据流转
+
+上游节点的输出会通过 `inputs` 字典注入到下游节点，key 为上游节点的 `node_kind`：
+
+```python
+# 在 process() 中获取上游数据
+prior_data = inputs.get("split_shots", {})   # 上游 split_shots 节点的输出
+clips = prior_data.get("clips", [])
+```
+
+### 节点状态对象 NodeState
+
+```python
+node_state.session_id        # 当前 session ID
+node_state.artifact_id       # 当前执行的 artifact ID（唯一标识本次调用）
+node_state.llm               # LLM 客户端（可用于调用语言模型）
+node_state.node_summary      # 用于记录日志和进度
+
+# 日志记录方法
+node_state.node_summary.info_for_user("显示给用户看的进度信息")
+node_state.node_summary.info_for_llm("给 LLM 看的执行信息")
+node_state.node_summary.add_warning("警告信息", artifact_id=node_state.artifact_id)
+node_state.node_summary.add_error("错误信息", artifact_id=node_state.artifact_id)
+```
+
+---
+
+## 添加新节点的完整步骤
+
+### Step 1：在 `node_schema.py` 定义 Input/Output Schema
+
+打开 `src/open_storyline/nodes/node_schema.py`，参考已有模型添加：
+
+```python
+# ===== 新节点的 Input Schema =====
+class MyNewNodeInput(BaseInput):
+    mode: Literal["auto", "skip", "default"] = Field(
+        default="auto",
+        description="auto: 执行核心逻辑; skip: 跳过; default: 使用默认行为"
+    )
+    # 添加节点特有的参数
+    my_param: str = Field(
+        default="",
+        description="用户对该节点的自定义参数说明"
+    )
+
+# ===== 新节点的 Output Schema（可选，用于文档和类型提示） =====
+class MyNewNodeOutput(BaseModel):
+    result: List[SomeType] = Field(
+        default_factory=list,
+        description="节点输出结果"
+    )
+```
+
+**注意事项：**
+- `Input` 类必须继承 `BaseInput`（已包含 `mode` 字段）或 `BaseModel`
+- 字段 `description` 会直接透传为 MCP Tool 的参数说明，写清楚对 LLM 很重要
+- 如果节点不需要特殊输入参数，可以直接用 `class MyNewNodeInput(BaseInput): ...`（三个点代表空 body）
+
+---
+
+### Step 2：创建节点实现文件
+
+在 `src/open_storyline/nodes/core_nodes/` 下新建文件，例如 `my_new_node.py`：
+
+```python
+from typing import Any, ClassVar, Dict, List, Type
+from pydantic import BaseModel
+
+from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
+from open_storyline.nodes.node_schema import MyNewNodeInput
+from open_storyline.nodes.node_state import NodeState
+from open_storyline.utils.register import NODE_REGISTRY
+
+
+@NODE_REGISTRY.register()           # 必须：注册到全局注册表
+class MyNewNode(BaseNode):
+
+    # 必须：节点元数据
+    meta = NodeMeta(
+        name="my_new_node",         # MCP Tool 名称（snake_case，唯一）
+        description=(               # MCP Tool 描述，LLM 根据此决定何时调用
+            "这个节点的功能说明，写清楚它做什么、"
+            "什么情况下应该调用它"
+        ),
+        node_id="my_new_node",      # 节点唯一 ID（通常与 name 相同）
+        node_kind="my_new_node",    # 节点类型（同类替代节点共享同一 kind）
+        require_prior_kind=[        # 执行 process() 时必须已完成的前置节点 kind
+            "filter_clips",
+        ],
+        default_require_prior_kind=[  # 执行 default_process() 时的前置依赖
+            "filter_clips",
+        ],
+        next_available_node=[       # 执行完本节点后，下游可选的节点 ID
+            "plan_timeline",
+        ],
+        priority=5,                 # 同 kind 中多个节点时的优先级（越大越优先）
+    )
+
+    input_schema: ClassVar[Type[BaseModel]] = MyNewNodeInput
+
+    async def default_process(
+        self, node_state: NodeState, inputs: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """
+        mode != "auto" 时执行。
+        通常：跳过处理，透传上游数据或返回空结果。
+        """
+        node_state.node_summary.info_for_user(
+            f"[{self.meta.node_id}] 跳过，使用默认结果"
+        )
+        return {"result": []}
+
+    async def process(
+        self, node_state: NodeState, inputs: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """
+        mode == "auto" 时执行。核心业务逻辑在此实现。
+        """
+        # 1. 获取上游数据（key 为上游节点的 node_kind）
+        prior_clips = inputs.get("filter_clips", {}).get("clip_captions", [])
+
+        node_state.node_summary.info_for_user(
+            f"[{self.meta.node_id}] 开始处理，共 {len(prior_clips)} 个 clip"
+        )
+
+        # 2. 你的核心处理逻辑
+        result = []
+        for clip in prior_clips:
+            # ... 处理每个 clip
+            result.append({...})
+
+        node_state.node_summary.info_for_user(
+            f"[{self.meta.node_id}] 处理完成，输出 {len(result)} 条结果"
+        )
+
+        return {"result": result}
+```
+
+**NodeMeta 字段说明：**
+
+| 字段 | 类型 | 说明 |
+|------|------|------|
+| `name` | str | MCP Tool 的调用名，snake_case，全局唯一 |
+| `description` | str | LLM 看到的工具描述，决定何时调用 |
+| `node_id` | str | 节点唯一标识，通常与 name 相同 |
+| `node_kind` | str | 节点功能类别，同类节点共享 kind（用于依赖解析） |
+| `require_prior_kind` | List[str] | `process()` 的前置依赖 kind 列表 |
+| `default_require_prior_kind` | List[str] | `default_process()` 的前置依赖 kind 列表 |
+| `next_available_node` | List[str] | 执行完后下游可选节点的 node_id 列表 |
+| `priority` | int | 同 kind 下多实现时的优先级，默认 5 |
+
+---
+
+### Step 3：在 `config.toml` 中注册节点
+
+打开根目录的 `config.toml`，找到 `[local_mcp_server]` 部分，将新节点类名加入 `available_nodes`：
+
+```toml
+[local_mcp_server]
+available_node_pkgs = [
+    "open_storyline.nodes.core_nodes"    # 扫描这个包下的所有模块
+]
+available_nodes = [
+    "LoadMediaNode",
+    "SplitShotsNode",
+    # ... 其他现有节点 ...
+    "MyNewNode",                          # 新增这行
+]
+```
+
+**机制说明：**
+1. 系统启动时会扫描 `available_node_pkgs` 中的所有模块，触发 `@NODE_REGISTRY.register()` 自动注册
+2. 然后只实例化 `available_nodes` 中列出的类，将其包装为 MCP Tool
+3. 新节点文件放在 `core_nodes/` 下会被自动扫描到，不需要手动 import
+
+---
+
+### Step 4（可选）：添加节点独立配置
+
+如果节点需要从 `config.toml` 读取参数（如模型路径、阈值等），在 `config.py` 中新增 Config 类：
+
+```python
+# src/open_storyline/config.py
+
+class MyNewNodeConfig(ConfigBaseModel):
+    threshold: float = Field(default=0.5, description="处理阈值")
+    model_path: Path = Field(..., description="模型权重路径")
+
+class Settings(ConfigBaseModel):
+    # ... 现有字段 ...
+    my_new_node: MyNewNodeConfig    # 新增这行
+```
+
+然后在 `config.toml` 添加对应 section：
+
+```toml
+[my_new_node]
+threshold = 0.5
+model_path = "./models/my_model.pth"
+```
+
+在节点实现中通过 `self.server_cfg` 访问：
+
+```python
+def __init__(self, server_cfg: Settings) -> None:
+    super().__init__(server_cfg)
+    self.threshold = self.server_cfg.my_new_node.threshold
+    self.model_path = self.server_cfg.my_new_node.model_path
+```
+
+---
+
+## 完整示例：最简单的透传节点
+
+下面是一个完整可运行的最简节点，直接透传上游数据：
+
+```python
+# src/open_storyline/nodes/core_nodes/passthrough_node.py
+
+from typing import Any, ClassVar, Dict, Type
+from pydantic import BaseModel
+
+from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
+from open_storyline.nodes.node_schema import BaseInput
+from open_storyline.nodes.node_state import NodeState
+from open_storyline.utils.register import NODE_REGISTRY
+
+
+class PassthroughNodeInput(BaseInput):
+    ...  # 只继承 mode 字段，无额外参数
+
+
+@NODE_REGISTRY.register()
+class PassthroughNode(BaseNode):
+    meta = NodeMeta(
+        name="passthrough",
+        description="透传节点：直接将上游数据传递给下游，不做任何处理",
+        node_id="passthrough",
+        node_kind="passthrough",
+        require_prior_kind=["filter_clips"],
+        default_require_prior_kind=["filter_clips"],
+        next_available_node=["plan_timeline"],
+    )
+    input_schema: ClassVar[Type[BaseModel]] = PassthroughNodeInput
+
+    async def default_process(self, node_state: NodeState, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        return await self.process(node_state, inputs)
+
+    async def process(self, node_state: NodeState, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        clips = inputs.get("filter_clips", {}).get("clip_captions", [])
+        node_state.node_summary.info_for_user(f"透传 {len(clips)} 个 clip")
+        return {"clip_captions": clips}
+```
+
+---
+
+## 环境说明
+
+本项目使用 **conda 虚拟环境**（不是 venv），环境名为 `storyline3`。所有 Python 命令需在该环境下运行：
+
+```bash
+# 验证节点可以正常导入
+conda run -n storyline3 bash -c "PYTHONPATH=src python -c 'from open_storyline.nodes.core_nodes.my_new_node import MyNewNode; print(MyNewNode.meta.name)'"
+
+# 验证完整注册流程（模拟服务启动时的扫描）
+conda run -n storyline3 bash -c "PYTHONPATH=src python -c \"
+from open_storyline.utils.register import NODE_REGISTRY
+NODE_REGISTRY.scan_package('open_storyline.nodes.core_nodes')
+print('MyNewNode in registry:', 'MyNewNode' in NODE_REGISTRY.list())
+\""
+```
+
+---
+
+## 节点调试技巧
+
+### 开启 developer_mode
+
+`config.toml` 中设置：
+```toml
+[developer]
+developer_mode = true
+```
+
+开启后，节点异常会返回完整 traceback 而非简短错误信息。
+
+### 查看节点执行历史
+
+通过 MCP Tool `read_node_history` 可以读取任意 artifact_id 对应的执行结果：
+
+```
+read_node_history(query_artifact_id="xxx-artifact-id")
+```
+
+### LLM 调用示例
+
+如果节点需要调用 LLM，通过 `node_state.llm` 使用：
+
+```python
+from open_storyline.mcp.sampling_requester import LLMClient
+
+async def process(self, node_state: NodeState, inputs: Dict[str, Any]) -> Dict[str, Any]:
+    llm: LLMClient = inputs.get("llm") or node_state.llm
+    
+    response = await llm.ainvoke([
+        {"role": "user", "content": "你的 prompt"}
+    ])
+    result_text = response.content
+    ...
+```
+
+---
+
+## 常见错误排查
+
+| 错误现象 | 可能原因 | 解决方案 |
+|----------|----------|----------|
+| 节点未出现在 MCP Tool 列表 | 类名未加入 `available_nodes` | 检查 `config.toml` |
+| `KeyError: 'MyNewNode'` | 类名拼写错误或文件未被扫描到 | 检查文件在 `core_nodes/` 目录下，且类名与 config 一致 |
+| `require_prior_kind` 依赖未满足 | 上游节点未执行 | 确认前置节点已执行并有输出 |
+| `inputs.get("xxx")` 返回 None | node_kind 写错，或上游节点输出 key 不对 | 打印 `inputs.keys()` 检查实际 key |
+| `Settings` 初始化报错 | 新增了 Config 字段但 config.toml 未对应添加 | 补充 `config.toml` 中的 section |
+
+---
+
+## 已有节点的节点类型（node_kind）速查
+
+| node_kind | 节点类 | 功能 |
+|-----------|--------|------|
+| `load_media` | LoadMediaNode | 加载本地媒体文件 |
+| `load_media` | SearchMediaNode | 从 Pexels 搜索媒体（同 kind） |
+| `split_shots` | SplitShotsNode | 镜头分割（TransNetV2） |
+| `asr_node` | LocalASRNode | 语音识别 |
+| `speech_rough_cut` | SpeechRoughCutNode | 语音粗剪 |
+| `understand_clips` | UnderstandClipsNode | VLM 视觉理解 |
+| `filter_clips` | FilterClipsNode | 智能筛选 |
+| `group_clips` | GroupClipsNode | 镜头分组 |
+| `generate_script` | GenerateScriptNode | 脚本/字幕生成 |
+| `tts` | GenerateVoiceoverNode | TTS 配音生成 |
+| `tts` | VoiceCloneMinimaxNode | MiniMax 音色克隆 + TTS（与上行同 kind，可互替） |
+| `select_bgm` | SelectBGMNode | 背景音乐选择 |
+| `plan_timeline` | PlanTimelineProNode | 时间线规划 |
+| `plan_timeline_ai_transition` | PlanTimelineAITransitionNode | AI 转场时间线 |
+| `generate_ai_transition` | GenerateAITransitionNode | AI 转场生成 |
+| `render_video` | RenderVideoNode | 最终渲染输出 |
+
+---
+
+## 特殊主题：节点接收音频文件输入
+
+### 音频文件在系统中的流转路径
+
+```
+用户上传音频（Web UI）
+  → 存入 session media_dir（绝对路径）
+  → scan_media_dir 返回统计 + 绝对路径列表（注入到 Agent 的 system message）
+  → Agent 知道有哪些音频文件可用
+  → Agent 在 tool call 参数中填写 clone_audio=[{"path": "/abs/path/audio.mp3"}]
+  → 拦截器 inject_media_content_before 检测到 voice_clone_minimax 且 clone_audio 未传时
+      → 自动从 media_dir 扫描音频文件并注入 clone_audio
+  → BaseNode.load_inputs_from_client 处理 clone_audio 字段（base64/path 双模式）
+  → node.process() 中 clone_audio_list[0]["path"] 是 server 本地可读路径
+```
+
+### 为什么音频文件不经过 load_media 节点？
+
+`load_media` 节点只处理视频和图片（VIDEO_EXTS / IMAGE_EXTS），音频文件会被 skip。
+音频文件不需要 `load_media` 处理，因为：
+- 视频/图片需要元数据提取（duration、fps、width×height）
+- 音频只是一个文件路径，直接传给 TTS/音色克隆 API 即可
+
+### 如果你的新节点也需要接收音频输入
+
+1. **Input Schema** 中声明 `List[Dict[str, Any]]` 字段：
+
+```python
+class MyAudioNodeInput(BaseInput):
+    audio_files: List[Dict[str, Any]] = Field(
+        default_factory=list,
+        description="音频文件列表，格式：[{'path': '/abs/path/audio.mp3'}]"
+    )
+```
+
+2. **拦截器自动注入**（可选）：如果希望系统自动从 media_dir 找音频注入，
+   在 `node_interceptors.py` 的 `inject_media_content_before` 中参考
+   `voice_clone_minimax` 的处理方式，添加 `node_id == 'my_audio_node'` 分支。
+
+3. **节点 process() 中读取**：`BaseNode.load_inputs_from_client` 自动处理 `List[dict]`
+   字段的 base64/path 双模式，process() 里直接 `inputs.get("audio_files")` 即可，
+   `item["path"]` 已经是 server 本地可 open() 的路径。
+
+### scan_media_dir 的行为
+
+`src/open_storyline/utils/media_handler.py` 中的 `scan_media_dir` 会统计：
+- 图片数量
+- 视频数量
+- 音频文件的绝对路径列表（`.mp3`/`.wav`/`.m4a`）
+
+这些统计信息会注入到 Agent 的 system message，让 Agent 知道用户上传了哪些媒体。

agent_fastapi.py

@@ -309,6 +309,8 @@ def detect_media_kind(filename: str) -> str:
         return "image"
     if ext in {".mp4", ".mov", ".avi", ".mkv", ".webm"}:
         return "video"
+    if ext in {".mp3", ".wav", ".m4a"}:
+        return "audio"
     return "unknown"
 
 
@@ -3244,6 +3246,20 @@ async def ws_chat(ws: WebSocket, session_id: str):
                             [SystemMessage(content="\n\n".join(_system_parts))] if _system_parts else []
                         ) + _non_system
 
+                        # Strip reasoning_content from AIMessage history before sending to LLM.
+                        # Some providers (e.g. domestic OpenAI-compatible APIs in thinking mode)
+                        # return reasoning_content in additional_kwargs but do not accept it back,
+                        # causing a 400 error on subsequent turns.
+                        for _m in _merged_messages:
+                            if (
+                                isinstance(_m, AIMessage)
+                                and "reasoning_content" in _m.additional_kwargs
+                            ):
+                                _m.additional_kwargs = {
+                                    k: v for k, v in _m.additional_kwargs.items()
+                                    if k != "reasoning_content"
+                                }
+
                         async def pump_agent():
                             nonlocal new_messages
                             try:

config.toml

@@ -47,7 +47,8 @@ available_nodes = [
     "LoadMediaNode", "SearchMediaNode", "SplitShotsNode", "LocalASRNode", "SpeechRoughCutNode", "GenerateAITransitionNode", 
     "UnderstandClipsNode", "FilterClipsNode", "GroupClipsNode", "GenerateScriptNode", "ScriptTemplateRecomendation",
     "GenerateVoiceoverNode", "SelectBGMNode", "RecommendTransitionNode", "RecommendTextNode",
-    "PlanTimelineProNode", "PlanTimelineAITransitionNode", "RenderVideoNode"
+    "PlanTimelineProNode", "PlanTimelineAITransitionNode", "RenderVideoNode",
+    "VoiceCloneMinimaxNode"
 ]
 
 # =========== skills ==========

src/open_storyline/mcp/hooks/node_interceptors.py

@@ -22,6 +22,7 @@ logger = get_logger(__name__)
 
 # Hosts that indicate Agent and MCP server are on the same machine (path-only, no base64). 0.0.0.0 for Docker.
 _LOCAL_CONNECT_HOSTS = frozenset({"127.0.0.1", "localhost", "::1", "0.0.0.0"})
+_AUDIO_EXTS = {".mp3", ".wav", ".m4a"}
 
 
 def should_inline_media_as_base64(server_cfg=None) -> bool:
@@ -315,6 +316,11 @@ class ToolInterceptor:
                     # Collect dependencies again
                     collect_result = meta_collector.check_excutable(session_id, store, require_kind)
                     load_collected_data(collect_result['collected_node'], input_data, store)
+                if node_id == 'voice_clone_minimax' and not request.args.get('clone_audio'):
+                    audio_files = [str(p.resolve()) for p in Path(context.media_dir).iterdir() if p.is_file() and p.suffix.lower() in _AUDIO_EXTS]
+                    if audio_files:
+                        input_data['clone_audio'] = audio_files[0]
+                        logger.info(f'[voice_clone_minimax] Auto-injected clone_audio: {audio_files[0]}')
             else:
                 input_data['artifacts_dir'] = store.artifacts_dir
 
@@ -441,10 +447,13 @@ class ToolInterceptor:
         Interceptor: Injects runtime.context.tts_config parameters into request.args before invoking voiceover/TTS tools.
         - tts_config: {"provider": "bytedance", "bytedance": {...}, "azure": {...}, ...}
         """
+        tool_name = str(getattr(request, 'name', '') or '')
+        if not any(kw in tool_name for kw in ('voiceover', 'voice_clone_minimax')):
+            return await handler(request)
         return await ToolInterceptor._inject_provider_config(
             request,
             handler,
-            tool_name_keyword="voiceover",
+            tool_name_keyword="",  # already filtered above
             context_attr="tts_config",
             default_provider="minimax",
         )

src/open_storyline/nodes/core_nodes/voice_clone_minimax.py

@@ -0,0 +1,481 @@
+"""
+MiniMax Voice Clone + Voiceover Node
+=====================================
+Replaces generate_voiceover when the user wants to narrate with a cloned voice.
+
+Pipeline position
+-----------------
+  group_clips ──┐
+                ├──► voice_clone_minimax ──► plan_timeline ──► render_video
+  generate_script ─┘
+
+This node shares node_kind="tts" with GenerateVoiceoverNode so plan_timeline
+(which require_prior_kind=["tts"]) can treat them interchangeably.
+
+What it does
+------------
+1. Clone voice  – upload clone_audio (and optional prompt_audio) to MiniMax,
+                  get back a persistent voice_id.
+2. Generate TTS – for every group_script, call MiniMax T2A v2 with the cloned
+                  voice_id, save each wav, return the standard voiceover list.
+
+Output format  (identical to GenerateVoiceoverNode)
+------------------------------------------------------
+{
+  "voiceover": [
+    {"voiceover_id": "voiceover_0001", "group_id": "group_0001",
+     "path": "/…/voiceover_0001_<ts>.wav", "duration": 3500},
+    …
+  ]
+}
+
+File input format (clone_audio / prompt_audio)
+-----------------------------------------------
+List-of-dicts, same convention as load_media.inputs, so both local-path and
+remote base64 transport are handled transparently by BaseNode.load_inputs_from_client:
+
+  Local:  [{"path": "/abs/path/audio.mp3"}]
+  Remote: [{"path": "audio.mp3", "base64": "<b64>", "md5": "<md5>"}]
+
+After load_inputs_from_client, item["path"] is already a server-local path.
+
+API reference: https://platform.minimaxi.com/docs/guides/speech-voice-clone
+"""
+
+from __future__ import annotations
+
+import asyncio
+import binascii
+import time
+import uuid
+from pathlib import Path
+from typing import Any, ClassVar, Dict, List, Optional, Type
+
+import requests
+from pydantic import BaseModel
+
+from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta
+from open_storyline.nodes.node_schema import VoiceCloneMinimaxInput
+from open_storyline.nodes.node_state import NodeState
+from open_storyline.utils.logging import get_logger
+from open_storyline.utils.register import NODE_REGISTRY
+
+logger = get_logger(__name__)
+
+_DEFAULT_BASE_URL = "https://api.minimaxi.com"
+_MILLISECONDS_PER_SECOND = 1000.0
+
+
+@NODE_REGISTRY.register()
+class VoiceCloneMinimaxNode(BaseNode):
+    """
+    Clone a voice then generate voiceover for all script groups with that voice.
+
+    node_kind="tts" makes this node a drop-in replacement for GenerateVoiceoverNode
+    from the perspective of downstream nodes (plan_timeline, render_video).
+    """
+
+    meta = NodeMeta(
+        name="voice_clone_minimax",
+        description=(
+            "Clone a voice using MiniMax's Voice Clone API, then generate voiceover "
+            "for every script group using the cloned voice. "
+            "Use this instead of generate_voiceover when the user wants narration in "
+            "their own voice or a specific person's voice. "
+            "Requires clone_audio (mp3/m4a/wav, 10s–5min, ≤20MB). "
+            "Optionally accepts prompt_audio (<8s) to improve clone quality. "
+            "Output format is identical to generate_voiceover and feeds directly into plan_timeline."
+        ),
+        node_id="voice_clone_minimax",
+        node_kind="tts",  # same kind as GenerateVoiceoverNode
+        require_prior_kind=["group_clips", "generate_script"],
+        default_require_prior_kind=["group_clips", "generate_script"],
+        next_available_node=["plan_timeline", "select_bgm"],
+        priority=5,
+    )
+
+    input_schema: ClassVar[Type[BaseModel]] = VoiceCloneMinimaxInput
+
+    # ------------------------------------------------------------------
+    # Public entry points
+    # ------------------------------------------------------------------
+
+    async def default_process(
+        self, node_state: NodeState, inputs: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """Skip mode: return empty voiceover list (same shape as GenerateVoiceoverNode)."""
+        node_state.node_summary.info_for_user(
+            "[voice_clone_minimax] Skipped — no voiceover generated."
+        )
+        return {"voiceover": []}
+
+    async def process(
+        self, node_state: NodeState, inputs: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        """Auto mode: clone voice, then generate TTS for every group script."""
+
+        # ---- 1. Resolve credentials ---------------------------------------
+        api_key, base_url = self._resolve_credentials(inputs, node_state)
+
+        # ---- 2. Get group scripts from upstream generate_script -----------
+        group_scripts = (inputs.get("generate_script") or {}).get("group_scripts") or []
+        if not isinstance(group_scripts, list) or not group_scripts:
+            node_state.node_summary.info_for_user(
+                "[voice_clone_minimax] No group_scripts found — skipping voiceover generation."
+            )
+            return {"voiceover": []}
+
+        # ---- 3. Extract clone_audio (already decoded by load_inputs_from_client) --
+        clone_audio_list: List[Dict[str, Any]] = inputs.get("clone_audio") or []
+        if not clone_audio_list:
+            raise ValueError(
+                "clone_audio is required. "
+                'Pass [{"path": "/abs/path/audio.mp3"}] in local mode, or '
+                '[{"path": "filename.mp3", "base64": "...", "md5": "..."}] in remote/web mode.'
+            )
+        clone_audio_file = Path(clone_audio_list[0]["path"])
+        if not clone_audio_file.exists():
+            raise FileNotFoundError(
+                f"Clone audio file not found on server: {clone_audio_file}"
+            )
+
+        # ---- 4. Extract optional prompt_audio -----------------------------
+        prompt_audio_list: List[Dict[str, Any]] = inputs.get("prompt_audio") or []
+        prompt_audio_file: Optional[Path] = None
+        if prompt_audio_list:
+            p = Path(prompt_audio_list[0]["path"])
+            if p.exists():
+                prompt_audio_file = p
+            else:
+                node_state.node_summary.add_warning(
+                    f"[voice_clone_minimax] prompt_audio not found, skipping: {p}",
+                    artifact_id=node_state.artifact_id,
+                )
+
+        # ---- 5. Determine voice_id ----------------------------------------
+        custom_voice_id = (inputs.get("voice_id") or "").strip()
+        if not custom_voice_id:
+            custom_voice_id = f"cloned_{int(time.time())}_{uuid.uuid4().hex[:6]}"
+            node_state.node_summary.info_for_user(
+                f"[voice_clone_minimax] No voice_id provided — auto-generated: {custom_voice_id}"
+            )
+
+        # ---- 6. Prepare output directory ----------------------------------
+        output_dir = self._prepare_output_directory(node_state)
+
+        # ---- 7. Clone voice (blocking HTTP, run in thread) ----------------
+        cloned_voice_id = await asyncio.to_thread(
+            self._clone_voice_sync,
+            api_key=api_key,
+            base_url=base_url,
+            clone_audio_file=clone_audio_file,
+            voice_id=custom_voice_id,
+            prompt_audio_file=prompt_audio_file,
+            prompt_text=(inputs.get("prompt_text") or "").strip(),
+            node_state=node_state,
+        )
+
+        # ---- 8. Generate TTS for every group script -----------------------
+        model = (inputs.get("model") or "speech-02-hd").strip()
+        speed = float(inputs.get("speed") or 1.0)
+        ts_ms = int(time.time() * 1000)
+        voiceover: List[Dict[str, Any]] = []
+
+        for i, group in enumerate(group_scripts, start=1):
+            group_id = (group or {}).get("group_id", "")
+            raw_text = (group or {}).get("raw_text", "")
+
+            if not group_id:
+                raise ValueError(f"Missing group_id in group_scripts[{i}]: {group}")
+            if not isinstance(raw_text, str) or not raw_text.strip():
+                raise ValueError(
+                    f"raw_text is empty for group_id={group_id}, cannot generate speech."
+                )
+
+            voiceover_id = f"voiceover_{i:04d}"
+            wav_path = output_dir / f"{voiceover_id}_{ts_ms}.wav"
+
+            await asyncio.to_thread(
+                self._tts_minimax_sync,
+                api_key=api_key,
+                base_url=base_url,
+                text=raw_text,
+                voice_id=cloned_voice_id,
+                model=model,
+                speed=speed,
+                wav_path=wav_path,
+            )
+
+            duration = self._audio_duration_ms(wav_path)
+            voiceover.append(
+                {
+                    "voiceover_id": voiceover_id,
+                    "group_id": group_id,
+                    "path": str(wav_path),
+                    "duration": duration,
+                }
+            )
+            node_state.node_summary.info_for_user(
+                f"[voice_clone_minimax] Generated {voiceover_id} ({duration}ms)",
+                preview_urls=[str(wav_path)],
+            )
+
+        node_state.node_summary.info_for_user(
+            f"[voice_clone_minimax] Done. voice_id={cloned_voice_id}, "
+            f"{len(voiceover)} voiceover segment(s) generated."
+        )
+        return {"voiceover": voiceover}
+
+    # ------------------------------------------------------------------
+    # Step 1: Voice cloning
+    # ------------------------------------------------------------------
+
+    def _clone_voice_sync(
+        self,
+        *,
+        api_key: str,
+        base_url: str,
+        clone_audio_file: Path,
+        voice_id: str,
+        prompt_audio_file: Optional[Path],
+        prompt_text: str,
+        node_state: NodeState,
+    ) -> str:
+        """Upload audio, call /v1/voice_clone, return the confirmed voice_id."""
+
+        headers_auth = {"Authorization": f"Bearer {api_key}"}
+        upload_url = base_url.rstrip("/") + "/v1/files/upload"
+
+        # Upload clone audio
+        node_state.node_summary.info_for_user(
+            f"[voice_clone_minimax] Uploading clone audio: {clone_audio_file.name}"
+        )
+        file_id = self._upload_file(
+            upload_url=upload_url,
+            headers=headers_auth,
+            file_path=clone_audio_file,
+            purpose="voice_clone",
+        )
+        node_state.node_summary.info_for_user(
+            f"[voice_clone_minimax] Clone audio uploaded, file_id={file_id}"
+        )
+
+        # (Optional) Upload prompt audio
+        prompt_file_id: Optional[str] = None
+        if prompt_audio_file is not None:
+            node_state.node_summary.info_for_user(
+                f"[voice_clone_minimax] Uploading prompt audio: {prompt_audio_file.name}"
+            )
+            prompt_file_id = self._upload_file(
+                upload_url=upload_url,
+                headers=headers_auth,
+                file_path=prompt_audio_file,
+                purpose="prompt_audio",
+            )
+            node_state.node_summary.info_for_user(
+                f"[voice_clone_minimax] Prompt audio uploaded, file_id={prompt_file_id}"
+            )
+
+        # Call voice_clone API
+        clone_url = base_url.rstrip("/") + "/v1/voice_clone"
+        clone_payload: Dict[str, Any] = {
+            "file_id": file_id,
+            "voice_id": voice_id,
+        }
+        if prompt_file_id:
+            clone_payload["clone_prompt"] = {
+                "prompt_audio": prompt_file_id,
+                "prompt_text": prompt_text,
+            }
+
+        resp = requests.post(
+            clone_url,
+            headers={**headers_auth, "Content-Type": "application/json"},
+            json=clone_payload,
+            timeout=120,
+        )
+        resp.raise_for_status()
+        resp_json = resp.json()
+
+        base_resp = (resp_json or {}).get("base_resp") or {}
+        status_code = base_resp.get("status_code")
+        if status_code not in (0, None):
+            raise RuntimeError(
+                f"MiniMax voice_clone API error: status_code={status_code}, "
+                f"status_msg={base_resp.get('status_msg')}, response={resp_json}"
+            )
+
+        node_state.node_summary.info_for_user(
+            f"[voice_clone_minimax] Voice cloned successfully. voice_id={voice_id}"
+        )
+        return voice_id
+
+    # ------------------------------------------------------------------
+    # Step 2: TTS with cloned voice (MiniMax T2A v2)
+    # ------------------------------------------------------------------
+
+    def _tts_minimax_sync(
+        self,
+        *,
+        api_key: str,
+        base_url: str,
+        text: str,
+        voice_id: str,
+        model: str,
+        speed: float,
+        wav_path: Path,
+    ) -> None:
+        """Call MiniMax T2A v2 with the cloned voice_id and save wav to disk."""
+
+        api_url = base_url.rstrip("/") + "/v1/t2a_v2"
+        headers = {
+            "Authorization": f"Bearer {api_key}",
+            "Content-Type": "application/json",
+        }
+        body = {
+            "model": model,
+            "text": text,
+            "stream": False,
+            "output_format": "hex",
+            "voice_setting": {
+                "voice_id": voice_id,
+                "speed": max(0.5, min(2.0, speed)),
+                "vol": 1.0,
+                "pitch": 0,
+            },
+            "audio_setting": {
+                "sample_rate": 24000,
+                "bitrate": 128000,
+                "format": "wav",
+            },
+        }
+
+        resp = requests.post(api_url, headers=headers, json=body, timeout=120)
+        resp.raise_for_status()
+        resp_json = resp.json()
+
+        base_resp = (resp_json or {}).get("base_resp") or {}
+        if base_resp.get("status_code") not in (0, None):
+            raise RuntimeError(f"MiniMax TTS failed: {resp_json}")
+
+        data = (resp_json or {}).get("data") or {}
+        audio_field = data.get("audio")
+        if not audio_field:
+            raise RuntimeError(f"MiniMax TTS returned no audio data: {resp_json}")
+
+        if isinstance(audio_field, str) and audio_field.startswith("http"):
+            audio_resp = requests.get(audio_field, timeout=120)
+            audio_resp.raise_for_status()
+            wav_path.write_bytes(audio_resp.content)
+        else:
+            try:
+                wav_path.write_bytes(binascii.unhexlify(audio_field))
+            except Exception as e:
+                raise RuntimeError(
+                    f"MiniMax TTS hex decode failed: {e}, "
+                    f"audio_field[:64]={str(audio_field)[:64]}"
+                )
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    def _resolve_credentials(
+        self, inputs: Dict[str, Any], node_state: NodeState
+    ) -> tuple[str, str]:
+        """
+        Resolve MiniMax API key. Priority:
+          1. inputs["api_key"]
+          2. config.toml [generate_voiceover.providers.minimax].api_key
+          3. env MINIMAX_API_KEY / TTS_MINIMAX_API_KEY
+        """
+        import os
+
+        api_key = (inputs.get("api_key") or "").strip()
+
+        if not api_key:
+            try:
+                providers = (
+                    getattr(self.server_cfg.generate_voiceover, "providers", {}) or {}
+                )
+                api_key = (
+                    (providers.get("minimax") or {}).get("api_key") or ""
+                ).strip()
+            except Exception:
+                pass
+
+        if not api_key:
+            for env_var in ("MINIMAX_API_KEY", "TTS_MINIMAX_API_KEY"):
+                api_key = (os.getenv(env_var) or "").strip()
+                if api_key:
+                    break
+
+        if not api_key:
+            node_state.node_summary.info_for_llm(
+                "MiniMax API key is missing. Ask the user to provide it via the sidebar "
+                "or config.toml [generate_voiceover.providers.minimax] api_key."
+            )
+            raise ValueError(
+                "MiniMax API key not found. Set it in config.toml under "
+                "[generate_voiceover.providers.minimax] api_key, or via the "
+                "MINIMAX_API_KEY environment variable."
+            )
+
+        raw_url = (inputs.get("base_url") or "").strip()
+        if raw_url:
+            # Keep only scheme+host; strip any path that may come from TTS endpoint config
+            from urllib.parse import urlparse
+            _p = urlparse(raw_url)
+            base_url = f"{_p.scheme}://{_p.netloc}" if _p.netloc else raw_url
+        else:
+            base_url = _DEFAULT_BASE_URL
+        return api_key, base_url
+
+    def _upload_file(
+        self,
+        *,
+        upload_url: str,
+        headers: Dict[str, str],
+        file_path: Path,
+        purpose: str,
+    ) -> Any:
+        """Upload a file to MiniMax /v1/files/upload and return file_id."""
+        with open(file_path, "rb") as f:
+            resp = requests.post(
+                upload_url,
+                headers=headers,
+                data={"purpose": purpose},
+                files={"file": (file_path.name, f)},
+                timeout=120,
+            )
+        resp.raise_for_status()
+        resp_json = resp.json()
+
+        base_resp = (resp_json or {}).get("base_resp") or {}
+        if base_resp.get("status_code") not in (0, None):
+            raise RuntimeError(
+                f"MiniMax file upload failed: status_code={base_resp.get('status_code')}, "
+                f"status_msg={base_resp.get('status_msg')}"
+            )
+
+        file_id = (resp_json.get("file") or {}).get("file_id")
+        if not file_id:
+            raise RuntimeError(
+                f"MiniMax file upload returned no file_id. Response: {resp_json}"
+            )
+        return file_id
+
+    def _audio_duration_ms(self, audio_path: Path) -> int:
+        """Return audio duration in milliseconds."""
+        try:
+            import librosa
+
+            return int(
+                round(
+                    librosa.get_duration(path=str(audio_path))
+                    * _MILLISECONDS_PER_SECOND
+                )
+            )
+        except Exception as e:
+            logger.warning(f"Could not read duration of {audio_path}: {e}")
+            return 0

src/open_storyline/nodes/node_schema.py

@@ -178,6 +178,60 @@ class BaseInput(BaseModel):
     )
 
 
+class VoiceCloneMinimaxInput(BaseInput):
+    mode: Literal["auto", "skip", "default"] = Field(
+        default="auto",
+        description=(
+            "auto: Clone voice then generate voiceover for all script groups using the cloned voice; "
+            "skip/default: Skip cloning and return empty voiceover list."
+        ),
+    )
+    clone_audio: List[Dict[str, Any]] = Field(
+        default_factory=list,
+        description=(
+            "Audio file to clone. Pass exactly one item. "
+            'Local mode: [{"path": "/abs/path/audio.mp3"}]. '
+            'Remote/web mode: [{"path": "filename.mp3", "base64": "<b64>", "md5": "<md5>"}]. '
+            "Supported formats: mp3, m4a, wav. Duration: 10s-5min. Max size: 20MB."
+        ),
+    )
+    voice_id: str = Field(
+        default="",
+        description=(
+            "Custom voice_id to assign to the cloned voice (e.g. 'my_cloned_voice_001'). "
+            "Must be unique within your MiniMax account. "
+            "If empty, a unique id will be auto-generated."
+        ),
+    )
+    prompt_audio: List[Dict[str, Any]] = Field(
+        default_factory=list,
+        description=(
+            "Optional reference audio to improve clone quality. Pass at most one item, same format as clone_audio. "
+            "Duration < 8s. Max size: 20MB. Leave empty to skip."
+        ),
+    )
+    prompt_text: str = Field(
+        default="",
+        description=(
+            "Optional: transcript text matching prompt_audio, used together with prompt_audio "
+            "to guide the cloning. Leave empty if no prompt audio is provided."
+        ),
+    )
+    model: str = Field(
+        default="speech-02-hd",
+        description="MiniMax TTS model for voiceover generation. Options: speech-02-hd, speech-02-turbo, speech-2.6-hd.",
+    )
+    speed: float = Field(
+        default=1.0,
+        description="Speech speed multiplier, range 0.5-2.0.",
+    )
+    user_request: str = Field(
+        default="",
+        description="User's additional requirements for the voiceover style or delivery.",
+    )
+
+
+
 class LoadMediaInput(BaseInput):
     ...
 
@@ -319,6 +373,8 @@ class RecommendScriptTemplateInput(BaseInput):
 class GenerateVoiceoverOutput(BaseModel):
     voiceover: List[Voiceover] = Field(default_factory=list, description="Voiceover list")
 
+VoiceCloneMinimaxOutput = GenerateVoiceoverOutput
+
 
 class SelectBGMInput(BaseInput):
     mode: Literal["auto", "skip", "default"] = Field(

src/open_storyline/utils/media_handler.py

@@ -1,14 +1,17 @@
 import os
 from pathlib import Path
-from typing import Union
+from typing import List, Union
 
 _MEDIA_EXTS_IMG = {".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp"}
 _MEDIA_EXTS_VID = {".mp4", ".mov", ".avi", ".mkv", ".webm", ".m4v"}
+_MEDIA_EXTS_AUD = {".mp3", ".wav", ".m4a"}
+
 
 def scan_media_dir(media_dir: Union[Path, str]) -> dict:
     image_num, video_num = 0, 0
+    audio_paths: List[str] = []
     media_dir = Path(media_dir)
-    media_dir.mkdir(parents=True, exist_ok=True) 
+    media_dir.mkdir(parents=True, exist_ok=True)
 
     for path in media_dir.iterdir():
         name = path.name
@@ -23,8 +26,15 @@ def scan_media_dir(media_dir: Union[Path, str]) -> dict:
             image_num += 1
         elif ext in _MEDIA_EXTS_VID:
             video_num += 1
+        elif ext in _MEDIA_EXTS_AUD:
+            audio_paths.append(str(path.resolve()))
 
-    return {
+    result: dict = {
         "image number in user's media library": image_num,
         "video number in user's media library": video_num,
     }
+    if audio_paths:
+        result[
+            "audio files in user's media library (use absolute path as clone_audio when voice cloning)"
+        ] = audio_paths
+    return result

web/static/app.js

@@ -115,9 +115,9 @@ const __OS_I18N = {
     "toast.switch_while_streaming": "正在生成回复，暂时无法切换会话。请先等待完成或打断当前回复。",
     "toast.session_restore_unavailable": "暂时无法从服务器恢复会话（网络或服务繁忙）。请稍后刷新或重试；本地会话 ID 已保留。",
     "toast.uploading_interrupt_send": "素材正在上传中，暂时无法发送新消息。已为你打断当前回复；上传完成后再按 Enter 发送。",
-    "toast.media_all_filtered": "仅支持上传视频或图片文件。",
-    "toast.media_partial_filtered": "已过滤 {n} 个不支持的文件类型，仅上传视频/图片。",
-    "toast.audio_not_supported": "暂不支持音频文件上传（后端尚未支持音频处理）。",
+    "toast.media_all_filtered": "仅支持上传视频、图片或音频文件。",
+    "toast.media_partial_filtered": "已过滤 {n} 个不支持的文件类型，仅上传视频/图片/音频。",
+    "toast.audio_not_supported": "暂不支持该音频格式，请上传 mp3、wav 或 m4a 文件。",
 
     // tools
     "tool.card.default_name": "工具调用",
@@ -239,9 +239,9 @@ const __OS_I18N = {
     "toast.switch_while_streaming": "A reply is still being generated. Please wait or interrupt before switching chats.",
     "toast.session_restore_unavailable": "Could not restore the session from the server (network or temporary overload). Please retry later or refresh. Your local session id is kept.",
     "toast.uploading_interrupt_send": "Media is uploading, so a new message can't be sent yet. I interrupted the current reply; press Enter after the upload finishes.",
-    "toast.media_all_filtered": "Only video or image files are supported.",
-    "toast.media_partial_filtered": "{n} unsupported file(s) were filtered; only video/image files will be uploaded.",
-    "toast.audio_not_supported": "Audio uploads are not supported yet (backend audio processing is not available).",
+    "toast.media_all_filtered": "Only video, image, or audio files are supported.",
+    "toast.media_partial_filtered": "{n} unsupported file(s) were filtered; only video/image/audio files will be uploaded.",
+    "toast.audio_not_supported": "This audio format is not supported. Please upload mp3, wav, or m4a files.",
     
     // tools
     "tool.card.default_name": "Tool call",
@@ -3873,35 +3873,49 @@ class App {
     let files = Array.isArray(rawFiles) ? rawFiles.slice() : Array.from(rawFiles || []);
     if (!files.length) return;
 
-    const isAudioFile = (f) => {
+    // 支持的音频后缀（用于 voice_clone_minimax 等节点）
+    const isSupportedAudioFile = (f) => {
       if (!f) return false;
       const type = String(f.type || "").toLowerCase();
-      if (type.startsWith("audio/")) return true;
+      if (type.startsWith("audio/")) {
+        // 仅放行常见格式，屏蔽小众格式（aac/flac/ogg/opus 后端暂不处理）
+        return /audio\/(mpeg|mp3|wav|x-wav|m4a|x-m4a|mp4)/.test(type);
+      }
       const name = String(f.name || "").toLowerCase();
-      return /\.(mp3|wav|m4a|aac|flac|ogg|opus)$/.test(name);
+      return /\.(mp3|wav|m4a)$/.test(name);
     };
 
-    // 仅允许视频/图片（音频暂不支持：后端没有处理逻辑）
+    // 不支持的音频格式（给出专门提示）
+    const isUnsupportedAudioFile = (f) => {
+      if (!f) return false;
+      const type = String(f.type || "").toLowerCase();
+      if (type.startsWith("audio/") && !isSupportedAudioFile(f)) return true;
+      const name = String(f.name || "").toLowerCase();
+      return /\.(aac|flac|ogg|opus)$/.test(name);
+    };
+
+    // 仅允许视频/图片/支持的音频格式
     const isSupportedMediaFile = (f) => {
       if (!f) return false;
       const type = String(f.type || "").toLowerCase();
       if (type.startsWith("video/") || type.startsWith("image/")) {
         return true;
       }
+      if (isSupportedAudioFile(f)) return true;
       // 对部分没有正确 MIME 的文件，fallback 到后缀判断
       const name = String(f.name || "").toLowerCase();
       return /\.(mp4|mov|m4v|avi|mkv|webm|flv|wmv|jpg|jpeg|png|gif|webp|bmp|tiff)$/.test(name);
     };
 
     const beforeCount = files.length;
-    const audioCount = files.filter(isAudioFile).length;
-    files = files.filter((f) => isSupportedMediaFile(f) && !isAudioFile(f));
+    const unsupportedAudioCount = files.filter(isUnsupportedAudioFile).length;
+    files = files.filter(isSupportedMediaFile);
     const filteredCount = beforeCount - files.length;
 
     if (!files.length) {
       // 全部被过滤，直接提示并返回
       try {
-        if (audioCount > 0) {
+        if (unsupportedAudioCount > 0) {
           this.ui.showToastI18n("toast.audio_not_supported", {});
         } else {
           this.ui.showToastI18n("toast.media_all_filtered", {});