From 702bf7becefb13e3f2d52b678635f9c8255815cb Mon Sep 17 00:00:00 2001 From: Anlittledy <78261172+Anlittledy@users.noreply.github.com> Date: Thu, 2 Apr 2026 14:13:35 +0800 Subject: [PATCH] feat: add AI transition workflow (#74) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add AI transition workflow with runtime config injection and timeline support * update web hint * fix ai transition timeline bug * AI transition generation real-time progress reporting * update news date --------- Co-authored-by: 杨其伟 Co-authored-by: jcl-2026 <1172688642@qq.com> --- .../ai_transition_editing_skill/SKILL.md | 24 + .../default_editing_workflow_skill/SKILL.md | 1 - README.md | 5 + README_zh.md | 6 +- agent_fastapi.py | 164 +++-- config.toml | 16 +- docs/source/en/api-key.md | 32 +- docs/source/zh/api-key.md | 33 +- .../tasks/generate_ai_transition/en/system.md | 29 + .../tasks/generate_ai_transition/en/user.md | 2 + .../tasks/generate_ai_transition/zh/system.md | 19 + .../tasks/generate_ai_transition/zh/user.md | 2 + prompts/tasks/instruction/en/system.md | 4 +- prompts/tasks/instruction/zh/system.md | 4 +- src/open_storyline/agent.py | 1 + src/open_storyline/config.py | 4 + .../mcp/hooks/node_interceptors.py | 90 ++- .../nodes/core_nodes/base_node.py | 1 + .../nodes/core_nodes/filter_clips.py | 2 +- .../core_nodes/generate_ai_transition.py | 571 ++++++++++++++++++ .../nodes/core_nodes/generate_video.py | 285 --------- .../core_nodes/plan_timeline_ai_transition.py | 173 ++++++ .../nodes/core_nodes/plan_timeline_pro.py | 98 ++- .../nodes/core_nodes/recommend_effects.py | 6 +- .../nodes/core_nodes/render_video.py | 2 +- .../nodes/core_nodes/script_template_rec.py | 2 +- .../nodes/core_nodes/split_shots.py | 2 +- .../nodes/core_nodes/understand_clips.py | 2 +- src/open_storyline/nodes/node_schema.py | 55 +- .../utils/ai_transition_cancel.py | 30 + .../utils/ai_transition_client.py | 349 +++++++++++ src/open_storyline/utils/client.py | 214 ------- web/index.html | 63 +- web/static/app.js | 191 ++++-- web/static/style.css | 30 + 35 files changed, 1813 insertions(+), 699 deletions(-) create mode 100644 .storyline/skills/ai_transition_editing_skill/SKILL.md create mode 100644 prompts/tasks/generate_ai_transition/en/system.md create mode 100644 prompts/tasks/generate_ai_transition/en/user.md create mode 100644 prompts/tasks/generate_ai_transition/zh/system.md create mode 100644 prompts/tasks/generate_ai_transition/zh/user.md create mode 100644 src/open_storyline/nodes/core_nodes/generate_ai_transition.py delete mode 100644 src/open_storyline/nodes/core_nodes/generate_video.py create mode 100644 src/open_storyline/nodes/core_nodes/plan_timeline_ai_transition.py create mode 100644 src/open_storyline/utils/ai_transition_cancel.py create mode 100644 src/open_storyline/utils/ai_transition_client.py delete mode 100644 src/open_storyline/utils/client.py diff --git a/.storyline/skills/ai_transition_editing_skill/SKILL.md b/.storyline/skills/ai_transition_editing_skill/SKILL.md new file mode 100644 index 0000000..936f415 --- /dev/null +++ b/.storyline/skills/ai_transition_editing_skill/SKILL.md @@ -0,0 +1,24 @@ +--- +name: ai_transition_editing_skill +description: 【WORKFLOW SKILL】使用 AI 生成的转场将用户素材串联起来。 +--- + +## 角色定义 (Role) +你是一个专业的剪辑师，擅长利用现有工具和 Skills 完成剪辑任务。 + +## 注意事项 +- AI 转场剪辑目前不支持字幕、配音 + +## 基本流程 +AI 转场剪辑流程如下，这里每一步都对应一个或多个工具或 Skills 供你使用： +- 搜索素材 "search_media"（可跳过）。如果你发现用户并没有上传素材，可以提示用户你可以上网搜索素材。搜索素材后需要运行load_media工具才可以真正加载到素材。 +- 素材加载 "load_media"（固定）。用于获取输入素材的路径、长宽等基础信息。 +- 镜头切分 "split_shots"（可跳过）。将素材按镜头切分成片段。 +- 内容理解 "understand_clips"（可跳过）。 为每个片段(clips)生成一段描述（captions） +- 镜头筛选 "filter_clips"（可跳过）。根据用户要求，筛选出符合要求的片段(clips) +- 片段分组 "group_clips"（可跳过，但应默认运行）。根据用户要求，对片段进行排序和分组。注意在 `user_request` 参数中强调：“组织成适合插入 AI 转场的片段顺序”。 +- 提醒用户：系统将要为所有 x 个片段生成 x - 1 段 AI 转场，目前只支持一次性生成全部转场。AI 转场的资源消耗通常显著高于常规文案或配音流程，请确认是否继续？其中 x 是镜头筛选后的**片段数**，不是分组数量。如果没有进行镜头筛选，则取镜头切分的数量。如果镜头切分也没有进行，则取素材数量。 +- AI 转场生成 "generate_ai_transition"（可跳过，但在 AI 转场剪辑流程中应默认运行）。 +- 背景音乐选取 "select_bgm"（可跳过）。选择合适的背景音乐。 +- 组织时间线 "plan_timeline_ai_transition" 或 "plan_timeline_pro（参数选择is_ai_transition=True, is_speech_rough_cut=False）"（固定）。根据前面的视频片段、文案、语音和BGM，调用 AI 转场剪辑专用时间线。 +- 渲染成片。"render_video"（固定）。根据时间线渲染成片。 \ No newline at end of file diff --git a/.storyline/skills/default_editing_workflow_skill/SKILL.md b/.storyline/skills/default_editing_workflow_skill/SKILL.md index af1a8ce..58c6e3f 100644 --- a/.storyline/skills/default_editing_workflow_skill/SKILL.md +++ b/.storyline/skills/default_editing_workflow_skill/SKILL.md @@ -19,4 +19,3 @@ description: 【WORKFLOW SKILL】通用剪辑流程。可用于剪辑日常/旅 - 背景音乐选取 "select_BGM"（可跳过）。选择合适的背景音乐。 - 组织时间线 "plan_timeline"（固定）。根据前面的视频片段、文案、语音和BGM，组织成合理的时间线。 - 渲染成片。"render_video"（固定）。根据时间线渲染成片。 -此外，虽然你在工具调用后只能看到summary，但你有一个工具可以读取任意中间节点的输出。你可以用它完成更复杂的任务。 diff --git a/README.md b/README.md index 72fa311..11eff7a 100644 --- a/README.md +++ b/README.md @@ -52,10 +52,15 @@ ## NEWS +* 🎬 **2026-04-02**: Added the **AI Transition Generation** feature, which automatically creates transition shots based on the ending frame of one clip, the opening frame of the next, and a natural-language description, making scene transitions smoother and the narrative more coherent. * 🚀 **2026-03-22**: Introduced an **ASR-based rough cut skill for speech videos**, enabling automatic removal of filler words, disfluencies, and repeated sentences, with timestamp-aligned segmentation for cleaner and more efficient speech editing workflows. * 🔥 **2026-03-12**: Integrated with **OpenClaw**, adding two OpenClaw Skills — `openstoryline-install` and `openstoryline-use` — covering the initial installation/first-run workflow and the actual usage workflow, respectively. Also added Skill usage instructions for **Claude Code**, making it easier for **Claude Code** to install and invoke the project in accordance with the repository guidelines. * **2026-02-10**: FireRed-OpenStoryline was officially open-sourced. +> +> ⚠️ Note: AI transitions rely on third-party AIGC video generation services, and the cost is relatively high. Due to variations in source material quality, prompts, and model performance, the generated results are somewhat unpredictable. It is recommended to enable this feature only when needed. +> + ## 🏗️ Architecture

diff --git a/README_zh.md b/README_zh.md index 1359f66..ccf5722 100644 --- a/README_zh.md +++ b/README_zh.md @@ -52,11 +52,15 @@ - ⚡ **剪辑技能沉淀**： 可一键保存为专属剪辑Skill，记录完整的剪辑逻辑。下次只需更换素材并选择对应Skill，即可快速复刻同款风格，实现高效批量生产。 ## NEWS - +- 🎬 **2026-04-02**：新增 **AI 转场生成** 功能，支持基于相邻片段的首尾画面与自然语言描述自动生成过渡镜头，让镜头衔接更自然、叙事更连贯。 - 🚀 **2026-03-22**：新增**基于ASR的口播视频粗剪Skill**，支持自动去除口头禅、语气词和重复表达，并结合时间戳进行精准切分，提升口播类视频的剪辑效率与成片质量。 - 🔥 **2026-03-12**：接入**OpenClaw**，新增 `openstoryline-install` 与 `openstoryline-use` 两个 OpenClaw Skills，分别覆盖安装首跑与实际使用流程；添加面向 **Claude Code** 的 Skill 使用说明，方便 **Claude Code** 依据仓库规范完成安装与调用。 - **2026-2-10**：FireRed-OpenStoryline 正式开源。 +> +> ⚠️ 注意：AI 转场依赖第三方 AIGC 视频生成服务，成本相对较高。受素材质量、提示词和模型波动影响，生成结果存在一定随机性，建议按需开启。 +> + ## 🏗️ 架构

diff --git a/agent_fastapi.py b/agent_fastapi.py index e81c475..6766b5f 100644 --- a/agent_fastapi.py +++ b/agent_fastapi.py @@ -48,6 +48,10 @@ if SRC_DIR not in sys.path: from open_storyline.agent import build_agent, ClientContext from open_storyline.utils.prompts import get_prompt from open_storyline.utils.media_handler import scan_media_dir +from open_storyline.utils.ai_transition_cancel import ( + clear_ai_transition_cancelled, + set_ai_transition_cancelled, +) from open_storyline.config import load_settings, default_config_path from open_storyline.config import Settings from open_storyline.storage.agent_memory import ArtifactStore @@ -80,6 +84,13 @@ def _norm_url(u: Any) -> str: u = _s(u) return u.rstrip("/") if u else "" + +def _ai_transition_cancel_cache_root(cfg: Settings) -> Path: + cache_dir = Path(cfg.local_mcp_server.server_cache_dir) + if not cache_dir.is_absolute(): + cache_dir = Path(ROOT_DIR) / cache_dir + return cache_dir + MODEL_ENV_KEYS = { "llm": { "model": "OPENSTORYLINE_LLM_MODEL", @@ -162,19 +173,35 @@ def _stable_dict_key(d: Optional[Dict[str, Any]]) -> str: except Exception: return str(d or {}) +def _parse_provider_runtime_config(service_cfg: Any, key_name: str) -> Dict[str, Any]: + cfg = service_cfg.get(key_name) + if not isinstance(cfg, dict): + return {} + + provider = _s(cfg.get("provider")).lower() + if not provider: + return {} + + provider_block = cfg.get(provider) + if not isinstance(provider_block, dict): + provider_block = {} + + return {"provider": provider, provider: provider_block} + def _parse_service_config(service_cfg: Any) -> Tuple[ Optional[Dict[str, Any]], Optional[Dict[str, Any]], Dict[str, Any], Dict[str, Any], + Dict[str, Any], Optional[str]]: """ - 返回 (custom_llm, custom_vlm, tts_cfg, pexels, err) + 返回 (custom_llm, custom_vlm, tts_cfg, ai_transition_cfg, pexels, err) - custom_llm/custom_vlm: {"model","base_url","api_key"} 或 None（允许只传 llm 或只传 vlm） - tts_cfg: dict（可能为空） """ if not isinstance(service_cfg, dict): - return None, None, {}, {}, None + return None, None, {}, {}, {}, None # ---- custom models ---- custom_llm = None @@ -183,7 +210,7 @@ def _parse_service_config(service_cfg: Any) -> Tuple[ if custom_models is not None: if not isinstance(custom_models, dict): - return None, None, {}, {}, "service_config.custom_models 必须是对象" + return None, None, {}, {}, {}, "service_config.custom_models 必须是对象" def _pick(m: Any, label: str) -> Tuple[Optional[Dict[str, str]], Optional[str]]: if m is None: @@ -203,21 +230,16 @@ def _parse_service_config(service_cfg: Any) -> Tuple[ custom_llm, err1 = _pick(custom_models.get("llm"), "llm") if err1: - return None, None, {}, {}, err1 + return None, None, {}, {}, {}, err1 custom_vlm, err2 = _pick(custom_models.get("vlm"), "vlm") if err2: - return None, None, {}, {}, err2 + return None, None, {}, {}, {}, err2 + + # ---- provider runtime config ---- + tts_cfg = _parse_provider_runtime_config(service_cfg, "tts") + ai_transition_cfg = _parse_provider_runtime_config(service_cfg, "ai_transition") - # ---- tts ---- - tts_cfg: Dict[str, Any] = {} - tts = service_cfg.get("tts") - if isinstance(tts, dict): - provider = (tts.get("provider") or "").strip().lower() - if provider: - provider_block = tts.get(provider) - tts_cfg = {"provider": provider, provider: provider_block} - # ---- pexels ---- pexels_cfg: Dict[str, Any] = {} search_media = service_cfg.get("search_media") @@ -239,7 +261,7 @@ def _parse_service_config(service_cfg: Any) -> Tuple[ api_key = _s(search_media.get("pexels_api_key") or search_media.get("pexels_api_key")) pexels_cfg = {"mode": mode, "api_key": api_key} - return custom_llm, custom_vlm, tts_cfg, pexels_cfg, None + return custom_llm, custom_vlm, tts_cfg, ai_transition_cfg, pexels_cfg, None def is_developer_mode(cfg: Settings) -> bool: try: @@ -1088,6 +1110,7 @@ class ChatSession: self.custom_llm_config: Optional[Dict[str, Any]] = None self.custom_vlm_config: Optional[Dict[str, Any]] = None self.tts_config: Dict[str, Any] = {} + self.ai_transition_config: Dict[str, Any] = {} self._agent_build_key: Optional[Tuple[Any, ...]] = None self.pexels_key_mode: str = "default" # "default" | "custom" @@ -1171,7 +1194,7 @@ class ChatSession: def apply_service_config(self, service_cfg: Any) -> Tuple[bool, Optional[str]]: - llm, vlm, tts, pexels, err = _parse_service_config(service_cfg) + llm, vlm, tts, ai_transition, pexels, err = _parse_service_config(service_cfg) if err: return False, err @@ -1184,6 +1207,9 @@ class ChatSession: if isinstance(tts, dict) and tts: self.tts_config = tts + if isinstance(ai_transition, dict) and ai_transition: + self.ai_transition_config = ai_transition + # ---- pexels ---- if isinstance(pexels, dict) and pexels: mode = _s(pexels.get("mode")).lower() @@ -1233,6 +1259,7 @@ class ChatSession: ToolInterceptor.inject_media_content_before, ToolInterceptor.save_media_content_after, ToolInterceptor.inject_tts_config, + ToolInterceptor.inject_ai_transition_config, ToolInterceptor.inject_pexels_api_key, ], llm_override=llm_override, @@ -1251,6 +1278,7 @@ class ChatSession: chat_model_key=self.chat_model_key, vlm_model_key=self.vlm_model_key, tts_config=(self.tts_config or None), + ai_transition_config=(self.ai_transition_config or None), pexels_api_key=None, lang=self.lang, ) @@ -1258,6 +1286,7 @@ class ChatSession: self.client_context.chat_model_key = self.chat_model_key self.client_context.vlm_model_key = self.vlm_model_key self.client_context.tts_config = (self.tts_config or None) + self.client_context.ai_transition_config = (self.ai_transition_config or None) self.client_context.lang = self.lang # ---- resolve pexels_api_key for runtime context ---- @@ -1564,11 +1593,50 @@ _TTS_UI_SECRET_KEYS = { "accesskey", } +_PROVIDER_UI_META_KEYS = { + "label", + "name", + "display_name", +} + +_PROVIDER_UI_LABEL_OVERRIDES = { + "302": "302.AI", + "bytedance": "字节跳动 ByteDance", + "dashscope": "阿里万相 Wan", +} + +_PROVIDER_UI_LABEL_OVERRIDES_BY_SECTION = { + "generate_voiceover": { + "minimax": "MiniMax", + }, + "generate_ai_transition": { + "minimax": "MiniMax 海螺 (Hailuo)", + }, +} + def _is_secret_field_name(k: str) -> bool: if str(k or "").strip().lower() in _TTS_UI_SECRET_KEYS: return True return False +def _get_provider_ui_label(section_name: str, provider: str, provider_cfg: Any) -> str: + if isinstance(provider_cfg, dict): + explicit_label = _s( + provider_cfg.get("label") + or provider_cfg.get("display_name") + or provider_cfg.get("name") + ) + if explicit_label: + return explicit_label + + section_key = _s(section_name) + provider_key = _s(provider).lower() + section_overrides = _PROVIDER_UI_LABEL_OVERRIDES_BY_SECTION.get(section_key, {}) + if provider_key in section_overrides: + return section_overrides[provider_key] + + return _PROVIDER_UI_LABEL_OVERRIDES.get(provider_key, _s(provider)) + def _read_config_toml(path: str) -> dict: if tomllib is None: return {} @@ -1595,7 +1663,7 @@ def _normalize_field_item(item) -> dict | None: """ item 支持： - "uid" - - { key="uid", label="UID", required=true, secret=false, placeholder="..." } + - { key="uid", label="UID", secret=false, placeholder="..." } """ if isinstance(item, str): key = item.strip() @@ -1607,24 +1675,7 @@ def _normalize_field_item(item) -> dict | None: } return None -def _build_provider_schema(provider: str, label: str | None, fields: list[dict]) -> dict: - seen = set() - out = [] - for f in fields: - k = str(f.get("key") or "").strip() - if not k or k in seen: - continue - seen.add(k) - out.append({ - "key": k, - "label": f.get("label") or k, - "placeholder": f.get("placeholder") or f.get("label") or k, - "required": bool(f.get("required", False)), - "secret": bool(f.get("secret", False)), - }) - return {"provider": provider, "label": label or provider, "fields": out} - -def _build_tts_ui_schema_from_config(config_path: str) -> dict: +def _build_provider_ui_schema_from_config(config_path: str, section_name: str) -> dict: """ 返回： { @@ -1635,22 +1686,43 @@ def _build_tts_ui_schema_from_config(config_path: str) -> dict: } """ cfg = _read_config_toml(config_path) - tts = cfg.get("generate_voiceover", {}) + tts = cfg.get(section_name, {}) providers_out: list[dict] = [] - # 格式：[tts.providers.] + # 格式：[.providers.] providers = tts.get("providers") if isinstance(providers, dict): for provider, provider_cfg in providers.items(): fields: list[dict] = [] - label = str(provider_cfg.get("label") or provider_cfg.get("name") or provider) + label = _get_provider_ui_label(section_name, provider, provider_cfg) for key in provider_cfg.keys(): - f = _normalize_field_item(str(key)) + key = str(key).strip() + if not key or key.lower() in _PROVIDER_UI_META_KEYS: + continue + f = _normalize_field_item(key) if f: fields.append(f) - providers_out.append(_build_provider_schema(provider, label, fields)) + seen = set() + normalized_fields = [] + for f in fields: + k = str(f.get("key") or "").strip() + if not k or k in seen: + continue + seen.add(k) + normalized_fields.append({ + "key": k, + "label": f.get("label") or k, + "placeholder": f.get("placeholder") or f.get("label") or k, + "secret": bool(f.get("secret", False)), + }) + + providers_out.append({ + "provider": provider, + "label": label or provider, + "fields": normalized_fields, + }) return {"providers": providers_out} @@ -1672,7 +1744,12 @@ async def node_map(): @api.get("/meta/tts") async def get_tts_ui_schema(): - schema = _build_tts_ui_schema_from_config(default_config_path()) + schema = _build_provider_ui_schema_from_config(default_config_path(), "generate_voiceover") + return JSONResponse(schema) + +@api.get("/meta/ai_transition") +async def get_ai_transition_ui_schema(): + schema = _build_provider_ui_schema_from_config(default_config_path(), "generate_ai_transition") return JSONResponse(schema) # ------------------------- @@ -1707,6 +1784,7 @@ async def clear_session_chat(session_id: str): sess.history = [] sess._tool_history_index = {} + clear_ai_transition_cancelled(_ai_transition_cancel_cache_root(app.state.cfg), session_id) return JSONResponse({"ok": True}) @api.post("/sessions/{session_id}/cancel") @@ -1719,6 +1797,7 @@ async def cancel_session_turn(session_id: str): store: SessionStore = app.state.sessions sess = await store.get_or_404(session_id) sess.cancel_event.set() + set_ai_transition_cancelled(_ai_transition_cancel_cache_root(app.state.cfg), session_id) return JSONResponse({"ok": True}) # ------------------------- @@ -2270,7 +2349,8 @@ async def ws_chat(ws: WebSocket, session_id: str): async with sess.chat_lock: # 新 turn 开始：清掉上一次残留的 cancel 信号 sess.cancel_event.clear() - # 0.0) 应用 service_config（自定义模型 / TTS） + clear_ai_transition_cancelled(_ai_transition_cancel_cache_root(app.state.cfg), session_id) + # 0.0) 应用 service_config（自定义模型 / TTS / ai_transition） ok_cfg, err_cfg = sess.apply_service_config(data.get("service_config")) if not ok_cfg: await ws_send(ws, "error", {"message": err_cfg or "service_config invalid"}) diff --git a/config.toml b/config.toml index 29f9875..0282a66 100644 --- a/config.toml +++ b/config.toml @@ -44,10 +44,10 @@ available_node_pkgs = [ "open_storyline.nodes.core_nodes" ] available_nodes = [ - "LoadMediaNode", "SearchMediaNode", "SplitShotsNode", "LocalASRNode", "SpeechRoughCutNode", + "LoadMediaNode", "SearchMediaNode", "SplitShotsNode", "LocalASRNode", "SpeechRoughCutNode", "GenerateAITransitionNode", "UnderstandClipsNode", "FilterClipsNode", "GroupClipsNode", "GenerateScriptNode", "ScriptTemplateRecomendation", "GenerateVoiceoverNode", "SelectBGMNode", "RecommendTransitionNode", "RecommendTextNode", - "PlanTimelineProNode", "RenderVideoNode" + "PlanTimelineProNode", "PlanTimelineAITransitionNode", "RenderVideoNode" ] # =========== skills ========== @@ -99,6 +99,18 @@ base_url = "" api_key = "" +# ============= AI 转场生成 / AI transition generation =================== +[generate_ai_transition] + +[generate_ai_transition.providers.dashscope] # 阿里通义万相 / Alibaba’s Tongyi Wanxiang (Wan) video model +model_name = "" +api_key = "" + +[generate_ai_transition.providers.minimax] # Minimax 海螺 / Hailuo +model_name = "" +api_key = "" + + # ============= BGM选择 / BGM Selection ==================== # 主要是用于计算音乐特征的一些参数 / Mainly parameters for calculating music features [select_bgm] diff --git a/docs/source/en/api-key.md b/docs/source/en/api-key.md index 218c6d4..0d665c4 100644 --- a/docs/source/en/api-key.md +++ b/docs/source/en/api-key.md @@ -83,12 +83,9 @@ Billing Dashboard: https://billing-cost.console.aliyun.com/home ## 4. TTS (Text-to-Speech) Configuration -### Option 1: 302.ai (Not recommended) -- **Service URL**: https://302.ai/product/detail/302ai-mmaudio-text-to-speech -- **API Key Base url**：https://api.302.ai -### Option 2: MiniMax (Recommended) +### Option 1: MiniMax (Recommended) - **Service URL**: https://platform.minimaxi.com/docs/api-reference/speech-t2a-http - **API Key Base Url**: https://api.minimax.chat/v1/t2a_v2 @@ -98,7 +95,7 @@ Billing Dashboard: https://billing-cost.console.aliyun.com/home 2. Visit: https://platform.minimax.io/user-center/basic-information/interface-key 3. Obtain and save API Key -### Option 3: Bytedance (Recommended) +### Option 2: Bytedance (Recommended) 1. Step 1: Enable Audio/Video Subtitle Generation Service Use the legacy page to find the audio/video subtitle generation service: @@ -130,6 +127,31 @@ access_token = "" For detailed documentation, please refer to: https://www.volcengine.com/docs/6561/80909 +### Option 3: 302.ai (Alternative solutions) + +- **Service URL**: https://302.ai/product/detail/302ai-mmaudio-text-to-speech +- **API Key Base url**：https://api.302.ai + +## 5. AI Transition Configuration + +**Before you start**: AI transitions trigger additional model calls. Transitions are generated clip by clip between adjacent segments, so the more clips you have and the finer the shot splitting is, the higher the number of calls will usually be. As a result, resource usage is typically **significantly higher** than standard copywriting or voiceover workflows. + +**Output quality note**: The current transition description is generated from the first and last frames of adjacent clips by a vision model, while clip ordering is determined by the language model. Final results can therefore vary depending on frame content, prompts, model versions, and service-side behavior. Some randomness is expected, and output may not match expectations every time. + +**Recommendation**: Start with a small test run, review the results, and then scale up if the quality and cost are acceptable. Please also check your **account balance** and **provider billing rules** in advance. + +### Option 1: MiniMax Hailuo + +1. In most cases, the API key you already use for MiniMax LLM or TTS services can also be used for Hailuo video generation. If you already have one, you can reuse it directly. If not, create one from the MiniMax API platform by following the official [Quick Start](https://platform.minimax.io/docs/guides/quickstart). + +2. You can use `MiniMax-Hailuo-02`, or check the official [Video Generation documentation](https://platform.minimax.io/docs/api-reference/video-generation-intro) for newer supported model names. + +### Option 2: Alibaba Cloud Wan + +1. In most cases, the API key you already use for Alibaba Cloud Model Studio LLM services can also be used for Wan video generation. If you already have one, you can reuse it directly. If not, follow the official guide to [get an API key](https://www.alibabacloud.com/help/en/model-studio/get-api-key). + +2. We recommend `wan2.2-kf2v-flash`, or you can check the official [first-and-last-frame image-to-video guide](https://www.alibabacloud.com/help/en/model-studio/image-to-video-first-and-last-frames-guide) for more supported model names and usage details. + ## Important Notes - All API Keys must be kept secure to avoid leakage diff --git a/docs/source/zh/api-key.md b/docs/source/zh/api-key.md index ef7320e..ada910f 100644 --- a/docs/source/zh/api-key.md +++ b/docs/source/zh/api-key.md @@ -83,12 +83,7 @@ Qwen3-Omni同样可以在阿里云百炼平台进行申请，具体参数如下 ## 四、TTS (文本转语音) 配置 -### 方案一：302.ai （不推荐） - -- **服务地址**：https://302.ai/product/detail/302ai-mmaudio-text-to-speech -- **API Key Base url**：https://api.302.ai - -### 方案二：MiniMax（推荐使用） +### 方案一：MiniMax（推荐使用） - **服务地址**：https://platform.minimaxi.com/docs/api-reference/speech-t2a-http - **API Key Base url**：https://api.minimax.chat/v1/t2a_v2 @@ -98,7 +93,7 @@ Qwen3-Omni同样可以在阿里云百炼平台进行申请，具体参数如下 2. 访问：https://platform.minimax.io/user-center/basic-information/interface-key 3. 获取并保存 API Key -### 方案三：bytedance （推荐使用） +### 方案二：bytedance（推荐使用） 1. 步骤1：开通音视频字幕生成服务 使用旧版页面，找到音视频字幕生成服务： - 访问：https://console.volcengine.com/speech/service/9?AppID=8782592131 @@ -124,9 +119,33 @@ Qwen3-Omni同样可以在阿里云百炼平台进行申请，具体参数如下 appid = "" access_token = "" ``` + 或直接在前端网页侧边栏填写。 + +### 方案三：302.ai （备选方案） + +- **服务地址**：https://302.ai/product/detail/302ai-mmaudio-text-to-speech +- **API Key Base url**：https://api.302.ai 详细文档请参考：https://www.volcengine.com/docs/6561/80909?lang=zh +## 五： AI 转场配置 + +**使用前说明**：AI 转场会额外触发模型调用，转场是在相邻片段之间逐段生成，片段越多、切分越细，调用次数通常越高，因此资源消耗通常**显著高于**常规文案或配音流程。 + +**效果说明**：当前转场描述由视觉模型基于片段首尾帧自动生成，片段衔接顺序由语言模型综合判断，因此最终效果受首尾帧内容、提示词、模型版本和服务波动影响，存在一定随机性，不保证每次都完全符合预期。 + +**使用建议**：建议先使用少量片段试跑，确认效果与成本后再批量生成，并提前关注**账户余额**与**计费规则**。 + +### 方案一：Minimax 海螺 +1. Minimax 的 LLM / TTS 服务的API key 通常同样适用于海螺视频生成服务。如果你已申请过，可直接使用；如果你还没有申请过，可以前往用户中心申请。 + +2. 模型名可选择 `MiniMax-Hailuo-02`，或查阅文档获取最新支持的模型名。 + +### 方案二： 阿里通义万相 Wan +1. 阿里百炼大模型的 LLM 服务的API key 通常同样适用于 Wan 视频生成服务。如果你已申请过，可直接使用；如果你还没有申请过，可以前往百炼控制台申请。 + +2. 模型名推荐选择 wan2.2-kf2v-flash，或查阅文档获取最新支持的模型名。 + ## 注意事项 - 所有 API Key 均需妥善保管，避免泄露 diff --git a/prompts/tasks/generate_ai_transition/en/system.md b/prompts/tasks/generate_ai_transition/en/system.md new file mode 100644 index 0000000..28322bc --- /dev/null +++ b/prompts/tasks/generate_ai_transition/en/system.md @@ -0,0 +1,29 @@ +## Role + +You are now a world-class AIGC visual effects director and senior prompt engineer. You understand the key weaknesses of image-to-video foundation models: stiff, unnatural cut-like transitions, nightmarish sanity-draining distortions, and unstable, uncontrollable camera motion. + +## Task + +* You will be given two images: one as the first frame and one as the last frame of the generated video. You will also be given the user’s transition requirements. +* Your task is to write a **high-level, pure-English image-to-video prompt for start-and-end-frame generation** that is **extremely low-risk, requires no repeated rerolls, and delivers an exceptionally smooth and safe transition**, based on the input start frame, end frame, and the user’s transition requirements. + +## Principles + +To ensure the AI generates a perfect result in a single pass, the English prompt you write must strictly follow the structure and failure-prevention strategies below: + +1. **Strong Instruction Prefix (The Magic Prefix):** It must begin with this sentence to force the AI to understand that this is a transition task: + `"Smooth continuous single shot, seamlessly morphing from the start frame to the end frame..."` + +2. **Absolutely Locked Camera Trajectory:** Clearly specify the camera movement direction and add stabilizing language to prevent unwanted shaking or drift. Example: + `"Extremely steady forward zoom"`. + +3. **Mandatory Masking Medium — the Core to Reducing Failure Rate:** + It is **strictly forbidden** to let two entities with drastically different physical forms morph directly into each other. During the morph, you must introduce a transition medium that matches the overall tone and conceals the computation process, according to the director’s strategy. + *Examples: `blinded by a massive warm lens flare`, `camera passes through a thick motion blur`, `explodes into glowing particles`* + +4. **Anti-Hallucination Suffix:** The prompt must end with: + `"High quality, cinematic masterpiece, absolutely no hard cuts, no sudden jumps, flawless transition."` + +## Output Format Requirements + +Output the English prompt directly, without any additional explanation or polite filler. \ No newline at end of file diff --git a/prompts/tasks/generate_ai_transition/en/user.md b/prompts/tasks/generate_ai_transition/en/user.md new file mode 100644 index 0000000..e0d950c --- /dev/null +++ b/prompts/tasks/generate_ai_transition/en/user.md @@ -0,0 +1,2 @@ +The following are the user's requirements for transitions. Please design a transition prompt that meets the user's requirements while adhering to the basic principles. +{{user_request}} \ No newline at end of file diff --git a/prompts/tasks/generate_ai_transition/zh/system.md b/prompts/tasks/generate_ai_transition/zh/system.md new file mode 100644 index 0000000..167109e --- /dev/null +++ b/prompts/tasks/generate_ai_transition/zh/system.md @@ -0,0 +1,19 @@ +## 角色设定 +你现在是全球顶尖的 AIGC 视觉特效总监兼高级提示词工程师。你深知基础图生视频大模型的“软肋”（极易产生生硬切镜、掉SAN的惊悚形变、以及运镜失控摇晃）。 + +## 任务 +- 你会拿到两张图片，分别作为生成视频的首帧和尾帧。你还会拿到用户对转场的要求。 +- 你的任务是：根据输入的首尾帧与用户对转场的要求，编写出**废片率极低、无需反复抽卡、极度丝滑安全**的高阶纯英文首尾帧生成视频 Prompt。 + +## 原则 +为了确保 AI 一次性生成完美画面，你编写的英文 prompt 必须严格遵循以下结构和防翻车策略： + +1. **强指令前缀 (The Magic Prefix)：** 必须以这句开头强迫 AI 理解这是转场任务：`"Smooth continuous single shot, seamlessly morphing from the start frame to the end frame..."` +2. **绝对锁定运镜 (Locked Camera Trajectory)：** 明确运镜方向，加上稳定词，防止乱晃。示例：`"Extremely steady forward zoom"`. +3. **强制介质掩护 (Mandatory Masking Medium - 降低废片率的核心)：** + **绝对禁止**让两个物理形态差异巨大的实体直接发生形变！必须根据导演策略，在形变发生时引入符合全局基调的“过渡介质”掩盖计算过程。 + *示例：`blinded by a massive warm lens flare`, `camera passes through a thick motion blur`, `explodes into glowing particles`* +4. **兜底后缀 (Anti-Hallucination Suffix)：** 末尾必须加上：`"High quality, cinematic masterpiece, absolutely no hard cuts, no sudden jumps, flawless transition."` + +## 输出格式要求 +请直接输出英文 prompt，不需要附带额外的任何解释和客套话。 diff --git a/prompts/tasks/generate_ai_transition/zh/user.md b/prompts/tasks/generate_ai_transition/zh/user.md new file mode 100644 index 0000000..9cd67c1 --- /dev/null +++ b/prompts/tasks/generate_ai_transition/zh/user.md @@ -0,0 +1,2 @@ +以下是用户对转场的要求，请你在满足基本原则的情况下，设计出符合用户要求的转场prompt。 +{{user_request}} \ No newline at end of file diff --git a/prompts/tasks/instruction/en/system.md b/prompts/tasks/instruction/en/system.md index 224d0ed..19fc6a9 100644 --- a/prompts/tasks/instruction/en/system.md +++ b/prompts/tasks/instruction/en/system.md @@ -2,7 +2,8 @@ You are a video editing assistant. ## Skill Types and When to Use Them -- **【WORKFLOW SKILL】** is used to define the main flow of an editing task. When entering an editing task for the first time, first select and **invoke** the single most appropriate **【WORKFLOW SKILL】**, then present the editing plan to the user based on its contents. The actual editing steps may only be executed after the user confirms the plan. You must explicitly invoke the Skill; you may not work based only on its description. +* Before any actual editing begins, you **must** first select an appropriate **[WORKFLOW SKILL]**. +* A **[WORKFLOW SKILL]** defines the main workflow for a video editing task. When entering an editing task for the first time, you must first select and **invoke** the most appropriate **[WORKFLOW SKILL]**, then present the editing plan to the user based on its content. The actual editing steps may only be executed after the user has confirmed the plan. You must explicitly invoke that Skill; you are not allowed to work based only on its description. You must follow the instructions in the **[WORKFLOW SKILL]** completely when editing, and you are not allowed to call tools without a valid basis. - **【CAPABILITY SKILL】** is used to provide localized capability enhancements within the workflow, such as style imitation. It does not participate in the initial main workflow selection and is usually invoked on demand during the execution of a **【WORKFLOW SKILL】**. - **【META SKILL】** is used to create, modify, summarize, and manage skills. It does not directly handle the video editing workflow itself and must not be used as the default editing workflow. Only invoke a **【META SKILL】** when the user explicitly asks to create, modify, or manage a skill. @@ -20,6 +21,7 @@ You are a video editing assistant. - Unless the user explicitly wants to skip a certain step, when presenting the plan, **use as many tools as reasonably possible to enrich the video content**, unless the user explicitly states that they do not want a certain element. - Some steps depend on the results of earlier steps. You can find the specific dependency relationships in the tool descriptions. Check dependencies before calling a tool. The tools will locate dependency outputs on their own; you do not need to pass previous step results as tool parameters. If a tool requires input parameters, this will be separately specified in the tool description, and you should fill in appropriate values. - **Only call one tool at a time. Parallel tool calls are not allowed.** If multiple tools need to be called in sequence, after each tool call, briefly summarize the result of that tool call and your intent for the next step to the user, so the interaction feels more engaging, and then proceed to the next tool call. +- Although you can only see the summary after calling the tool, you have a `read_history` tool that can read the output of any intermediate node. You can use it to accomplish more complex tasks. ## Style Requirements - Use concise, conversational language. diff --git a/prompts/tasks/instruction/zh/system.md b/prompts/tasks/instruction/zh/system.md index 7b93db9..9fe9978 100644 --- a/prompts/tasks/instruction/zh/system.md +++ b/prompts/tasks/instruction/zh/system.md @@ -2,7 +2,8 @@ 你是一个视频剪辑助理。 ## Skill 类型与使用时机 -- 【WORKFLOW SKILL】用于定义一次剪辑任务的主流程。首次进入剪辑任务时，先选择并**调用**一个最合适的【WORKFLOW SKILL】，再根据其内容向用户列出剪辑计划，正式剪辑节点需在用户确认计划后再执行。你必须明确调用该 Skill，不允许仅根据 description 进行工作。。 +- 在正式剪辑开始之前，**必须**先选取一个合适的【WORKFLOW SKILL】。 +- 【WORKFLOW SKILL】用于定义一次剪辑任务的主流程。首次进入剪辑任务时，先选择并**调用**一个最合适的【WORKFLOW SKILL】，再根据其内容向用户列出剪辑计划，正式剪辑节点需在用户确认计划后再执行。你必须明确调用该 Skill，不允许仅根据 description 进行工作。你必须完全遵从【WORKFLOW SKILL】中的指示进行剪辑，不允许在没有依据的情况下调用工具。 - 【CAPABILITY SKILL】用于提供流程中的局部能力增强，例如文风仿写。它不参与首次主流程选择，通常只在【WORKFLOW SKILL】执行过程中按需调用。 - 【META SKILL】用于创建、修改、总结、管理 skill，本身不直接承担视频剪辑流程，也不作为默认剪辑流程使用。当用户明确要求创建、修改或管理 skill 时，才调用【META SKILL】。 @@ -20,6 +21,7 @@ - 除非用户明确想要跳过某个步骤，否则在列出计划时，**尽可能使用多的工具以丰富视频内容**，除非用户明确指出不要某个元素。 - 有些节点依赖前面节点的结果，具体的依赖关系你可以在工具描述中看到，请在工具调用前检查依赖。工具会自己寻找依赖的结果，你不需要将前面节点的结果输入到工具参数中。如果工具需要输入参数，会在工具描述中另加说明，请填入合适的参数。 - **每次只调用一个工具，不允许并行工具调用**。如果需要连续调用工具，每次调用完工具后，向用户简单总结本次工具调用的结果和下一步的意图，增强互动感，然后再进行下一次工具调用。 +- 虽然你在工具调用后只能看到summary，但你有一个`read_history`工具可以读取任意中间节点的输出。你可以用它完成更复杂的任务。 ## 风格要求 - 用简洁、口语化的语言 diff --git a/src/open_storyline/agent.py b/src/open_storyline/agent.py index 98431ce..e0bc4d9 100644 --- a/src/open_storyline/agent.py +++ b/src/open_storyline/agent.py @@ -138,6 +138,7 @@ class ClientContext: vlm_model_key: str = "" # VLM model key pexels_api_key: Optional[str] = None tts_config: Optional[dict] = None # TTS config at runtime + ai_transition_config: Optional[dict] = None # AI transition config at runtime llm_pool: dict[tuple[str, bool], ChatOpenAI] = field(default_factory=dict) lang: str = "zh" # Default language: Chinese diff --git a/src/open_storyline/config.py b/src/open_storyline/config.py index 380aa55..8f48812 100644 --- a/src/open_storyline/config.py +++ b/src/open_storyline/config.py @@ -157,6 +157,9 @@ class GenerateVoiceoverConfig(ConfigBaseModel): tts_provider_params_path: Path = Field(..., description="TTS provider config file path") providers: dict[str, dict[str, Any]] = Field(default_factory=dict) +class GenerateAITransitionConfig(ConfigBaseModel): + providers: dict[str, dict[str, Any]] = Field(default_factory=dict) + class SelectBGMConfig(ConfigBaseModel): sample_rate: int = 22050 hop_length: int = 2048 @@ -251,6 +254,7 @@ class Settings(ConfigBaseModel): group_clips: GroupClipsConfig = Field(default_factory=GroupClipsConfig) script_template: RecommendScriptTemplateConfig generate_voiceover: GenerateVoiceoverConfig + generate_ai_transition: GenerateAITransitionConfig select_bgm: SelectBGMConfig recommend_text: RecommendTextConfig plan_timeline: PlanTimelineConfig diff --git a/src/open_storyline/mcp/hooks/node_interceptors.py b/src/open_storyline/mcp/hooks/node_interceptors.py index c07f431..e1e670d 100644 --- a/src/open_storyline/mcp/hooks/node_interceptors.py +++ b/src/open_storyline/mcp/hooks/node_interceptors.py @@ -394,42 +394,74 @@ class ToolInterceptor: logger.error(f"Tool Call result: {result}") raise + @staticmethod + async def _inject_provider_config( + request, + handler, + *, + tool_name_keyword: str, + context_attr: str, + default_provider: str | None = None, + ): + try: + tool_name = str(getattr(request, "name", "") or "") + args = getattr(request, "args", None) + + if tool_name_keyword not in tool_name or not isinstance(args, dict): + return await handler(request) + + runtime = getattr(request, "runtime", None) + ctx = getattr(runtime, "context", None) if runtime else None + provider_cfg_all = getattr(ctx, context_attr, None) if ctx else None + if not isinstance(provider_cfg_all, dict): + return await handler(request) + + provider = str(provider_cfg_all.get("provider") or "").strip().lower() + if not provider: + if default_provider: + args.setdefault("provider", default_provider) + return await handler(request) + + args.setdefault("provider", provider) + + provider_cfg = provider_cfg_all.get(provider) + if isinstance(provider_cfg, dict): + for key, value in provider_cfg.items(): + if value is None: + continue + args.setdefault(key, str(value).strip()) + except Exception as e: + logger.warning(f"Failed to inject provider config ({context_attr}): {e}") + return await handler(request) + + @staticmethod async def inject_tts_config(request: MCPToolCallRequest, handler): """ Interceptor: Injects runtime.context.tts_config parameters into request.args before invoking voiceover/TTS tools. - tts_config: {"provider": "bytedance", "bytedance": {...}, "azure": {...}, ...} """ - try: - tool_name = str(getattr(request, "name", "") or "") - args = getattr(request, "args", None) + return await ToolInterceptor._inject_provider_config( + request, + handler, + tool_name_keyword="voiceover", + context_attr="tts_config", + default_provider="minimax", + ) - if "voiceover" not in tool_name or not isinstance(args, dict): - return await handler(request) - - runtime = getattr(request, "runtime", None) - ctx = getattr(runtime, "context", None) if runtime else None - tts_cfg = getattr(ctx, "tts_config", None) if ctx else None - if not isinstance(tts_cfg, dict): - return await handler(request) - - provider = str(tts_cfg.get("provider") or "").strip().lower() - - if not provider: - args.setdefault("provider", "302") - return await handler(request) - - args.setdefault("provider", provider) - - provider_cfg = tts_cfg.get(provider) - if isinstance(provider_cfg, dict): - for key, value in provider_cfg.items(): - if value is None: - continue - args.setdefault(key, str(value).strip()) - except Exception as e: - logger.warning(f"Failed to inject TTS config: {e}") - return await handler(request) + @staticmethod + async def inject_ai_transition_config(request: MCPToolCallRequest, handler): + """ + Interceptor: Injects runtime.context.ai_transition_config parameters into request.args + before invoking AI transition tools. + - ai_transition_config: {"provider": "dashscope", "dashscope": {...}, ...} + """ + return await ToolInterceptor._inject_provider_config( + request, + handler, + tool_name_keyword="generate_ai_transition", + context_attr="ai_transition_config", + ) @staticmethod async def inject_pexels_api_key(request: MCPToolCallRequest, handler): diff --git a/src/open_storyline/nodes/core_nodes/base_node.py b/src/open_storyline/nodes/core_nodes/base_node.py index f04fe7d..97ee44b 100644 --- a/src/open_storyline/nodes/core_nodes/base_node.py +++ b/src/open_storyline/nodes/core_nodes/base_node.py @@ -322,6 +322,7 @@ class BaseNode(ABC): 'isError': False } except Exception as e: + node_state.node_summary.add_error(str(e), artifact_id=node_state.artifact_id) if self.server_cfg.developer.developer_mode: traceback_info = ''.join(traceback.format_exception(e)) summary = { diff --git a/src/open_storyline/nodes/core_nodes/filter_clips.py b/src/open_storyline/nodes/core_nodes/filter_clips.py index 1bcecfb..cf187df 100644 --- a/src/open_storyline/nodes/core_nodes/filter_clips.py +++ b/src/open_storyline/nodes/core_nodes/filter_clips.py @@ -18,7 +18,7 @@ class FilterClipsNode(BaseNode): node_kind="filter_clips", require_prior_kind=['split_shots','understand_clips'], default_require_prior_kind=['split_shots','understand_clips'], - next_available_node=['group_clips', 'group_clips_pro'], + next_available_node=['group_clips'], ) input_schema = FilterClipsInput diff --git a/src/open_storyline/nodes/core_nodes/generate_ai_transition.py b/src/open_storyline/nodes/core_nodes/generate_ai_transition.py new file mode 100644 index 0000000..e13e817 --- /dev/null +++ b/src/open_storyline/nodes/core_nodes/generate_ai_transition.py @@ -0,0 +1,571 @@ +import base64 +from typing import Dict, Any, List, Union, Tuple, Optional +from pathlib import Path +import os +from io import BytesIO +from PIL import Image, ImageOps +from moviepy import VideoFileClip # MoviePy 2.x standard import + +from open_storyline.utils.register import NODE_REGISTRY +from open_storyline.utils.prompts import get_prompt +from open_storyline.utils.ai_transition_cancel import is_ai_transition_cancelled +from open_storyline.utils.ai_transition_client import VisionClientFactory +from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta +from open_storyline.nodes.node_state import NodeState +from open_storyline.nodes.node_schema import GenerateAITransitionInput + +def encode_image_to_data_url( + image: Image.Image, + format: str = "JPEG", + quality: int = 85, + max_long_edge: Optional[int] = None, +) -> str: + """ + Converts a PIL Image object into a Base64-encoded Data URL. + + Args: + image (Image.Image): The PIL Image instance to be encoded. + format (str): Image format for encoding ('JPEG', 'PNG', 'WEBP'). Defaults to 'JPEG'. + quality (int): Encoding quality for JPEG/WEBP (1-100). Higher is better quality but larger size. + max_long_edge (Optional[int]): If provided, downsample the image so its long edge + does not exceed this value while preserving aspect ratio. + + Returns: + str: A complete Data URL string (e.g., "data:image/jpeg;base64,..."). + """ + # 1. Optionally downsample the image to reduce payload size. + if max_long_edge and max_long_edge > 0: + width, height = image.size + long_edge = max(width, height) + if long_edge > max_long_edge: + scale = max_long_edge / float(long_edge) + new_size = ( + max(1, int(round(width * scale))), + max(1, int(round(height * scale))), + ) + image = image.resize(new_size, Image.Resampling.LANCZOS) + + # 2. Handle mode compatibility + # JPEG format does not support transparency (RGBA) or palette (P) modes. + # We must convert these to RGB to avoid "OSError: cannot write mode RGBA as JPEG". + save_format = format.upper() + if save_format == "JPEG": + if image.mode in ("RGBA", "P", "LA"): + image = image.convert("RGB") + mime_type = "image/jpeg" + elif save_format == "PNG": + mime_type = "image/png" + else: + mime_type = f"image/{save_format.lower()}" + + # 3. Save image to an in-memory byte buffer + # This avoids slow disk I/O and temporary file management. + buffered = BytesIO() + image.save( + buffered, + format=save_format, + quality=quality if save_format in ("JPEG", "WEBP") else None + ) + + # 4. Encode binary data to Base64 string + # getvalue() retrieves the bytes from the buffer, b64encode converts to base64 bytes, + # and decode('utf-8') converts it to a standard Python string. + base64_str = base64.b64encode(buffered.getvalue()).decode('utf-8') + + # 5. Format and return the standard Data URL pattern + return f"data:{mime_type};base64,{base64_str}" + + +@NODE_REGISTRY.register() +class GenerateAITransitionNode(BaseNode): + meta = NodeMeta( + name="generate_ai_transition", + description="Generate transition videos: Create transition videos for grouped video clips, generating an appropriate transition from the last frame of the previous clip to the first frame of the next clip based on user requirements.", + node_id="generate_ai_transition", + node_kind="generate_ai_transition", + require_prior_kind=["split_shots", "group_clips"], + default_require_prior_kind=['group_clips'], + next_available_node=["generate_script"], + ) + input_schema = GenerateAITransitionInput + VIDEO_EXTS = { + ".mp4", ".mov", ".mkv", ".avi" + } + IMAGE_EXTS = { + ".jpg", ".jpeg", ".png", ".webp", ".bmp" + } + + DEFAULT_TRANSITION_DURATION = 5 + SECOND_TO_MILLISECOND = 1000 + MAX_ASPECT_RATIO_FACTOR = 1.1 + def _raise_if_cancelled(self, node_state: NodeState) -> None: + if is_ai_transition_cancelled(self.server_cache_dir, node_state.session_id): + raise RuntimeError("generate_ai_transition cancelled by user") + + async def process(self, node_state: NodeState, inputs: Dict[str, Any]) -> Any: + group_clips = inputs.get("group_clips", {}) + split_shots = inputs.get("split_shots", {}) + groups = group_clips.get("groups", []) + clips = split_shots.get('clips', []) + + runtime_cfg = self._resolve_ai_transition_runtime_cfg(inputs) + provider = runtime_cfg["provider"] + api_key = runtime_cfg["api_key"] + model_name = runtime_cfg["model_name"] + transition_duration = inputs.get("duration") + resolution = inputs.get("resolution") + user_request = inputs.get("user_request", "以一镜到底的方式拍摄，场景丝滑过渡") + + clip_map = {clip['clip_id']: clip for clip in clips} + total_transitions = max(sum(len(group.get("clip_ids", [])) for group in groups) - 1, 0) + + await node_state.mcp_ctx.report_progress( + 0, + total_transitions, + "AI transition generation starting...", + ) + + node_cache_dir = self._prepare_output_directory(node_state) + + transition_info = {} + transition_index = 1 + transition_context = { + "clip_map": clip_map, + "node_state": node_state, + "node_cache_dir": node_cache_dir, + "provider": provider, + "api_key": api_key, + "model_name": model_name, + "transition_duration": transition_duration, + "resolution": resolution, + "user_request": user_request, + "total_transitions": total_transitions, + } + + for i, group in enumerate(groups): + group_clip_ids = group.get("clip_ids", []) + valid_group_clip_ids = [clip_id for clip_id in group_clip_ids if clip_id in clip_map] + if len(valid_group_clip_ids) != len(group_clip_ids): + missing_clip_ids = [clip_id for clip_id in group_clip_ids if clip_id not in clip_map] + node_state.node_summary.add_warning( + f"Clips <{missing_clip_ids}> not found in split_shots; they will be skipped in generate_ai_transition." + ) + + new_group_clip_ids = [] + transition_total_duration_sec = 0.0 + for clip_index, clip_id in enumerate(valid_group_clip_ids): + new_group_clip_ids.append(clip_id) + + if clip_index < len(valid_group_clip_ids) - 1: + next_clip_id = valid_group_clip_ids[clip_index + 1] + transition_result = await self._build_transition_clip( + from_clip_id=clip_id, + to_clip_id=next_clip_id, + transition_index=transition_index, + **transition_context, + ) + if transition_result: + transition_clip_id, transition_payload = transition_result + transition_info[transition_clip_id] = transition_payload + transition_index += 1 + new_group_clip_ids.append(transition_clip_id) + transition_total_duration_sec += self._transition_payload_duration_seconds(transition_payload) + + if i < len(groups) - 1 and valid_group_clip_ids: + next_group = groups[i + 1] + next_group_clip_ids = [clip_id for clip_id in next_group.get("clip_ids", []) if clip_id in clip_map] + if next_group_clip_ids: + transition_result = await self._build_transition_clip( + from_clip_id=valid_group_clip_ids[-1], + to_clip_id=next_group_clip_ids[0], + transition_index=transition_index, + **transition_context, + ) + if transition_result: + transition_clip_id, transition_payload = transition_result + transition_info[transition_clip_id] = transition_payload + transition_index += 1 + new_group_clip_ids.append(transition_clip_id) + transition_total_duration_sec += self._transition_payload_duration_seconds(transition_payload) + + base_duration_sec = self._parse_duration_seconds(group.get("duration", 0.0)) + group["clip_ids"] = new_group_clip_ids + group["duration"] = f"{base_duration_sec + transition_total_duration_sec:.1f}s" + + group_clips["groups"] = groups + group_clips["transition_info"] = transition_info + if total_transitions > 0: + await node_state.mcp_ctx.report_progress( + total_transitions, + total_transitions, + "AI transition generation finished", + ) + return group_clips + + async def default_process(self, node_state: NodeState, inputs: Dict[str, Any]) -> Any: + return inputs.get("group_clips", {}) + + def _is_complete_provider_cfg(self, cfg: Dict[str, Any], required_keys: list[str]) -> bool: + return all(cfg.get(k) not in (None, "") for k in required_keys) + + def _get_provider_cfg(self, provider_name: str) -> Dict[str, Any]: + providers = getattr(self.server_cfg.generate_ai_transition, "providers", None) or {} + cfg = providers.get(provider_name) + if not isinstance(cfg, dict): + raise ValueError(f"provider={provider_name} not configured in config.toml") + + return cfg + + def _resolve_ai_transition_runtime_cfg(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + provider = str(inputs.get("provider") or "").strip().lower() or "minimax" + config_cfg = self._get_provider_cfg(provider) + + required_keys = list(config_cfg.keys()) + + frontend_cfg = {k: inputs.get(k) for k in required_keys} + + if self._is_complete_provider_cfg(frontend_cfg, required_keys): + final_cfg = frontend_cfg + elif self._is_complete_provider_cfg(config_cfg, required_keys): + final_cfg = config_cfg + else: + missing = [k for k in required_keys if config_cfg.get(k) in (None, "")] + raise ValueError( + f"provider={provider} missing required fields: {missing}. " + f"Please configure in sidebar or config.toml." + ) + + return {"provider": provider, **final_cfg} + + async def _build_transition_clip( + self, + *, + from_clip_id: str, + to_clip_id: str, + clip_map: Dict[str, Any], + node_state: NodeState, + node_cache_dir: Path, + provider: str, + api_key: str, + model_name: str, + transition_duration: Optional[int], + resolution: Optional[str], + transition_index: int, + total_transitions: int, + user_request: str, + ) -> Optional[Tuple[str, Dict[str, Any]]]: + self._raise_if_cancelled(node_state) + prev_clip = clip_map.get(from_clip_id) + next_clip = clip_map.get(to_clip_id) + if not prev_clip or not next_clip: + node_state.node_summary.add_warning( + f"Clips <{from_clip_id}, {to_clip_id}> not found in split_shots; skipping transition generation." + ) + return None + + prev_clip_size = self._extract_clip_size(prev_clip) + next_clip_size = self._extract_clip_size(next_clip) + if not self._is_aspect_ratio_compatible(prev_clip_size, next_clip_size): + node_state.node_summary.info_for_user( + f"Skipped AI transition for clips <{from_clip_id}, {to_clip_id}> " + f"because aspect ratios differ too much: " + f"{self._format_size(prev_clip_size)} -> {self._format_size(next_clip_size)}." + ) + return None + + prev_frames = self._load_clip(prev_clip.get("path")) + next_frames = self._load_clip(next_clip.get("path")) + + first_frame = prev_frames[-1] + last_frame = next_frames[0] + + aligned_first_frame, aligned_last_frame, _, _ = self._preprocess_first_last_frame( + first_frame, + last_frame, + ) + + llm = node_state.llm + + meta_system_prompt = get_prompt("generate_ai_transition.system", lang=node_state.lang) + meta_user_prompt = get_prompt("generate_ai_transition.user", lang=node_state.lang, user_request=user_request) + + prompt = await llm.complete( + system_prompt=meta_system_prompt, + user_prompt=meta_user_prompt, + media=[ + {"url": encode_image_to_data_url(aligned_first_frame, quality=80, max_long_edge=768)}, + {"url": encode_image_to_data_url(aligned_last_frame, quality=80, max_long_edge=768)}, + ], + temperature=0.3, + top_p=0.9, + max_tokens=1024, + model_preferences=None + ) + self._raise_if_cancelled(node_state) + + gen_video_path, _, effective_duration = self._generate_video( + provider=provider, + api_key=api_key, + model_name=model_name, + prompt=prompt, + first_frame_data_url=encode_image_to_data_url(aligned_first_frame), + last_frame_data_url=encode_image_to_data_url(aligned_last_frame), + duration=transition_duration, + resolution=resolution, + output_dir=node_cache_dir, + cancel_checker=lambda: is_ai_transition_cancelled(self.server_cache_dir, node_state.session_id), + ) + + with VideoFileClip(str(gen_video_path)) as generated_clip: + fps = float(generated_clip.fps or 0) + width, height = map(int, generated_clip.size) + duration_ms = int(round((generated_clip.duration or effective_duration or self.DEFAULT_TRANSITION_DURATION) * self.SECOND_TO_MILLISECOND)) + await node_state.mcp_ctx.report_progress( + transition_index, + total_transitions or transition_index, + f"AI transition {transition_index}/{total_transitions or transition_index} generated", + ) + node_state.node_summary.info_for_user( + f"AI transition for clips <{from_clip_id}, {to_clip_id}> succeeded", + preview_urls=[gen_video_path] + ) + + transition_clip_id = f"transition_{transition_index:04d}" + return transition_clip_id, { + "fps": fps, + "path": gen_video_path, + "source_ref": { + "duration_ms": duration_ms, + "width": width, + "height": height, + } + } + + def _extract_clip_size(self, clip: Dict[str, Any]) -> Optional[Tuple[int, int]]: + source_ref = clip.get("source_ref") or {} + size = clip.get("size") + + width = source_ref.get("width") + height = source_ref.get("height") + if width and height: + return int(width), int(height) + + if isinstance(size, (list, tuple)) and len(size) >= 2 and size[0] and size[1]: + return int(size[0]), int(size[1]) + + return None + + def _is_aspect_ratio_compatible( + self, + first_size: Optional[Tuple[int, int]], + second_size: Optional[Tuple[int, int]], + ) -> bool: + if not first_size or not second_size: + return False + + first_ratio = self._aspect_ratio(first_size) + second_ratio = self._aspect_ratio(second_size) + if first_ratio is None or second_ratio is None: + return False + + ratio_factor = max(first_ratio, second_ratio) / min(first_ratio, second_ratio) + return ratio_factor <= self.MAX_ASPECT_RATIO_FACTOR + + def _aspect_ratio(self, size: Tuple[int, int]) -> Optional[float]: + width, height = size + if width <= 0 or height <= 0: + return None + return width / height + + def _format_size(self, size: Optional[Tuple[int, int]]) -> str: + if not size: + return "unknown" + return f"{size[0]}x{size[1]}" + + def _parse_duration_seconds(self, duration: Any) -> float: + if isinstance(duration, (int, float)): + return float(duration) + if isinstance(duration, str): + normalized = duration.strip().lower() + if normalized.endswith("s"): + normalized = normalized[:-1] + try: + return float(normalized) + except ValueError: + return 0.0 + return 0.0 + + def _transition_payload_duration_seconds(self, transition_payload: Dict[str, Any]) -> float: + source_ref = transition_payload.get("source_ref") or {} + duration_ms = source_ref.get("duration_ms", 0) + try: + return max(0.0, float(duration_ms) / self.SECOND_TO_MILLISECOND) + except (TypeError, ValueError): + return 0.0 + + def _load_clip( + self, + image_or_video_path: Union[str, Path] + ) -> List[Image.Image]: + """ + Loads media frames using PIL for images and MoviePy 2.2.1 for videos. + + Args: + image_or_video_path: Path to the media file. + + Returns: + List[Image.Image]: A list of frames as PIL Image objects in RGB mode. + """ + path = Path(image_or_video_path) + if not path.exists(): + raise FileNotFoundError(f"Media file not found: {path}") + + ext = path.suffix.lower() + frames: List[Image.Image] = [] + + # --- Process Images --- + if ext in self.IMAGE_EXTS: + try: + # Open with PIL and force RGB mode + with Image.open(path) as img: + frames.append(ImageOps.exif_transpose(img).convert("RGB")) + except Exception as e: + raise RuntimeError(f"Failed to load image via PIL: {path}. Error: {e}") + + # --- Process Videos --- + elif ext in self.VIDEO_EXTS: + try: + # VideoFileClip in v2.x works best within a context manager + with VideoFileClip(str(path)) as clip: + # iter_frames yields RGB numpy arrays by default + for frame_array in clip.iter_frames(): + # Image.fromarray converts the numpy array (RGB) to a PIL object + frames.append(Image.fromarray(frame_array)) + except Exception as e: + raise RuntimeError(f"MoviePy failed to decode video: {path}. Error: {e}") + + if not frames: + raise RuntimeError(f"Extraction resulted in an empty frame list for: {path}") + + else: + raise ValueError(f"File extension {ext} is not supported by this processor.") + + return frames + + def _preprocess_first_last_frame( + self, + first_frame: Image.Image, + last_frame: Image.Image, + target_width: Optional[int] = None, + target_height: Optional[int] = None + ) -> Tuple[Image.Image, Image.Image, Dict[str, Any], Dict[str, Any]]: + """ + Normalizes color modes and aligns both frames to a target resolution. + + Args: + first_frame (Image.Image): The starting frame. + last_frame (Image.Image): The ending frame. + target_width (Optional[int]): Desired output width. Defaults to first_frame width. + target_height (Optional[int]): Desired output height. Defaults to first_frame height. + + Returns: + Tuple[Image.Image, Image.Image, Dict, Dict]: + (Aligned First Frame, Aligned Last Frame, First Frame Meta, Last Frame Meta) + """ + + # 1. Determine Target Resolution + # Use provided dimensions or fallback to the first frame's original size + if target_width and target_height: + target_size = (target_width, target_height) + else: + target_size = first_frame.size + + # 2. Color Mode Normalization (RGB) + # Required for API compatibility (removes Alpha/transparency channels) + def normalize_img(img: Image.Image) -> Image.Image: + return img.convert("RGB") if img.mode != "RGB" else img + + first_frame = normalize_img(first_frame) + last_frame = normalize_img(last_frame) + + # 3. Helper for Resizing and Metadata Logging + def process_frame(img: Image.Image, role: str) -> Tuple[Image.Image, Dict[str, Any]]: + meta = { + "role": role, + "original_size": img.size, + "target_size": target_size, + "transformations": [] + } + + if img.size != target_size: + src_w, src_h = img.size + dst_w, dst_h = target_size + scale = max(dst_w / src_w, dst_h / src_h) + resized_size = ( + max(1, int(round(src_w * scale))), + max(1, int(round(src_h * scale))), + ) + + resized_img = img.resize(resized_size, Image.Resampling.LANCZOS) + crop_box = ( + max(0, (resized_size[0] - dst_w) // 2), + max(0, (resized_size[1] - dst_h) // 2), + max(0, (resized_size[0] - dst_w) // 2) + dst_w, + max(0, (resized_size[1] - dst_h) // 2) + dst_h, + ) + img = resized_img.crop(crop_box) + + meta["transformations"].append({ + "type": "resize_with_center_crop", + "method": "LANCZOS", + "resized_size": resized_size, + "target": target_size, + "crop_box": crop_box, + }) + else: + meta["transformations"].append({"type": "none", "reason": "already_correct_size"}) + + return img, meta + + # 4. Execute Processing for both frames + aligned_first_frame, first_frame_meta = process_frame(first_frame, "first") + aligned_last_frame, last_frame_meta = process_frame(last_frame, "last") + + # Final pixel-perfect validation + assert aligned_first_frame.size == aligned_last_frame.size == target_size + + return aligned_first_frame, aligned_last_frame, first_frame_meta, last_frame_meta + + def _generate_video( + self, + provider, + api_key, + model_name, + prompt, + first_frame_data_url, + last_frame_data_url, + output_dir, + duration=None, + resolution=None, + cancel_checker=None, + ) -> Tuple[str, Dict[str, Any], int]: + client = VisionClientFactory.create( + provider=provider, + api_key=api_key, + cancel_checker=cancel_checker, + ) + effective_duration = int(duration) if duration is not None else int(client.duration) + + gen_video_path, response = client.generate( + task_type="video_generation", + model=model_name, + prompt=prompt, + first_frame=first_frame_data_url, + last_frame=last_frame_data_url, + resolution=resolution, + duration=duration, + prompt_optimizer=True, + output_dir=output_dir, + ) + + return gen_video_path, response, effective_duration diff --git a/src/open_storyline/nodes/core_nodes/generate_video.py b/src/open_storyline/nodes/core_nodes/generate_video.py deleted file mode 100644 index c9d3fb5..0000000 --- a/src/open_storyline/nodes/core_nodes/generate_video.py +++ /dev/null @@ -1,285 +0,0 @@ -import base64 -from typing import Dict, Any, List, Union, Tuple, Optional -from pathlib import Path -import os -from io import BytesIO -from PIL import Image -from moviepy import VideoFileClip # MoviePy 2.x standard import - -from open_storyline.utils.register import NODE_REGISTRY -from open_storyline.utils.prompts import get_prompt -from open_storyline.utils.client import VisionClientFactory -from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta -from open_storyline.nodes.node_state import NodeState -from open_storyline.nodes.node_schema import AIGCTransitionInput - -def encode_image_to_data_url(image: Image.Image, format: str = "JPEG", quality: int = 85) -> str: - """ - Converts a PIL Image object into a Base64-encoded Data URL. - - Args: - image (Image.Image): The PIL Image instance to be encoded. - format (str): Image format for encoding ('JPEG', 'PNG', 'WEBP'). Defaults to 'JPEG'. - quality (int): Encoding quality for JPEG/WEBP (1-100). Higher is better quality but larger size. - - Returns: - str: A complete Data URL string (e.g., "data:image/jpeg;base64,..."). - """ - # 1. Handle mode compatibility - # JPEG format does not support transparency (RGBA) or palette (P) modes. - # We must convert these to RGB to avoid "OSError: cannot write mode RGBA as JPEG". - save_format = format.upper() - if save_format == "JPEG": - if image.mode in ("RGBA", "P", "LA"): - image = image.convert("RGB") - mime_type = "image/jpeg" - elif save_format == "PNG": - mime_type = "image/png" - else: - mime_type = f"image/{save_format.lower()}" - - # 2. Save image to an in-memory byte buffer - # This avoids slow disk I/O and temporary file management. - buffered = BytesIO() - image.save( - buffered, - format=save_format, - quality=quality if save_format in ("JPEG", "WEBP") else None - ) - - # 3. Encode binary data to Base64 string - # getvalue() retrieves the bytes from the buffer, b64encode converts to base64 bytes, - # and decode('utf-8') converts it to a standard Python string. - base64_str = base64.b64encode(buffered.getvalue()).decode('utf-8') - - # 4. Format and return the standard Data URL pattern - return f"data:{mime_type};base64,{base64_str}" - - -@NODE_REGISTRY.register() -class FirstLastFrameToVideoNode(BaseNode): - meta = NodeMeta( - name="aigc_flf2v", - description="Generate transition videos: Create transition videos for grouped video clips, generating an appropriate transition from the last frame of the previous clip to the first frame of the next clip based on user requirements.", - node_id="aigc_flf2v", - node_kind="aigc_flf2v", - require_prior_kind=["split_shots", "group_clips"], - ) - input_schema = AIGCTransitionInput - VIDEO_EXTS = { - ".mp4", ".mov", ".mkv", ".avi" - } - IMAGE_EXTS = { - ".jpg", ".jpeg", ".png", ".webp", ".bmp" - } - - async def process(self, node_state: NodeState, inputs: Dict[str, Any]) -> Any: - group_clips = inputs.get("group_clips", {}) - split_shots = inputs.get("split_shots", {}) - groups = group_clips.get("groups", []) - clips = split_shots.get('clips', []) - provider = inputs.get("provider", "minimax") - api_key = inputs.get("api_key", "") - model_name = inputs.get("model_name", "MiniMax-Hailuo-02") - - clip_map = {clip['clip_id']: clip for clip in clips} - - node_cache_dir = self._prepare_output_directory(node_state) - - transitions_result = [] - for i in range(len(groups) - 1): - prev_group = groups[i] - next_group = groups[i+1] - - last_clip_id_of_prev = prev_group['clip_ids'][-1] - first_clip_id_of_next = next_group['clip_ids'][0] - - prev_clip = clip_map.get(last_clip_id_of_prev) - next_clip = clip_map.get(first_clip_id_of_next) - - if not prev_clip or not next_clip: - node_state.node_summary.add_warning(f"Clips <{prev_group['clip_ids']}, {next_group['clip_ids']}> not found in split_shots; skipping transition generation.") - transitions_result.append({}) - continue - - prev_frames = self._load_clip(prev_clip.get('path')) - next_frames = self._load_clip(next_clip.get('path')) - prev_clip_type = 'V' if len(prev_frames) > 1 else 'I' - next_clip_type = 'V' if len(next_frames) > 1 else 'I' - - first_frame = prev_frames[-1] - last_frame = next_frames[0] - - aligned_first_frame, aligned_last_frame, first_frame_meta, last_frame_meta = self._preprocess_first_last_frame(first_frame, last_frame) - - gen_video_path, response = self._generate_video( - provider=provider, - api_key=api_key, - model_name=model_name, - prompt="以一镜到底的方式拍摄，场景丝滑过渡，摄像机快速推进", - first_frame_data_url=encode_image_to_data_url(aligned_first_frame), - last_frame_data_url=encode_image_to_data_url(aligned_last_frame), - output_dir=node_cache_dir - ) - transitions_result.append({ - 'from_group': prev_group['group_id'], - 'to_group': next_group['group_id'], - 'from_clip': last_clip_id_of_prev, - 'to_clip': first_clip_id_of_next, - 'transition_video_path': gen_video_path, - 'transition_type': f"{prev_clip_type}{next_clip_type}", - 'extra_info': { - "first_frame_meta": first_frame_meta, - "last_frame_meta": last_frame_meta, - "raw_request_result": response, - } - }) - return {"aigc_flf2v": transitions_result} - - async def default_process(self, node_state: NodeState, inputs: Dict[str, Any]) -> Any: - return await self.process(node_state, inputs) - - def _load_clip( - self, - image_or_video_path: Union[str, Path] - ) -> List[Image]: - """ - Loads media frames using PIL for images and MoviePy 2.2.1 for videos. - - Args: - image_or_video_path: Path to the media file. - - Returns: - List[Image.Image]: A list of frames as PIL Image objects in RGB mode. - """ - path = Path(image_or_video_path) - if not path.exists(): - raise FileNotFoundError(f"Media file not found: {path}") - - ext = path.suffix.lower() - frames: List[Image.Image] = [] - - # --- Process Images --- - if ext in self.IMAGE_EXTS: - try: - # Open with PIL and force RGB mode - with Image.open(path) as img: - frames.append(img.convert("RGB")) - except Exception as e: - raise RuntimeError(f"Failed to load image via PIL: {path}. Error: {e}") - - # --- Process Videos --- - elif ext in self.VIDEO_EXTS: - try: - # VideoFileClip in v2.x works best within a context manager - with VideoFileClip(str(path)) as clip: - # iter_frames yields RGB numpy arrays by default - for frame_array in clip.iter_frames(): - # Image.fromarray converts the numpy array (RGB) to a PIL object - frames.append(Image.fromarray(frame_array)) - except Exception as e: - raise RuntimeError(f"MoviePy failed to decode video: {path}. Error: {e}") - - if not frames: - raise RuntimeError(f"Extraction resulted in an empty frame list for: {path}") - - else: - raise ValueError(f"File extension {ext} is not supported by this processor.") - - return frames - - def _preprocess_first_last_frame( - self, - first_frame: Image.Image, - last_frame: Image.Image, - target_width: Optional[int] = None, - target_height: Optional[int] = None - ) -> Tuple[Image.Image, Image.Image, Dict[str, Any], Dict[str, Any]]: - """ - Normalizes color modes and aligns both frames to a target resolution. - - Args: - first_frame (Image.Image): The starting frame. - last_frame (Image.Image): The ending frame. - target_width (Optional[int]): Desired output width. Defaults to first_frame width. - target_height (Optional[int]): Desired output height. Defaults to first_frame height. - - Returns: - Tuple[Image.Image, Image.Image, Dict, Dict]: - (Aligned First Frame, Aligned Last Frame, First Frame Meta, Last Frame Meta) - """ - - # 1. Determine Target Resolution - # Use provided dimensions or fallback to the first frame's original size - if target_width and target_height: - target_size = (target_width, target_height) - else: - target_size = first_frame.size - - # 2. Color Mode Normalization (RGB) - # Required for API compatibility (removes Alpha/transparency channels) - def normalize_img(img: Image.Image) -> Image.Image: - return img.convert("RGB") if img.mode != "RGB" else img - - first_frame = normalize_img(first_frame) - last_frame = normalize_img(last_frame) - - # 3. Helper for Resizing and Metadata Logging - def process_frame(img: Image.Image, role: str) -> Tuple[Image.Image, Dict[str, Any]]: - meta = { - "role": role, - "original_size": img.size, - "target_size": target_size, - "transformations": [] - } - - if img.size != target_size: - # High-quality resampling for AI generation stability - img = img.resize(target_size, Image.Resampling.LANCZOS) - meta["transformations"].append({ - "type": "resize", - "method": "LANCZOS", - "target": target_size - }) - else: - meta["transformations"].append({"type": "none", "reason": "already_correct_size"}) - - return img, meta - - # 4. Execute Processing for both frames - aligned_first_frame, first_frame_meta = process_frame(first_frame, "first") - aligned_last_frame, last_frame_meta = process_frame(last_frame, "last") - - # Final pixel-perfect validation - assert aligned_first_frame.size == aligned_last_frame.size == target_size - - return aligned_first_frame, aligned_last_frame, first_frame_meta, last_frame_meta - - def _generate_video( - self, - provider, - api_key, - model_name, - prompt, - first_frame_data_url, - last_frame_data_url, - output_dir - ) -> Tuple[str, Dict[str, Any]]: - client = VisionClientFactory.create( - provider=provider, - api_key=api_key, - ) - - gen_video_path, response = client.generate( - task_type="video_generation", - model=model_name, - prompt=prompt, - first_frame=first_frame_data_url, - last_frame=last_frame_data_url, - resolution="480P", - duration=5, - prompt_optimizer=True, - output_dir=output_dir, - ) - - return gen_video_path, response diff --git a/src/open_storyline/nodes/core_nodes/plan_timeline_ai_transition.py b/src/open_storyline/nodes/core_nodes/plan_timeline_ai_transition.py new file mode 100644 index 0000000..3819d21 --- /dev/null +++ b/src/open_storyline/nodes/core_nodes/plan_timeline_ai_transition.py @@ -0,0 +1,173 @@ +from typing import List, Dict, Tuple, Union, Any +from src.open_storyline.config import Settings +from open_storyline.nodes.node_state import NodeState +from open_storyline.nodes.core_nodes.base_node import BaseNode, NodeMeta +from open_storyline.nodes.node_schema import PlanTimelineAITransitionInput +from open_storyline.utils.register import NODE_REGISTRY + + +@NODE_REGISTRY.register() +class PlanTimelineAITransitionNode(BaseNode): + + meta = NodeMeta( + name="plan_timeline_ai_transition", + description=( + "Create a coherent timeline for AI generated . " + ), + node_id="plan_timeline_ai_transition", + node_kind="plan_timeline", + require_prior_kind=["split_shots", "generate_ai_transition", "music_rec"], + default_require_prior_kind=["split_shots", "generate_ai_transition", "music_rec"], + next_available_node=["render_video"], + ) + + input_schema = PlanTimelineAITransitionInput + + + def __init__(self, server_cfg: Settings) -> None: + super().__init__(server_cfg) + + async def default_process( + self, + node_state: NodeState, + inputs: Dict[str, Any], + ) -> Any: + return await self.process(node_state, inputs) + + async def process(self, node_state: NodeState, inputs: Dict[str, Any]) -> Any: + clips = inputs.get("split_shots", {}).get("clips", []) + generate_ai_transition = inputs.get("generate_ai_transition", {}) + groups = generate_ai_transition.get("groups", []) + transition_info = generate_ai_transition.get("transition_info", {}) + music = inputs.get("music_rec", {}).get("bgm", {}) + + image_duration_ms = inputs.get("image_duration_ms", 2000) + clips_by_clip_id = {clip.get("clip_id", ""): clip for clip in clips} + + current_cursor = 0 + video_segments = [] + bgm_segments = [] + + for group in groups: + clip_ids = group.get("clip_ids") + for clip in clip_ids: + if self._is_transition_clip_id(clip): + current_transition_info = transition_info.get(clip, {}) + current_transition_duration = current_transition_info.get("source_ref", {}).get("duration_ms", 0) + video_segments.append({ + "clip_id": clip, + "group_id": group.get("group_id"), + "kind": "video", + "fps": current_transition_info.get("fps", 0), + "size": [ + current_transition_info.get("source_ref", {}).get("width", 0), + current_transition_info.get("source_ref", {}).get("height", 0), + ], + "source_path": current_transition_info.get("path", ""), + "source_window": { + "start": 0, + "end": current_transition_duration, + "duration": current_transition_duration + }, + "timeline_window": { + "start": current_cursor, + "end": current_cursor + current_transition_duration, + "duration": current_transition_duration + }, + "playback_rate": 1.0 + }) + current_cursor += current_transition_duration + else: + current_clip_info = clips_by_clip_id.get(clip, {}) + current_clip_kind = current_clip_info.get("kind") + current_clip_source_ref = current_clip_info.get("source_ref", {}) + current_clip_duration = image_duration_ms if current_clip_info.get("kind") == "image" else current_clip_info.get("source_ref", {}).get("duration", 0) + video_segments.append({ + "clip_id": clip, + "group_id": group.get("group_id"), + "kind": current_clip_kind, + "fps": current_clip_info.get("fps", 0), + "size": [ + current_clip_info.get("source_ref", {}).get("width", 0), + current_clip_info.get("source_ref", {}).get("height", 0), + ], + "source_path": current_clip_info.get("path", ""), + "source_window": { + "start": 0, + "end": current_clip_duration, + "duration": current_clip_duration + }, + "timeline_window": { + "start": current_cursor, + "end": current_cursor + current_clip_duration, + "duration": current_clip_duration + }, + "playback_rate": 1.0 + }) + current_cursor += current_clip_duration + + bgm_segments = self._build_bgm_track( + background_music=music, + total_duration_ms=current_cursor, + ) + + return { + "tracks": { + "video": video_segments, + "subtitles": [], + "voiceover": [], + "bgm": bgm_segments, + } + } + + def _is_transition_clip_id(self, clip_id: Any) -> bool: + return isinstance(clip_id, str) and clip_id.startswith("transition_") + + def _build_bgm_track( + self, + *, + background_music: dict[str, Any] | None, + total_duration_ms: int, + ) -> List[Dict[str, Any]]: + bgm_segments: List[Dict[str, Any]] = [] + if not background_music: + return bgm_segments + + music_duration_ms = int(background_music.get("duration", 0)) + if music_duration_ms <= 0: + return bgm_segments + + timeline_cursor_ms: int = 0 + source_cursor_ms: int = 0 + loop_index = 0 + + while timeline_cursor_ms < total_duration_ms: + remaining_timeline_ms = total_duration_ms - timeline_cursor_ms + remaining_source_ms = max(0, music_duration_ms - source_cursor_ms) + + if remaining_source_ms <= 0: + source_cursor_ms = 0 + loop_index += 1 + continue + + segment_duration_ms = min(remaining_timeline_ms, remaining_source_ms) + if segment_duration_ms <= 0: + break + + bgm_segments.append( + { + "bgm_id": background_music.get("bgm_id"), + "path": background_music.get("path"), + "source_window": {"start": source_cursor_ms, "end": source_cursor_ms + segment_duration_ms}, + "loop_idx": loop_index, + } + ) + + timeline_cursor_ms += segment_duration_ms + source_cursor_ms += segment_duration_ms + + if timeline_cursor_ms < total_duration_ms and source_cursor_ms >= music_duration_ms: + source_cursor_ms = 0 + loop_index += 1 + + return bgm_segments diff --git a/src/open_storyline/nodes/core_nodes/plan_timeline_pro.py b/src/open_storyline/nodes/core_nodes/plan_timeline_pro.py index 517ab60..7970307 100644 --- a/src/open_storyline/nodes/core_nodes/plan_timeline_pro.py +++ b/src/open_storyline/nodes/core_nodes/plan_timeline_pro.py @@ -31,7 +31,7 @@ class TimeLine: ''' Re-edit meterial durations according to tts duration or beats. ''' - if 'is_speech_rough_cut' in kwargs and kwargs['is_speech_rough_cut'] is True: + if kwargs.get("is_speech_rough_cut", False) or kwargs.get("is_ai_transition", False): return 0, meterial_durations, [1.0 for _ in meterial_durations], [0 for _ in meterial_durations] min_single_text_duration, max_text_duration = cfg.min_single_text_duration, cfg.max_text_duration @@ -46,7 +46,6 @@ class TimeLine: else: new_meterial_durations = meterial_durations time_margins = [0 for _ in range(len(meterial_durations))] - node_state.node_summary.add_error(f"Check config, one of `is_use_beats` and `is_use_tts` must be true.") else: if music: # get beats @@ -57,7 +56,6 @@ class TimeLine: else: new_meterial_durations = meterial_durations time_margins = [0 for _ in range(len(meterial_durations))] - node_state.node_summary.add_error(f"Check config, one of `is_use_beats` and `is_use_tts` must be true.") # edit speed speeds = [1.0 if old_duration > new_duration or _type == 'img' else old_duration / new_duration for _type, old_duration, new_duration in zip(types, meterial_durations, new_meterial_durations)] @@ -355,8 +353,8 @@ class PlanTimelineProNode(BaseNode): ), node_id="plan_timeline_pro", node_kind="plan_timeline", - require_prior_kind=["split_shots", "speech_rough_cut", "group_clips", "generate_script", "tts", "music_rec"], - default_require_prior_kind=["split_shots", "group_clips", "generate_script", "tts", "music_rec"], + require_prior_kind=["split_shots", "speech_rough_cut", "group_clips", "generate_ai_transition", "generate_script", "tts", "music_rec"], + default_require_prior_kind=["split_shots", "group_clips", "generate_ai_transition", "generate_script", "tts", "music_rec"], next_available_node=["render_video"], ) @@ -376,10 +374,10 @@ class PlanTimelineProNode(BaseNode): return await self.process(node_state, inputs) async def process(self, node_state: NodeState, inputs: Dict[str, Any]) -> Any: - music = inputs.pop("music", None) tts_res = inputs.pop("tts_res", None) is_speech_rough_cut = inputs.get("is_speech_rough_cut", False) + is_ai_transition = inputs.get("is_ai_transition", False) # Processing clip durations music_offset, new_meterial_durations, speeds, time_margins = self.timeline_client.edit_meterial_timeline( @@ -395,6 +393,7 @@ class PlanTimelineProNode(BaseNode): title_clip_duration=inputs.get('title_clip_duration', 0), is_on_beats=inputs.get('is_on_beats', False), is_speech_rough_cut=is_speech_rough_cut, + is_ai_transition=is_ai_transition, ) # Processing tts durations @@ -439,7 +438,7 @@ class PlanTimelineProNode(BaseNode): 'text_clip_maps': text_clip_maps, 'tts_start_timestamps': tts_start_timestamps, } - + def _combine_tool_outputs(self, node_state, outputs): """ Change output format. @@ -560,11 +559,13 @@ class PlanTimelineProNode(BaseNode): split_shots = inputs.get("split_shots", {}) group_clips = inputs.get("group_clips", {}) + generate_ai_transition = inputs.get("generate_ai_transition", {}) generate_script = inputs.get("generate_script", {}) - music = inputs.get("music_rec", None).get("bgm", {}) + music = (inputs.get("music_rec") or {}).get("bgm", {}) tts_res = inputs.get("tts", {}).get("voiceover", []) use_beats = inputs.get("use_beats", False) is_speech_rough_cut = inputs.get("is_speech_rough_cut", False) + is_ai_transition = inputs.get("is_ai_transition", False) speech_rough_cut = inputs.get("speech_rough_cut") texts, types = [], [] clips, clip_ids, clip_idxes = [], [], [] @@ -574,6 +575,85 @@ class PlanTimelineProNode(BaseNode): text_indices_map = {} tts_group_ids, voiceover_ids, tts_durations, tts_paths = [], [], [], [] + if is_ai_transition is True: + transition_info = generate_ai_transition.get("transition_info", {}) + groups = generate_ai_transition.get("groups", []) + image_duration_ms = int(inputs.get("image_duration_ms", 3000) or 3000) + start_times, fps, sizes = [], [], [] + clips_by_clip_id = {clip.get("clip_id", ""): clip for clip in split_shots.get("clips", [])} + + for group in groups: + group_id = group.get("group_id", "") + for clip_id in group.get("clip_ids", []) or []: + if isinstance(clip_id, str) and clip_id.startswith("transition_"): + transition_clip = transition_info.get(clip_id, {}) + transition_source_ref = transition_clip.get("source_ref", {}) + transition_duration = int(transition_source_ref.get("duration_ms", 0) or 0) + if transition_duration <= 0: + continue + + clip_ids.append(clip_id) + clip_group_ids.append(group_id) + clips.append(transition_clip.get("path", "")) + types.append("video") + clip_durations.append(transition_duration) + start_times.append(0) + fps.append(transition_clip.get("fps", 0)) + sizes.append([ + transition_source_ref.get("width", 576), + transition_source_ref.get("height", 1024), + ]) + continue + + clip_info = clips_by_clip_id.get(clip_id, {}) + if not clip_info: + continue + + clip_source_ref = clip_info.get("source_ref", {}) + clip_kind = clip_info.get("kind", "") + clip_duration = image_duration_ms if clip_kind == "image" else int(clip_source_ref.get("duration", 0) or 0) + if clip_duration <= 0: + continue + + clip_ids.append(clip_id) + clip_group_ids.append(group_id) + clips.append(clip_info.get("path", "")) + types.append(clip_kind) + clip_durations.append(clip_duration) + start_times.append(0 if clip_kind == "image" else int(clip_source_ref.get("start", 0) or 0)) + fps.append(clip_info.get("fps", None)) + sizes.append([ + clip_source_ref.get("width", 576), + clip_source_ref.get("height", 1024), + ]) + + return { + 'is_ai_transition': True, + 'types': types, + 'texts': [], + 'text_unit_ids': [], + 'text_group_ids': [], + 'text_index_in_group': [], + 'clips': clips, + 'clip_ids': clip_ids, + 'clip_group_ids': clip_group_ids, + 'fps': fps, + 'sizes': sizes, + 'clip_durations': clip_durations, + 'start_times': start_times, + 'text_indices_map': {}, + 'music': music, + 'tts_res': [], + 'tts_group_ids': [], + 'voiceover_ids': [], + 'tts_durations': [], + 'tts_paths': [], + 'is_on_beats': False, + 'is_speech_rough_cut': False, + 'speech_rough_cut': None, + 'title_clip_duration': 0, + } + if is_speech_rough_cut is True and speech_rough_cut is None: node_state.node_summary.add_error("The input clips are from speech rough cut, but no clips info is found in the input. Check whether the prior node `speech_rough_cut` output is correct.") raise ValueError("The input clips are from speech rough cut, but no clips info is found in the input. Check whether the prior node `speech_rough_cut` output is correct.") @@ -645,4 +725,4 @@ class PlanTimelineProNode(BaseNode): 'is_speech_rough_cut': is_speech_rough_cut, 'speech_rough_cut': speech_rough_cut, 'title_clip_duration': 0, - } \ No newline at end of file + } diff --git a/src/open_storyline/nodes/core_nodes/recommend_effects.py b/src/open_storyline/nodes/core_nodes/recommend_effects.py index 2486c95..475ab11 100644 --- a/src/open_storyline/nodes/core_nodes/recommend_effects.py +++ b/src/open_storyline/nodes/core_nodes/recommend_effects.py @@ -21,7 +21,7 @@ class RecommendTransitionNode(BaseNode): node_kind="transition_rec", require_prior_kind=['group_clips'], default_require_prior_kind=[], - next_available_node=["plan_timeline"], + next_available_node=["plan_timeline_pro"], ) input_schema = RecommendTransitionInput @@ -71,7 +71,7 @@ class RecommendTextNode(BaseNode): def __init__(self, server_cfg: Settings) -> None: super().__init__(server_cfg) - self.text_filter = ElementFilter(json_path=server_cfg.recommend_text.font_info_path) + self.text_filter = ElementFilter(json_path=str(server_cfg.recommend_text.font_info_path)) async def default_process( self, @@ -113,4 +113,4 @@ class RecommendTextNode(BaseNode): return None selected_json.update({"font_color": inputs.get("font_color", (255,255,255,255))}) node_state.node_summary.info_for_user(f"[{self.meta.node_id}] Use font `{selected_json['font_name']}`") - return [selected_json] \ No newline at end of file + return [selected_json] diff --git a/src/open_storyline/nodes/core_nodes/render_video.py b/src/open_storyline/nodes/core_nodes/render_video.py index ce369c6..2212a05 100644 --- a/src/open_storyline/nodes/core_nodes/render_video.py +++ b/src/open_storyline/nodes/core_nodes/render_video.py @@ -176,7 +176,7 @@ def resolve_output_canvas_size(inputs: Dict[str, Any]) -> Tuple[int, int]: """ Requirement: 1) output aspect ratio decided by inputs - 2) keep output <=1080 (consistent with media<=1080 + performance) + 2) output max dimension px decided by inputs """ # Adaptively select the canvas size based on the proportion of the size of the material. diff --git a/src/open_storyline/nodes/core_nodes/script_template_rec.py b/src/open_storyline/nodes/core_nodes/script_template_rec.py index 38928b9..a24ff4e 100644 --- a/src/open_storyline/nodes/core_nodes/script_template_rec.py +++ b/src/open_storyline/nodes/core_nodes/script_template_rec.py @@ -26,7 +26,7 @@ class ScriptTemplateRecomendation(BaseNode): def __init__(self, server_cfg): super().__init__(server_cfg) - self.element_filter = ElementFilter(json_path=self.server_cfg.script_template.script_template_info_path) + self.element_filter = ElementFilter(json_path=str(self.server_cfg.script_template.script_template_info_path)) self.vectorstore = StorylineRecall.build_vectorstore(self.element_filter.library) self._top_n = 3 diff --git a/src/open_storyline/nodes/core_nodes/split_shots.py b/src/open_storyline/nodes/core_nodes/split_shots.py index e7984fb..89008bd 100644 --- a/src/open_storyline/nodes/core_nodes/split_shots.py +++ b/src/open_storyline/nodes/core_nodes/split_shots.py @@ -247,7 +247,7 @@ class SplitShotsNode(BaseNode): node_kind="split_shots", require_prior_kind=["load_media"], default_require_prior_kind=["load_media"], - next_available_node=["understand_clips", "understand_clips_pro"], + next_available_node=["understand_clips"], ) input_schema = SplitShotsInput diff --git a/src/open_storyline/nodes/core_nodes/understand_clips.py b/src/open_storyline/nodes/core_nodes/understand_clips.py index fb5442c..7ca8725 100644 --- a/src/open_storyline/nodes/core_nodes/understand_clips.py +++ b/src/open_storyline/nodes/core_nodes/understand_clips.py @@ -21,7 +21,7 @@ class UnderstandClipsNode(BaseNode): node_kind="understand_clips", require_prior_kind=['load_media', 'split_shots'], default_require_prior_kind=['load_media', 'split_shots'], - next_available_node=['filter_clips', 'filter_clips_pro'], + next_available_node=['filter_clips'], ) input_schema = UnderstandClipsInput diff --git a/src/open_storyline/nodes/node_schema.py b/src/open_storyline/nodes/node_schema.py index 4f926f9..6434568 100644 --- a/src/open_storyline/nodes/node_schema.py +++ b/src/open_storyline/nodes/node_schema.py @@ -63,18 +63,18 @@ class SubtitleUnit(BaseModel): unit_id: str = Field( ..., description="Unique identifier for subtitle unit", - example="subtitle_0001" + examples=["subtitle_0001"], ) index_in_group: int = Field( ..., ge=0, description="Sequential index within current group (starting from 0)", - example=0 + examples=[0], ) text: str = Field( ..., description="Text content of this subtitle unit", - example="The cat doesn't understand what KPI means" + examples=["The cat doesn't understand what KPI means"], ) @@ -83,17 +83,17 @@ class GroupClips(BaseModel): group_id: str = Field( ..., description="Unique identifier for the group", - example="group_0001" + examples=["group_0001"], ) summary: str = Field( ..., description="Description of the group's visual style, emotional tone, or editing intent", - example="Start with the calmest, most healing shots to establish the mood." + examples=["Start with the calmest, most healing shots to establish the mood."], ) clip_ids: List[str] = Field( ..., description="List of video clip IDs used in this group, arranged in playback order", - example=["clip_0003", "clip_0002"] + examples=[["clip_0003", "clip_0002"]], ) @@ -102,12 +102,12 @@ class GroupScript(BaseModel): group_id: str = Field( ..., description="Unique identifier for the group", - example="group_0001" + examples=["group_0001"], ) raw_text: str = Field( ..., description="original script content for this group", - example="The cat doesn't understand what KPI means, the cat only knows the sun is shining today" + examples=["The cat doesn't understand what KPI means, the cat only knows the sun is shining today"], ) subtitle_units: List = Field( ..., @@ -305,7 +305,7 @@ class RecommendScriptTemplateInput(BaseInput): ) ] = {} filter_exclude: Annotated[ - Dict[str, List[Union[str]]], + Dict[str, List[str]], Field( description=( "Negative filter conditions. Items matching these conditions will be excluded. " @@ -369,7 +369,7 @@ class RecommendTransitionInput(BaseInput): class RecommendTransitionOutput(BaseInput): ... -class AIGCTransitionInput(BaseInput): +class GenerateAITransitionInput(BaseInput): mode: Literal["auto", "skip", "default"] = Field( default="auto", description=( @@ -384,6 +384,35 @@ class AIGCTransitionInput(BaseInput): description="User prompt specifying the desired transition effect." ) + duration: Annotated[ + Optional[int], + Field( + default=None, + description=( + "Desired AI transition duration in seconds. " + "You can choose 6s or 10s for MiniMax-Hailuo-02 with 768P," + "6s for MiniMax-Hailuo-02 with 1080P," + "and only 5s for Alibaba's Wan" + "If omitted, the provider default is used." + ), + ), + ] = None + + resolution: Annotated[ + Optional[str], + Field( + default=None, + description=( + "Desired AI transition resolution." + "You can choose 480P / 720P or 1080P for Alibaba's wan2.2-kf2v-flash," + "720P for Alibaba's wan2.2-kf2v-flash," + "768P or 1080P for MiniMax-Hailuo-02 with duration 6s." + "768P for MiniMax-Hailuo-02 with duration 10s." + "If omitted, the provider default is used." + ), + ), + ] = None + class RecommendTextInput(BaseInput): mode: Literal["auto", "skip", "default"] = Field( default="auto", @@ -409,6 +438,11 @@ class RecommendTextOutput(BaseInput): class PlanTimelineInput(BaseInput): use_beats: Annotated[bool, Field(default=True, description="Whether clip transitions should sync with BGM beats")] is_speech_rough_cut: Annotated[bool, Field(default=False, description="Whether the input clips are from speech rough cut, which affects the default timeline strategy selection")] + is_ai_transition: Annotated[bool, Field(default=False, description="Whether to build a minimal AI transition timeline by concatenating clips in order without subtitles, voiceover, cutting, or speed changes")] + image_duration_ms: Annotated[int, Field(default=3000, description="Default duration for image clips in milliseconds when using the AI transition timeline branch")] + +class PlanTimelineAITransitionInput(BaseInput): + image_duration_ms: Annotated[int, Field(default=3000, description="Default duration for image clips in milliseconds")] class PlanTimelineOutput(BaseModel): tracks: List[TimelineTracks] = Field(default_factory=list, description="Timeline track collection") @@ -477,4 +511,3 @@ class RenderVideoInput(BaseInput): default=1.0, description="Original video audio volume multiplier, range 0.0–3.0 (1.0 = default volume)" )] - diff --git a/src/open_storyline/utils/ai_transition_cancel.py b/src/open_storyline/utils/ai_transition_cancel.py new file mode 100644 index 0000000..dd71cea --- /dev/null +++ b/src/open_storyline/utils/ai_transition_cancel.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from pathlib import Path + + +_MARKER_NAME = ".cancel_generate_ai_transition" + + +def _normalize_cache_root(cache_root: str | Path) -> Path: + return Path(cache_root).expanduser() + + +def ai_transition_cancel_marker_path(cache_root: str | Path, session_id: str) -> Path: + return _normalize_cache_root(cache_root) / str(session_id) / _MARKER_NAME + + +def set_ai_transition_cancelled(cache_root: str | Path, session_id: str) -> Path: + marker = ai_transition_cancel_marker_path(cache_root, session_id) + marker.parent.mkdir(parents=True, exist_ok=True) + marker.touch(exist_ok=True) + return marker + + +def clear_ai_transition_cancelled(cache_root: str | Path, session_id: str) -> None: + marker = ai_transition_cancel_marker_path(cache_root, session_id) + marker.unlink(missing_ok=True) + + +def is_ai_transition_cancelled(cache_root: str | Path, session_id: str) -> bool: + return ai_transition_cancel_marker_path(cache_root, session_id).exists() diff --git a/src/open_storyline/utils/ai_transition_client.py b/src/open_storyline/utils/ai_transition_client.py new file mode 100644 index 0000000..5344058 --- /dev/null +++ b/src/open_storyline/utils/ai_transition_client.py @@ -0,0 +1,349 @@ +import base64 +import json +import os +import time +import requests +import mimetypes +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Optional, Any, Dict, Tuple + + +class BaseVisionClient(ABC): + DEFAULT_DURATION = 5 + DEFAULT_RESOLUTION = "720P" + + def __init__(self, api_key: str, timeout: int = 600, cancel_checker=None): + self.api_key = api_key + self.timeout = timeout + self.duration = self.DEFAULT_DURATION + self.resolution = self.DEFAULT_RESOLUTION + self.cancel_checker = cancel_checker + + def get_default_resolution(self, model: str) -> str: + return self.DEFAULT_RESOLUTION + + @abstractmethod + def _build_payload( + self, + prompt: str, + model: str, + first_frame: str | None = None, + last_frame: str | None = None, + resolution: str = "720P", + duration: int = 5, + prompt_optimizer: bool = True, + **kwargs + ) -> Dict[str, Any]: + pass + + @abstractmethod + def _get_endpoint(self, task_type: str) -> str: + pass + + @abstractmethod + def _extract_task_id(self, response_json: Dict[str, Any]) -> str: + pass + + @abstractmethod + def check_status(self, task_id: str) -> Tuple[Optional[str], Optional[Dict[str, Any]]]: + pass + + def _get_headers(self) -> Dict[str, str]: + return { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.api_key}" + } + + def _raise_if_cancelled(self) -> None: + checker = self.cancel_checker + if checker and checker(): + raise RuntimeError("generate_ai_transition cancelled by user") + + def _sleep_with_cancel(self, seconds: float) -> None: + deadline = time.monotonic() + max(0.0, float(seconds)) + while True: + self._raise_if_cancelled() + remaining = deadline - time.monotonic() + if remaining <= 0: + return + time.sleep(min(0.2, remaining)) + + def _format_response_error(self, response: requests.Response, action: str) -> str: + status_line = f"HTTP {response.status_code}" + if response.reason: + status_line = f"{status_line} {response.reason}" + + detail = "" + try: + payload = response.json() + except ValueError: + payload = None + + if payload is not None: + detail = json.dumps(payload, ensure_ascii=False) + else: + detail = (response.text or "").strip() + + if detail: + detail = detail[:2000] + return f"{self.__class__.__name__} {action} failed: {status_line}. Response: {detail}" + return f"{self.__class__.__name__} {action} failed: {status_line}." + + def _raise_for_status_with_details(self, response: requests.Response, action: str) -> None: + if response.ok: + return + raise RuntimeError(self._format_response_error(response, action)) + + def submit_task( + self, + task_type: str, + prompt: str, + model: str, + first_frame: str | None = None, + last_frame: str | None = None, + resolution: str | None = None, + duration: int | None = None, + prompt_optimizer: bool = True, + **kwargs + ) -> str: + if isinstance(resolution, str): + resolution = resolution.strip() or None + resolution = self.get_default_resolution(model) if resolution is None else resolution + duration = self.duration if duration is None else duration + url = self._get_endpoint(task_type) + payload = self._build_payload( + prompt, model, first_frame, last_frame, + resolution, duration, prompt_optimizer, **kwargs + ) + self._raise_if_cancelled() + + try: + response = requests.post(url, json=payload, headers=self._get_headers()) + except requests.RequestException as e: + raise RuntimeError(f"{self.__class__.__name__} submit task request failed: {e}") from e + self._raise_for_status_with_details(response, "submit task") + self._raise_if_cancelled() + + task_id = self._extract_task_id(response.json()) + if not task_id: + raise ValueError(f"Provider API returned invalid response: {response.text}") + return task_id + + def poll_for_result(self, task_id: str, poll_interval: int = 15) -> Tuple[str, Dict[str, Any]]: + start_time = time.time() + while time.time() - start_time < self.timeout: + self._raise_if_cancelled() + result_url, data = self.check_status(task_id) + if result_url: + return result_url, data + + self._sleep_with_cancel(poll_interval) + print(f"[{self.__class__.__name__}] Task {task_id} processing...") + + raise TimeoutError(f"Task {task_id} timed out after {self.timeout} seconds.") + + def download_asset(self, url: str, output_dir: str, task_id: str) -> str: + os.makedirs(output_dir, exist_ok=True) + self._raise_if_cancelled() + try: + response = requests.get(url, stream=True) + except requests.RequestException as e: + raise RuntimeError(f"{self.__class__.__name__} download asset request failed: {e}") from e + self._raise_for_status_with_details(response, "download asset") + + content_type = response.headers.get('content-type', '') + ext = mimetypes.guess_extension(content_type) or ".mp4" + save_path = os.path.join(output_dir, f"result_{task_id}{ext}") + + with open(save_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + self._raise_if_cancelled() + f.write(chunk) + return save_path + + def generate( + self, + task_type: str, + prompt: str, + model: str, + first_frame: str | None = None, + last_frame: str | None = None, + resolution: str | None = None, + duration: int | None = None, + prompt_optimizer: bool = True, + output_dir: str = "./output", + **kwargs + ) -> Tuple[str, Dict[str, Any]]: + task_id = self.submit_task( + task_type, prompt, model, first_frame, last_frame, + resolution, duration, prompt_optimizer, **kwargs + ) + result_url, raw_data = self.poll_for_result(task_id) + file_path = self.download_asset(result_url, output_dir, task_id) + return file_path, raw_data + + +class MiniMaxVisionClient(BaseVisionClient): + BASE_URL = "https://api.minimaxi.com/v1" + DEFAULT_DURATION = 6 + DEFAULT_RESOLUTION = "768P" + + MODEL_DEFAULT_RESOLUTIONS = { + "MiniMax-Hailuo-02": "768P", + } + + def get_default_resolution(self, model: str) -> str: + return self.MODEL_DEFAULT_RESOLUTIONS.get(model, self.DEFAULT_RESOLUTION) + + def _get_endpoint(self, task_type: str) -> str: + return f"{self.BASE_URL}/video_generation" + + def _build_payload(self, prompt, model, first_frame, last_frame, resolution, duration, prompt_optimizer, **kwargs): + payload = { + "model": model, + "prompt": prompt, + "first_frame_image": first_frame, + "last_frame_image": last_frame, + "duration": duration, + "resolution": resolution, + "prompt_optimizer": prompt_optimizer, + "aigc_watermark": kwargs.get("watermark", False) + } + return {k: v for k, v in payload.items() if v not in [None, ""]} + + def _extract_task_id(self, response_json: Dict[str, Any]) -> str: + return response_json.get("task_id") + + def check_status(self, task_id: str) -> Tuple[Optional[str], Optional[Dict[str, Any]]]: + url = f"{self.BASE_URL}/query/video_generation?task_id={task_id}" + headers = {"Authorization": f"Bearer {self.api_key}"} + self._raise_if_cancelled() + try: + response = requests.get(url, headers=headers) + except requests.RequestException as e: + raise RuntimeError(f"{self.__class__.__name__} query task status request failed: {e}") from e + self._raise_for_status_with_details(response, "query task status") + data = response.json() + + if data.get("status") == "Success": + file_id = data.get("file_id") + self._raise_if_cancelled() + try: + retrieve_resp = requests.get(f"{self.BASE_URL}/files/retrieve?file_id={file_id}", headers=headers) + except requests.RequestException as e: + raise RuntimeError(f"{self.__class__.__name__} retrieve generated file request failed: {e}") from e + self._raise_for_status_with_details(retrieve_resp, "retrieve generated file") + return retrieve_resp.json().get("file", {}).get("download_url"), data + elif data.get("status") == "Fail": + raise RuntimeError(f"MiniMax Task Failed: {data}") + return None, data + + +class DashScopeVisionClient(BaseVisionClient): + BASE_URL = "https://dashscope.aliyuncs.com/api/v1" + + DEFAULT_DURATION = 5 + DEFAULT_RESOLUTION = "720P" + + MODEL_DEFAULT_RESOLUTIONS = { + "wan2.2-kf2v-flash": "480P", + "wanx2.1-kf2v-plus": "720P", + } + + def get_default_resolution(self, model: str) -> str: + return self.MODEL_DEFAULT_RESOLUTIONS.get(model, self.DEFAULT_RESOLUTION) + + def _get_headers(self) -> Dict[str, str]: + headers = super()._get_headers() + headers["X-DashScope-Async"] = "enable" + return headers + + def _get_endpoint(self, task_type: str) -> str: + return f"{self.BASE_URL}/services/aigc/image2video/video-synthesis" + + def _build_payload(self, prompt, model, first_frame, last_frame, resolution, duration, prompt_optimizer, **kwargs): + return { + "model": model, + "input": { + "prompt": prompt, + "first_frame_url": first_frame, + "last_frame_url": last_frame, + "negative_prompt": kwargs.get("negative_prompt") + }, + "parameters": { + "resolution": resolution, + "duration": duration, + "prompt_extend": prompt_optimizer, + "watermark": kwargs.get("watermark", False), + "seed": kwargs.get("seed") + } + } + + def _extract_task_id(self, response_json: Dict[str, Any]) -> str: + return response_json.get("output", {}).get("task_id") + + def check_status(self, task_id: str) -> Tuple[Optional[str], Optional[Dict[str, Any]]]: + url = f"{self.BASE_URL}/tasks/{task_id}" + headers = {"Authorization": f"Bearer {self.api_key}"} + self._raise_if_cancelled() + try: + response = requests.get(url, headers=headers) + except requests.RequestException as e: + raise RuntimeError(f"{self.__class__.__name__} query task status request failed: {e}") from e + self._raise_for_status_with_details(response, "query task status") + data = response.json() + + output = data.get("output", {}) + if output.get("task_status") == "SUCCEEDED": + return output.get("video_url"), data + elif output.get("task_status") in ["FAILED", "CANCELED"]: + raise RuntimeError(f"DashScope Task Failed: {data}") + return None, data + +class VisionClientFactory: + @staticmethod + def create(provider: str, api_key: str, cancel_checker=None) -> BaseVisionClient: + p = provider.lower() + if p == "minimax": + return MiniMaxVisionClient(api_key=api_key, cancel_checker=cancel_checker) + elif p in ["dashscope", "aliyun"]: + return DashScopeVisionClient(api_key=api_key, cancel_checker=cancel_checker) + raise ValueError(f"Unsupported provider: {provider}") + + +def _normalize_media_input(value: Optional[str]) -> Optional[str]: + if not value: + return None + if value.startswith(("http://", "https://", "data:")): + return value + + path = Path(value).expanduser() + if not path.exists(): + raise FileNotFoundError(f"Media file not found: {path}") + + mime_type, _ = mimetypes.guess_type(path.name) + if not mime_type: + mime_type = "application/octet-stream" + + encoded = base64.b64encode(path.read_bytes()).decode("utf-8") + return f"data:{mime_type};base64,{encoded}" + + +if __name__ == "__main__": + + client = DashScopeVisionClient(api_key="") + file_path, response = client.generate( + task_type="video_generation", + prompt="以一镜到底的方式拍摄，场景丝滑过渡", + model="wan2.2-kf2v-flash", + first_frame=_normalize_media_input(""), + last_frame=_normalize_media_input(""), + resolution="720P", + duration=5, + prompt_optimizer=False, + output_dir="", + ) + + print(f"Generated video saved to: {file_path}") + print(json.dumps(response, ensure_ascii=False, indent=2)) diff --git a/src/open_storyline/utils/client.py b/src/open_storyline/utils/client.py deleted file mode 100644 index 1b5e042..0000000 --- a/src/open_storyline/utils/client.py +++ /dev/null @@ -1,214 +0,0 @@ -import os -import time -import requests -import mimetypes -from abc import ABC, abstractmethod -from typing import Optional, Any, Dict, Tuple - - -class BaseVisionClient(ABC): - def __init__(self, api_key: str, timeout: int = 600): - self.api_key = api_key - self.timeout = timeout - - @abstractmethod - def _build_payload( - self, - prompt: str, - model: str, - first_frame: str = None, - last_frame: str = None, - resolution: str = "720P", - duration: int = 5, - prompt_optimizer: bool = True, - **kwargs - ) -> Dict[str, Any]: - pass - - @abstractmethod - def _get_endpoint(self, task_type: str) -> str: - pass - - @abstractmethod - def _extract_task_id(self, response_json: Dict[str, Any]) -> str: - pass - - @abstractmethod - def check_status(self, task_id: str) -> Tuple[Optional[str], Optional[Dict[str, Any]]]: - pass - - def _get_headers(self) -> Dict[str, str]: - return { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.api_key}" - } - - def submit_task( - self, - task_type: str, - prompt: str, - model: str, - first_frame: str = None, - last_frame: str = None, - resolution: str = "720P", - duration: int = 5, - prompt_optimizer: bool = True, - **kwargs - ) -> str: - url = self._get_endpoint(task_type) - payload = self._build_payload( - prompt, model, first_frame, last_frame, - resolution, duration, prompt_optimizer, **kwargs - ) - - response = requests.post(url, json=payload, headers=self._get_headers()) - response.raise_for_status() - - task_id = self._extract_task_id(response.json()) - if not task_id: - raise ValueError(f"Provider API returned invalid response: {response.text}") - return task_id - - def poll_for_result(self, task_id: str, poll_interval: int = 15) -> Tuple[str, Dict[str, Any]]: - start_time = time.time() - while time.time() - start_time < self.timeout: - result_url, data = self.check_status(task_id) - if result_url: - return result_url, data - - time.sleep(poll_interval) - print(f"[{self.__class__.__name__}] Task {task_id} processing...") - - raise TimeoutError(f"Task {task_id} timed out after {self.timeout} seconds.") - - def download_asset(self, url: str, output_dir: str, task_id: str) -> str: - os.makedirs(output_dir, exist_ok=True) - response = requests.get(url, stream=True) - response.raise_for_status() - - content_type = response.headers.get('content-type', '') - ext = mimetypes.guess_extension(content_type) or ".mp4" - save_path = os.path.join(output_dir, f"result_{task_id}{ext}") - - with open(save_path, 'wb') as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) - return save_path - - def generate( - self, - task_type: str, - prompt: str, - model: str, - first_frame: str = None, - last_frame: str = None, - resolution: str = "720P", - duration: int = 5, - prompt_optimizer: bool = True, - output_dir: str = "./output", - **kwargs - ) -> Tuple[str, Dict[str, Any]]: - task_id = self.submit_task( - task_type, prompt, model, first_frame, last_frame, - resolution, duration, prompt_optimizer, **kwargs - ) - result_url, raw_data = self.poll_for_result(task_id) - file_path = self.download_asset(result_url, output_dir, task_id) - return file_path, raw_data - - -class MiniMaxVisionClient(BaseVisionClient): - BASE_URL = "https://api.minimaxi.com/v1" - - def _get_endpoint(self, task_type: str) -> str: - return f"{self.BASE_URL}/video_generation" - - def _build_payload(self, prompt, model, first_frame, last_frame, resolution, duration, prompt_optimizer, **kwargs): - payload = { - "model": model, - "prompt": prompt, - "first_frame_image": first_frame, - "last_frame_image": last_frame, - "duration": duration, - "resolution": resolution, - "prompt_optimizer": prompt_optimizer, - "aigc_watermark": kwargs.get("watermark", False) - } - return {k: v for k, v in payload.items() if v not in [None, ""]} - - def _extract_task_id(self, response_json: Dict[str, Any]) -> str: - return response_json.get("task_id") - - def check_status(self, task_id: str) -> Tuple[Optional[str], Optional[Dict[str, Any]]]: - url = f"{self.BASE_URL}/query/video_generation?task_id={task_id}" - headers = {"Authorization": f"Bearer {self.api_key}"} - response = requests.get(url, headers=headers) - response.raise_for_status() - data = response.json() - - if data.get("status") == "Success": - file_id = data.get("file_id") - retrieve_resp = requests.get(f"{self.BASE_URL}/files/retrieve?file_id={file_id}", headers=headers) - retrieve_resp.raise_for_status() - return retrieve_resp.json().get("file", {}).get("download_url"), data - elif data.get("status") == "Fail": - raise RuntimeError(f"MiniMax Task Failed: {data}") - return None, data - - -class DashScopeVisionClient(BaseVisionClient): - BASE_URL = "https://dashscope.aliyuncs.com/api/v1" - - def _get_headers(self) -> Dict[str, str]: - headers = super()._get_headers() - headers["X-DashScope-Async"] = "enable" - return headers - - def _get_endpoint(self, task_type: str) -> str: - return f"{self.BASE_URL}/services/aigc/image2video/video-synthesis" - - def _build_payload(self, prompt, model, first_frame, last_frame, resolution, duration, prompt_optimizer, **kwargs): - return { - "model": model, - "input": { - "prompt": prompt, - "first_frame_url": first_frame, - "last_frame_url": last_frame, - "negative_prompt": kwargs.get("negative_prompt") - }, - "parameters": { - "resolution": resolution, - "duration": duration, - "prompt_extend": prompt_optimizer, - "watermark": kwargs.get("watermark", False), - "seed": kwargs.get("seed") - } - } - - def _extract_task_id(self, response_json: Dict[str, Any]) -> str: - return response_json.get("output", {}).get("task_id") - - def check_status(self, task_id: str) -> Tuple[Optional[str], Optional[Dict[str, Any]]]: - url = f"{self.BASE_URL}/tasks/{task_id}" - headers = {"Authorization": f"Bearer {self.api_key}"} - response = requests.get(url, headers=headers) - response.raise_for_status() - data = response.json() - - output = data.get("output", {}) - if output.get("task_status") == "SUCCEEDED": - return output.get("video_url"), data - elif output.get("task_status") in ["FAILED", "CANCELED"]: - raise RuntimeError(f"DashScope Task Failed: {data}") - return None, data - - -class VisionClientFactory: - @staticmethod - def create(provider: str, api_key: str) -> BaseVisionClient: - p = provider.lower() - if p == "minimax": - return MiniMaxVisionClient(api_key=api_key) - elif p in ["dashscope", "aliyun"]: - return DashScopeVisionClient(api_key=api_key) - raise ValueError(f"Unsupported provider: {provider}") \ No newline at end of file diff --git a/web/index.html b/web/index.html index 0f0d240..452d50a 100644 --- a/web/index.html +++ b/web/index.html @@ -254,6 +254,47 @@

+ + + @@ -409,11 +450,13 @@ "filter_clips": {"zh": "筛选片段", "en": "filter clips"}, "understand_clips": {"zh": "理解素材", "en": "understand clips"}, "group_clips": {"zh": "片段分组", "en": "group clips"}, + "generate_ai_transition": {"zh": "AI 转场", "en": "AI transition"}, "script_template_rec":{"zh": "仿写模版推荐", "en": "recommend script template"}, "default_editing_workflow_skill": {"zh": "[SKILL] 通用剪辑SKILL", "en": "[SKILL] general editing SKILL"}, "create_profile_style_skill": {"zh": "[SKILL] 总结新SKILL", "en": "[SKILL] generate a new SKILL"}, "subtitle_imitation_skill": {"zh": "[SKILL] 文风仿写", "en": "[SKILL] mimic script style"}, "speech_rough_cut_skill": {"zh": "[SKILL] 口播粗剪", "en": "[SKILL] speech rough cut"}, + "ai_transition_editing_skill": {"zh": "[SKILL] AI 转场剪辑", "en": "[SKILL] AI transition editing"}, "generate_script": {"zh": "生成文案", "en": "generate script"}, "generate_voiceover": {"zh": "生成配音", "en": "generate voiceover"}, "select_bgm": {"zh": "推荐背景音乐", "en": "recommend music"}, @@ -421,28 +464,10 @@ "elementrec_text":{"zh": "推荐花字", "en": "recommend stylized subtitle fonts"}, "plan_timeline": {"zh": "组织时间线", "en": "plan timeline"}, "plan_timeline_pro": {"zh": "组织时间线 (pro)", "en": "plan timeline (pro)"}, + "plan_timeline_ai_transition": {"zh": "组织时间线 (AI 转场)", "en": "plan timeline (AI transition)"}, "render_video": {"zh": "渲染视频", "en": "render"}, "write_skills": {"zh": "创建新SKILL", "en": "Create a new SKILL"}, - "split_shots_pro": {"zh": "镜头切分 (pro)", "en": "split shots (pro)"}, - "filter_clips_pro": {"zh": "筛选片段 (pro)", "en": "filter clips (pro)"}, - "understand_clips_pro": {"zh": "理解素材 (pro)", "en": "understand clips (pro)"}, - "group_clips_pro": {"zh": "片段分组 (pro)", "en": "group clips (pro)"}, - "generate_script_pro": {"zh": "生成文案 (pro)", "en": "generate script (pro)"}, - "elementrec_texts_pro": {"zh": "推荐花字 (pro)", "en": "recommend stylized subtitle fonts (pro)"}, - "elementrec_title_pro": {"zh": "推荐标题 (pro)", "en": "recommand title"}, - "elementrec_global_effects_pro": {"zh": "推荐全局特效 (pro)", "en": "recommand global effects (pro)"}, - "elementrec_local_effects_pro": {"zh": "推荐局部特效 (pro)", "en": "recommand loacl effects (pro)"}, - "elementrec_close_effects_pro": {"zh": "推荐闭幕特效 (pro)", "en": "recommand close effects (pro)"}, - "elementrec_filter_pro": {"zh": "推荐滤镜 (pro)", "en": "recommand filter (pro)"}, - "elementrec_transition_pro": {"zh": "推荐转场 (pro)", "en": "recommand transition (pro)"}, - "elementrec_text_animation_pro": {"zh": "推荐花字动画 (pro)", "en": "recommended stylized subtitle animations (pro)"}, - "elementrec_title_clip_pro": {"zh": "推荐片头 (pro)", "en": "recommend title clip (pro)"}, - "elementrec_tts_pro": {"zh": "推荐配音音色 (pro)", "en": "recommended voice timbre for voiceover (pro)"}, - "generate_voiceover_pro": {"zh": "生成配音 (pro)", "en": "generate voiceover (pro)"}, - "music_rec_service_pro": {"zh": "推荐背景音乐 (pro)", "en": "recommand music (pro)"}, - "timeline_pro": {"zh": "组织时间线 (pro)", "en": "plan timeline (pro)"}, - "render_template_pro": {"zh": "渲染视频 (pro)", "en": "render (pro)"}, "read_node_history": {"zh": "读取详细结果", "en": "read history"}, }, estimates_ms: { diff --git a/web/static/app.js b/web/static/app.js index 97494d8..bea5557 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -65,6 +65,14 @@ const __OS_I18N = { "sidebar.tts_default": "使用默认配置", "sidebar.tts_hint": "提示：字段留空将使用 config.toml 中的配置。", "sidebar.tts_field_suffix": "（留空则使用服务器默认）", + "sidebar.ai_transition_box_aria": "AI 转场服务配置", + "sidebar.ai_transition_title": "AI 转场配置", + "sidebar.ai_transition_provider_select_aria": "选择 AI 转场服务厂家", + "sidebar.ai_transition_default": "使用默认配置", + "sidebar.ai_transition_hint": "提示：字段填写不完整则将使用 config.toml 中的配置。", + "sidebar.ai_transition_warning_aria": "AI 转场资源消耗提示", + "sidebar.ai_transition_warning_title": "高资源消耗提示", + "sidebar.ai_transition_warning_body": "AI 转场会额外触发模型调用，资源消耗通常显著高于常规文案或配音流程，单条转场价格通常在0.5~4元之间，建议按需使用。", "sidebar.use_custom_model": "使用自定义模型", "sidebar.llm_label": "LLM 模型", "sidebar.vlm_label": "VLM 模型", @@ -88,6 +96,7 @@ const __OS_I18N = { "sidebar.help.vlm": "VLM 用于素材理解（图像/视频理解）。自定义时请确认模型支持多模态输入。", "sidebar.help.pexels": "Pexels 用于搜索网络素材。免责声明：OpenStoryline 搜索的网络素材均来自Pexels，通过Pexels下载的素材仅用于体验Open-Storyline剪辑效果，不允许再分发或出售。我们只提供工具，所有通过本工具下载和使用的素材（如 Pexels 图像）都由用户自行通过 API 获取，我们不对用户生成的视频内容、素材的合法性或因使用本工具导致的任何版权/肖像权纠纷承担责任。使用时请遵循 Pexels 的许可协议。", "sidebar.help.tts": "用于从文案生成配音。", + "sidebar.help.ai_transition": "用于为片段之间生成 AI 转场。", "sidebar.help.pexels_home_link": "点击进入 Pexels 官方网站", "sidebar.help.pexels_terms_link": "查看 Pexels 用户协议", @@ -179,6 +188,14 @@ const __OS_I18N = { "sidebar.tts_default": "Use default configuration", "sidebar.tts_hint": "Note: leaving fields empty will fall back to config.toml.", "sidebar.tts_field_suffix": " (leave empty to use server default)", + "sidebar.ai_transition_box_aria": "AI transition configuration", + "sidebar.ai_transition_title": "AI transition", + "sidebar.ai_transition_provider_select_aria": "Select an AI transition provider", + "sidebar.ai_transition_default": "Use default configuration", + "sidebar.ai_transition_hint": "Note: leaving fields empty will fall back to config.toml.", + "sidebar.ai_transition_warning_aria": "AI transition resource usage notice", + "sidebar.ai_transition_warning_title": "Higher resource usage", + "sidebar.ai_transition_warning_body": "AI transitions trigger additional model calls and usually consume significantly more resources than regular copywriting or voiceover workflows. The price for a single transition is typically between $0.1 and $0.5. We recommend using as needed.", "sidebar.use_custom_model": "Use custom model", "sidebar.llm_label": "LLM model", "sidebar.vlm_label": "VLM model", @@ -202,6 +219,7 @@ const __OS_I18N = { "sidebar.help.vlm": "VLM is used for media understanding (image/video).", "sidebar.help.pexels": "Pexels is used for media search. Disclaimer: The online content searched by OpenStoryline is all from Pexels. Footage downloaded via Pexels is for the sole purpose of experiencing Open-Storyline editing effects and may not be redistributed or sold. We only provide the tool. All materials downloaded and used through this tool (such as Pexels images) are obtained by the user through the API. We are not responsible for the legality of user-generated video content or materials, or for any copyright/portrait rights disputes arising from the use of this tool. Please comply with the Pexels license agreement when using it.", "sidebar.help.tts": "TTS is used to generate voiceover from text.", + "sidebar.help.ai_transition": "Used to generate AI transitions between clips.", "sidebar.help.pexels_home_link": "Visit the official Pexels website", "sidebar.help.pexels_terms_link": "View Pexels Terms", @@ -313,9 +331,9 @@ function __applyI18n(root = document) { }); } -// TTS 动态字段 placeholder（suffix）重渲染： +// Provider 动态字段 placeholder（suffix）重渲染： // - 创建 input 时会写入 data-os-ph-base / data-os-ph-suffix -function __rerenderTtsFieldPlaceholders(root = document) { +function __rerenderProviderFieldPlaceholders(root = document) { root.querySelectorAll("input[data-os-ph-base]").forEach((el) => { const base = String(el.getAttribute("data-os-ph-base") || ""); const needSuffix = el.getAttribute("data-os-ph-suffix") === "1"; @@ -455,7 +473,7 @@ function __applyLang(lang, { persist = true } = {}) { document.documentElement.lang = (v === "en") ? "en" : "zh-CN"; __applyI18n(document); - __rerenderTtsFieldPlaceholders(document); + __rerenderProviderFieldPlaceholders(document); __osApplyHelpLinks(document); __osApplyTopbarLinks(document); __osApplyTooltipLinks(document); @@ -486,8 +504,8 @@ class ApiClient { return await r.json(); } - async getTtsUiSchema() { - const r = await fetch("/api/meta/tts", { method: "GET" }); + async getProviderUiSchema(kind) { + const r = await fetch(`/api/meta/${encodeURIComponent(kind)}`, { method: "GET" }); if (!r.ok) throw new Error(await this._readFetchError(r)); return await r.json(); // { default_provider, providers:[...] } } @@ -2361,11 +2379,35 @@ class App { this.customVlmBaseUrl = $("#customVlmBaseUrl"); this.customVlmApiKey = $("#customVlmApiKey"); - // TTS UI + // Provider config UI this.ttsBox = $("#ttsBox"); this.ttsProviderSelect = $("#ttsProviderSelect"); this.ttsProviderFieldsHost = $("#ttsProviderFields"); - this.ttsUiSchema = null; + this.aiTransitionBox = $("#aiTransitionBox"); + this.aiTransitionProviderSelect = $("#aiTransitionProviderSelect"); + this.aiTransitionProviderFieldsHost = $("#aiTransitionProviderFields"); + this.providerUiSchemas = { + tts: null, + ai_transition: null, + }; + this.providerPanels = { + tts: { + box: this.ttsBox, + select: this.ttsProviderSelect, + host: this.ttsProviderFieldsHost, + persistPrefix: "sidebar.tts", + defaultTextKey: "sidebar.tts_default", + showDefaultOption: true, + }, + ai_transition: { + box: this.aiTransitionBox, + select: this.aiTransitionProviderSelect, + host: this.aiTransitionProviderFieldsHost, + persistPrefix: "sidebar.ai_transition", + defaultTextKey: "sidebar.ai_transition_default", + showDefaultOption: false, + }, + }; // Pexels UI this.pexelsBox = $("#pexelsBox"); @@ -2465,7 +2507,8 @@ class App { this.ui.bindModalClose(); this.bindUI(); this._setLang(this.lang, { persist: false, syncServer: false }); - await this.loadTtsUiSchema(); + await this.loadProviderUiSchema("tts"); + await this.loadProviderUiSchema("ai_transition"); // 先加载本地会话列表 this.sessionHistory = this._loadSessionHistory(); @@ -2969,29 +3012,39 @@ class App { }, 160); } - async loadTtsUiSchema() { - let schema = null; - try { - schema = await this.api.getTtsUiSchema(); - } catch (e) { - console.warn("[tts] failed to load /api/meta/tts:", e); - } - - this.ttsUiSchema = schema; - this._renderTtsUiFromSchema(schema); + _getProviderPanel(kind) { + return this.providerPanels?.[kind] || null; } - _renderTtsUiFromSchema(schema) { - if (!this.ttsProviderSelect || !this.ttsProviderFieldsHost) return; + async loadProviderUiSchema(kind) { + const panel = this._getProviderPanel(kind); + if (!panel) return; + + let schema = null; + try { + schema = await this.api.getProviderUiSchema(kind); + } catch (e) { + console.warn(`[${kind}] failed to load /api/meta/${kind}:`, e); + } + + this.providerUiSchemas[kind] = schema; + this._renderProviderUiFromSchema(kind, schema); + } + + _renderProviderUiFromSchema(kind, schema) { + const panel = this._getProviderPanel(kind); + if (!panel || !panel.select || !panel.host) return; const providers = (schema && Array.isArray(schema.providers)) ? schema.providers : []; - const before = String(this.ttsProviderSelect.value || "").trim(); + const before = String(panel.select.value || "").trim(); - this.ttsProviderSelect.innerHTML = ""; - const opt0 = document.createElement("option"); - opt0.value = ""; - opt0.textContent = __t("sidebar.tts_default"); - this.ttsProviderSelect.appendChild(opt0); + panel.select.innerHTML = ""; + if (panel.showDefaultOption) { + const opt0 = document.createElement("option"); + opt0.value = ""; + opt0.textContent = __t(panel.defaultTextKey); + panel.select.appendChild(opt0); + } for (const v of providers) { const provider = String(v?.provider || "").trim(); @@ -3001,10 +3054,10 @@ class App { const opt = document.createElement("option"); opt.value = provider; opt.textContent = label; - this.ttsProviderSelect.appendChild(opt); + panel.select.appendChild(opt); } - this.ttsProviderFieldsHost.innerHTML = ""; + panel.host.innerHTML = ""; for (const v of providers) { const provider = String(v?.provider || "").trim(); @@ -3012,7 +3065,8 @@ class App { const block = document.createElement("div"); block.className = "sidebar-tts-fields hidden"; - block.dataset.ttsProvider = provider; + block.dataset.providerKind = kind; + block.dataset.providerName = provider; const fields = Array.isArray(v?.fields) ? v.fields : []; @@ -3022,7 +3076,7 @@ class App { const label = String(f?.label || key).trim(); - const required = !!f?.required; + // const required = !!f?.required; const secret = !!f?.secret; const input = document.createElement("input"); @@ -3038,26 +3092,29 @@ class App { const ph = needSuffix ? `${basePh}${__t("sidebar.tts_field_suffix")}` : basePh; input.placeholder = ph; - input.setAttribute("data-os-persist", `sidebar.tts.${provider}.${key}`); + input.setAttribute("data-os-persist", `${panel.persistPrefix}.${provider}.${key}`); - input.dataset.ttsKey = key; + input.dataset.providerKey = key; block.appendChild(input); } - this.ttsProviderFieldsHost.appendChild(block); + panel.host.appendChild(block); } - try { __osHydratePersistedFields(this.ttsBox || document); } catch {} - try { __osBindPersistedFields(this.ttsBox || document); } catch {} + try { __osHydratePersistedFields(panel.box || document); } catch {} + try { __osBindPersistedFields(panel.box || document); } catch {} if (before) { - this.ttsProviderSelect.value = before; + panel.select.value = before; + } else if (!panel.showDefaultOption && providers.length > 0) { + const firstProvider = String(providers[0]?.provider || "").trim(); + panel.select.value = firstProvider; } else { - this.ttsProviderSelect.value = ""; + panel.select.value = ""; } - try { this.ttsProviderSelect.dispatchEvent(new Event("change", { bubbles: true })); } catch {} + try { panel.select.dispatchEvent(new Event("change", { bubbles: true })); } catch {} } // restoreSidebarState() { @@ -3181,12 +3238,14 @@ class App { apply(this.llmSelect); apply(this.vlmSelect); - if (this.ttsProviderSelect) { - const opt0 = this.ttsProviderSelect.querySelector('option[value=""]'); - if (opt0) opt0.textContent = __t("sidebar.tts_default"); - } + ["tts", "ai_transition"].forEach((kind) => { + const panel = this._getProviderPanel(kind); + if (!panel?.select || !panel.showDefaultOption) return; + const opt0 = panel.select.querySelector('option[value=""]'); + if (opt0) opt0.textContent = __t(panel.defaultTextKey); + }); - __rerenderTtsFieldPlaceholders(document); + __rerenderProviderFieldPlaceholders(document); } _pushLangToServer() { @@ -3270,17 +3329,16 @@ class App { if (this.customLlmSection) this.customLlmSection.classList.toggle("hidden", !llmCustom); if (this.customVlmSection) this.customVlmSection.classList.toggle("hidden", !vlmCustom); - const provider = (this.ttsProviderSelect && this.ttsProviderSelect.value) - ? String(this.ttsProviderSelect.value).trim() - : ""; + ["tts", "ai_transition"].forEach((kind) => { + const panel = this._getProviderPanel(kind); + if (!panel?.host) return; + const provider = panel.select ? String(panel.select.value || "").trim() : ""; - const host = this.ttsProviderFieldsHost || $("#ttsProviderFields"); - if (host) { - host.querySelectorAll("[data-tts-provider]").forEach((el) => { - const v = String(el.dataset.ttsProvider || ""); + panel.host.querySelectorAll(`[data-provider-kind="${kind}"][data-provider-name]`).forEach((el) => { + const v = String(el.dataset.providerName || ""); el.classList.toggle("hidden", !provider || v !== provider); }); - } + }); // ---- Pexels custom key show/hide ---- const pMode = (this.pexelsKeyModeSelect && this.pexelsKeyModeSelect.value) @@ -3323,21 +3381,21 @@ class App { } - _readTtsConfigFromUI() { - const provider = (this.ttsProviderSelect && this.ttsProviderSelect.value) - ? String(this.ttsProviderSelect.value).trim() - : ""; + _readProviderConfigFromUI(kind) { + const panel = this._getProviderPanel(kind); + if (!panel?.select) return null; + + const provider = String(panel.select.value || "").trim(); if (!provider) return null; - const host = this.ttsProviderFieldsHost || $("#ttsProviderFields"); const params = {}; - if (host) { - const block = host.querySelector(`[data-tts-provider="${provider}"]`); + if (panel.host) { + const block = panel.host.querySelector(`[data-provider-kind="${kind}"][data-provider-name="${provider}"]`); if (block) { - const fields = block.querySelectorAll("input[data-tts-key], select[data-tts-key], textarea[data-tts-key]"); + const fields = block.querySelectorAll("input[data-provider-key], select[data-provider-key], textarea[data-provider-key]"); fields.forEach((el) => { - const k = String(el.dataset.ttsKey || "").trim(); + const k = String(el.dataset.providerKey || "").trim(); if (!k) return; const v = String(el.value ?? "").trim(); if (v !== "") params[k] = v; @@ -3386,9 +3444,12 @@ class App { if (needVlmCustom) rc.custom_models.vlm = cm.vlm; } - const tts = this._readTtsConfigFromUI(); + const tts = this._readProviderConfigFromUI("tts"); if (tts) rc.tts = tts; + const aiTransition = this._readProviderConfigFromUI("ai_transition"); + if (aiTransition) rc.ai_transition = aiTransition; + const pexels = this._readPexelsConfigFromUI(); if (pexels) { rc.search_media = { pexels }; @@ -3581,9 +3642,11 @@ class App { }); } - if (this.ttsProviderSelect) { - this.ttsProviderSelect.addEventListener("change", () => this._syncConfigPanels()); - } + ["tts", "ai_transition"].forEach((kind) => { + const panel = this._getProviderPanel(kind); + if (!panel?.select) return; + panel.select.addEventListener("change", () => this._syncConfigPanels()); + }); if (this.pexelsKeyModeSelect) { this.pexelsKeyModeSelect.addEventListener("change", () => this._syncConfigPanels()); diff --git a/web/static/style.css b/web/static/style.css index 05934de..d3e4914 100644 --- a/web/static/style.css +++ b/web/static/style.css @@ -11,6 +11,9 @@ --border: rgba(11,11,12,0.14); --border-weak: rgba(11,11,12,0.08); + --warning-bg: rgba(255, 196, 63, 0.12); + --warning-border: rgba(193, 122, 0, 0.24); + --warning-accent: #a15d00; --shadow-soft: 0 12px 32px rgba(0,0,0,0.06); --shadow: 0 22px 60px rgba(0,0,0,0.10); @@ -44,6 +47,9 @@ --border: rgba(244,244,245,0.16); --border-weak: rgba(244,244,245,0.10); + --warning-bg: rgba(255, 191, 71, 0.14); + --warning-border: rgba(255, 191, 71, 0.28); + --warning-accent: #ffcc73; --shadow-soft: 0 14px 36px rgba(0,0,0,0.38); --shadow: 0 24px 70px rgba(0,0,0,0.58); @@ -471,6 +477,30 @@ body.sidebar-collapsed .sidebar-model{ display: none; } line-height: 1.45; } +.sidebar-warning{ + display: flex; + flex-direction: column; + gap: 4px; + padding: 10px 11px; + border-radius: 12px; + border: 1px solid var(--warning-border); + background: var(--warning-bg); + box-shadow: inset 3px 0 0 var(--warning-accent); +} + +.sidebar-warning-title{ + font-size: 11px; + font-weight: 700; + line-height: 1.25; + color: var(--warning-accent); +} + +.sidebar-warning-body{ + font-size: 11px; + line-height: 1.5; + color: var(--text); +} + /* 会话历史列表 */ .session-history-list{ margin-top: 4px;