diff --git a/astrbot/core/astr_main_agent.py b/astrbot/core/astr_main_agent.py index d0c615824..ed20512e6 100644 --- a/astrbot/core/astr_main_agent.py +++ b/astrbot/core/astr_main_agent.py @@ -21,7 +21,7 @@ from astrbot.core.astr_agent_hooks import MAIN_AGENT_HOOKS from astrbot.core.astr_agent_run_util import AgentRunner from astrbot.core.astr_agent_tool_exec import FunctionToolExecutor from astrbot.core.conversation_mgr import Conversation -from astrbot.core.message.components import File, Image, Reply +from astrbot.core.message.components import File, Image, Record, Reply from astrbot.core.persona_error_reply import ( extract_persona_custom_error_message_from_persona, set_persona_custom_error_message_on_event, @@ -419,6 +419,18 @@ def _append_quoted_image_attachment(req: ProviderRequest, image_path: str) -> No ) +def _append_audio_attachment(req: ProviderRequest, audio_path: str) -> None: + req.extra_user_content_parts.append( + TextPart(text=f"[Audio Attachment: path {audio_path}]") + ) + + +def _append_quoted_audio_attachment(req: ProviderRequest, audio_path: str) -> None: + req.extra_user_content_parts.append( + TextPart(text=f"[Audio Attachment in quoted message: path {audio_path}]") + ) + + def _get_quoted_message_parser_settings( provider_settings: dict[str, object] | None, ) -> QuotedMessageParserSettings: @@ -704,12 +716,25 @@ def _modalities_fix(provider: Provider, req: ProviderRequest) -> None: "Provider %s does not support image, using placeholder.", provider ) image_count = len(req.image_urls) - placeholder = " ".join(["[图片]"] * image_count) + placeholder = " ".join(["[Image]"] * image_count) if req.prompt: req.prompt = f"{placeholder} {req.prompt}" else: req.prompt = placeholder req.image_urls = [] + if req.audio_urls: + provider_cfg = provider.provider_config.get("modalities", ["audio"]) + if "audio" not in provider_cfg: + logger.debug( + "Provider %s does not support audio, using placeholder.", provider + ) + audio_count = len(req.audio_urls) + placeholder = " ".join(["[Audio]"] * audio_count) + if req.prompt: + req.prompt = f"{placeholder} {req.prompt}" + else: + req.prompt = placeholder + req.audio_urls = [] if req.func_tool: provider_cfg = provider.provider_config.get("modalities", ["tool_use"]) if "tool_use" not in provider_cfg: @@ -730,11 +755,13 @@ def _sanitize_context_by_modalities( if not modalities or not isinstance(modalities, list): return supports_image = bool("image" in modalities) + supports_audio = bool("audio" in modalities) supports_tool_use = bool("tool_use" in modalities) - if supports_image and supports_tool_use: + if supports_image and supports_audio and supports_tool_use: return sanitized_contexts: list[dict] = [] removed_image_blocks = 0 + removed_audio_blocks = 0 removed_tool_messages = 0 removed_tool_calls = 0 for msg in req.contexts: @@ -753,20 +780,28 @@ def _sanitize_context_by_modalities( removed_tool_calls += 1 new_msg.pop("tool_calls", None) new_msg.pop("tool_call_id", None) - if not supports_image: + + if not supports_image or not supports_audio: content = new_msg.get("content") if isinstance(content, list): filtered_parts: list = [] - removed_any_image = False + removed_any_multimodal = False for part in content: if isinstance(part, dict): part_type = str(part.get("type", "")).lower() - if part_type in {"image_url", "image"}: - removed_any_image = True + if not supports_image and part_type in {"image_url", "image"}: + removed_any_multimodal = True removed_image_blocks += 1 continue + if not supports_audio and part_type in { + "audio_url", + "input_audio", + }: + removed_any_multimodal = True + removed_audio_blocks += 1 + continue filtered_parts.append(part) - if removed_any_image: + if removed_any_multimodal: new_msg["content"] = filtered_parts if role == "assistant": content = new_msg.get("content") @@ -777,10 +812,19 @@ def _sanitize_context_by_modalities( if isinstance(content, str) and (not content.strip()): continue sanitized_contexts.append(new_msg) - if removed_image_blocks or removed_tool_messages or removed_tool_calls: + + if ( + removed_image_blocks + or removed_audio_blocks + or removed_tool_messages + or removed_tool_calls + ): logger.debug( - "sanitize_context_by_modalities applied: removed_image_blocks=%s, removed_tool_messages=%s, removed_tool_calls=%s", + "sanitize_context_by_modalities applied: " + "removed_image_blocks=%s, removed_audio_blocks=%s, " + "removed_tool_messages=%s, removed_tool_calls=%s", removed_image_blocks, + removed_audio_blocks, removed_tool_messages, removed_tool_calls, ) @@ -969,6 +1013,7 @@ async def build_main_agent( req = ProviderRequest() req.prompt = "" req.image_urls = [] + req.audio_urls = [] if sel_model := event.get_extra("selected_model"): req.model = sel_model if config.provider_wake_prefix and ( @@ -988,6 +1033,10 @@ async def build_main_agent( req.extra_user_content_parts.append( TextPart(text=f"[Image Attachment: path {image_path}]") ) + elif isinstance(comp, Record): + audio_path = await comp.convert_to_file_path() + req.audio_urls.append(audio_path) + _append_audio_attachment(req, audio_path) elif isinstance(comp, File): file_path = await comp.get_file() file_name = comp.name or os.path.basename(file_path) @@ -1017,6 +1066,10 @@ async def build_main_agent( event.track_temporary_local_file(image_path) req.image_urls.append(image_path) _append_quoted_image_attachment(req, image_path) + elif isinstance(reply_comp, Record): + audio_path = await reply_comp.convert_to_file_path() + req.audio_urls.append(audio_path) + _append_quoted_audio_attachment(req, audio_path) elif isinstance(reply_comp, File): file_path = await reply_comp.get_file() file_name = reply_comp.name or os.path.basename(file_path) @@ -1074,12 +1127,15 @@ async def build_main_agent( if isinstance(req.contexts, str): req.contexts = json.loads(req.contexts) req.image_urls = normalize_and_dedupe_strings(req.image_urls) + req.audio_urls = normalize_and_dedupe_strings(req.audio_urls) + if config.file_extract_enabled: try: await _apply_file_extract(event, req, config) except Exception as exc: logger.error("Error occurred while applying file extract: %s", exc) - if not req.prompt and (not req.image_urls): + + if not req.prompt and not req.image_urls and not req.audio_urls: if not event.get_group_id() and req.extra_user_content_parts: req.prompt = "" else: diff --git a/astrbot/core/config/default.py b/astrbot/core/config/default.py index dec1c9285..911d561bd 100644 --- a/astrbot/core/config/default.py +++ b/astrbot/core/config/default.py @@ -1247,6 +1247,18 @@ CONFIG_METADATA_2 = { "proxy": "", "custom_headers": {}, }, + "LongCat": { + "id": "longcat", + "provider": "longcat", + "type": "longcat_chat_completion", + "provider_type": "chat_completion", + "enable": True, + "key": [], + "api_base": "https://api.longcat.chat/openai", + "timeout": 120, + "proxy": "", + "custom_headers": {}, + }, "AIHubMix": { "id": "aihubmix", "provider": "aihubmix", @@ -1761,6 +1773,7 @@ CONFIG_METADATA_2 = { "enable": True, "rerank_api_key": "", "rerank_api_base": "http://127.0.0.1:8000", + "rerank_api_suffix": "/v1/rerank", "rerank_model": "BAAI/bge-reranker-base", "timeout": 20, }, @@ -1789,6 +1802,19 @@ CONFIG_METADATA_2 = { "return_documents": False, "instruct": "", }, + "NVIDIA Rerank": { + "id": "nvidia_rerank", + "type": "nvidia_rerank", + "provider": "nvidia", + "provider_type": "rerank", + "enable": True, + "nvidia_rerank_api_key": "", + "nvidia_rerank_api_base": "https://ai.api.nvidia.com/v1/retrieval", + "nvidia_rerank_model": "nv-rerank-qa-mistral-4b:1", + "nvidia_rerank_model_endpoint": "/reranking", + "timeout": 20, + "nvidia_rerank_truncate": "", + }, "Xinference STT": { "id": "xinference_stt", "type": "xinference_stt", @@ -1826,7 +1852,12 @@ CONFIG_METADATA_2 = { "rerank_api_base": { "description": "重排序模型 API Base URL", "type": "string", - "hint": "AstrBot 会在请求时在末尾加上 /v1/rerank。", + "hint": "最终请求路径由 Base URL 和路径后缀拼接而成(默认为 /v1/rerank)。", + }, + "rerank_api_suffix": { + "description": "API URL 路径后缀", + "type": "string", + "hint": "追加到 base_url 后的路径,如 /v1/rerank。留空则不追加。", }, "rerank_api_key": { "description": "API Key", @@ -1852,12 +1883,40 @@ CONFIG_METADATA_2 = { "type": "bool", "hint": "如果模型当前未在 Xinference 服务中运行,是否尝试自动启动它。在生产环境中建议关闭。", }, + "nvidia_rerank_api_base": { + "description": "API Base URL", + "type": "string", + }, + "nvidia_rerank_api_key": { + "description": "API Key", + "type": "string", + }, + "nvidia_rerank_model": { + "description": "重排序模型名称", + "type": "string", + "hint": "请参照NVIDIA Docs中模型名称填写。", + }, + "nvidia_rerank_model_endpoint": { + "description": "自定义模型端点", + "type": "string", + "hint": "自定义URL末尾端点,默认为 /reranking", + }, + "nvidia_rerank_truncate": { + "description": "文本截断策略", + "type": "string", + "hint": "当输入文本过长时,是否截断输入以适应模型的最大上下文长度。", + "options": [ + "", + "NONE", + "END", + ], + }, "modalities": { "description": "模型能力", "type": "list", "items": {"type": "string"}, - "options": ["text", "image", "tool_use"], - "labels": ["文本", "图像", "工具使用"], + "options": ["text", "image", "audio", "tool_use"], + "labels": ["文本", "图像", "音频", "工具使用"], "render_type": "checkbox", "hint": "模型支持的模态。如所填写的模型不支持图像,请取消勾选图像。", }, diff --git a/astrbot/core/message/components.py b/astrbot/core/message/components.py index fe689249b..f40cab05c 100644 --- a/astrbot/core/message/components.py +++ b/astrbot/core/message/components.py @@ -74,12 +74,6 @@ class ComponentType(str, Enum): Music = "Music" Json = "Json" Unknown = "Unknown" - WechatEmoji = "WechatEmoji" # Wechat 下的 emoji 表情包 - # Discord-specific component types - DiscordEmbed = "DiscordEmbed" - DiscordButton = "DiscordButton" - DiscordReference = "DiscordReference" - DiscordView = "DiscordView" class BaseMessageComponent(BaseModel): @@ -106,7 +100,6 @@ class BaseMessageComponent(BaseModel): class Plain(BaseMessageComponent): type: ComponentType = ComponentType.Plain text: str - convert: bool | None = True def __init__(self, text: str, convert: bool = True, **_) -> None: super().__init__(text=text, convert=convert, **_) @@ -129,11 +122,7 @@ class Face(BaseMessageComponent): class Record(BaseMessageComponent): type: ComponentType = ComponentType.Record file: str | None = "" - magic: bool | None = False url: str | None = "" - cache: bool | None = True - proxy: bool | None = True - timeout: int | None = 0 # Original text content (e.g. TTS source text), used as caption in fallback scenarios text: str | None = None # 额外 @@ -239,7 +228,6 @@ class Video(BaseMessageComponent): type: ComponentType = ComponentType.Video file: str cover: str | None = "" - c: int | None = 2 # 额外 path: str | None = "" @@ -416,14 +404,9 @@ class Image(BaseMessageComponent): type: ComponentType = ComponentType.Image file: str | None = "" _type: str | None = "" - subType: int | None = 0 url: str | None = "" - cache: bool | None = True - id: int | None = 40000 - c: int | None = 2 # 额外 path: str | None = "" - file_unique: str | None = "" # 某些平台可能有图片缓存的唯一标识 def __init__(self, file: str | None, **_) -> None: super().__init__(file=file, **_) @@ -854,16 +837,6 @@ class File(BaseMessageComponent): } -class WechatEmoji(BaseMessageComponent): - type: ComponentType = ComponentType.WechatEmoji - md5: str | None = "" - md5_len: int | None = 0 - cdnurl: str | None = "" - - def __init__(self, **_) -> None: - super().__init__(**_) - - ComponentTypes = { # Basic Message Segments "plain": Plain, @@ -889,5 +862,4 @@ ComponentTypes = { "nodes": Nodes, "json": Json, "unknown": Unknown, - "WechatEmoji": WechatEmoji, } diff --git a/astrbot/core/pipeline/preprocess_stage/stage.py b/astrbot/core/pipeline/preprocess_stage/stage.py index ce4aff15f..d726b5897 100644 --- a/astrbot/core/pipeline/preprocess_stage/stage.py +++ b/astrbot/core/pipeline/preprocess_stage/stage.py @@ -7,6 +7,7 @@ from astrbot.core.message.components import Image, Plain, Record from astrbot.core.pipeline.context import PipelineContext from astrbot.core.pipeline.stage import Stage, register_stage from astrbot.core.platform.astr_message_event import AstrMessageEvent +from astrbot.core.utils.media_utils import ensure_wav @register_stage @@ -62,6 +63,21 @@ class PreProcessStage(Stage): logger.debug(f"路径映射: {url} -> {component.url}") message_chain[idx] = component + # In here, we convert all Record components to wav format and update the file path. + message_chain = event.get_messages() + for idx, component in enumerate(message_chain): + if isinstance(component, Record): + try: + original_path = await component.convert_to_file_path() + record_path = await ensure_wav(original_path) + if record_path != original_path: + event.track_temporary_local_file(record_path) + component.file = record_path + component.path = record_path + message_chain[idx] = component + except Exception as e: + logger.warning(f"Voice processing failed: {e}") + # STT if self.stt_settings.get("enable", False): # TODO: 独立 diff --git a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py index 00adcf780..d4f172a9c 100644 --- a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py +++ b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py @@ -14,7 +14,7 @@ from astrbot.core.astr_main_agent import ( MainAgentBuildResult, build_main_agent, ) -from astrbot.core.message.components import File, Image +from astrbot.core.message.components import File, Image, Record, Video from astrbot.core.message.message_event_result import ( MessageChain, MessageEventResult, @@ -147,7 +147,8 @@ class InternalAgentSubStage(Stage): has_provider_request = event.get_extra("provider_request") is not None has_valid_message = bool(event.message_str and event.message_str.strip()) has_media_content = any( - isinstance(comp, Image | File) for comp in event.message_obj.message + isinstance(comp, (Image, File, Record, Video)) + for comp in event.message_obj.message ) if ( not has_provider_request diff --git a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/third_party.py b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/third_party.py index ad37d44a6..dd91334f1 100644 --- a/astrbot/core/pipeline/process_stage/method/agent_sub_stages/third_party.py +++ b/astrbot/core/pipeline/process_stage/method/agent_sub_stages/third_party.py @@ -9,7 +9,7 @@ from astrbot.core.agent.runners.deerflow.constants import ( DEERFLOW_PROVIDER_TYPE, ) from astrbot.core.astr_agent_hooks import MAIN_AGENT_HOOKS -from astrbot.core.message.components import Image +from astrbot.core.message.components import Image, Record from astrbot.core.message.message_event_result import ( MessageChain, MessageEventResult, @@ -313,8 +313,11 @@ class ThirdPartyAgentSubStage(Stage): if isinstance(comp, Image): image_path = await comp.convert_to_base64() req.image_urls.append(image_path) + elif isinstance(comp, Record): + audio_path = await comp.convert_to_file_path() + req.audio_urls.append(audio_path) - if not req.prompt and not req.image_urls: + if not req.prompt and not req.image_urls and not req.audio_urls: return custom_error_message = await self._resolve_persona_custom_error_message(event) diff --git a/astrbot/core/pipeline/respond/stage.py b/astrbot/core/pipeline/respond/stage.py index df24a913a..179700a7a 100644 --- a/astrbot/core/pipeline/respond/stage.py +++ b/astrbot/core/pipeline/respond/stage.py @@ -15,46 +15,37 @@ from astrbot.core.utils.path_util import path_Mapping @register_stage class RespondStage(Stage): - @staticmethod - def _has_meaningful_content(comp: BaseMessageComponent) -> bool: - if isinstance(comp, Comp.Plain | Comp.Unknown): - return bool(comp.text.strip()) - if isinstance(comp, Comp.Face): - return comp.id is not None - if isinstance(comp, Comp.Record | Comp.Video | Comp.Image): - return bool(comp.file) - if isinstance(comp, Comp.At): - return bool(comp.qq) or bool(comp.name) - if isinstance(comp, Comp.Reply): - return bool(comp.id) and comp.sender_id is not None - if isinstance(comp, Comp.Poke): - return comp.target_id() is not None - if isinstance(comp, Comp.Node): - return bool(comp.content) - if isinstance(comp, Comp.Nodes): - return bool(comp.nodes) - if isinstance(comp, Comp.File): - return bool(comp.file_ or comp.url) - if isinstance(comp, Comp.WechatEmoji): - return comp.md5 is not None - if isinstance(comp, Comp.Json): - return bool(comp.data) - if isinstance(comp, Comp.Share): - return bool(comp.url) or bool(comp.title) - if isinstance(comp, Comp.Music): - return bool( - (comp.id and comp._type and comp._type != "custom") - or (comp._type == "custom" and comp.url and comp.audio and comp.title) - ) - if isinstance(comp, Comp.Forward): - return bool(comp.id) - if isinstance(comp, Comp.Location): - return comp.lat is not None and comp.lon is not None - if isinstance(comp, Comp.Contact): - return bool(comp._type and comp.id) - if isinstance(comp, Comp.Shake | Comp.Dice | Comp.RPS): - return True - return False + # 组件类型到其非空判断函数的映射 + _component_validators = { + Comp.Plain: lambda comp: bool( + comp.text and comp.text.strip(), + ), # 纯文本消息需要strip + Comp.Face: lambda comp: comp.id is not None, # QQ表情 + Comp.Record: lambda comp: bool(comp.file), # 语音 + Comp.Video: lambda comp: bool(comp.file), # 视频 + Comp.At: lambda comp: bool(comp.qq) or bool(comp.name), # @ + Comp.Image: lambda comp: bool(comp.file), # 图片 + Comp.Reply: lambda comp: bool(comp.id) and comp.sender_id is not None, # 回复 + Comp.Poke: lambda comp: comp.target_id() is not None, # 戳一戳 + Comp.Node: lambda comp: bool(comp.content), # 转发节点 + Comp.Nodes: lambda comp: bool(comp.nodes), # 多个转发节点 + Comp.File: lambda comp: bool(comp.file_ or comp.url), + Comp.Json: lambda comp: bool(comp.data), # Json 卡片 + Comp.Share: lambda comp: bool(comp.url) or bool(comp.title), + Comp.Music: lambda comp: ( + (comp.id and comp._type and comp._type != "custom") + or (comp._type == "custom" and comp.url and comp.audio and comp.title) + ), # 音乐分享 + Comp.Forward: lambda comp: bool(comp.id), # 合并转发 + Comp.Location: lambda comp: bool( + comp.lat is not None and comp.lon is not None + ), # 位置 + Comp.Contact: lambda comp: bool(comp._type and comp.id), # 推荐好友 or 群 + Comp.Shake: lambda _: True, # 窗口抖动(戳一戳) + Comp.Dice: lambda _: True, # 掷骰子魔法表情 + Comp.RPS: lambda _: True, # 猜拳魔法表情 + Comp.Unknown: lambda comp: bool(comp.text and comp.text.strip()), + } async def initialize(self, ctx: PipelineContext) -> None: self.ctx = ctx diff --git a/astrbot/core/pipeline/waking_check/stage.py b/astrbot/core/pipeline/waking_check/stage.py index 54ec0bf8b..b40359129 100644 --- a/astrbot/core/pipeline/waking_check/stage.py +++ b/astrbot/core/pipeline/waking_check/stage.py @@ -21,6 +21,7 @@ UNIQUE_SESSION_ID_BUILDERS: dict[str, Callable[[AstrMessageEvent], str | None]] "qq_official_webhook": lambda e: e.get_sender_id(), "lark": lambda e: f"{e.get_sender_id()}%{e.get_group_id()}", "misskey": lambda e: f"{e.get_session_id()}_{e.get_sender_id()}", + "matrix": lambda e: f"{e.get_sender_id()}_{e.get_group_id() or e.get_session_id()}", } diff --git a/astrbot/core/platform/astr_message_event.py b/astrbot/core/platform/astr_message_event.py index 5521ba5b4..59818260e 100644 --- a/astrbot/core/platform/astr_message_event.py +++ b/astrbot/core/platform/astr_message_event.py @@ -419,6 +419,7 @@ class AstrMessageEvent(abc.ABC): tool_set: ToolSet | None = None, session_id: str = "", image_urls: list[str] | None = None, + audio_urls: list[str] | None = None, contexts: list | None = None, system_prompt: str = "", conversation: Conversation | None = None, @@ -437,7 +438,9 @@ class AstrMessageEvent(abc.ABC): image_urls: 可以是 base64:// 或者 http:// 开头的图片链接,也可以是本地图片路径。 - contexts: 当指定 contexts 时,将会使用 contexts 作为上下文。如果同时传入了 conversation,将会忽略 conversation。 + audio_urls: 音频 URL 列表,也支持本地路径。 + + contexts: 当指定 contexts 时,将会使用 contexts 作为上下文。如果同时传入了 conversation,将会忽略 conversation。 func_tool_manager: [Deprecated] 函数工具管理器,用于调用函数工具。用 self.context.get_llm_tool_manager() 获取。已过时,请使用 tool_set 参数代替。 @@ -446,6 +449,8 @@ class AstrMessageEvent(abc.ABC): """ if image_urls is None: image_urls = [] + if audio_urls is None: + audio_urls = [] if contexts is None: contexts = [] if len(contexts) > 0 and conversation: @@ -457,6 +462,7 @@ class AstrMessageEvent(abc.ABC): prompt=prompt, session_id=session_id, image_urls=image_urls, + audio_urls=audio_urls, # func_tool=func_tool_manager, func_tool=tool_set, contexts=contexts, diff --git a/astrbot/core/provider/entities.py b/astrbot/core/provider/entities.py index f3ceaebdc..cdcb731f7 100644 --- a/astrbot/core/provider/entities.py +++ b/astrbot/core/provider/entities.py @@ -3,8 +3,11 @@ from __future__ import annotations import base64 import enum import json +import uuid from dataclasses import dataclass, field +from pathlib import Path from typing import Any +from urllib.parse import urlparse import aiofiles from anthropic.types import Message as AnthropicMessage @@ -22,7 +25,8 @@ from astrbot.core.agent.message import ( from astrbot.core.agent.tool import ToolSet from astrbot.core.db.po import Conversation from astrbot.core.message.message_event_result import MessageChain -from astrbot.core.utils.io import download_image_by_url +from astrbot.core.utils.astrbot_path import get_astrbot_temp_path +from astrbot.core.utils.io import download_file, download_image_by_url class ProviderType(enum.Enum): @@ -94,6 +98,8 @@ class ProviderRequest: """会话 ID""" image_urls: list[str] = field(default_factory=list) """图片 URL 列表""" + audio_urls: list[str] = field(default_factory=list) + """音频 URL 列表,也支持本地路径""" extra_user_content_parts: list[ContentPart] = field(default_factory=list) """额外的用户消息内容部分列表,用于在用户消息后添加额外的内容块(如系统提醒、指令等)。支持 dict 或 ContentPart 对象""" func_tool: ToolSet | None = None @@ -116,6 +122,7 @@ class ProviderRequest: return ( f"ProviderRequest(prompt={self.prompt}, session_id={self.session_id}, " f"image_count={len(self.image_urls or [])}, " + f"audio_count={len(self.audio_urls or [])}, " f"func_tool={self.func_tool}, " f"contexts={self._print_friendly_context()}, " f"system_prompt={self.system_prompt}, " @@ -134,9 +141,12 @@ class ProviderRequest: self.tool_calls_result.append(tool_calls_result) def _print_friendly_context(self): - """打印友好的消息上下文。将 image_url 的值替换为 """ + """打印友好的消息上下文。将多模态内容折叠为简短标记。""" if not self.contexts: - return f"prompt: {self.prompt}, image_count: {len(self.image_urls or [])}" + return ( + f"prompt: {self.prompt}, image_count: {len(self.image_urls or [])}, " + f"audio_count: {len(self.audio_urls or [])}" + ) result_parts = [] @@ -149,6 +159,7 @@ class ProviderRequest: elif isinstance(content, list): msg_parts = [] image_count = 0 + audio_count = 0 for item in content: item_type = item.get("type", "") @@ -157,19 +168,26 @@ class ProviderRequest: msg_parts.append(item.get("text", "")) elif item_type == "image_url": image_count += 1 + elif item_type == "audio_url": + audio_count += 1 if image_count > 0: if msg_parts: msg_parts.append(f"[+{image_count} images]") else: msg_parts.append(f"[{image_count} images]") + if audio_count > 0: + if msg_parts: + msg_parts.append(f"[+{audio_count} audios]") + else: + msg_parts.append(f"[{audio_count} audios]") result_parts.append(f"{role}: {''.join(msg_parts)}") return "\n".join(result_parts) - async def assemble_context(self) -> dict[str, str | list[dict[str, object]]]: - """将请求(prompt 和 image_urls)包装成 OpenAI 的消息格式。""" + async def assemble_context(self) -> dict: + """将请求(prompt、image_urls 和 audio_urls)包装成统一消息格式。""" # 构建内容块列表 content_blocks: list[dict[str, object]] = [] @@ -179,6 +197,9 @@ class ProviderRequest: elif self.image_urls: # 如果没有文本但有图片,添加占位文本 content_blocks.append({"type": "text", "text": "[图片]"}) + elif self.audio_urls: + # 如果没有文本但有音频,添加占位文本 + content_blocks.append({"type": "text", "text": "[音频]"}) # 2. 额外的内容块(系统提醒、指令等) if self.extra_user_content_parts: @@ -204,11 +225,56 @@ class ProviderRequest: {"type": "image_url", "image_url": {"url": image_data}}, ) - # 只有当只有一个来自 prompt 的文本块且没有额外内容块时,才降级为简单格式以保持向后兼容 + # 4. 音频内容 + if self.audio_urls: + for audio_url in self.audio_urls: + if audio_url.startswith("http"): + parsed_url = urlparse(audio_url) + suffix = Path(parsed_url.path).suffix + temp_dir = Path(get_astrbot_temp_path()) + temp_dir.mkdir(parents=True, exist_ok=True) + temp_audio_path = ( + temp_dir / f"provider_request_audio_{uuid.uuid4().hex}{suffix}" + ) + try: + await download_file(audio_url, str(temp_audio_path)) + audio_data = await self._encode_audio_bs64( + str(temp_audio_path), + source_ref=audio_url, + ) + finally: + try: + temp_audio_path.unlink(missing_ok=True) + except Exception as exc: + logger.warning( + "Failed to cleanup %s: %s", + temp_audio_path, + exc, + ) + elif audio_url.startswith("file:///"): + audio_path = audio_url.replace("file:///", "") + audio_data = await self._encode_audio_bs64( + audio_path, + source_ref=audio_url, + ) + else: + audio_data = await self._encode_audio_bs64( + audio_url, + source_ref=audio_url, + ) + if not audio_data: + logger.warning(f"音频 {audio_url} 得到的结果为空,将忽略。") + continue + content_blocks.append( + {"type": "audio_url", "audio_url": {"url": audio_data}}, + ) + + # 只有当只有一个来自 prompt 的文本块且没有额外内容块时,才降级为简单格式以保持向后兼容 if ( len(content_blocks) == 1 and not self.extra_user_content_parts and not self.image_urls + and not self.audio_urls ): first_block = content_blocks[0] if first_block.get("type") == "text": @@ -226,7 +292,21 @@ class ProviderRequest: async with aiofiles.open(image_url, "rb") as f: image_bs64 = base64.b64encode(await f.read()).decode("utf-8") return "data:image/jpeg;base64," + image_bs64 - return "" + + async def _encode_audio_bs64( + self, + audio_path: str, + source_ref: str | None = None, + ) -> str: + """将音频转换为 base64""" + mime_type = "audio/wav" + + if audio_path.startswith("base64://"): + return audio_path.replace("base64://", f"data:{mime_type};base64,", 1) + + with open(audio_path, "rb") as f: + audio_bs64 = base64.b64encode(f.read()).decode("utf-8") + return f"data:{mime_type};base64," + audio_bs64 @dataclass diff --git a/astrbot/core/provider/manager.py b/astrbot/core/provider/manager.py index cb6b3f662..86899287c 100644 --- a/astrbot/core/provider/manager.py +++ b/astrbot/core/provider/manager.py @@ -256,6 +256,8 @@ class ProviderManager: provider = self.provider_insts[0] if self.provider_insts else None elif provider_type == ProviderType.SPEECH_TO_TEXT: provider_id = config["provider_stt_settings"].get("provider_id") + if not config["provider_stt_settings"].get("enable"): + return None if not provider_id: return None provider = self.inst_map.get(provider_id) @@ -265,6 +267,8 @@ class ProviderManager: ) elif provider_type == ProviderType.TEXT_TO_SPEECH: provider_id = config["provider_tts_settings"].get("provider_id") + if not config["provider_tts_settings"].get("enable"): + return None if not provider_id: return None provider = self.inst_map.get(provider_id) @@ -369,6 +373,8 @@ class ProviderManager: from .sources.openai_source import ( ProviderOpenAIOfficial as ProviderOpenAIOfficial, ) + case "longcat_chat_completion": + from .sources.longcat_source import ProviderLongCat as ProviderLongCat case "zhipu_chat_completion": from .sources.zhipu_source import ProviderZhipu as ProviderZhipu case "groq_chat_completion": @@ -483,6 +489,10 @@ class ProviderManager: from .sources.bailian_rerank_source import ( BailianRerankProvider as BailianRerankProvider, ) + case "nvidia_rerank": + from .sources.nvidia_rerank_source import ( + NvidiaRerankProvider as NvidiaRerankProvider, + ) def get_merged_provider_config(self, provider_config: dict) -> dict: """获取 provider 配置和 provider_source 配置合并后的结果 diff --git a/astrbot/core/provider/provider.py b/astrbot/core/provider/provider.py index 2238e8300..7943927ad 100644 --- a/astrbot/core/provider/provider.py +++ b/astrbot/core/provider/provider.py @@ -93,6 +93,7 @@ class Provider(AbstractProvider): prompt: str | None = None, session_id: str | None = None, image_urls: list[str] | None = None, + audio_urls: list[str] | None = None, func_tool: ToolSet | None = None, contexts: list[Message] | list[dict] | None = None, system_prompt: str | None = None, @@ -108,6 +109,7 @@ class Provider(AbstractProvider): prompt: 提示词,和 contexts 二选一使用,如果都指定,则会将 prompt(以及可能的 image_urls) 作为最新的一条记录添加到 contexts 中 session_id: 会话 ID(此属性已经被废弃) image_urls: 图片 URL 列表 + audio_urls: 音频 URL 列表,也支持本地路径 tools: tool set contexts: 上下文,和 prompt 二选一使用 tool_calls_result: 回传给 LLM 的工具调用结果。参考: https://platform.openai.com/docs/guides/function-calling @@ -116,8 +118,9 @@ class Provider(AbstractProvider): kwargs: 其他参数 Notes: - - 如果传入了 image_urls,将会在对话时附上图片。如果模型不支持图片输入,将会抛出错误。 - - 如果传入了 tools,将会使用 tools 进行 Function-calling。如果模型不支持 Function-calling,将会抛出错误。 + - 如果传入了 image_urls,将会在对话时附上图片。如果模型不支持图片输入,将会抛出错误。 + - 如果传入了 audio_urls,将会在对话时附上音频。如果模型不支持音频输入,将会抛出错误或降级处理。 + - 如果传入了 tools,将会使用 tools 进行 Function-calling。如果模型不支持 Function-calling,将会抛出错误。 """ ... @@ -127,6 +130,7 @@ class Provider(AbstractProvider): prompt: str | None = None, session_id: str | None = None, image_urls: list[str] | None = None, + audio_urls: list[str] | None = None, func_tool: ToolSet | None = None, contexts: list[Message] | list[dict] | None = None, system_prompt: str | None = None, @@ -142,6 +146,7 @@ class Provider(AbstractProvider): prompt: 提示词,和 contexts 二选一使用,如果都指定,则会将 prompt(以及可能的 image_urls) 作为最新的一条记录添加到 contexts 中 session_id: 会话 ID(此属性已经被废弃) image_urls: 图片 URL 列表 + audio_urls: 音频 URL 列表,也支持本地路径 tools: tool set contexts: 上下文,和 prompt 二选一使用 tool_calls_result: 回传给 LLM 的工具调用结果。参考: https://platform.openai.com/docs/guides/function-calling @@ -150,8 +155,9 @@ class Provider(AbstractProvider): kwargs: 其他参数 Notes: - - 如果传入了 image_urls,将会在对话时附上图片。如果模型不支持图片输入,将会抛出错误。 - - 如果传入了 tools,将会使用 tools 进行 Function-calling。如果模型不支持 Function-calling,将会抛出错误。 + - 如果传入了 image_urls,将会在对话时附上图片。如果模型不支持图片输入,将会抛出错误。 + - 如果传入了 audio_urls,将会在对话时附上音频。如果模型不支持音频输入,将会抛出错误或降级处理。 + - 如果传入了 tools,将会使用 tools 进行 Function-calling。如果模型不支持 Function-calling,将会抛出错误。 """ if False: diff --git a/astrbot/core/provider/sources/anthropic_source.py b/astrbot/core/provider/sources/anthropic_source.py index 735c1ba45..1f3f9c36d 100644 --- a/astrbot/core/provider/sources/anthropic_source.py +++ b/astrbot/core/provider/sources/anthropic_source.py @@ -13,7 +13,7 @@ from anthropic.types.usage import Usage from astrbot import logger from astrbot.api.provider import Provider -from astrbot.core.agent.message import ContentPart, ImageURLPart, TextPart +from astrbot.core.agent.message import AudioURLPart, ContentPart, ImageURLPart, TextPart from astrbot.core.exceptions import EmptyModelOutputError from astrbot.core.provider.entities import LLMResponse, TokenUsage from astrbot.core.provider.func_tool_manager import ToolSet @@ -253,6 +253,13 @@ class ProviderAnthropic(Provider): logger.warning( f"Unsupported image URL format for Anthropic: {url[:50]}..." ) + elif part.get("type") == "audio_url": + converted_content.append( + { + "type": "text", + "text": "[Audio Attachment]", + } + ) else: converted_content.append(part) new_messages.append( @@ -534,6 +541,7 @@ class ProviderAnthropic(Provider): prompt=None, session_id=None, image_urls=None, + audio_urls=None, func_tool=None, contexts=None, system_prompt=None, @@ -548,7 +556,10 @@ class ProviderAnthropic(Provider): new_record = None if prompt is not None: new_record = await self.assemble_context( - prompt, image_urls, extra_user_content_parts + prompt or "", + image_urls, + audio_urls, + extra_user_content_parts, ) context_query = self._ensure_message_to_dicts(contexts) if new_record: @@ -594,6 +605,7 @@ class ProviderAnthropic(Provider): prompt=None, session_id=None, image_urls=None, + audio_urls=None, func_tool=None, contexts=None, system_prompt=None, @@ -608,7 +620,10 @@ class ProviderAnthropic(Provider): new_record = None if prompt is not None: new_record = await self.assemble_context( - prompt, image_urls, extra_user_content_parts + prompt or "", + image_urls, + audio_urls, + extra_user_content_parts, ) context_query = self._ensure_message_to_dicts(contexts) if new_record: @@ -659,6 +674,7 @@ class ProviderAnthropic(Provider): self, text: str, image_urls: list[str] | None = None, + audio_urls: list[str] | None = None, extra_user_content_parts: list[ContentPart] | None = None, ): """组装上下文,支持文本和图片""" @@ -696,8 +712,10 @@ class ProviderAnthropic(Provider): if text: content.append({"type": "text", "text": text}) elif image_urls: - # 如果没有文本但有图片,添加占位文本 - content.append({"type": "text", "text": "[图片]"}) + # 如果没有文本但有图片,添加占位文本 + content.append({"type": "text", "text": "[Image]"}) + elif audio_urls: + content.append({"type": "text", "text": "[Audio]"}) elif extra_user_content_parts: # 如果只有额外内容块,也需要添加占位文本 content.append({"type": "text", "text": " "}) @@ -711,6 +729,8 @@ class ProviderAnthropic(Provider): image_dict = await resolve_image_url(block.image_url.url) if image_dict: content.append(image_dict) + elif isinstance(block, AudioURLPart): + content.append({"type": "text", "text": "[Audio]"}) else: raise ValueError(f"不支持的额外内容块类型: {type(block)}") @@ -720,12 +740,16 @@ class ProviderAnthropic(Provider): image_dict = await resolve_image_url(image_url) if image_dict: content.append(image_dict) + if audio_urls: + for _audio_path in audio_urls: + content.append({"type": "text", "text": "[Audio]"}) # 如果只有主文本且没有额外内容块和图片,返回简单格式以保持向后兼容 if ( text and not extra_user_content_parts and not image_urls + and not audio_urls and len(content) == 1 and content[0]["type"] == "text" ): diff --git a/astrbot/core/provider/sources/gemini_source.py b/astrbot/core/provider/sources/gemini_source.py index ba7d478b3..96e404510 100644 --- a/astrbot/core/provider/sources/gemini_source.py +++ b/astrbot/core/provider/sources/gemini_source.py @@ -3,8 +3,11 @@ import base64 import json import logging import random +import uuid from collections.abc import AsyncGenerator -from typing import ClassVar, Literal +from pathlib import Path +from typing import Literal, cast +from urllib.parse import urlparse import aiofiles from google import genai @@ -14,13 +17,14 @@ from google.genai.errors import APIError import astrbot.core.message.components as Comp from astrbot import logger from astrbot.api.provider import Provider -from astrbot.core.agent.message import ContentPart, ImageURLPart, Message, TextPart -from astrbot.core.agent.tool import ToolSet +from astrbot.core.agent.message import AudioURLPart, ContentPart, ImageURLPart, TextPart from astrbot.core.exceptions import EmptyModelOutputError from astrbot.core.message.message_event_result import MessageChain -from astrbot.core.provider.entities import LLMResponse, TokenUsage, ToolCallsResult -from astrbot.core.provider.register import register_provider_adapter -from astrbot.core.utils.io import download_image_by_url +from astrbot.core.provider.entities import LLMResponse, TokenUsage +from astrbot.core.provider.func_tool_manager import ToolSet +from astrbot.core.utils.astrbot_path import get_astrbot_temp_path +from astrbot.core.utils.io import download_file, download_image_by_url +from astrbot.core.utils.media_utils import ensure_wav from astrbot.core.utils.network_utils import is_connection_error, log_connection_failure @@ -263,6 +267,12 @@ class ProviderGoogleGenAI(Provider): image_bytes = base64.b64decode(url.split(",", 1)[1]) return types.Part.from_bytes(data=image_bytes, mime_type=mime_type) + def process_audio_url(audio_url_dict: dict) -> types.Part: + url = audio_url_dict["url"] + mime_type = url.split(":")[1].split(";")[0] + audio_bytes = base64.b64decode(url.split(",", 1)[1]) + return types.Part.from_bytes(data=audio_bytes, mime_type=mime_type) + def append_or_extend( contents: list[types.Content], part: list[types.Part], @@ -286,9 +296,15 @@ class ProviderGoogleGenAI(Provider): if role == "user": if isinstance(content, list): parts = [ - types.Part.from_text(text=item["text"] or " ") - if item["type"] == "text" - else process_image_url(item["image_url"]) + ( + types.Part.from_text(text=item["text"] or " ") + if item["type"] == "text" + else ( + process_image_url(item["image_url"]) + if item["type"] == "image_url" + else process_audio_url(item["audio_url"]) + ) + ) for item in content ] else: @@ -395,14 +411,23 @@ class ProviderGoogleGenAI(Provider): ) def _process_content_parts( - self, candidate: types.Candidate, llm_response: LLMResponse + self, + candidate: types.Candidate, + llm_response: LLMResponse, + *, + validate_output: bool = True, ) -> MessageChain: """处理内容部分并构建消息链""" if not candidate.content: logger.warning(f"收到的 candidate.content 为空: {candidate}") - raise EmptyModelOutputError( - f"Gemini candidate content is empty. finish_reason={candidate.finish_reason}" - ) + if validate_output: + raise EmptyModelOutputError( + "Gemini candidate content is empty. " + f"finish_reason={candidate.finish_reason}" + ) + llm_response.result_chain = MessageChain(chain=[]) + return llm_response.result_chain + finish_reason = candidate.finish_reason result_parts: list[types.Part] | None = candidate.content.parts if finish_reason == types.FinishReason.SAFETY: @@ -418,9 +443,15 @@ class ProviderGoogleGenAI(Provider): raise Exception("模型生成内容违反 Gemini 平台政策") if not result_parts: logger.warning(f"收到的 candidate.content.parts 为空: {candidate}") - raise EmptyModelOutputError( - f"Gemini candidate content parts are empty. finish_reason={candidate.finish_reason}" - ) + if validate_output: + raise EmptyModelOutputError( + "Gemini candidate content parts are empty. " + f"finish_reason={candidate.finish_reason}" + ) + llm_response.result_chain = MessageChain(chain=[]) + return llm_response.result_chain + + # 提取 reasoning content reasoning = self._extract_reasoning_content(candidate) if reasoning: llm_response.reasoning_content = reasoning @@ -466,11 +497,12 @@ class ProviderGoogleGenAI(Provider): llm_response.reasoning_signature = base64.b64encode(ts).decode("utf-8") chain_result = MessageChain(chain=chain) llm_response.result_chain = chain_result - self._ensure_usable_response( - llm_response, - response_id=None, - finish_reason=str(finish_reason) if finish_reason is not None else None, - ) + if validate_output: + self._ensure_usable_response( + llm_response, + response_id=None, + finish_reason=str(finish_reason) if finish_reason is not None else None, + ) return chain_result async def _query(self, payloads: dict, tools: ToolSet | None) -> LLMResponse: @@ -600,7 +632,9 @@ class ProviderGoogleGenAI(Provider): llm_response = LLMResponse("assistant", is_chunk=False) llm_response.raw_completion = chunk llm_response.result_chain = self._process_content_parts( - chunk.candidates[0], llm_response + chunk.candidates[0], + llm_response, + validate_output=False, ) llm_response.id = chunk.response_id if chunk.usage_metadata: @@ -624,7 +658,9 @@ class ProviderGoogleGenAI(Provider): final_response = LLMResponse("assistant", is_chunk=False) final_response.raw_completion = chunk final_response.result_chain = self._process_content_parts( - chunk.candidates[0], final_response + chunk.candidates[0], + final_response, + validate_output=False, ) final_response.id = chunk.response_id if chunk.usage_metadata: @@ -647,15 +683,16 @@ class ProviderGoogleGenAI(Provider): async def text_chat( self, - prompt: str | None = None, - session_id: str | None = None, - image_urls: list[str] | None = None, - func_tool: ToolSet | None = None, - contexts: list[Message] | list[dict] | None = None, - system_prompt: str | None = None, - tool_calls_result: ToolCallsResult | list[ToolCallsResult] | None = None, - model: str | None = None, - extra_user_content_parts: list[ContentPart] | None = None, + prompt=None, + session_id=None, + image_urls=None, + audio_urls=None, + func_tool=None, + contexts=None, + system_prompt=None, + tool_calls_result=None, + model=None, + extra_user_content_parts=None, tool_choice: Literal["auto", "required"] = "auto", **kwargs, ) -> LLMResponse: @@ -664,7 +701,10 @@ class ProviderGoogleGenAI(Provider): new_record = None if prompt is not None: new_record = await self.assemble_context( - prompt, image_urls, extra_user_content_parts + prompt or "", + image_urls, + audio_urls, + extra_user_content_parts, ) context_query = self._ensure_message_to_dicts(contexts) if new_record: @@ -698,15 +738,16 @@ class ProviderGoogleGenAI(Provider): async def text_chat_stream( self, - prompt: str | None = None, - session_id: str | None = None, - image_urls: list[str] | None = None, - func_tool: ToolSet | None = None, - contexts: list[Message] | list[dict] | None = None, - system_prompt: str | None = None, - tool_calls_result: ToolCallsResult | list[ToolCallsResult] | None = None, - model: str | None = None, - extra_user_content_parts: list[ContentPart] | None = None, + prompt=None, + session_id=None, + image_urls=None, + audio_urls=None, + func_tool=None, + contexts=None, + system_prompt=None, + tool_calls_result=None, + model=None, + extra_user_content_parts=None, tool_choice: Literal["auto", "required"] = "auto", **kwargs, ) -> AsyncGenerator[LLMResponse, None]: @@ -715,7 +756,10 @@ class ProviderGoogleGenAI(Provider): new_record = None if prompt is not None: new_record = await self.assemble_context( - prompt, image_urls, extra_user_content_parts + prompt or "", + image_urls, + audio_urls, + extra_user_content_parts, ) context_query = self._ensure_message_to_dicts(contexts) if new_record: @@ -775,6 +819,7 @@ class ProviderGoogleGenAI(Provider): self, text: str, image_urls: list[str] | None = None, + audio_urls: list[str] | None = None, extra_user_content_parts: list[ContentPart] | None = None, ): """组装上下文。""" @@ -793,11 +838,52 @@ class ProviderGoogleGenAI(Provider): return None return {"type": "image_url", "image_url": {"url": image_data}} + async def resolve_audio_part(audio_path: str) -> dict | None: + if audio_path.startswith("http"): + suffix = Path(urlparse(audio_path).path).suffix or ".wav" + temp_dir = Path(get_astrbot_temp_path()) + temp_dir.mkdir(parents=True, exist_ok=True) + resolved_path = str( + temp_dir / f"provider_audio_{uuid.uuid4().hex}{suffix}" + ) + await download_file(audio_path, resolved_path) + elif audio_path.startswith("file:///"): + resolved_path = audio_path.replace("file:///", "") + else: + resolved_path = audio_path + + suffix = Path(resolved_path).suffix.lower() + if suffix != ".mp3": + resolved_path = await ensure_wav(resolved_path) + suffix = ".wav" + + try: + audio_bytes = Path(resolved_path).read_bytes() + except OSError as exc: + logger.warning( + f"Failed to read audio file {resolved_path}, skipping. Error: {exc}" + ) + return None + + mime_type = { + ".wav": "audio/wav", + ".mp3": "audio/mp3", + }.get(suffix, "audio/wav") + audio_data = base64.b64encode(audio_bytes).decode("utf-8") + return { + "type": "audio_url", + "audio_url": {"url": f"data:{mime_type};base64,{audio_data}"}, + } + + # 构建内容块列表 content_blocks = [] if text: content_blocks.append({"type": "text", "text": text}) elif image_urls: - content_blocks.append({"type": "text", "text": "[图片]"}) + # 如果没有文本但有图片,添加占位文本 + content_blocks.append({"type": "text", "text": "[Image]"}) + elif audio_urls: + content_blocks.append({"type": "text", "text": "[Audio]"}) elif extra_user_content_parts: content_blocks.append({"type": "text", "text": " "}) if extra_user_content_parts: @@ -808,6 +894,10 @@ class ProviderGoogleGenAI(Provider): image_part = await resolve_image_part(part.image_url.url) if image_part: content_blocks.append(image_part) + elif isinstance(part, AudioURLPart): + audio_part = await resolve_audio_part(part.audio_url.url) + if audio_part: + content_blocks.append(audio_part) else: raise ValueError(f"不支持的额外内容块类型: {type(part)}") if image_urls: @@ -815,12 +905,21 @@ class ProviderGoogleGenAI(Provider): image_part = await resolve_image_part(image_url) if image_part: content_blocks.append(image_part) + + if audio_urls: + for audio_path in audio_urls: + audio_part = await resolve_audio_part(audio_path) + if audio_part: + content_blocks.append(audio_part) + + # 如果只有主文本且没有额外内容块和图片,返回简单格式以保持向后兼容 if ( text - and (not extra_user_content_parts) - and (not image_urls) - and (len(content_blocks) == 1) - and (content_blocks[0]["type"] == "text") + and not extra_user_content_parts + and not image_urls + and not audio_urls + and len(content_blocks) == 1 + and content_blocks[0]["type"] == "text" ): return {"role": "user", "content": content_blocks[0]["text"]} return {"role": "user", "content": content_blocks} diff --git a/astrbot/core/provider/sources/longcat_source.py b/astrbot/core/provider/sources/longcat_source.py new file mode 100644 index 000000000..e251fb310 --- /dev/null +++ b/astrbot/core/provider/sources/longcat_source.py @@ -0,0 +1,23 @@ +from ..register import register_provider_adapter +from .openai_source import ProviderOpenAIOfficial + + +@register_provider_adapter( + "longcat_chat_completion", "LongCat Chat Completion Provider Adapter" +) +class ProviderLongCat(ProviderOpenAIOfficial): + def __init__( + self, + provider_config: dict, + provider_settings: dict, + ) -> None: + api_base = (provider_config.get("api_base", "") or "").strip() + if not api_base: + provider_config["api_base"] = "https://api.longcat.chat/openai/v1" + else: + normalized_api_base = api_base.rstrip("/") + if normalized_api_base.endswith("/openai"): + normalized_api_base = f"{normalized_api_base}/v1" + provider_config["api_base"] = normalized_api_base + + super().__init__(provider_config, provider_settings) diff --git a/astrbot/core/provider/sources/nvidia_rerank_source.py b/astrbot/core/provider/sources/nvidia_rerank_source.py new file mode 100644 index 000000000..c168da4a6 --- /dev/null +++ b/astrbot/core/provider/sources/nvidia_rerank_source.py @@ -0,0 +1,164 @@ +import aiohttp + +from astrbot import logger + +from ..entities import ProviderType, RerankResult +from ..provider import RerankProvider +from ..register import register_provider_adapter + + +@register_provider_adapter( + "nvidia_rerank", "NVIDIA Rerank 适配器", provider_type=ProviderType.RERANK +) +class NvidiaRerankProvider(RerankProvider): + def __init__(self, provider_config: dict, provider_settings: dict) -> None: + super().__init__(provider_config, provider_settings) + self.api_key = provider_config.get("nvidia_rerank_api_key", "") + self.base_url = provider_config.get( + "nvidia_rerank_api_base", "https://ai.api.nvidia.com/v1/retrieval" + ).rstrip("/") + self.timeout = provider_config.get("timeout", 20) + self.model = provider_config.get( + "nvidia_rerank_model", "nv-rerank-qa-mistral-4b:1" + ) + self.model_endpoint = provider_config.get( + "nvidia_rerank_model_endpoint", "/reranking" + ) + self.truncate = provider_config.get("nvidia_rerank_truncate", "") + + self.client = None + self.set_model(self.model) + + async def _get_client(self): + if self.client is None or self.client.closed: + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + "Accept": "application/json", + } + self.client = aiohttp.ClientSession( + headers=headers, timeout=aiohttp.ClientTimeout(total=self.timeout) + ) + return self.client + + def _get_endpoint(self) -> str: + """ + 构建完整API URL。 + + 根据 Nvidia Rerank API 文档来看,当前URL存在不同模型格式不一致的问题。 + 这里针对模型名做一个基础判断用以适配,后续要等Nvidia统一API格式后再做调整。 + + 例: + 模型: nv-rerank-qa-mistral-4b:1 + URL: .../v1/retrieval/nvidia/reranking + + 模型: nvidia/llama-nemotron-rerank-1b-v2 + URL: .../v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking + """ + + model_path = "nvidia" + logger.debug(f"[NVIDIA Rerank] Building endpoint for model: {self.model}") + if "/" in self.model: + """遵循NVIDIA API的URL规则,替换模型名中特殊字符""" + model_path = self.model.strip("/").replace(".", "_") + endpoint = self.model_endpoint.lstrip("/") + return f"{self.base_url}/{model_path}/{endpoint}" + + def _build_payload(self, query: str, documents: list[str]) -> dict: + """构建请求载荷""" + payload = { + "model": self.model, + "query": {"text": query}, + "passages": [{"text": doc} for doc in documents], + } + if self.truncate: + payload["truncate"] = self.truncate + return payload + + def _parse_results( + self, response_data: dict, top_n: int | None + ) -> list[RerankResult]: + """解析响应数据""" + results = response_data.get("rankings", []) + if not results: + logger.warning(f"[NVIDIA Rerank] Empty response: {response_data}") + return [] + + rerank_results = [] + for idx, item in enumerate(results): + try: + index = item.get("index", idx) + score = item.get("relevance_score", item.get("logit", 0.0)) + rerank_results.append( + RerankResult(index=index, relevance_score=float(score)) + ) + except Exception as e: + logger.warning( + f"[NVIDIA Rerank] Result parsing error: {e}, Data={item}" + ) + + rerank_results.sort(key=lambda x: x.relevance_score, reverse=True) + + if top_n is not None and top_n > 0: + return rerank_results[:top_n] + return rerank_results + + def _log_usage(self, data: dict) -> None: + usage = data.get("usage", {}) + total_tokens = usage.get("total_tokens", 0) + if total_tokens > 0: + logger.debug(f"[NVIDIA Rerank] Token Usage: {total_tokens}") + + async def rerank( + self, + query: str, + documents: list[str], + top_n: int | None = None, + ) -> list[RerankResult]: + client = await self._get_client() + if not client or client.closed: + logger.error("[NVIDIA Rerank] Client session not initialized or closed") + return [] + + if not documents or not query.strip(): + logger.warning( + "[NVIDIA Rerank] Input data is invalid, query or documents are empty" + ) + return [] + + try: + payload = self._build_payload(query, documents) + request_url = self._get_endpoint() + + async with client.post(request_url, json=payload) as response: + if response.status != 200: + try: + response_data = await response.json() + error_detail = response_data.get( + "detail", response_data.get("message", "Unknown Error") + ) + + except Exception: + error_detail = await response.text() + response_data = {"message": error_detail} + + logger.error(f"[NVIDIA Rerank] API Error Response: {response_data}") + raise Exception(f"HTTP {response.status} - {error_detail}") + + response_data = await response.json() + logger.debug(f"[NVIDIA Rerank] API Response: {response_data}") + results = self._parse_results(response_data, top_n) + self._log_usage(response_data) + return results + + except aiohttp.ClientError as e: + logger.error(f"[NVIDIA Rerank] Network error: {e}") + raise Exception(f"Network error: {e}") from e + except Exception as e: + logger.error(f"[NVIDIA Rerank] Error: {e}") + raise Exception(f"Rerank error: {e}") from e + + async def terminate(self) -> None: + if self.client and not self.client.closed: + await self.client.close() + self.client = None diff --git a/astrbot/core/provider/sources/openai_source.py b/astrbot/core/provider/sources/openai_source.py index 71bf2b588..55735605d 100644 --- a/astrbot/core/provider/sources/openai_source.py +++ b/astrbot/core/provider/sources/openai_source.py @@ -5,6 +5,7 @@ import inspect import json import random import re +import uuid from collections.abc import AsyncGenerator from io import BytesIO from pathlib import Path @@ -24,13 +25,20 @@ from PIL import UnidentifiedImageError import astrbot.core.message.components as Comp from astrbot import logger from astrbot.api.provider import Provider -from astrbot.core.agent.message import ContentPart, ImageURLPart, Message, TextPart +from astrbot.core.agent.message import ( + AudioURLPart, + ContentPart, + ImageURLPart, + Message, + TextPart, +) from astrbot.core.agent.tool import ToolSet from astrbot.core.exceptions import EmptyModelOutputError from astrbot.core.message.message_event_result import MessageChain from astrbot.core.provider.entities import LLMResponse, TokenUsage, ToolCallsResult -from astrbot.core.provider.register import register_provider_adapter -from astrbot.core.utils.io import download_image_by_url +from astrbot.core.utils.astrbot_path import get_astrbot_temp_path +from astrbot.core.utils.io import download_file, download_image_by_url +from astrbot.core.utils.media_utils import ensure_wav from astrbot.core.utils.network_utils import ( create_proxy_client, is_connection_error, @@ -135,7 +143,10 @@ class ProviderOpenAIOfficial(Provider): if not isinstance(content, list): continue for item in content: - if isinstance(item, dict) and item.get("type") == "image_url": + if isinstance(item, dict) and item.get("type") in { + "image_url", + "audio_url", + }: return True return False @@ -284,24 +295,103 @@ class ProviderOpenAIOfficial(Provider): image_detail = None return url, image_detail - async def _transform_content_part(self, part: dict) -> dict: - url, image_detail = self._extract_image_part_info(part) - if not url: - return part + def _extract_audio_part_info(self, part: dict) -> str | None: + if not isinstance(part, dict) or part.get("type") != "audio_url": + return None + audio_url_data = part.get("audio_url") + if not isinstance(audio_url_data, dict): + logger.warning("音频内容块格式无效,将保留原始内容。") + return None + + url = audio_url_data.get("url") + if not isinstance(url, str) or not url: + logger.warning("音频内容块缺少有效路径,将保留原始内容。") + return None + + return url + + async def _audio_ref_to_local_path(self, audio_ref: str) -> tuple[str, list[Path]]: + cleanup_paths: list[Path] = [] + if audio_ref.startswith("http"): + suffix = Path(urlparse(audio_ref).path).suffix or ".wav" + temp_dir = Path(get_astrbot_temp_path()) + temp_dir.mkdir(parents=True, exist_ok=True) + target_path = temp_dir / f"provider_audio_{uuid.uuid4().hex}{suffix}" + await download_file(audio_ref, str(target_path)) + cleanup_paths.append(target_path) + return str(target_path), cleanup_paths + if audio_ref.startswith("file://"): + return self._file_uri_to_path(audio_ref), cleanup_paths + return audio_ref, cleanup_paths + + async def _resolve_audio_part(self, audio_ref: str) -> dict | None: + cleanup_paths: list[Path] = [] try: - resolved_part = await self._resolve_image_part( - url, image_detail=image_detail - ) + audio_path, cleanup_paths = await self._audio_ref_to_local_path(audio_ref) + suffix = Path(audio_path).suffix.lower() + if suffix == ".mp3": + audio_format = "mp3" + else: + converted_audio_path = await ensure_wav(audio_path) + if converted_audio_path != audio_path: + cleanup_paths.append(Path(converted_audio_path)) + audio_path = converted_audio_path + audio_format = "wav" + audio_bytes = Path(audio_path).read_bytes() except Exception as exc: - logger.warning( - "图片 %s 预处理失败,将保留原始内容。错误: %s", - url, - exc, - ) + logger.warning("音频 %s 预处理失败,将忽略。错误: %s", audio_ref, exc) + return None + finally: + for cleanup_path in cleanup_paths: + try: + cleanup_path.unlink(missing_ok=True) + except Exception as cleanup_exc: + logger.warning( + "Failed to cleanup %s: %s", + cleanup_path, + cleanup_exc, + ) + + return { + "type": "input_audio", + "input_audio": { + "data": base64.b64encode(audio_bytes).decode("utf-8"), + "format": audio_format, + }, + } + + async def _transform_content_part(self, part: dict) -> dict: + if not isinstance(part, dict): return part - return resolved_part or part + if part.get("type") == "image_url": + url, image_detail = self._extract_image_part_info(part) + if not url: + return part + + try: + resolved_part = await self._resolve_image_part( + url, image_detail=image_detail + ) + except Exception as exc: + logger.warning( + "图片 %s 预处理失败,将保留原始内容。错误: %s", + url, + exc, + ) + return part + + return resolved_part or part + + if part.get("type") == "audio_url": + audio_ref = self._extract_audio_part_info(part) + if not audio_ref: + return part + resolved_part = await self._resolve_audio_part(audio_ref) + return resolved_part or part + + return part async def _materialize_message_image_parts(self, message: dict) -> dict: content = message.get("content") @@ -532,6 +622,7 @@ class ProviderOpenAIOfficial(Provider): **payloads, stream=True, extra_body=extra_body, + stream_options={"include_usage": True}, ) llm_response = LLMResponse("assistant", is_chunk=True) @@ -539,12 +630,10 @@ class ProviderOpenAIOfficial(Provider): state = ChatCompletionStreamState() async for chunk in stream: - if not chunk.choices: - continue - choice = chunk.choices[0] - delta = choice.delta + choice = chunk.choices[0] if chunk.choices else None + delta = choice.delta if choice else None - if dtcs := delta.tool_calls: + if delta and (dtcs := delta.tool_calls): for idx, tc in enumerate(dtcs): # siliconflow workaround if tc.function and tc.function.arguments: @@ -574,7 +663,7 @@ class ProviderOpenAIOfficial(Provider): _y = True if chunk.usage: llm_response.usage = self._extract_usage(chunk.usage) - elif choice_usage := getattr(choice, "usage", None): + elif choice and (choice_usage := getattr(choice, "usage", None)): # Workaround for some providers that only return usage in choices[].usage, e.g. MoonshotAI # See https://github.com/AstrBotDevs/AstrBot/issues/6614 llm_response.usage = self._extract_usage(choice_usage) @@ -822,6 +911,7 @@ class ProviderOpenAIOfficial(Provider): self, prompt: str | None, image_urls: list[str] | None = None, + audio_urls: list[str] | None = None, contexts: list[dict] | list[Message] | None = None, system_prompt: str | None = None, tool_calls_result: ToolCallsResult | list[ToolCallsResult] | None = None, @@ -835,7 +925,10 @@ class ProviderOpenAIOfficial(Provider): new_record = None if prompt is not None: new_record = await self.assemble_context( - prompt, image_urls, extra_user_content_parts + prompt or "", + image_urls, + audio_urls, + extra_user_content_parts, ) context_query = copy.deepcopy(self._ensure_message_to_dicts(contexts)) if new_record: @@ -1022,6 +1115,7 @@ class ProviderOpenAIOfficial(Provider): prompt=None, session_id=None, image_urls=None, + audio_urls=None, func_tool=None, contexts=None, system_prompt=None, @@ -1034,6 +1128,7 @@ class ProviderOpenAIOfficial(Provider): payloads, context_query = await self._prepare_chat_payload( prompt, image_urls, + audio_urls, contexts, system_prompt, tool_calls_result, @@ -1093,6 +1188,7 @@ class ProviderOpenAIOfficial(Provider): prompt=None, session_id=None, image_urls=None, + audio_urls=None, func_tool=None, contexts=None, system_prompt=None, @@ -1106,6 +1202,7 @@ class ProviderOpenAIOfficial(Provider): payloads, context_query = await self._prepare_chat_payload( prompt, image_urls, + audio_urls, contexts, system_prompt, tool_calls_result, @@ -1190,6 +1287,7 @@ class ProviderOpenAIOfficial(Provider): self, text: str, image_urls: list[str] | None = None, + audio_urls: list[str] | None = None, extra_user_content_parts: list[ContentPart] | None = None, ) -> dict: """组装成符合 OpenAI 格式的 role 为 user 的消息段""" @@ -1201,8 +1299,10 @@ class ProviderOpenAIOfficial(Provider): if text: content_blocks.append({"type": "text", "text": text}) elif image_urls: - # 如果没有文本但有图片,添加占位文本 - content_blocks.append({"type": "text", "text": "[图片]"}) + # 如果没有文本但有图片,添加占位文本 + content_blocks.append({"type": "text", "text": "[Image]"}) + elif audio_urls: + content_blocks.append({"type": "text", "text": "[Audio]"}) elif extra_user_content_parts: # 如果只有额外内容块,也需要添加占位文本 content_blocks.append({"type": "text", "text": " "}) @@ -1218,6 +1318,10 @@ class ProviderOpenAIOfficial(Provider): ) if image_part: content_blocks.append(image_part) + elif isinstance(part, AudioURLPart): + audio_part = await self._resolve_audio_part(part.audio_url.url) + if audio_part: + content_blocks.append(audio_part) else: raise ValueError(f"不支持的额外内容块类型: {type(part)}") @@ -1228,11 +1332,18 @@ class ProviderOpenAIOfficial(Provider): if image_part: content_blocks.append(image_part) - # 如果只有主文本且没有额外内容块和图片,返回简单格式以保持向后兼容 + if audio_urls: + for audio_path in audio_urls: + audio_part = await self._resolve_audio_part(audio_path) + if audio_part: + content_blocks.append(audio_part) + + # 如果只有主文本且没有额外内容块和图片,返回简单格式以保持向后兼容 if ( text and not extra_user_content_parts and not image_urls + and not audio_urls and len(content_blocks) == 1 and content_blocks[0]["type"] == "text" ): diff --git a/astrbot/core/provider/sources/vllm_rerank_source.py b/astrbot/core/provider/sources/vllm_rerank_source.py index f140d2c4c..18bc7110c 100644 --- a/astrbot/core/provider/sources/vllm_rerank_source.py +++ b/astrbot/core/provider/sources/vllm_rerank_source.py @@ -19,6 +19,11 @@ class VLLMRerankProvider(RerankProvider): self.auth_key = provider_config.get("rerank_api_key", "") self.base_url = provider_config.get("rerank_api_base", "http://127.0.0.1:8000") self.base_url = self.base_url.rstrip("/") + self.api_suffix = provider_config.get("rerank_api_suffix", "/v1/rerank") + if self.api_suffix is None: + self.api_suffix = "/v1/rerank" + if self.api_suffix and not self.api_suffix.startswith("/"): + self.api_suffix = "/" + self.api_suffix self.timeout = provider_config.get("timeout", 20) self.model = provider_config.get("rerank_model", "BAAI/bge-reranker-base") @@ -44,8 +49,9 @@ class VLLMRerankProvider(RerankProvider): if top_n is not None: payload["top_n"] = top_n assert self.client is not None + rerank_url = f"{self.base_url}{self.api_suffix}" async with self.client.post( - f"{self.base_url}/v1/rerank", + rerank_url, json=payload, ) as response: response_data = await response.json() diff --git a/astrbot/core/star/context.py b/astrbot/core/star/context.py index 314e12aca..4cb71d19b 100644 --- a/astrbot/core/star/context.py +++ b/astrbot/core/star/context.py @@ -124,6 +124,7 @@ class Context: chat_provider_id: str, prompt: str | None = None, image_urls: list[str] | None = None, + audio_urls: list[str] | None = None, tools: ToolSet | None = None, system_prompt: str | None = None, contexts: list[Message] | None = None, @@ -137,6 +138,7 @@ class Context: chat_provider_id: The chat provider ID to use. prompt: The prompt to send to the LLM, if `contexts` and `prompt` are both provided, `prompt` will be appended as the last user message image_urls: List of image URLs to include in the prompt, if `contexts` and `prompt` are both provided, `image_urls` will be appended to the last user message + audio_urls: List of audio URLs or local paths to include in the prompt, if `contexts` and `prompt` are both provided, `audio_urls` will be appended to the last user message tools: ToolSet of tools available to the LLM system_prompt: System prompt to guide the LLM's behavior, if provided, it will always insert as the first system message in the context contexts: context messages for the LLM @@ -152,6 +154,7 @@ class Context: llm_resp = await prov.text_chat( prompt=prompt, image_urls=image_urls, + audio_urls=audio_urls, func_tool=tools, contexts=contexts, system_prompt=system_prompt, @@ -166,6 +169,7 @@ class Context: chat_provider_id: str, prompt: str | None = None, image_urls: list[str] | None = None, + audio_urls: list[str] | None = None, tools: ToolSet | None = None, system_prompt: str | None = None, contexts: list[Message] | None = None, @@ -185,6 +189,7 @@ class Context: chat_provider_id: The chat provider ID to use. prompt: The prompt to send to the LLM, if `contexts` and `prompt` are both provided, `prompt` will be appended as the last user message image_urls: List of image URLs to include in the prompt, if `contexts` and `prompt` are both provided, `image_urls` will be appended to the last user message + audio_urls: List of audio URLs or local paths to include in the prompt, if `contexts` and `prompt` are both provided, `audio_urls` will be appended to the last user message tools: ToolSet of tools available to the LLM system_prompt: System prompt to guide the LLM's behavior, if provided, it will always insert as the first system message in the context contexts: context messages for the LLM @@ -221,6 +226,7 @@ class Context: request = ProviderRequest( prompt=prompt, image_urls=image_urls or [], + audio_urls=audio_urls or [], func_tool=tools, contexts=context_, system_prompt=system_prompt or "", diff --git a/astrbot/core/star/filter/platform_adapter_type.py b/astrbot/core/star/filter/platform_adapter_type.py index 3ac8019ef..55043cbe3 100644 --- a/astrbot/core/star/filter/platform_adapter_type.py +++ b/astrbot/core/star/filter/platform_adapter_type.py @@ -22,6 +22,7 @@ class PlatformAdapterType(enum.Flag): SATORI = enum.auto() MISSKEY = enum.auto() LINE = enum.auto() + MATRIX = enum.auto() ALL = ( AIOCQHTTP | QQOFFICIAL @@ -38,6 +39,7 @@ class PlatformAdapterType(enum.Flag): | SATORI | MISSKEY | LINE + | MATRIX ) @@ -57,6 +59,7 @@ ADAPTER_NAME_2_TYPE = { "satori": PlatformAdapterType.SATORI, "misskey": PlatformAdapterType.MISSKEY, "line": PlatformAdapterType.LINE, + "matrix": PlatformAdapterType.MATRIX, } diff --git a/astrbot/core/utils/media_utils.py b/astrbot/core/utils/media_utils.py index 85788f4eb..16ae20b7b 100644 --- a/astrbot/core/utils/media_utils.py +++ b/astrbot/core/utils/media_utils.py @@ -231,8 +231,8 @@ async def convert_audio_format( Args: audio_path: 原始音频文件路径 - output_format: 目标格式,例如 amr / ogg - output_path: 输出文件路径,如果为None则自动生成 + output_format: 目标格式,例如 amr / ogg / opus / wav + output_path: 输出文件路径,如果为None则自动生成 Returns: 转换后的音频文件路径 @@ -250,6 +250,8 @@ async def convert_audio_format( args.extend(["-ac", "1", "-ar", "8000", "-ab", "12.2k"]) elif output_format == "ogg": args.extend(["-acodec", "libopus", "-ac", "1", "-ar", "16000"]) + elif output_format == "opus": + args.extend(["-acodec", "libopus", "-ac", "1", "-ar", "16000"]) args.append(output_path) try: @@ -291,11 +293,67 @@ async def convert_audio_to_wav(audio_path: str, output_path: str | None = None) ) +async def ensure_wav(audio_path: str, output_path: str | None = None) -> str: + """Ensure the audio path points to wav format by extension/guess and convert when needed. + + If the file appears to already be wav, return it directly to avoid extra conversion. + """ + + if not audio_path: + return audio_path + + if _get_audio_magic_type(audio_path) == "wav": + return audio_path + + return await convert_audio_to_wav(audio_path, output_path) + + +def _get_audio_magic_type(audio_path: str) -> str: + """Detect common audio formats from magic bytes.""" + try: + with open(audio_path, "rb") as f: + header = f.read(64) + except FileNotFoundError: + logger.warning(f"[Media Utils] wav check file not found: {audio_path}") + return "" + except Exception as e: + logger.warning(f"[Media Utils] wav check failed: {audio_path}, error: {e}") + return "" + + if len(header) < 12: + return "" + + if header[:4] == b"RIFF" and header[8:12] == b"WAVE": + return "wav" + + if header[:4] == b"#!AM": + return "amr" + + if header[:4] == b"OggS": + if b"OpusHead" in header: + return "opus" + return "ogg" + + if header[:3] == b"fLa": + return "flac" + + if header[:3] == b"ID3" or header[:2] == b"\xff\xfb": + return "mp3" + + if header[:4] == b"ftyp" and b"mp4" in header[:8]: + return "mp4" + + if header[:8] == b"#!SILK_V3": + return "silk" + + return "" + + async def extract_video_cover( video_path: str, output_path: str | None = None, ) -> str: - """从视频中提取封面图(JPG)。""" + """从视频中提取封面图(JPG)""" if output_path is None: temp_dir = anyio.Path(get_astrbot_temp_path()) await temp_dir.mkdir(parents=True, exist_ok=True) diff --git a/changelogs/v4.22.3.md b/changelogs/v4.22.3.md new file mode 100644 index 000000000..6f265e613 --- /dev/null +++ b/changelogs/v4.22.3.md @@ -0,0 +1,68 @@ +- [更新日志(简体中文)](#chinese) +- [Changelog(English)](#english) + + + +## What's Changed + +### 新增 + +- 新增 `vllm_rerank` 提供商支持可配置的 `rerank_api_suffix`,可灵活适配不同 API 路径。([#7278](https://github.com/AstrBotDevs/AstrBot/pull/7278)) +- 适配 llama.cpp 的 token 用量提取支持,增强 token 统计与计费分析能力。([#7358](https://github.com/AstrBotDevs/AstrBot/pull/7358)) +- WebUI 捆绑 Monaco Editor 的 worker 文件,防止从 CDN 下载失败。([#7249](https://github.com/AstrBotDevs/AstrBot/pull/7249)) +- 全新的模型/平台总览统计面板。([#7152](https://github.com/AstrBotDevs/AstrBot/pull/7152)) + +### 优化 + +- 优化 OpenAI 消息处理,若 `content` 列表为空时将其置为 `None`,并降低空内容下的异常率。([#6551](https://github.com/AstrBotDevs/AstrBot/pull/6551)) +- 优化 provider page 布局与样式,提升整体设置页面排版一致性。([#7248](https://github.com/AstrBotDevs/AstrBot/pull/7248)) +- 用户已登录时将直接跳转欢迎页,减少再次进入登录页的步骤。([commit](https://github.com/AstrBotDevs/AstrBot/commit/9896b48c)) + +### 修复 + +- 修复 Windows 下 MCP runner 未继承完整系统环境变量导致启动异常的问题。([#7054](https://github.com/AstrBotDevs/AstrBot/pull/7054)) +- 修复 Gemini 思考片段泄露到用户可见消息链路的问题。([#7196](https://github.com/AstrBotDevs/AstrBot/pull/7196)) +- 修复 Gemini(OpenAI 兼容 API) 下,`function_response` 格式要求导致的 `400 Invalid argument` 错误。([#7216](https://github.com/AstrBotDevs/AstrBot/pull/7216) +- 修复知识库初始化失败时的错误提示与恢复问题。([#7243](https://github.com/AstrBotDevs/AstrBot/pull/7243)) +- 修复 Bailian Rerank 对新旧响应格式兼容性,避免解析失败。([#7217](https://github.com/AstrBotDevs/AstrBot/pull/7217)) +- Docker 镜像补充 AMR 编码器依赖,修复部分部署场景下的音频转码缺失问题。([#7247](https://github.com/AstrBotDevs/AstrBot/pull/7247)) +- 修复 pip 安装缺少 `pysocks` 依赖导致的 SOCKS5 代理不可用问题。([#7221](https://github.com/AstrBotDevs/AstrBot/pull/7221)) +- 修复 `cached_tokens` 在 `_extract_usage` 中的处理逻辑,避免 token 统计错乱。([#6719](https://github.com/AstrBotDevs/AstrBot/pull/6719)) +- 增强 SSE 断开保护,压缩传输下主动发送心跳,减少 WebChat 误断连。([#7003](https://github.com/AstrBotDevs/AstrBot/pull/7003)) +- 修复严格 API 下空 assistant 消息导致的 400 错误。([#7202](https://github.com/AstrBotDevs/AstrBot/pull/7202)) +- 新增二维码功能缺失的 `qrcode` 依赖修复,恢复二维码生成功能。([commit](https://github.com/AstrBotDevs/AstrBot/commit/5886c437)) +- 修复配置器数字输入组件的交互体验问题,避免异常输入场景下的处理偏差。([#7153](https://github.com/AstrBotDevs/AstrBot/pull/7153)) +- 修复 `Record.path` 在 Pydantic v2 下被错误视为必填字段的问题。([#7048](https://github.com/AstrBotDevs/AstrBot/pull/7048)) + + +## What's Changed (EN) + +### New Features + +- Added configurable `rerank_api_suffix` for the `vllm_rerank` provider to support custom API paths.([#7278](https://github.com/AstrBotDevs/AstrBot/pull/7278)) +- Added token usage extraction support for llama.cpp to improve token accounting.([#7358](https://github.com/AstrBotDevs/AstrBot/pull/7358)) +- Bundled Monaco Editor workers in WebUI so code editing does not depend on external CDN downloads.([#7249](https://github.com/AstrBotDevs/AstrBot/pull/7249)) +- Added a new statistics overview for model and platform usage.([#7152](https://github.com/AstrBotDevs/AstrBot/pull/7152)) +- Redirected logged-in users to the welcome page instead of the login page.([commit](https://github.com/AstrBotDevs/AstrBot/commit/9896b48c)) + +### Improvements + +- Improved OpenAI message handling by setting message `content` to `None` when content lists are empty, reducing empty-payload edge cases.([#6551](https://github.com/AstrBotDevs/AstrBot/pull/6551)) +- Improved provider page layout and styling to keep settings pages consistent.([#7248](https://github.com/AstrBotDevs/AstrBot/pull/7248)) +- Refined connection-success handling and messaging when users are already logged in.([commit](https://github.com/AstrBotDevs/AstrBot/commit/2de23184)) + +### Bug Fixes + +- Inherited all system environment variables in Windows MCP runner to fix environment-dependent launch issues.([#7054](https://github.com/AstrBotDevs/AstrBot/pull/7054)) +- Filtered Gemini thinking parts from user-facing message chains.([#7196](https://github.com/AstrBotDevs/AstrBot/pull/7196)) +- Fixed Gemini `function_response` compatibility to prevent `400 Invalid argument` responses.([#7216](https://github.com/AstrBotDevs/AstrBot/pull/7216)) +- Improved knowledge base initialization error handling.([#7243](https://github.com/AstrBotDevs/AstrBot/pull/7243)) +- Fixed Bailian rerank parsing for both old and new response formats.([#7217](https://github.com/AstrBotDevs/AstrBot/pull/7217)) +- Added AMR encoder support in Docker builds to fix missing audio codec support.([#7247](https://github.com/AstrBotDevs/AstrBot/pull/7247)) +- Added the `pysocks` dependency so pip installs can use SOCKS5 proxy correctly.([#7221](https://github.com/AstrBotDevs/AstrBot/pull/7221)) +- Fixed `_extract_usage` cached token handling.([#6719](https://github.com/AstrBotDevs/AstrBot/pull/6719)) +- Sent SSE heartbeat packets to prevent WebChat disconnection under compression.([#7003](https://github.com/AstrBotDevs/AstrBot/pull/7003)) +- Fixed `400` errors caused by empty assistant messages on strict APIs.([#7202](https://github.com/AstrBotDevs/AstrBot/pull/7202)) +- Added missing `qrcode` dependency to restore QR code generation.([commit](https://github.com/AstrBotDevs/AstrBot/commit/5886c437)) +- Fixed number input UX in config renderer.([#7153](https://github.com/AstrBotDevs/AstrBot/pull/7153)) +- Fixed `Record.path` being incorrectly treated as required under Pydantic v2.([#7048](https://github.com/AstrBotDevs/AstrBot/pull/7048)) diff --git a/dashboard/src/assets/images/platform_logos/matrix.svg b/dashboard/src/assets/images/platform_logos/matrix.svg new file mode 100644 index 000000000..b8e64eccc --- /dev/null +++ b/dashboard/src/assets/images/platform_logos/matrix.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/dashboard/src/assets/mdi-subset/materialdesignicons-subset.css b/dashboard/src/assets/mdi-subset/materialdesignicons-subset.css index 3baa7d48e..da960e963 100644 --- a/dashboard/src/assets/mdi-subset/materialdesignicons-subset.css +++ b/dashboard/src/assets/mdi-subset/materialdesignicons-subset.css @@ -1,4 +1,4 @@ -/* Auto-generated MDI subset – 255 icons */ +/* Auto-generated MDI subset – 256 icons */ /* Do not edit manually. Run: pnpm run subset-icons */ @font-face { @@ -684,6 +684,10 @@ content: "\F0375"; } +.mdi-music-note-outline::before { + content: "\F0F74"; +} + .mdi-note-text-outline::before { content: "\F11D7"; } diff --git a/dashboard/src/assets/mdi-subset/materialdesignicons-webfont-subset.woff b/dashboard/src/assets/mdi-subset/materialdesignicons-webfont-subset.woff index 1c3a7f027..2027f555a 100644 Binary files a/dashboard/src/assets/mdi-subset/materialdesignicons-webfont-subset.woff and b/dashboard/src/assets/mdi-subset/materialdesignicons-webfont-subset.woff differ diff --git a/dashboard/src/assets/mdi-subset/materialdesignicons-webfont-subset.woff2 b/dashboard/src/assets/mdi-subset/materialdesignicons-webfont-subset.woff2 index 923b8a10a..79b3946fb 100644 Binary files a/dashboard/src/assets/mdi-subset/materialdesignicons-webfont-subset.woff2 and b/dashboard/src/assets/mdi-subset/materialdesignicons-webfont-subset.woff2 differ diff --git a/dashboard/src/components/chat/ProviderConfigDialog.vue b/dashboard/src/components/chat/ProviderConfigDialog.vue index 16d54b1ba..85bad2d93 100644 --- a/dashboard/src/components/chat/ProviderConfigDialog.vue +++ b/dashboard/src/components/chat/ProviderConfigDialog.vue @@ -121,12 +121,13 @@ + >>>>>>> origin ======= + + ======= + + >>>>>>> origin
- - - - - - - - - {{ provider.id }} - - - {{ provider.model }} - - - - - - - - - - - - - - -
- No available models -
-
-
-
+ + + + + + + + + {{ provider.id }} + + {{ provider.model }} + + + + + + + + + + + + + + + + + +
+ No available models +
+
+
+
+