Merge dev and fix conflicts

astrbot/core/astr_main_agent.py

@@ -21,7 +21,7 @@ from astrbot.core.astr_agent_hooks import MAIN_AGENT_HOOKS
 from astrbot.core.astr_agent_run_util import AgentRunner
 from astrbot.core.astr_agent_tool_exec import FunctionToolExecutor
 from astrbot.core.conversation_mgr import Conversation
-from astrbot.core.message.components import File, Image, Reply
+from astrbot.core.message.components import File, Image, Record, Reply
 from astrbot.core.persona_error_reply import (
     extract_persona_custom_error_message_from_persona,
     set_persona_custom_error_message_on_event,
@@ -419,6 +419,18 @@ def _append_quoted_image_attachment(req: ProviderRequest, image_path: str) -> No
     )
 
 
+def _append_audio_attachment(req: ProviderRequest, audio_path: str) -> None:
+    req.extra_user_content_parts.append(
+        TextPart(text=f"[Audio Attachment: path {audio_path}]")
+    )
+
+
+def _append_quoted_audio_attachment(req: ProviderRequest, audio_path: str) -> None:
+    req.extra_user_content_parts.append(
+        TextPart(text=f"[Audio Attachment in quoted message: path {audio_path}]")
+    )
+
+
 def _get_quoted_message_parser_settings(
     provider_settings: dict[str, object] | None,
 ) -> QuotedMessageParserSettings:
@@ -704,12 +716,25 @@ def _modalities_fix(provider: Provider, req: ProviderRequest) -> None:
                 "Provider %s does not support image, using placeholder.", provider
             )
             image_count = len(req.image_urls)
-            placeholder = " ".join(["[图片]"] * image_count)
+            placeholder = " ".join(["[Image]"] * image_count)
             if req.prompt:
                 req.prompt = f"{placeholder} {req.prompt}"
             else:
                 req.prompt = placeholder
             req.image_urls = []
+    if req.audio_urls:
+        provider_cfg = provider.provider_config.get("modalities", ["audio"])
+        if "audio" not in provider_cfg:
+            logger.debug(
+                "Provider %s does not support audio, using placeholder.", provider
+            )
+            audio_count = len(req.audio_urls)
+            placeholder = " ".join(["[Audio]"] * audio_count)
+            if req.prompt:
+                req.prompt = f"{placeholder} {req.prompt}"
+            else:
+                req.prompt = placeholder
+            req.audio_urls = []
     if req.func_tool:
         provider_cfg = provider.provider_config.get("modalities", ["tool_use"])
         if "tool_use" not in provider_cfg:
@@ -730,11 +755,13 @@ def _sanitize_context_by_modalities(
     if not modalities or not isinstance(modalities, list):
         return
     supports_image = bool("image" in modalities)
+    supports_audio = bool("audio" in modalities)
     supports_tool_use = bool("tool_use" in modalities)
-    if supports_image and supports_tool_use:
+    if supports_image and supports_audio and supports_tool_use:
         return
     sanitized_contexts: list[dict] = []
     removed_image_blocks = 0
+    removed_audio_blocks = 0
     removed_tool_messages = 0
     removed_tool_calls = 0
     for msg in req.contexts:
@@ -753,20 +780,28 @@ def _sanitize_context_by_modalities(
                     removed_tool_calls += 1
                 new_msg.pop("tool_calls", None)
                 new_msg.pop("tool_call_id", None)
-        if not supports_image:
+
+        if not supports_image or not supports_audio:
             content = new_msg.get("content")
             if isinstance(content, list):
                 filtered_parts: list = []
-                removed_any_image = False
+                removed_any_multimodal = False
                 for part in content:
                     if isinstance(part, dict):
                         part_type = str(part.get("type", "")).lower()
-                        if part_type in {"image_url", "image"}:
-                            removed_any_image = True
+                        if not supports_image and part_type in {"image_url", "image"}:
+                            removed_any_multimodal = True
                             removed_image_blocks += 1
                             continue
+                        if not supports_audio and part_type in {
+                            "audio_url",
+                            "input_audio",
+                        }:
+                            removed_any_multimodal = True
+                            removed_audio_blocks += 1
+                            continue
                     filtered_parts.append(part)
-                if removed_any_image:
+                if removed_any_multimodal:
                     new_msg["content"] = filtered_parts
         if role == "assistant":
             content = new_msg.get("content")
@@ -777,10 +812,19 @@ def _sanitize_context_by_modalities(
                 if isinstance(content, str) and (not content.strip()):
                     continue
         sanitized_contexts.append(new_msg)
-    if removed_image_blocks or removed_tool_messages or removed_tool_calls:
+
+    if (
+        removed_image_blocks
+        or removed_audio_blocks
+        or removed_tool_messages
+        or removed_tool_calls
+    ):
         logger.debug(
-            "sanitize_context_by_modalities applied: removed_image_blocks=%s, removed_tool_messages=%s, removed_tool_calls=%s",
+            "sanitize_context_by_modalities applied: "
+            "removed_image_blocks=%s, removed_audio_blocks=%s, "
+            "removed_tool_messages=%s, removed_tool_calls=%s",
             removed_image_blocks,
+            removed_audio_blocks,
             removed_tool_messages,
             removed_tool_calls,
         )
@@ -969,6 +1013,7 @@ async def build_main_agent(
             req = ProviderRequest()
             req.prompt = ""
             req.image_urls = []
+            req.audio_urls = []
             if sel_model := event.get_extra("selected_model"):
                 req.model = sel_model
             if config.provider_wake_prefix and (
@@ -988,6 +1033,10 @@ async def build_main_agent(
                     req.extra_user_content_parts.append(
                         TextPart(text=f"[Image Attachment: path {image_path}]")
                     )
+                elif isinstance(comp, Record):
+                    audio_path = await comp.convert_to_file_path()
+                    req.audio_urls.append(audio_path)
+                    _append_audio_attachment(req, audio_path)
                 elif isinstance(comp, File):
                     file_path = await comp.get_file()
                     file_name = comp.name or os.path.basename(file_path)
@@ -1017,6 +1066,10 @@ async def build_main_agent(
                                 event.track_temporary_local_file(image_path)
                             req.image_urls.append(image_path)
                             _append_quoted_image_attachment(req, image_path)
+                        elif isinstance(reply_comp, Record):
+                            audio_path = await reply_comp.convert_to_file_path()
+                            req.audio_urls.append(audio_path)
+                            _append_quoted_audio_attachment(req, audio_path)
                         elif isinstance(reply_comp, File):
                             file_path = await reply_comp.get_file()
                             file_name = reply_comp.name or os.path.basename(file_path)
@@ -1074,12 +1127,15 @@ async def build_main_agent(
     if isinstance(req.contexts, str):
         req.contexts = json.loads(req.contexts)
     req.image_urls = normalize_and_dedupe_strings(req.image_urls)
+    req.audio_urls = normalize_and_dedupe_strings(req.audio_urls)
+
     if config.file_extract_enabled:
         try:
             await _apply_file_extract(event, req, config)
         except Exception as exc:
             logger.error("Error occurred while applying file extract: %s", exc)
-    if not req.prompt and (not req.image_urls):
+
+    if not req.prompt and not req.image_urls and not req.audio_urls:
         if not event.get_group_id() and req.extra_user_content_parts:
             req.prompt = "<attachment>"
         else:

astrbot/core/config/default.py

@@ -1247,6 +1247,18 @@ CONFIG_METADATA_2 = {
                         "proxy": "",
                         "custom_headers": {},
                     },
+                    "LongCat": {
+                        "id": "longcat",
+                        "provider": "longcat",
+                        "type": "longcat_chat_completion",
+                        "provider_type": "chat_completion",
+                        "enable": True,
+                        "key": [],
+                        "api_base": "https://api.longcat.chat/openai",
+                        "timeout": 120,
+                        "proxy": "",
+                        "custom_headers": {},
+                    },
                     "AIHubMix": {
                         "id": "aihubmix",
                         "provider": "aihubmix",
@@ -1761,6 +1773,7 @@ CONFIG_METADATA_2 = {
                         "enable": True,
                         "rerank_api_key": "",
                         "rerank_api_base": "http://127.0.0.1:8000",
+                        "rerank_api_suffix": "/v1/rerank",
                         "rerank_model": "BAAI/bge-reranker-base",
                         "timeout": 20,
                     },
@@ -1789,6 +1802,19 @@ CONFIG_METADATA_2 = {
                         "return_documents": False,
                         "instruct": "",
                     },
+                    "NVIDIA Rerank": {
+                        "id": "nvidia_rerank",
+                        "type": "nvidia_rerank",
+                        "provider": "nvidia",
+                        "provider_type": "rerank",
+                        "enable": True,
+                        "nvidia_rerank_api_key": "",
+                        "nvidia_rerank_api_base": "https://ai.api.nvidia.com/v1/retrieval",
+                        "nvidia_rerank_model": "nv-rerank-qa-mistral-4b:1",
+                        "nvidia_rerank_model_endpoint": "/reranking",
+                        "timeout": 20,
+                        "nvidia_rerank_truncate": "",
+                    },
                     "Xinference STT": {
                         "id": "xinference_stt",
                         "type": "xinference_stt",
@@ -1826,7 +1852,12 @@ CONFIG_METADATA_2 = {
                     "rerank_api_base": {
                         "description": "重排序模型 API Base URL",
                         "type": "string",
-                        "hint": "AstrBot 会在请求时在末尾加上 /v1/rerank。",
+                        "hint": "最终请求路径由 Base URL 和路径后缀拼接而成（默认为 /v1/rerank）。",
+                    },
+                    "rerank_api_suffix": {
+                        "description": "API URL 路径后缀",
+                        "type": "string",
+                        "hint": "追加到 base_url 后的路径，如 /v1/rerank。留空则不追加。",
                     },
                     "rerank_api_key": {
                         "description": "API Key",
@@ -1852,12 +1883,40 @@ CONFIG_METADATA_2 = {
                         "type": "bool",
                         "hint": "如果模型当前未在 Xinference 服务中运行，是否尝试自动启动它。在生产环境中建议关闭。",
                     },
+                    "nvidia_rerank_api_base": {
+                        "description": "API Base URL",
+                        "type": "string",
+                    },
+                    "nvidia_rerank_api_key": {
+                        "description": "API Key",
+                        "type": "string",
+                    },
+                    "nvidia_rerank_model": {
+                        "description": "重排序模型名称",
+                        "type": "string",
+                        "hint": "请参照NVIDIA Docs中模型名称填写。",
+                    },
+                    "nvidia_rerank_model_endpoint": {
+                        "description": "自定义模型端点",
+                        "type": "string",
+                        "hint": "自定义URL末尾端点，默认为 /reranking",
+                    },
+                    "nvidia_rerank_truncate": {
+                        "description": "文本截断策略",
+                        "type": "string",
+                        "hint": "当输入文本过长时，是否截断输入以适应模型的最大上下文长度。",
+                        "options": [
+                            "",
+                            "NONE",
+                            "END",
+                        ],
+                    },
                     "modalities": {
                         "description": "模型能力",
                         "type": "list",
                         "items": {"type": "string"},
-                        "options": ["text", "image", "tool_use"],
-                        "labels": ["文本", "图像", "工具使用"],
+                        "options": ["text", "image", "audio", "tool_use"],
+                        "labels": ["文本", "图像", "音频", "工具使用"],
                         "render_type": "checkbox",
                         "hint": "模型支持的模态。如所填写的模型不支持图像，请取消勾选图像。",
                     },

astrbot/core/message/components.py

@@ -74,12 +74,6 @@ class ComponentType(str, Enum):
     Music = "Music"
     Json = "Json"
     Unknown = "Unknown"
-    WechatEmoji = "WechatEmoji"  # Wechat 下的 emoji 表情包
-    # Discord-specific component types
-    DiscordEmbed = "DiscordEmbed"
-    DiscordButton = "DiscordButton"
-    DiscordReference = "DiscordReference"
-    DiscordView = "DiscordView"
 
 
 class BaseMessageComponent(BaseModel):
@@ -106,7 +100,6 @@ class BaseMessageComponent(BaseModel):
 class Plain(BaseMessageComponent):
     type: ComponentType = ComponentType.Plain
     text: str
-    convert: bool | None = True
 
     def __init__(self, text: str, convert: bool = True, **_) -> None:
         super().__init__(text=text, convert=convert, **_)
@@ -129,11 +122,7 @@ class Face(BaseMessageComponent):
 class Record(BaseMessageComponent):
     type: ComponentType = ComponentType.Record
     file: str | None = ""
-    magic: bool | None = False
     url: str | None = ""
-    cache: bool | None = True
-    proxy: bool | None = True
-    timeout: int | None = 0
     # Original text content (e.g. TTS source text), used as caption in fallback scenarios
     text: str | None = None
     # 额外
@@ -239,7 +228,6 @@ class Video(BaseMessageComponent):
     type: ComponentType = ComponentType.Video
     file: str
     cover: str | None = ""
-    c: int | None = 2
     # 额外
     path: str | None = ""
 
@@ -416,14 +404,9 @@ class Image(BaseMessageComponent):
     type: ComponentType = ComponentType.Image
     file: str | None = ""
     _type: str | None = ""
-    subType: int | None = 0
     url: str | None = ""
-    cache: bool | None = True
-    id: int | None = 40000
-    c: int | None = 2
     # 额外
     path: str | None = ""
-    file_unique: str | None = ""  # 某些平台可能有图片缓存的唯一标识
 
     def __init__(self, file: str | None, **_) -> None:
         super().__init__(file=file, **_)
@@ -854,16 +837,6 @@ class File(BaseMessageComponent):
         }
 
 
-class WechatEmoji(BaseMessageComponent):
-    type: ComponentType = ComponentType.WechatEmoji
-    md5: str | None = ""
-    md5_len: int | None = 0
-    cdnurl: str | None = ""
-
-    def __init__(self, **_) -> None:
-        super().__init__(**_)
-
-
 ComponentTypes = {
     # Basic Message Segments
     "plain": Plain,
@@ -889,5 +862,4 @@ ComponentTypes = {
     "nodes": Nodes,
     "json": Json,
     "unknown": Unknown,
-    "WechatEmoji": WechatEmoji,
 }

astrbot/core/pipeline/preprocess_stage/stage.py

@@ -7,6 +7,7 @@ from astrbot.core.message.components import Image, Plain, Record
 from astrbot.core.pipeline.context import PipelineContext
 from astrbot.core.pipeline.stage import Stage, register_stage
 from astrbot.core.platform.astr_message_event import AstrMessageEvent
+from astrbot.core.utils.media_utils import ensure_wav
 
 
 @register_stage
@@ -62,6 +63,21 @@ class PreProcessStage(Stage):
                             logger.debug(f"路径映射: {url} -> {component.url}")
                     message_chain[idx] = component
 
+        # In here, we convert all Record components to wav format and update the file path.
+        message_chain = event.get_messages()
+        for idx, component in enumerate(message_chain):
+            if isinstance(component, Record):
+                try:
+                    original_path = await component.convert_to_file_path()
+                    record_path = await ensure_wav(original_path)
+                    if record_path != original_path:
+                        event.track_temporary_local_file(record_path)
+                    component.file = record_path
+                    component.path = record_path
+                    message_chain[idx] = component
+                except Exception as e:
+                    logger.warning(f"Voice processing failed: {e}")
+
         # STT
         if self.stt_settings.get("enable", False):
             # TODO: 独立

astrbot/core/pipeline/process_stage/method/agent_sub_stages/internal.py

@@ -14,7 +14,7 @@ from astrbot.core.astr_main_agent import (
     MainAgentBuildResult,
     build_main_agent,
 )
-from astrbot.core.message.components import File, Image
+from astrbot.core.message.components import File, Image, Record, Video
 from astrbot.core.message.message_event_result import (
     MessageChain,
     MessageEventResult,
@@ -147,7 +147,8 @@ class InternalAgentSubStage(Stage):
             has_provider_request = event.get_extra("provider_request") is not None
             has_valid_message = bool(event.message_str and event.message_str.strip())
             has_media_content = any(
-                isinstance(comp, Image | File) for comp in event.message_obj.message
+                isinstance(comp, (Image, File, Record, Video))
+                for comp in event.message_obj.message
             )
             if (
                 not has_provider_request

astrbot/core/pipeline/process_stage/method/agent_sub_stages/third_party.py

@@ -9,7 +9,7 @@ from astrbot.core.agent.runners.deerflow.constants import (
     DEERFLOW_PROVIDER_TYPE,
 )
 from astrbot.core.astr_agent_hooks import MAIN_AGENT_HOOKS
-from astrbot.core.message.components import Image
+from astrbot.core.message.components import Image, Record
 from astrbot.core.message.message_event_result import (
     MessageChain,
     MessageEventResult,
@@ -313,8 +313,11 @@ class ThirdPartyAgentSubStage(Stage):
             if isinstance(comp, Image):
                 image_path = await comp.convert_to_base64()
                 req.image_urls.append(image_path)
+            elif isinstance(comp, Record):
+                audio_path = await comp.convert_to_file_path()
+                req.audio_urls.append(audio_path)
 
-        if not req.prompt and not req.image_urls:
+        if not req.prompt and not req.image_urls and not req.audio_urls:
             return
 
         custom_error_message = await self._resolve_persona_custom_error_message(event)

astrbot/core/pipeline/respond/stage.py

@@ -15,46 +15,37 @@ from astrbot.core.utils.path_util import path_Mapping
 
 @register_stage
 class RespondStage(Stage):
-    @staticmethod
-    def _has_meaningful_content(comp: BaseMessageComponent) -> bool:
-        if isinstance(comp, Comp.Plain | Comp.Unknown):
-            return bool(comp.text.strip())
-        if isinstance(comp, Comp.Face):
-            return comp.id is not None
-        if isinstance(comp, Comp.Record | Comp.Video | Comp.Image):
-            return bool(comp.file)
-        if isinstance(comp, Comp.At):
-            return bool(comp.qq) or bool(comp.name)
-        if isinstance(comp, Comp.Reply):
-            return bool(comp.id) and comp.sender_id is not None
-        if isinstance(comp, Comp.Poke):
-            return comp.target_id() is not None
-        if isinstance(comp, Comp.Node):
-            return bool(comp.content)
-        if isinstance(comp, Comp.Nodes):
-            return bool(comp.nodes)
-        if isinstance(comp, Comp.File):
-            return bool(comp.file_ or comp.url)
-        if isinstance(comp, Comp.WechatEmoji):
-            return comp.md5 is not None
-        if isinstance(comp, Comp.Json):
-            return bool(comp.data)
-        if isinstance(comp, Comp.Share):
-            return bool(comp.url) or bool(comp.title)
-        if isinstance(comp, Comp.Music):
-            return bool(
-                (comp.id and comp._type and comp._type != "custom")
-                or (comp._type == "custom" and comp.url and comp.audio and comp.title)
-            )
-        if isinstance(comp, Comp.Forward):
-            return bool(comp.id)
-        if isinstance(comp, Comp.Location):
-            return comp.lat is not None and comp.lon is not None
-        if isinstance(comp, Comp.Contact):
-            return bool(comp._type and comp.id)
-        if isinstance(comp, Comp.Shake | Comp.Dice | Comp.RPS):
-            return True
-        return False
+    # 组件类型到其非空判断函数的映射
+    _component_validators = {
+        Comp.Plain: lambda comp: bool(
+            comp.text and comp.text.strip(),
+        ),  # 纯文本消息需要strip
+        Comp.Face: lambda comp: comp.id is not None,  # QQ表情
+        Comp.Record: lambda comp: bool(comp.file),  # 语音
+        Comp.Video: lambda comp: bool(comp.file),  # 视频
+        Comp.At: lambda comp: bool(comp.qq) or bool(comp.name),  # @
+        Comp.Image: lambda comp: bool(comp.file),  # 图片
+        Comp.Reply: lambda comp: bool(comp.id) and comp.sender_id is not None,  # 回复
+        Comp.Poke: lambda comp: comp.target_id() is not None,  # 戳一戳
+        Comp.Node: lambda comp: bool(comp.content),  # 转发节点
+        Comp.Nodes: lambda comp: bool(comp.nodes),  # 多个转发节点
+        Comp.File: lambda comp: bool(comp.file_ or comp.url),
+        Comp.Json: lambda comp: bool(comp.data),  # Json 卡片
+        Comp.Share: lambda comp: bool(comp.url) or bool(comp.title),
+        Comp.Music: lambda comp: (
+            (comp.id and comp._type and comp._type != "custom")
+            or (comp._type == "custom" and comp.url and comp.audio and comp.title)
+        ),  # 音乐分享
+        Comp.Forward: lambda comp: bool(comp.id),  # 合并转发
+        Comp.Location: lambda comp: bool(
+            comp.lat is not None and comp.lon is not None
+        ),  # 位置
+        Comp.Contact: lambda comp: bool(comp._type and comp.id),  # 推荐好友 or 群
+        Comp.Shake: lambda _: True,  # 窗口抖动（戳一戳）
+        Comp.Dice: lambda _: True,  # 掷骰子魔法表情
+        Comp.RPS: lambda _: True,  # 猜拳魔法表情
+        Comp.Unknown: lambda comp: bool(comp.text and comp.text.strip()),
+    }
 
     async def initialize(self, ctx: PipelineContext) -> None:
         self.ctx = ctx

astrbot/core/pipeline/waking_check/stage.py

@@ -21,6 +21,7 @@ UNIQUE_SESSION_ID_BUILDERS: dict[str, Callable[[AstrMessageEvent], str | None]]
     "qq_official_webhook": lambda e: e.get_sender_id(),
     "lark": lambda e: f"{e.get_sender_id()}%{e.get_group_id()}",
     "misskey": lambda e: f"{e.get_session_id()}_{e.get_sender_id()}",
+    "matrix": lambda e: f"{e.get_sender_id()}_{e.get_group_id() or e.get_session_id()}",
 }
 
 

astrbot/core/platform/astr_message_event.py

@@ -419,6 +419,7 @@ class AstrMessageEvent(abc.ABC):
         tool_set: ToolSet | None = None,
         session_id: str = "",
         image_urls: list[str] | None = None,
+        audio_urls: list[str] | None = None,
         contexts: list | None = None,
         system_prompt: str = "",
         conversation: Conversation | None = None,
@@ -437,7 +438,9 @@ class AstrMessageEvent(abc.ABC):
 
         image_urls: 可以是 base64:// 或者 http:// 开头的图片链接,也可以是本地图片路径｡
 
-        contexts: 当指定 contexts 时,将会使用 contexts 作为上下文｡如果同时传入了 conversation,将会忽略 conversation｡
+        audio_urls: 音频 URL 列表，也支持本地路径。
+
+        contexts: 当指定 contexts 时，将会使用 contexts 作为上下文。如果同时传入了 conversation，将会忽略 conversation。
 
         func_tool_manager: [Deprecated] 函数工具管理器,用于调用函数工具｡用 self.context.get_llm_tool_manager() 获取｡已过时,请使用 tool_set 参数代替｡
 
@@ -446,6 +449,8 @@ class AstrMessageEvent(abc.ABC):
         """
         if image_urls is None:
             image_urls = []
+        if audio_urls is None:
+            audio_urls = []
         if contexts is None:
             contexts = []
         if len(contexts) > 0 and conversation:
@@ -457,6 +462,7 @@ class AstrMessageEvent(abc.ABC):
             prompt=prompt,
             session_id=session_id,
             image_urls=image_urls,
+            audio_urls=audio_urls,
             # func_tool=func_tool_manager,
             func_tool=tool_set,
             contexts=contexts,

astrbot/core/provider/entities.py

@@ -3,8 +3,11 @@ from __future__ import annotations
 import base64
 import enum
 import json
+import uuid
 from dataclasses import dataclass, field
+from pathlib import Path
 from typing import Any
+from urllib.parse import urlparse
 
 import aiofiles
 from anthropic.types import Message as AnthropicMessage
@@ -22,7 +25,8 @@ from astrbot.core.agent.message import (
 from astrbot.core.agent.tool import ToolSet
 from astrbot.core.db.po import Conversation
 from astrbot.core.message.message_event_result import MessageChain
-from astrbot.core.utils.io import download_image_by_url
+from astrbot.core.utils.astrbot_path import get_astrbot_temp_path
+from astrbot.core.utils.io import download_file, download_image_by_url
 
 
 class ProviderType(enum.Enum):
@@ -94,6 +98,8 @@ class ProviderRequest:
     """会话 ID"""
     image_urls: list[str] = field(default_factory=list)
     """图片 URL 列表"""
+    audio_urls: list[str] = field(default_factory=list)
+    """音频 URL 列表，也支持本地路径"""
     extra_user_content_parts: list[ContentPart] = field(default_factory=list)
     """额外的用户消息内容部分列表,用于在用户消息后添加额外的内容块(如系统提醒､指令等)｡支持 dict 或 ContentPart 对象"""
     func_tool: ToolSet | None = None
@@ -116,6 +122,7 @@ class ProviderRequest:
         return (
             f"ProviderRequest(prompt={self.prompt}, session_id={self.session_id}, "
             f"image_count={len(self.image_urls or [])}, "
+            f"audio_count={len(self.audio_urls or [])}, "
             f"func_tool={self.func_tool}, "
             f"contexts={self._print_friendly_context()}, "
             f"system_prompt={self.system_prompt}, "
@@ -134,9 +141,12 @@ class ProviderRequest:
         self.tool_calls_result.append(tool_calls_result)
 
     def _print_friendly_context(self):
-        """打印友好的消息上下文｡将 image_url 的值替换为 <Image>"""
+        """打印友好的消息上下文。将多模态内容折叠为简短标记。"""
         if not self.contexts:
-            return f"prompt: {self.prompt}, image_count: {len(self.image_urls or [])}"
+            return (
+                f"prompt: {self.prompt}, image_count: {len(self.image_urls or [])}, "
+                f"audio_count: {len(self.audio_urls or [])}"
+            )
 
         result_parts = []
 
@@ -149,6 +159,7 @@ class ProviderRequest:
             elif isinstance(content, list):
                 msg_parts = []
                 image_count = 0
+                audio_count = 0
 
                 for item in content:
                     item_type = item.get("type", "")
@@ -157,19 +168,26 @@ class ProviderRequest:
                         msg_parts.append(item.get("text", ""))
                     elif item_type == "image_url":
                         image_count += 1
+                    elif item_type == "audio_url":
+                        audio_count += 1
 
                 if image_count > 0:
                     if msg_parts:
                         msg_parts.append(f"[+{image_count} images]")
                     else:
                         msg_parts.append(f"[{image_count} images]")
+                if audio_count > 0:
+                    if msg_parts:
+                        msg_parts.append(f"[+{audio_count} audios]")
+                    else:
+                        msg_parts.append(f"[{audio_count} audios]")
 
                 result_parts.append(f"{role}: {''.join(msg_parts)}")
 
         return "\n".join(result_parts)
 
-    async def assemble_context(self) -> dict[str, str | list[dict[str, object]]]:
-        """将请求(prompt 和 image_urls)包装成 OpenAI 的消息格式｡"""
+    async def assemble_context(self) -> dict:
+        """将请求(prompt、image_urls 和 audio_urls)包装成统一消息格式。"""
         # 构建内容块列表
         content_blocks: list[dict[str, object]] = []
 
@@ -179,6 +197,9 @@ class ProviderRequest:
         elif self.image_urls:
             # 如果没有文本但有图片,添加占位文本
             content_blocks.append({"type": "text", "text": "[图片]"})
+        elif self.audio_urls:
+            # 如果没有文本但有音频，添加占位文本
+            content_blocks.append({"type": "text", "text": "[音频]"})
 
         # 2. 额外的内容块(系统提醒､指令等)
         if self.extra_user_content_parts:
@@ -204,11 +225,56 @@ class ProviderRequest:
                     {"type": "image_url", "image_url": {"url": image_data}},
                 )
 
-        # 只有当只有一个来自 prompt 的文本块且没有额外内容块时,才降级为简单格式以保持向后兼容
+        # 4. 音频内容
+        if self.audio_urls:
+            for audio_url in self.audio_urls:
+                if audio_url.startswith("http"):
+                    parsed_url = urlparse(audio_url)
+                    suffix = Path(parsed_url.path).suffix
+                    temp_dir = Path(get_astrbot_temp_path())
+                    temp_dir.mkdir(parents=True, exist_ok=True)
+                    temp_audio_path = (
+                        temp_dir / f"provider_request_audio_{uuid.uuid4().hex}{suffix}"
+                    )
+                    try:
+                        await download_file(audio_url, str(temp_audio_path))
+                        audio_data = await self._encode_audio_bs64(
+                            str(temp_audio_path),
+                            source_ref=audio_url,
+                        )
+                    finally:
+                        try:
+                            temp_audio_path.unlink(missing_ok=True)
+                        except Exception as exc:
+                            logger.warning(
+                                "Failed to cleanup %s: %s",
+                                temp_audio_path,
+                                exc,
+                            )
+                elif audio_url.startswith("file:///"):
+                    audio_path = audio_url.replace("file:///", "")
+                    audio_data = await self._encode_audio_bs64(
+                        audio_path,
+                        source_ref=audio_url,
+                    )
+                else:
+                    audio_data = await self._encode_audio_bs64(
+                        audio_url,
+                        source_ref=audio_url,
+                    )
+                if not audio_data:
+                    logger.warning(f"音频 {audio_url} 得到的结果为空，将忽略。")
+                    continue
+                content_blocks.append(
+                    {"type": "audio_url", "audio_url": {"url": audio_data}},
+                )
+
+        # 只有当只有一个来自 prompt 的文本块且没有额外内容块时，才降级为简单格式以保持向后兼容
         if (
             len(content_blocks) == 1
             and not self.extra_user_content_parts
             and not self.image_urls
+            and not self.audio_urls
         ):
             first_block = content_blocks[0]
             if first_block.get("type") == "text":
@@ -226,7 +292,21 @@ class ProviderRequest:
         async with aiofiles.open(image_url, "rb") as f:
             image_bs64 = base64.b64encode(await f.read()).decode("utf-8")
             return "data:image/jpeg;base64," + image_bs64
-        return ""
+
+    async def _encode_audio_bs64(
+        self,
+        audio_path: str,
+        source_ref: str | None = None,
+    ) -> str:
+        """将音频转换为 base64"""
+        mime_type = "audio/wav"
+
+        if audio_path.startswith("base64://"):
+            return audio_path.replace("base64://", f"data:{mime_type};base64,", 1)
+
+        with open(audio_path, "rb") as f:
+            audio_bs64 = base64.b64encode(f.read()).decode("utf-8")
+            return f"data:{mime_type};base64," + audio_bs64
 
 
 @dataclass

astrbot/core/provider/manager.py

@@ -256,6 +256,8 @@ class ProviderManager:
                     provider = self.provider_insts[0] if self.provider_insts else None
             elif provider_type == ProviderType.SPEECH_TO_TEXT:
                 provider_id = config["provider_stt_settings"].get("provider_id")
+                if not config["provider_stt_settings"].get("enable"):
+                    return None
                 if not provider_id:
                     return None
                 provider = self.inst_map.get(provider_id)
@@ -265,6 +267,8 @@ class ProviderManager:
                     )
             elif provider_type == ProviderType.TEXT_TO_SPEECH:
                 provider_id = config["provider_tts_settings"].get("provider_id")
+                if not config["provider_tts_settings"].get("enable"):
+                    return None
                 if not provider_id:
                     return None
                 provider = self.inst_map.get(provider_id)
@@ -369,6 +373,8 @@ class ProviderManager:
                 from .sources.openai_source import (
                     ProviderOpenAIOfficial as ProviderOpenAIOfficial,
                 )
+            case "longcat_chat_completion":
+                from .sources.longcat_source import ProviderLongCat as ProviderLongCat
             case "zhipu_chat_completion":
                 from .sources.zhipu_source import ProviderZhipu as ProviderZhipu
             case "groq_chat_completion":
@@ -483,6 +489,10 @@ class ProviderManager:
                 from .sources.bailian_rerank_source import (
                     BailianRerankProvider as BailianRerankProvider,
                 )
+            case "nvidia_rerank":
+                from .sources.nvidia_rerank_source import (
+                    NvidiaRerankProvider as NvidiaRerankProvider,
+                )
 
     def get_merged_provider_config(self, provider_config: dict) -> dict:
         """获取 provider 配置和 provider_source 配置合并后的结果

astrbot/core/provider/provider.py

@@ -93,6 +93,7 @@ class Provider(AbstractProvider):
         prompt: str | None = None,
         session_id: str | None = None,
         image_urls: list[str] | None = None,
+        audio_urls: list[str] | None = None,
         func_tool: ToolSet | None = None,
         contexts: list[Message] | list[dict] | None = None,
         system_prompt: str | None = None,
@@ -108,6 +109,7 @@ class Provider(AbstractProvider):
             prompt: 提示词,和 contexts 二选一使用,如果都指定,则会将 prompt(以及可能的 image_urls) 作为最新的一条记录添加到 contexts 中
             session_id: 会话 ID(此属性已经被废弃)
             image_urls: 图片 URL 列表
+            audio_urls: 音频 URL 列表，也支持本地路径
             tools: tool set
             contexts: 上下文,和 prompt 二选一使用
             tool_calls_result: 回传给 LLM 的工具调用结果｡参考: https://platform.openai.com/docs/guides/function-calling
@@ -116,8 +118,9 @@ class Provider(AbstractProvider):
             kwargs: 其他参数
 
         Notes:
-            - 如果传入了 image_urls,将会在对话时附上图片｡如果模型不支持图片输入,将会抛出错误｡
-            - 如果传入了 tools,将会使用 tools 进行 Function-calling｡如果模型不支持 Function-calling,将会抛出错误｡
+            - 如果传入了 image_urls，将会在对话时附上图片。如果模型不支持图片输入，将会抛出错误。
+            - 如果传入了 audio_urls，将会在对话时附上音频。如果模型不支持音频输入，将会抛出错误或降级处理。
+            - 如果传入了 tools，将会使用 tools 进行 Function-calling。如果模型不支持 Function-calling，将会抛出错误。
 
         """
         ...
@@ -127,6 +130,7 @@ class Provider(AbstractProvider):
         prompt: str | None = None,
         session_id: str | None = None,
         image_urls: list[str] | None = None,
+        audio_urls: list[str] | None = None,
         func_tool: ToolSet | None = None,
         contexts: list[Message] | list[dict] | None = None,
         system_prompt: str | None = None,
@@ -142,6 +146,7 @@ class Provider(AbstractProvider):
             prompt: 提示词,和 contexts 二选一使用,如果都指定,则会将 prompt(以及可能的 image_urls) 作为最新的一条记录添加到 contexts 中
             session_id: 会话 ID(此属性已经被废弃)
             image_urls: 图片 URL 列表
+            audio_urls: 音频 URL 列表，也支持本地路径
             tools: tool set
             contexts: 上下文,和 prompt 二选一使用
             tool_calls_result: 回传给 LLM 的工具调用结果｡参考: https://platform.openai.com/docs/guides/function-calling
@@ -150,8 +155,9 @@ class Provider(AbstractProvider):
             kwargs: 其他参数
 
         Notes:
-            - 如果传入了 image_urls,将会在对话时附上图片｡如果模型不支持图片输入,将会抛出错误｡
-            - 如果传入了 tools,将会使用 tools 进行 Function-calling｡如果模型不支持 Function-calling,将会抛出错误｡
+            - 如果传入了 image_urls，将会在对话时附上图片。如果模型不支持图片输入，将会抛出错误。
+            - 如果传入了 audio_urls，将会在对话时附上音频。如果模型不支持音频输入，将会抛出错误或降级处理。
+            - 如果传入了 tools，将会使用 tools 进行 Function-calling。如果模型不支持 Function-calling，将会抛出错误。
 
         """
         if False:

astrbot/core/provider/sources/anthropic_source.py

@@ -13,7 +13,7 @@ from anthropic.types.usage import Usage
 
 from astrbot import logger
 from astrbot.api.provider import Provider
-from astrbot.core.agent.message import ContentPart, ImageURLPart, TextPart
+from astrbot.core.agent.message import AudioURLPart, ContentPart, ImageURLPart, TextPart
 from astrbot.core.exceptions import EmptyModelOutputError
 from astrbot.core.provider.entities import LLMResponse, TokenUsage
 from astrbot.core.provider.func_tool_manager import ToolSet
@@ -253,6 +253,13 @@ class ProviderAnthropic(Provider):
                                 logger.warning(
                                     f"Unsupported image URL format for Anthropic: {url[:50]}..."
                                 )
+                        elif part.get("type") == "audio_url":
+                            converted_content.append(
+                                {
+                                    "type": "text",
+                                    "text": "[Audio Attachment]",
+                                }
+                            )
                         else:
                             converted_content.append(part)
                     new_messages.append(
@@ -534,6 +541,7 @@ class ProviderAnthropic(Provider):
         prompt=None,
         session_id=None,
         image_urls=None,
+        audio_urls=None,
         func_tool=None,
         contexts=None,
         system_prompt=None,
@@ -548,7 +556,10 @@ class ProviderAnthropic(Provider):
         new_record = None
         if prompt is not None:
             new_record = await self.assemble_context(
-                prompt, image_urls, extra_user_content_parts
+                prompt or "",
+                image_urls,
+                audio_urls,
+                extra_user_content_parts,
             )
         context_query = self._ensure_message_to_dicts(contexts)
         if new_record:
@@ -594,6 +605,7 @@ class ProviderAnthropic(Provider):
         prompt=None,
         session_id=None,
         image_urls=None,
+        audio_urls=None,
         func_tool=None,
         contexts=None,
         system_prompt=None,
@@ -608,7 +620,10 @@ class ProviderAnthropic(Provider):
         new_record = None
         if prompt is not None:
             new_record = await self.assemble_context(
-                prompt, image_urls, extra_user_content_parts
+                prompt or "",
+                image_urls,
+                audio_urls,
+                extra_user_content_parts,
             )
         context_query = self._ensure_message_to_dicts(contexts)
         if new_record:
@@ -659,6 +674,7 @@ class ProviderAnthropic(Provider):
         self,
         text: str,
         image_urls: list[str] | None = None,
+        audio_urls: list[str] | None = None,
         extra_user_content_parts: list[ContentPart] | None = None,
     ):
         """组装上下文,支持文本和图片"""
@@ -696,8 +712,10 @@ class ProviderAnthropic(Provider):
         if text:
             content.append({"type": "text", "text": text})
         elif image_urls:
-            # 如果没有文本但有图片,添加占位文本
-            content.append({"type": "text", "text": "[图片]"})
+            # 如果没有文本但有图片，添加占位文本
+            content.append({"type": "text", "text": "[Image]"})
+        elif audio_urls:
+            content.append({"type": "text", "text": "[Audio]"})
         elif extra_user_content_parts:
             # 如果只有额外内容块,也需要添加占位文本
             content.append({"type": "text", "text": " "})
@@ -711,6 +729,8 @@ class ProviderAnthropic(Provider):
                     image_dict = await resolve_image_url(block.image_url.url)
                     if image_dict:
                         content.append(image_dict)
+                elif isinstance(block, AudioURLPart):
+                    content.append({"type": "text", "text": "[Audio]"})
                 else:
                     raise ValueError(f"不支持的额外内容块类型: {type(block)}")
 
@@ -720,12 +740,16 @@ class ProviderAnthropic(Provider):
                 image_dict = await resolve_image_url(image_url)
                 if image_dict:
                     content.append(image_dict)
+        if audio_urls:
+            for _audio_path in audio_urls:
+                content.append({"type": "text", "text": "[Audio]"})
 
         # 如果只有主文本且没有额外内容块和图片,返回简单格式以保持向后兼容
         if (
             text
             and not extra_user_content_parts
             and not image_urls
+            and not audio_urls
             and len(content) == 1
             and content[0]["type"] == "text"
         ):

astrbot/core/provider/sources/gemini_source.py

@@ -3,8 +3,11 @@ import base64
 import json
 import logging
 import random
+import uuid
 from collections.abc import AsyncGenerator
-from typing import ClassVar, Literal
+from pathlib import Path
+from typing import Literal, cast
+from urllib.parse import urlparse
 
 import aiofiles
 from google import genai
@@ -14,13 +17,14 @@ from google.genai.errors import APIError
 import astrbot.core.message.components as Comp
 from astrbot import logger
 from astrbot.api.provider import Provider
-from astrbot.core.agent.message import ContentPart, ImageURLPart, Message, TextPart
-from astrbot.core.agent.tool import ToolSet
+from astrbot.core.agent.message import AudioURLPart, ContentPart, ImageURLPart, TextPart
 from astrbot.core.exceptions import EmptyModelOutputError
 from astrbot.core.message.message_event_result import MessageChain
-from astrbot.core.provider.entities import LLMResponse, TokenUsage, ToolCallsResult
-from astrbot.core.provider.register import register_provider_adapter
-from astrbot.core.utils.io import download_image_by_url
+from astrbot.core.provider.entities import LLMResponse, TokenUsage
+from astrbot.core.provider.func_tool_manager import ToolSet
+from astrbot.core.utils.astrbot_path import get_astrbot_temp_path
+from astrbot.core.utils.io import download_file, download_image_by_url
+from astrbot.core.utils.media_utils import ensure_wav
 from astrbot.core.utils.network_utils import is_connection_error, log_connection_failure
 
 
@@ -263,6 +267,12 @@ class ProviderGoogleGenAI(Provider):
             image_bytes = base64.b64decode(url.split(",", 1)[1])
             return types.Part.from_bytes(data=image_bytes, mime_type=mime_type)
 
+        def process_audio_url(audio_url_dict: dict) -> types.Part:
+            url = audio_url_dict["url"]
+            mime_type = url.split(":")[1].split(";")[0]
+            audio_bytes = base64.b64decode(url.split(",", 1)[1])
+            return types.Part.from_bytes(data=audio_bytes, mime_type=mime_type)
+
         def append_or_extend(
             contents: list[types.Content],
             part: list[types.Part],
@@ -286,9 +296,15 @@ class ProviderGoogleGenAI(Provider):
             if role == "user":
                 if isinstance(content, list):
                     parts = [
-                        types.Part.from_text(text=item["text"] or " ")
-                        if item["type"] == "text"
-                        else process_image_url(item["image_url"])
+                        (
+                            types.Part.from_text(text=item["text"] or " ")
+                            if item["type"] == "text"
+                            else (
+                                process_image_url(item["image_url"])
+                                if item["type"] == "image_url"
+                                else process_audio_url(item["audio_url"])
+                            )
+                        )
                         for item in content
                     ]
                 else:
@@ -395,14 +411,23 @@ class ProviderGoogleGenAI(Provider):
         )
 
     def _process_content_parts(
-        self, candidate: types.Candidate, llm_response: LLMResponse
+        self,
+        candidate: types.Candidate,
+        llm_response: LLMResponse,
+        *,
+        validate_output: bool = True,
     ) -> MessageChain:
         """处理内容部分并构建消息链"""
         if not candidate.content:
             logger.warning(f"收到的 candidate.content 为空: {candidate}")
-            raise EmptyModelOutputError(
-                f"Gemini candidate content is empty. finish_reason={candidate.finish_reason}"
-            )
+            if validate_output:
+                raise EmptyModelOutputError(
+                    "Gemini candidate content is empty. "
+                    f"finish_reason={candidate.finish_reason}"
+                )
+            llm_response.result_chain = MessageChain(chain=[])
+            return llm_response.result_chain
+
         finish_reason = candidate.finish_reason
         result_parts: list[types.Part] | None = candidate.content.parts
         if finish_reason == types.FinishReason.SAFETY:
@@ -418,9 +443,15 @@ class ProviderGoogleGenAI(Provider):
                 raise Exception("模型生成内容违反 Gemini 平台政策")
         if not result_parts:
             logger.warning(f"收到的 candidate.content.parts 为空: {candidate}")
-            raise EmptyModelOutputError(
-                f"Gemini candidate content parts are empty. finish_reason={candidate.finish_reason}"
-            )
+            if validate_output:
+                raise EmptyModelOutputError(
+                    "Gemini candidate content parts are empty. "
+                    f"finish_reason={candidate.finish_reason}"
+                )
+            llm_response.result_chain = MessageChain(chain=[])
+            return llm_response.result_chain
+
+        # 提取 reasoning content
         reasoning = self._extract_reasoning_content(candidate)
         if reasoning:
             llm_response.reasoning_content = reasoning
@@ -466,11 +497,12 @@ class ProviderGoogleGenAI(Provider):
                 llm_response.reasoning_signature = base64.b64encode(ts).decode("utf-8")
         chain_result = MessageChain(chain=chain)
         llm_response.result_chain = chain_result
-        self._ensure_usable_response(
-            llm_response,
-            response_id=None,
-            finish_reason=str(finish_reason) if finish_reason is not None else None,
-        )
+        if validate_output:
+            self._ensure_usable_response(
+                llm_response,
+                response_id=None,
+                finish_reason=str(finish_reason) if finish_reason is not None else None,
+            )
         return chain_result
 
     async def _query(self, payloads: dict, tools: ToolSet | None) -> LLMResponse:
@@ -600,7 +632,9 @@ class ProviderGoogleGenAI(Provider):
                 llm_response = LLMResponse("assistant", is_chunk=False)
                 llm_response.raw_completion = chunk
                 llm_response.result_chain = self._process_content_parts(
-                    chunk.candidates[0], llm_response
+                    chunk.candidates[0],
+                    llm_response,
+                    validate_output=False,
                 )
                 llm_response.id = chunk.response_id
                 if chunk.usage_metadata:
@@ -624,7 +658,9 @@ class ProviderGoogleGenAI(Provider):
                     final_response = LLMResponse("assistant", is_chunk=False)
                     final_response.raw_completion = chunk
                     final_response.result_chain = self._process_content_parts(
-                        chunk.candidates[0], final_response
+                        chunk.candidates[0],
+                        final_response,
+                        validate_output=False,
                     )
                     final_response.id = chunk.response_id
                     if chunk.usage_metadata:
@@ -647,15 +683,16 @@ class ProviderGoogleGenAI(Provider):
 
     async def text_chat(
         self,
-        prompt: str | None = None,
-        session_id: str | None = None,
-        image_urls: list[str] | None = None,
-        func_tool: ToolSet | None = None,
-        contexts: list[Message] | list[dict] | None = None,
-        system_prompt: str | None = None,
-        tool_calls_result: ToolCallsResult | list[ToolCallsResult] | None = None,
-        model: str | None = None,
-        extra_user_content_parts: list[ContentPart] | None = None,
+        prompt=None,
+        session_id=None,
+        image_urls=None,
+        audio_urls=None,
+        func_tool=None,
+        contexts=None,
+        system_prompt=None,
+        tool_calls_result=None,
+        model=None,
+        extra_user_content_parts=None,
         tool_choice: Literal["auto", "required"] = "auto",
         **kwargs,
     ) -> LLMResponse:
@@ -664,7 +701,10 @@ class ProviderGoogleGenAI(Provider):
         new_record = None
         if prompt is not None:
             new_record = await self.assemble_context(
-                prompt, image_urls, extra_user_content_parts
+                prompt or "",
+                image_urls,
+                audio_urls,
+                extra_user_content_parts,
             )
         context_query = self._ensure_message_to_dicts(contexts)
         if new_record:
@@ -698,15 +738,16 @@ class ProviderGoogleGenAI(Provider):
 
     async def text_chat_stream(
         self,
-        prompt: str | None = None,
-        session_id: str | None = None,
-        image_urls: list[str] | None = None,
-        func_tool: ToolSet | None = None,
-        contexts: list[Message] | list[dict] | None = None,
-        system_prompt: str | None = None,
-        tool_calls_result: ToolCallsResult | list[ToolCallsResult] | None = None,
-        model: str | None = None,
-        extra_user_content_parts: list[ContentPart] | None = None,
+        prompt=None,
+        session_id=None,
+        image_urls=None,
+        audio_urls=None,
+        func_tool=None,
+        contexts=None,
+        system_prompt=None,
+        tool_calls_result=None,
+        model=None,
+        extra_user_content_parts=None,
         tool_choice: Literal["auto", "required"] = "auto",
         **kwargs,
     ) -> AsyncGenerator[LLMResponse, None]:
@@ -715,7 +756,10 @@ class ProviderGoogleGenAI(Provider):
         new_record = None
         if prompt is not None:
             new_record = await self.assemble_context(
-                prompt, image_urls, extra_user_content_parts
+                prompt or "",
+                image_urls,
+                audio_urls,
+                extra_user_content_parts,
             )
         context_query = self._ensure_message_to_dicts(contexts)
         if new_record:
@@ -775,6 +819,7 @@ class ProviderGoogleGenAI(Provider):
         self,
         text: str,
         image_urls: list[str] | None = None,
+        audio_urls: list[str] | None = None,
         extra_user_content_parts: list[ContentPart] | None = None,
     ):
         """组装上下文｡"""
@@ -793,11 +838,52 @@ class ProviderGoogleGenAI(Provider):
                 return None
             return {"type": "image_url", "image_url": {"url": image_data}}
 
+        async def resolve_audio_part(audio_path: str) -> dict | None:
+            if audio_path.startswith("http"):
+                suffix = Path(urlparse(audio_path).path).suffix or ".wav"
+                temp_dir = Path(get_astrbot_temp_path())
+                temp_dir.mkdir(parents=True, exist_ok=True)
+                resolved_path = str(
+                    temp_dir / f"provider_audio_{uuid.uuid4().hex}{suffix}"
+                )
+                await download_file(audio_path, resolved_path)
+            elif audio_path.startswith("file:///"):
+                resolved_path = audio_path.replace("file:///", "")
+            else:
+                resolved_path = audio_path
+
+            suffix = Path(resolved_path).suffix.lower()
+            if suffix != ".mp3":
+                resolved_path = await ensure_wav(resolved_path)
+                suffix = ".wav"
+
+            try:
+                audio_bytes = Path(resolved_path).read_bytes()
+            except OSError as exc:
+                logger.warning(
+                    f"Failed to read audio file {resolved_path}, skipping. Error: {exc}"
+                )
+                return None
+
+            mime_type = {
+                ".wav": "audio/wav",
+                ".mp3": "audio/mp3",
+            }.get(suffix, "audio/wav")
+            audio_data = base64.b64encode(audio_bytes).decode("utf-8")
+            return {
+                "type": "audio_url",
+                "audio_url": {"url": f"data:{mime_type};base64,{audio_data}"},
+            }
+
+        # 构建内容块列表
         content_blocks = []
         if text:
             content_blocks.append({"type": "text", "text": text})
         elif image_urls:
-            content_blocks.append({"type": "text", "text": "[图片]"})
+            # 如果没有文本但有图片，添加占位文本
+            content_blocks.append({"type": "text", "text": "[Image]"})
+        elif audio_urls:
+            content_blocks.append({"type": "text", "text": "[Audio]"})
         elif extra_user_content_parts:
             content_blocks.append({"type": "text", "text": " "})
         if extra_user_content_parts:
@@ -808,6 +894,10 @@ class ProviderGoogleGenAI(Provider):
                     image_part = await resolve_image_part(part.image_url.url)
                     if image_part:
                         content_blocks.append(image_part)
+                elif isinstance(part, AudioURLPart):
+                    audio_part = await resolve_audio_part(part.audio_url.url)
+                    if audio_part:
+                        content_blocks.append(audio_part)
                 else:
                     raise ValueError(f"不支持的额外内容块类型: {type(part)}")
         if image_urls:
@@ -815,12 +905,21 @@ class ProviderGoogleGenAI(Provider):
                 image_part = await resolve_image_part(image_url)
                 if image_part:
                     content_blocks.append(image_part)
+
+        if audio_urls:
+            for audio_path in audio_urls:
+                audio_part = await resolve_audio_part(audio_path)
+                if audio_part:
+                    content_blocks.append(audio_part)
+
+        # 如果只有主文本且没有额外内容块和图片，返回简单格式以保持向后兼容
         if (
             text
-            and (not extra_user_content_parts)
-            and (not image_urls)
-            and (len(content_blocks) == 1)
-            and (content_blocks[0]["type"] == "text")
+            and not extra_user_content_parts
+            and not image_urls
+            and not audio_urls
+            and len(content_blocks) == 1
+            and content_blocks[0]["type"] == "text"
         ):
             return {"role": "user", "content": content_blocks[0]["text"]}
         return {"role": "user", "content": content_blocks}

astrbot/core/provider/sources/longcat_source.py

@@ -0,0 +1,23 @@
+from ..register import register_provider_adapter
+from .openai_source import ProviderOpenAIOfficial
+
+
+@register_provider_adapter(
+    "longcat_chat_completion", "LongCat Chat Completion Provider Adapter"
+)
+class ProviderLongCat(ProviderOpenAIOfficial):
+    def __init__(
+        self,
+        provider_config: dict,
+        provider_settings: dict,
+    ) -> None:
+        api_base = (provider_config.get("api_base", "") or "").strip()
+        if not api_base:
+            provider_config["api_base"] = "https://api.longcat.chat/openai/v1"
+        else:
+            normalized_api_base = api_base.rstrip("/")
+            if normalized_api_base.endswith("/openai"):
+                normalized_api_base = f"{normalized_api_base}/v1"
+            provider_config["api_base"] = normalized_api_base
+
+        super().__init__(provider_config, provider_settings)

astrbot/core/provider/sources/nvidia_rerank_source.py

@@ -0,0 +1,164 @@
+import aiohttp
+
+from astrbot import logger
+
+from ..entities import ProviderType, RerankResult
+from ..provider import RerankProvider
+from ..register import register_provider_adapter
+
+
+@register_provider_adapter(
+    "nvidia_rerank", "NVIDIA Rerank 适配器", provider_type=ProviderType.RERANK
+)
+class NvidiaRerankProvider(RerankProvider):
+    def __init__(self, provider_config: dict, provider_settings: dict) -> None:
+        super().__init__(provider_config, provider_settings)
+        self.api_key = provider_config.get("nvidia_rerank_api_key", "")
+        self.base_url = provider_config.get(
+            "nvidia_rerank_api_base", "https://ai.api.nvidia.com/v1/retrieval"
+        ).rstrip("/")
+        self.timeout = provider_config.get("timeout", 20)
+        self.model = provider_config.get(
+            "nvidia_rerank_model", "nv-rerank-qa-mistral-4b:1"
+        )
+        self.model_endpoint = provider_config.get(
+            "nvidia_rerank_model_endpoint", "/reranking"
+        )
+        self.truncate = provider_config.get("nvidia_rerank_truncate", "")
+
+        self.client = None
+        self.set_model(self.model)
+
+    async def _get_client(self):
+        if self.client is None or self.client.closed:
+            headers = {
+                "Authorization": f"Bearer {self.api_key}",
+                "Content-Type": "application/json",
+                "Accept": "application/json",
+            }
+            self.client = aiohttp.ClientSession(
+                headers=headers, timeout=aiohttp.ClientTimeout(total=self.timeout)
+            )
+        return self.client
+
+    def _get_endpoint(self) -> str:
+        """
+        构建完整API URL。
+
+        根据 Nvidia Rerank API 文档来看，当前URL存在不同模型格式不一致的问题。
+        这里针对模型名做一个基础判断用以适配，后续要等Nvidia统一API格式后再做调整。
+
+        例：
+        模型： nv-rerank-qa-mistral-4b:1
+        URL: .../v1/retrieval/nvidia/reranking
+
+        模型： nvidia/llama-nemotron-rerank-1b-v2
+        URL: .../v1/retrieval/nvidia/llama-nemotron-rerank-1b-v2/reranking
+        """
+
+        model_path = "nvidia"
+        logger.debug(f"[NVIDIA Rerank] Building endpoint for model: {self.model}")
+        if "/" in self.model:
+            """遵循NVIDIA API的URL规则，替换模型名中特殊字符"""
+            model_path = self.model.strip("/").replace(".", "_")
+        endpoint = self.model_endpoint.lstrip("/")
+        return f"{self.base_url}/{model_path}/{endpoint}"
+
+    def _build_payload(self, query: str, documents: list[str]) -> dict:
+        """构建请求载荷"""
+        payload = {
+            "model": self.model,
+            "query": {"text": query},
+            "passages": [{"text": doc} for doc in documents],
+        }
+        if self.truncate:
+            payload["truncate"] = self.truncate
+        return payload
+
+    def _parse_results(
+        self, response_data: dict, top_n: int | None
+    ) -> list[RerankResult]:
+        """解析响应数据"""
+        results = response_data.get("rankings", [])
+        if not results:
+            logger.warning(f"[NVIDIA Rerank] Empty response: {response_data}")
+            return []
+
+        rerank_results = []
+        for idx, item in enumerate(results):
+            try:
+                index = item.get("index", idx)
+                score = item.get("relevance_score", item.get("logit", 0.0))
+                rerank_results.append(
+                    RerankResult(index=index, relevance_score=float(score))
+                )
+            except Exception as e:
+                logger.warning(
+                    f"[NVIDIA Rerank] Result parsing error: {e}, Data={item}"
+                )
+
+        rerank_results.sort(key=lambda x: x.relevance_score, reverse=True)
+
+        if top_n is not None and top_n > 0:
+            return rerank_results[:top_n]
+        return rerank_results
+
+    def _log_usage(self, data: dict) -> None:
+        usage = data.get("usage", {})
+        total_tokens = usage.get("total_tokens", 0)
+        if total_tokens > 0:
+            logger.debug(f"[NVIDIA Rerank] Token Usage: {total_tokens}")
+
+    async def rerank(
+        self,
+        query: str,
+        documents: list[str],
+        top_n: int | None = None,
+    ) -> list[RerankResult]:
+        client = await self._get_client()
+        if not client or client.closed:
+            logger.error("[NVIDIA Rerank] Client session not initialized or closed")
+            return []
+
+        if not documents or not query.strip():
+            logger.warning(
+                "[NVIDIA Rerank] Input data is invalid, query or documents are empty"
+            )
+            return []
+
+        try:
+            payload = self._build_payload(query, documents)
+            request_url = self._get_endpoint()
+
+            async with client.post(request_url, json=payload) as response:
+                if response.status != 200:
+                    try:
+                        response_data = await response.json()
+                        error_detail = response_data.get(
+                            "detail", response_data.get("message", "Unknown Error")
+                        )
+
+                    except Exception:
+                        error_detail = await response.text()
+                        response_data = {"message": error_detail}
+
+                    logger.error(f"[NVIDIA Rerank] API Error Response: {response_data}")
+                    raise Exception(f"HTTP {response.status} - {error_detail}")
+
+                response_data = await response.json()
+                logger.debug(f"[NVIDIA Rerank] API Response: {response_data}")
+                results = self._parse_results(response_data, top_n)
+                self._log_usage(response_data)
+                return results
+
+        except aiohttp.ClientError as e:
+            logger.error(f"[NVIDIA Rerank] Network error: {e}")
+            raise Exception(f"Network error: {e}") from e
+        except Exception as e:
+            logger.error(f"[NVIDIA Rerank] Error: {e}")
+            raise Exception(f"Rerank error: {e}") from e
+
+    async def terminate(self) -> None:
+        if self.client and not self.client.closed:
+            await self.client.close()
+            self.client = None

astrbot/core/provider/sources/openai_source.py

@@ -5,6 +5,7 @@ import inspect
 import json
 import random
 import re
+import uuid
 from collections.abc import AsyncGenerator
 from io import BytesIO
 from pathlib import Path
@@ -24,13 +25,20 @@ from PIL import UnidentifiedImageError
 import astrbot.core.message.components as Comp
 from astrbot import logger
 from astrbot.api.provider import Provider
-from astrbot.core.agent.message import ContentPart, ImageURLPart, Message, TextPart
+from astrbot.core.agent.message import (
+    AudioURLPart,
+    ContentPart,
+    ImageURLPart,
+    Message,
+    TextPart,
+)
 from astrbot.core.agent.tool import ToolSet
 from astrbot.core.exceptions import EmptyModelOutputError
 from astrbot.core.message.message_event_result import MessageChain
 from astrbot.core.provider.entities import LLMResponse, TokenUsage, ToolCallsResult
-from astrbot.core.provider.register import register_provider_adapter
-from astrbot.core.utils.io import download_image_by_url
+from astrbot.core.utils.astrbot_path import get_astrbot_temp_path
+from astrbot.core.utils.io import download_file, download_image_by_url
+from astrbot.core.utils.media_utils import ensure_wav
 from astrbot.core.utils.network_utils import (
     create_proxy_client,
     is_connection_error,
@@ -135,7 +143,10 @@ class ProviderOpenAIOfficial(Provider):
             if not isinstance(content, list):
                 continue
             for item in content:
-                if isinstance(item, dict) and item.get("type") == "image_url":
+                if isinstance(item, dict) and item.get("type") in {
+                    "image_url",
+                    "audio_url",
+                }:
                     return True
         return False
 
@@ -284,24 +295,103 @@ class ProviderOpenAIOfficial(Provider):
             image_detail = None
         return url, image_detail
 
-    async def _transform_content_part(self, part: dict) -> dict:
-        url, image_detail = self._extract_image_part_info(part)
-        if not url:
-            return part
+    def _extract_audio_part_info(self, part: dict) -> str | None:
+        if not isinstance(part, dict) or part.get("type") != "audio_url":
+            return None
 
+        audio_url_data = part.get("audio_url")
+        if not isinstance(audio_url_data, dict):
+            logger.warning("音频内容块格式无效，将保留原始内容。")
+            return None
+
+        url = audio_url_data.get("url")
+        if not isinstance(url, str) or not url:
+            logger.warning("音频内容块缺少有效路径，将保留原始内容。")
+            return None
+
+        return url
+
+    async def _audio_ref_to_local_path(self, audio_ref: str) -> tuple[str, list[Path]]:
+        cleanup_paths: list[Path] = []
+        if audio_ref.startswith("http"):
+            suffix = Path(urlparse(audio_ref).path).suffix or ".wav"
+            temp_dir = Path(get_astrbot_temp_path())
+            temp_dir.mkdir(parents=True, exist_ok=True)
+            target_path = temp_dir / f"provider_audio_{uuid.uuid4().hex}{suffix}"
+            await download_file(audio_ref, str(target_path))
+            cleanup_paths.append(target_path)
+            return str(target_path), cleanup_paths
+        if audio_ref.startswith("file://"):
+            return self._file_uri_to_path(audio_ref), cleanup_paths
+        return audio_ref, cleanup_paths
+
+    async def _resolve_audio_part(self, audio_ref: str) -> dict | None:
+        cleanup_paths: list[Path] = []
         try:
-            resolved_part = await self._resolve_image_part(
-                url, image_detail=image_detail
-            )
+            audio_path, cleanup_paths = await self._audio_ref_to_local_path(audio_ref)
+            suffix = Path(audio_path).suffix.lower()
+            if suffix == ".mp3":
+                audio_format = "mp3"
+            else:
+                converted_audio_path = await ensure_wav(audio_path)
+                if converted_audio_path != audio_path:
+                    cleanup_paths.append(Path(converted_audio_path))
+                audio_path = converted_audio_path
+                audio_format = "wav"
+            audio_bytes = Path(audio_path).read_bytes()
         except Exception as exc:
-            logger.warning(
-                "图片 %s 预处理失败，将保留原始内容。错误: %s",
-                url,
-                exc,
-            )
+            logger.warning("音频 %s 预处理失败，将忽略。错误: %s", audio_ref, exc)
+            return None
+        finally:
+            for cleanup_path in cleanup_paths:
+                try:
+                    cleanup_path.unlink(missing_ok=True)
+                except Exception as cleanup_exc:
+                    logger.warning(
+                        "Failed to cleanup %s: %s",
+                        cleanup_path,
+                        cleanup_exc,
+                    )
+
+        return {
+            "type": "input_audio",
+            "input_audio": {
+                "data": base64.b64encode(audio_bytes).decode("utf-8"),
+                "format": audio_format,
+            },
+        }
+
+    async def _transform_content_part(self, part: dict) -> dict:
+        if not isinstance(part, dict):
             return part
 
-        return resolved_part or part
+        if part.get("type") == "image_url":
+            url, image_detail = self._extract_image_part_info(part)
+            if not url:
+                return part
+
+            try:
+                resolved_part = await self._resolve_image_part(
+                    url, image_detail=image_detail
+                )
+            except Exception as exc:
+                logger.warning(
+                    "图片 %s 预处理失败，将保留原始内容。错误: %s",
+                    url,
+                    exc,
+                )
+                return part
+
+            return resolved_part or part
+
+        if part.get("type") == "audio_url":
+            audio_ref = self._extract_audio_part_info(part)
+            if not audio_ref:
+                return part
+            resolved_part = await self._resolve_audio_part(audio_ref)
+            return resolved_part or part
+
+        return part
 
     async def _materialize_message_image_parts(self, message: dict) -> dict:
         content = message.get("content")
@@ -532,6 +622,7 @@ class ProviderOpenAIOfficial(Provider):
             **payloads,
             stream=True,
             extra_body=extra_body,
+            stream_options={"include_usage": True},
         )
 
         llm_response = LLMResponse("assistant", is_chunk=True)
@@ -539,12 +630,10 @@ class ProviderOpenAIOfficial(Provider):
         state = ChatCompletionStreamState()
 
         async for chunk in stream:
-            if not chunk.choices:
-                continue
-            choice = chunk.choices[0]
-            delta = choice.delta
+            choice = chunk.choices[0] if chunk.choices else None
+            delta = choice.delta if choice else None
 
-            if dtcs := delta.tool_calls:
+            if delta and (dtcs := delta.tool_calls):
                 for idx, tc in enumerate(dtcs):
                     # siliconflow workaround
                     if tc.function and tc.function.arguments:
@@ -574,7 +663,7 @@ class ProviderOpenAIOfficial(Provider):
                 _y = True
             if chunk.usage:
                 llm_response.usage = self._extract_usage(chunk.usage)
-            elif choice_usage := getattr(choice, "usage", None):
+            elif choice and (choice_usage := getattr(choice, "usage", None)):
                 # Workaround for some providers that only return usage in choices[].usage, e.g. MoonshotAI
                 # See https://github.com/AstrBotDevs/AstrBot/issues/6614
                 llm_response.usage = self._extract_usage(choice_usage)
@@ -822,6 +911,7 @@ class ProviderOpenAIOfficial(Provider):
         self,
         prompt: str | None,
         image_urls: list[str] | None = None,
+        audio_urls: list[str] | None = None,
         contexts: list[dict] | list[Message] | None = None,
         system_prompt: str | None = None,
         tool_calls_result: ToolCallsResult | list[ToolCallsResult] | None = None,
@@ -835,7 +925,10 @@ class ProviderOpenAIOfficial(Provider):
         new_record = None
         if prompt is not None:
             new_record = await self.assemble_context(
-                prompt, image_urls, extra_user_content_parts
+                prompt or "",
+                image_urls,
+                audio_urls,
+                extra_user_content_parts,
             )
         context_query = copy.deepcopy(self._ensure_message_to_dicts(contexts))
         if new_record:
@@ -1022,6 +1115,7 @@ class ProviderOpenAIOfficial(Provider):
         prompt=None,
         session_id=None,
         image_urls=None,
+        audio_urls=None,
         func_tool=None,
         contexts=None,
         system_prompt=None,
@@ -1034,6 +1128,7 @@ class ProviderOpenAIOfficial(Provider):
         payloads, context_query = await self._prepare_chat_payload(
             prompt,
             image_urls,
+            audio_urls,
             contexts,
             system_prompt,
             tool_calls_result,
@@ -1093,6 +1188,7 @@ class ProviderOpenAIOfficial(Provider):
         prompt=None,
         session_id=None,
         image_urls=None,
+        audio_urls=None,
         func_tool=None,
         contexts=None,
         system_prompt=None,
@@ -1106,6 +1202,7 @@ class ProviderOpenAIOfficial(Provider):
         payloads, context_query = await self._prepare_chat_payload(
             prompt,
             image_urls,
+            audio_urls,
             contexts,
             system_prompt,
             tool_calls_result,
@@ -1190,6 +1287,7 @@ class ProviderOpenAIOfficial(Provider):
         self,
         text: str,
         image_urls: list[str] | None = None,
+        audio_urls: list[str] | None = None,
         extra_user_content_parts: list[ContentPart] | None = None,
     ) -> dict:
         """组装成符合 OpenAI 格式的 role 为 user 的消息段"""
@@ -1201,8 +1299,10 @@ class ProviderOpenAIOfficial(Provider):
         if text:
             content_blocks.append({"type": "text", "text": text})
         elif image_urls:
-            # 如果没有文本但有图片,添加占位文本
-            content_blocks.append({"type": "text", "text": "[图片]"})
+            # 如果没有文本但有图片，添加占位文本
+            content_blocks.append({"type": "text", "text": "[Image]"})
+        elif audio_urls:
+            content_blocks.append({"type": "text", "text": "[Audio]"})
         elif extra_user_content_parts:
             # 如果只有额外内容块,也需要添加占位文本
             content_blocks.append({"type": "text", "text": " "})
@@ -1218,6 +1318,10 @@ class ProviderOpenAIOfficial(Provider):
                     )
                     if image_part:
                         content_blocks.append(image_part)
+                elif isinstance(part, AudioURLPart):
+                    audio_part = await self._resolve_audio_part(part.audio_url.url)
+                    if audio_part:
+                        content_blocks.append(audio_part)
                 else:
                     raise ValueError(f"不支持的额外内容块类型: {type(part)}")
 
@@ -1228,11 +1332,18 @@ class ProviderOpenAIOfficial(Provider):
                 if image_part:
                     content_blocks.append(image_part)
 
-        # 如果只有主文本且没有额外内容块和图片,返回简单格式以保持向后兼容
+        if audio_urls:
+            for audio_path in audio_urls:
+                audio_part = await self._resolve_audio_part(audio_path)
+                if audio_part:
+                    content_blocks.append(audio_part)
+
+        # 如果只有主文本且没有额外内容块和图片，返回简单格式以保持向后兼容
         if (
             text
             and not extra_user_content_parts
             and not image_urls
+            and not audio_urls
             and len(content_blocks) == 1
             and content_blocks[0]["type"] == "text"
         ):

astrbot/core/provider/sources/vllm_rerank_source.py

@@ -19,6 +19,11 @@ class VLLMRerankProvider(RerankProvider):
         self.auth_key = provider_config.get("rerank_api_key", "")
         self.base_url = provider_config.get("rerank_api_base", "http://127.0.0.1:8000")
         self.base_url = self.base_url.rstrip("/")
+        self.api_suffix = provider_config.get("rerank_api_suffix", "/v1/rerank")
+        if self.api_suffix is None:
+            self.api_suffix = "/v1/rerank"
+        if self.api_suffix and not self.api_suffix.startswith("/"):
+            self.api_suffix = "/" + self.api_suffix
         self.timeout = provider_config.get("timeout", 20)
         self.model = provider_config.get("rerank_model", "BAAI/bge-reranker-base")
 
@@ -44,8 +49,9 @@ class VLLMRerankProvider(RerankProvider):
         if top_n is not None:
             payload["top_n"] = top_n
         assert self.client is not None
+        rerank_url = f"{self.base_url}{self.api_suffix}"
         async with self.client.post(
-            f"{self.base_url}/v1/rerank",
+            rerank_url,
             json=payload,
         ) as response:
             response_data = await response.json()

astrbot/core/star/context.py

@@ -124,6 +124,7 @@ class Context:
         chat_provider_id: str,
         prompt: str | None = None,
         image_urls: list[str] | None = None,
+        audio_urls: list[str] | None = None,
         tools: ToolSet | None = None,
         system_prompt: str | None = None,
         contexts: list[Message] | None = None,
@@ -137,6 +138,7 @@ class Context:
             chat_provider_id: The chat provider ID to use.
             prompt: The prompt to send to the LLM, if `contexts` and `prompt` are both provided, `prompt` will be appended as the last user message
             image_urls: List of image URLs to include in the prompt, if `contexts` and `prompt` are both provided, `image_urls` will be appended to the last user message
+            audio_urls: List of audio URLs or local paths to include in the prompt, if `contexts` and `prompt` are both provided, `audio_urls` will be appended to the last user message
             tools: ToolSet of tools available to the LLM
             system_prompt: System prompt to guide the LLM's behavior, if provided, it will always insert as the first system message in the context
             contexts: context messages for the LLM
@@ -152,6 +154,7 @@ class Context:
         llm_resp = await prov.text_chat(
             prompt=prompt,
             image_urls=image_urls,
+            audio_urls=audio_urls,
             func_tool=tools,
             contexts=contexts,
             system_prompt=system_prompt,
@@ -166,6 +169,7 @@ class Context:
         chat_provider_id: str,
         prompt: str | None = None,
         image_urls: list[str] | None = None,
+        audio_urls: list[str] | None = None,
         tools: ToolSet | None = None,
         system_prompt: str | None = None,
         contexts: list[Message] | None = None,
@@ -185,6 +189,7 @@ class Context:
             chat_provider_id: The chat provider ID to use.
             prompt: The prompt to send to the LLM, if `contexts` and `prompt` are both provided, `prompt` will be appended as the last user message
             image_urls: List of image URLs to include in the prompt, if `contexts` and `prompt` are both provided, `image_urls` will be appended to the last user message
+            audio_urls: List of audio URLs or local paths to include in the prompt, if `contexts` and `prompt` are both provided, `audio_urls` will be appended to the last user message
             tools: ToolSet of tools available to the LLM
             system_prompt: System prompt to guide the LLM's behavior, if provided, it will always insert as the first system message in the context
             contexts: context messages for the LLM
@@ -221,6 +226,7 @@ class Context:
         request = ProviderRequest(
             prompt=prompt,
             image_urls=image_urls or [],
+            audio_urls=audio_urls or [],
             func_tool=tools,
             contexts=context_,
             system_prompt=system_prompt or "",

astrbot/core/star/filter/platform_adapter_type.py

@@ -22,6 +22,7 @@ class PlatformAdapterType(enum.Flag):
     SATORI = enum.auto()
     MISSKEY = enum.auto()
     LINE = enum.auto()
+    MATRIX = enum.auto()
     ALL = (
         AIOCQHTTP
         | QQOFFICIAL
@@ -38,6 +39,7 @@ class PlatformAdapterType(enum.Flag):
         | SATORI
         | MISSKEY
         | LINE
+        | MATRIX
     )
 
 
@@ -57,6 +59,7 @@ ADAPTER_NAME_2_TYPE = {
     "satori": PlatformAdapterType.SATORI,
     "misskey": PlatformAdapterType.MISSKEY,
     "line": PlatformAdapterType.LINE,
+    "matrix": PlatformAdapterType.MATRIX,
 }
 
 

astrbot/core/utils/media_utils.py

@@ -231,8 +231,8 @@ async def convert_audio_format(
 
     Args:
         audio_path: 原始音频文件路径
-        output_format: 目标格式,例如 amr / ogg
-        output_path: 输出文件路径,如果为None则自动生成
+        output_format: 目标格式，例如 amr / ogg / opus / wav
+        output_path: 输出文件路径，如果为None则自动生成
 
     Returns:
         转换后的音频文件路径
@@ -250,6 +250,8 @@ async def convert_audio_format(
         args.extend(["-ac", "1", "-ar", "8000", "-ab", "12.2k"])
     elif output_format == "ogg":
         args.extend(["-acodec", "libopus", "-ac", "1", "-ar", "16000"])
+    elif output_format == "opus":
+        args.extend(["-acodec", "libopus", "-ac", "1", "-ar", "16000"])
     args.append(output_path)
 
     try:
@@ -291,11 +293,67 @@ async def convert_audio_to_wav(audio_path: str, output_path: str | None = None)
     )
 
 
+async def ensure_wav(audio_path: str, output_path: str | None = None) -> str:
+    """Ensure the audio path points to wav format by extension/guess and convert when needed.
+
+    If the file appears to already be wav, return it directly to avoid extra conversion.
+    """
+
+    if not audio_path:
+        return audio_path
+
+    if _get_audio_magic_type(audio_path) == "wav":
+        return audio_path
+
+    return await convert_audio_to_wav(audio_path, output_path)
+
+
+def _get_audio_magic_type(audio_path: str) -> str:
+    """Detect common audio formats from magic bytes."""
+    try:
+        with open(audio_path, "rb") as f:
+            header = f.read(64)
+    except FileNotFoundError:
+        logger.warning(f"[Media Utils] wav check file not found: {audio_path}")
+        return ""
+    except Exception as e:
+        logger.warning(f"[Media Utils] wav check failed: {audio_path}, error: {e}")
+        return ""
+
+    if len(header) < 12:
+        return ""
+
+    if header[:4] == b"RIFF" and header[8:12] == b"WAVE":
+        return "wav"
+
+    if header[:4] == b"#!AM":
+        return "amr"
+
+    if header[:4] == b"OggS":
+        if b"OpusHead" in header:
+            return "opus"
+        return "ogg"
+
+    if header[:3] == b"fLa":
+        return "flac"
+
+    if header[:3] == b"ID3" or header[:2] == b"\xff\xfb":
+        return "mp3"
+
+    if header[:4] == b"ftyp" and b"mp4" in header[:8]:
+        return "mp4"
+
+    if header[:8] == b"#!SILK_V3":
+        return "silk"
+
+    return ""
+
+
 async def extract_video_cover(
     video_path: str,
     output_path: str | None = None,
 ) -> str:
-    """从视频中提取封面图(JPG)｡"""
+    """从视频中提取封面图(JPG)"""
     if output_path is None:
         temp_dir = anyio.Path(get_astrbot_temp_path())
         await temp_dir.mkdir(parents=True, exist_ok=True)

changelogs/v4.22.3.md

@@ -0,0 +1,68 @@
+- [更新日志(简体中文)](#chinese)
+- [Changelog(English)](#english)
+
+<a id="chinese"></a>
+
+## What's Changed
+
+### 新增
+
+- 新增 `vllm_rerank` 提供商支持可配置的 `rerank_api_suffix`，可灵活适配不同 API 路径。（[#7278](https://github.com/AstrBotDevs/AstrBot/pull/7278)）
+- 适配 llama.cpp 的 token 用量提取支持，增强 token 统计与计费分析能力。（[#7358](https://github.com/AstrBotDevs/AstrBot/pull/7358)）
+- WebUI 捆绑 Monaco Editor 的 worker 文件，防止从 CDN 下载失败。（[#7249](https://github.com/AstrBotDevs/AstrBot/pull/7249)）
+- 全新的模型/平台总览统计面板。（[#7152](https://github.com/AstrBotDevs/AstrBot/pull/7152)）
+
+### 优化
+
+- 优化 OpenAI 消息处理，若 `content` 列表为空时将其置为 `None`，并降低空内容下的异常率。（[#6551](https://github.com/AstrBotDevs/AstrBot/pull/6551)）
+- 优化 provider page 布局与样式，提升整体设置页面排版一致性。（[#7248](https://github.com/AstrBotDevs/AstrBot/pull/7248)）
+- 用户已登录时将直接跳转欢迎页，减少再次进入登录页的步骤。（[commit](https://github.com/AstrBotDevs/AstrBot/commit/9896b48c)）
+
+### 修复
+
+- 修复 Windows 下 MCP runner 未继承完整系统环境变量导致启动异常的问题。（[#7054](https://github.com/AstrBotDevs/AstrBot/pull/7054)）
+- 修复 Gemini 思考片段泄露到用户可见消息链路的问题。（[#7196](https://github.com/AstrBotDevs/AstrBot/pull/7196)）
+- 修复 Gemini(OpenAI 兼容 API) 下，`function_response` 格式要求导致的 `400 Invalid argument` 错误。（[#7216](https://github.com/AstrBotDevs/AstrBot/pull/7216)
+- 修复知识库初始化失败时的错误提示与恢复问题。（[#7243](https://github.com/AstrBotDevs/AstrBot/pull/7243)）
+- 修复 Bailian Rerank 对新旧响应格式兼容性，避免解析失败。（[#7217](https://github.com/AstrBotDevs/AstrBot/pull/7217)）
+- Docker 镜像补充 AMR 编码器依赖，修复部分部署场景下的音频转码缺失问题。（[#7247](https://github.com/AstrBotDevs/AstrBot/pull/7247)）
+- 修复 pip 安装缺少 `pysocks` 依赖导致的 SOCKS5 代理不可用问题。（[#7221](https://github.com/AstrBotDevs/AstrBot/pull/7221)）
+- 修复 `cached_tokens` 在 `_extract_usage` 中的处理逻辑，避免 token 统计错乱。（[#6719](https://github.com/AstrBotDevs/AstrBot/pull/6719)）
+- 增强 SSE 断开保护，压缩传输下主动发送心跳，减少 WebChat 误断连。（[#7003](https://github.com/AstrBotDevs/AstrBot/pull/7003)）
+- 修复严格 API 下空 assistant 消息导致的 400 错误。（[#7202](https://github.com/AstrBotDevs/AstrBot/pull/7202)）
+- 新增二维码功能缺失的 `qrcode` 依赖修复，恢复二维码生成功能。（[commit](https://github.com/AstrBotDevs/AstrBot/commit/5886c437)）
+- 修复配置器数字输入组件的交互体验问题，避免异常输入场景下的处理偏差。（[#7153](https://github.com/AstrBotDevs/AstrBot/pull/7153)）
+- 修复 `Record.path` 在 Pydantic v2 下被错误视为必填字段的问题。（[#7048](https://github.com/AstrBotDevs/AstrBot/pull/7048)）
+
+<a id="english"></a>
+## What's Changed (EN)
+
+### New Features
+
+- Added configurable `rerank_api_suffix` for the `vllm_rerank` provider to support custom API paths.（[#7278](https://github.com/AstrBotDevs/AstrBot/pull/7278)）
+- Added token usage extraction support for llama.cpp to improve token accounting.（[#7358](https://github.com/AstrBotDevs/AstrBot/pull/7358)）
+- Bundled Monaco Editor workers in WebUI so code editing does not depend on external CDN downloads.（[#7249](https://github.com/AstrBotDevs/AstrBot/pull/7249)）
+- Added a new statistics overview for model and platform usage.（[#7152](https://github.com/AstrBotDevs/AstrBot/pull/7152)）
+- Redirected logged-in users to the welcome page instead of the login page.（[commit](https://github.com/AstrBotDevs/AstrBot/commit/9896b48c)）
+
+### Improvements
+
+- Improved OpenAI message handling by setting message `content` to `None` when content lists are empty, reducing empty-payload edge cases.（[#6551](https://github.com/AstrBotDevs/AstrBot/pull/6551)）
+- Improved provider page layout and styling to keep settings pages consistent.（[#7248](https://github.com/AstrBotDevs/AstrBot/pull/7248)）
+- Refined connection-success handling and messaging when users are already logged in.（[commit](https://github.com/AstrBotDevs/AstrBot/commit/2de23184)）
+
+### Bug Fixes
+
+- Inherited all system environment variables in Windows MCP runner to fix environment-dependent launch issues.（[#7054](https://github.com/AstrBotDevs/AstrBot/pull/7054)）
+- Filtered Gemini thinking parts from user-facing message chains.（[#7196](https://github.com/AstrBotDevs/AstrBot/pull/7196)）
+- Fixed Gemini `function_response` compatibility to prevent `400 Invalid argument` responses.（[#7216](https://github.com/AstrBotDevs/AstrBot/pull/7216)）
+- Improved knowledge base initialization error handling.（[#7243](https://github.com/AstrBotDevs/AstrBot/pull/7243)）
+- Fixed Bailian rerank parsing for both old and new response formats.（[#7217](https://github.com/AstrBotDevs/AstrBot/pull/7217)）
+- Added AMR encoder support in Docker builds to fix missing audio codec support.（[#7247](https://github.com/AstrBotDevs/AstrBot/pull/7247)）
+- Added the `pysocks` dependency so pip installs can use SOCKS5 proxy correctly.（[#7221](https://github.com/AstrBotDevs/AstrBot/pull/7221)）
+- Fixed `_extract_usage` cached token handling.（[#6719](https://github.com/AstrBotDevs/AstrBot/pull/6719)）
+- Sent SSE heartbeat packets to prevent WebChat disconnection under compression.（[#7003](https://github.com/AstrBotDevs/AstrBot/pull/7003)）
+- Fixed `400` errors caused by empty assistant messages on strict APIs.（[#7202](https://github.com/AstrBotDevs/AstrBot/pull/7202)）
+- Added missing `qrcode` dependency to restore QR code generation.（[commit](https://github.com/AstrBotDevs/AstrBot/commit/5886c437)）
+- Fixed number input UX in config renderer.（[#7153](https://github.com/AstrBotDevs/AstrBot/pull/7153)）
+- Fixed `Record.path` being incorrectly treated as required under Pydantic v2.（[#7048](https://github.com/AstrBotDevs/AstrBot/pull/7048)）

dashboard/src/assets/images/platform_logos/matrix.svg

@@ -0,0 +1 @@
+<svg width="75" height="32" xmlns="http://www.w3.org/2000/svg"><g fill="#000" fill-rule="nonzero"><path d="M.936.732V31.25H3.13v.732H.095V0h3.034v.732zm8.45 9.675v1.544h.044a4.461 4.461.0 011.487-1.368c.58-.323 1.245-.485 1.993-.485.72.0 1.377.14 1.972.42.595.279 1.047.771 1.355 1.477.338-.5.796-.941 1.377-1.323.58-.383 1.266-.574 2.06-.574.602.0 1.16.074 1.674.22.514.148.954.383 1.322.707.366.323.653.746.859 1.268.205.522.308 1.15.308 1.887v7.633H20.71v-6.464c0-.383-.015-.743-.044-1.082a2.305 2.305.0 00-.242-.882 1.473 1.473.0 00-.584-.596c-.257-.146-.606-.22-1.047-.22-.44.0-.796.085-1.068.253-.272.17-.485.39-.639.662a2.654 2.654.0 00-.308.927 7.074 7.074.0 00-.078 1.048v6.354h-3.128v-6.398c0-.338-.007-.673-.021-1.004a2.825 2.825.0 00-.188-.916 1.411 1.411.0 00-.55-.673c-.258-.168-.636-.253-1.135-.253a2.33 2.33.0 00-.584.1 1.94 1.94.0 00-.705.374c-.228.184-.422.449-.584.794-.161.346-.242.798-.242 1.357v6.619H6.434V10.407h2.952zm16.456 1.677a3.751 3.751.0 011.233-1.17 5.37 5.37.0 011.685-.629 9.579 9.579.0 011.884-.187c.573.0 1.153.04 1.74.121.588.081 1.124.24 1.609.475.484.235.88.562 1.19.981.308.42.462.975.462 1.666v5.934c0 .516.03 1.008.088 1.478.058.471.161.824.308 1.06H32.87a4.435 4.435.0 01-.22-1.104c-.5.515-1.087.876-1.762 1.081a7.084 7.084.0 01-2.071.31c-.544.0-1.05-.067-1.52-.2a3.472 3.472.0 01-1.234-.617 2.87 2.87.0 01-.826-1.059c-.199-.426-.298-.934-.298-1.522.0-.647.114-1.18.342-1.6.227-.419.52-.753.881-1.004.36-.25.771-.437 1.234-.562.462-.125.929-.224 1.399-.298.47-.073.932-.132 1.387-.176.456-.044.86-.11 1.212-.199.353-.088.631-.217.837-.386.206-.169.301-.415.287-.74.0-.337-.055-.606-.166-.804a1.217 1.217.0 00-.44-.464 1.737 1.737.0 00-.639-.22 5.292 5.292.0 00-.782-.055c-.617.0-1.101.132-1.454.397-.352.264-.558.706-.617 1.323h-3.128c.044-.735.227-1.345.55-1.83zm6.179 4.423a5.095 5.095.0 01-.639.165 9.68 9.68.0 01-.716.11c-.25.03-.5.067-.749.11a5.616 5.616.0 00-.694.177 2.057 2.057.0 00-.594.298c-.17.125-.305.284-.408.474-.103.192-.154.434-.154.728.0.28.051.515.154.706.103.192.242.342.419.453.176.11.381.187.617.231.234.044.477.066.726.066.617.0 1.094-.102 1.432-.309.338-.205.587-.452.75-.739.16-.286.26-.576.297-.87.036-.295.055-.53.055-.707v-1.17a1.4 1.4.0 01-.496.277zm11.863-6.1v2.096h-2.291v5.647c0 .53.088.883.264 1.059.176.177.529.265 1.057.265.177.0.345-.007.507-.022.161-.015.316-.037.463-.066v2.426a7.49 7.49.0 01-.882.089 21.67 21.67.0 01-.947.022c-.484.0-.944-.034-1.377-.1a3.233 3.233.0 01-1.145-.386 2.04 2.04.0 01-.782-.816c-.191-.353-.287-.816-.287-1.39v-6.728H36.57v-2.096h1.894v-3.42h3.129v3.42h2.29zm4.471.0v2.118h.044a3.907 3.907.0 011.454-1.754 4.213 4.213.0 011.036-.497 3.734 3.734.0 011.145-.176c.206.0.433.037.683.11v2.912a5.862 5.862.0 00-.528-.077 5.566 5.566.0 00-.595-.033c-.573.0-1.058.096-1.454.287a2.52 2.52.0 00-.958.783 3.143 3.143.0 00-.518 1.158 6.32 6.32.0 00-.154 1.434v5.14h-3.128V10.407h2.973zM54.039 8.642V6.06h3.128v2.582H54.04zm3.128 1.765v11.405H54.04V10.407h3.128zm1.63.0h3.569l2.005 2.978 1.982-2.978h3.459l-3.745 5.339 4.208 6.067h-3.57l-2.378-3.596-2.38 3.596h-3.502l4.097-6.001zM74.094 31.25V.732H71.9V0h3.035v31.982H71.9v-.732z"></path></g></svg>

dashboard/src/assets/mdi-subset/materialdesignicons-subset.css

@@ -1,4 +1,4 @@
-/* Auto-generated MDI subset – 255 icons */
+/* Auto-generated MDI subset – 256 icons */
 /* Do not edit manually. Run: pnpm run subset-icons */
 
 @font-face {
@@ -684,6 +684,10 @@
   content: "\F0375";
 }
 
+.mdi-music-note-outline::before {
+  content: "\F0F74";
+}
+
 .mdi-note-text-outline::before {
   content: "\F11D7";
 }

dashboard/src/components/chat/ProviderConfigDialog.vue

@@ -121,12 +121,13 @@
 
                 <!-- 模型配置 -->
                 <ProviderModelsPanel
-                  v-model:model-search="modelSearch"
                   :entries="filteredMergedModelEntries"
                   :available-count="availableModels.length"
+                  v-model:model-search="modelSearch"
                   :loading-models="loadingModels"
                   :is-source-modified="isSourceModified"
                   :supports-image-input="supportsImageInput"
+                  :supports-audio-input="supportsAudioInput"
                   :supports-tool-call="supportsToolCall"
                   :supports-reasoning="supportsReasoning"
                   :format-context-limit="formatContextLimit"
@@ -140,6 +141,51 @@
                   @delete-provider="deleteProvider"
                   @add-model-provider="addModelProvider"
                 />
+                >>>>>>> origin =======
+                <ProviderModelsPanel
+                  v-model:model-search="modelSearch"
+                  :entries="filteredMergedModelEntries"
+                  :available-count="availableModels.length"
+                  :loading-models="loadingModels"
+                  :is-source-modified="isSourceModified"
+                  :supports-image-input="supportsImageInput"
+                  :supports-audio-input="supportsAudioInput"
+                  :supports-tool-call="supportsToolCall"
+                  :supports-reasoning="supportsReasoning"
+                  :format-context-limit="formatContextLimit"
+                  :testing-providers="testingProviders"
+                  :tm="tm"
+                  @fetch-models="fetchAvailableModels"
+                  @open-manual-model="openManualModelDialog"
+                  @open-provider-edit="openProviderEdit"
+                  @toggle-provider-enable="toggleProviderEnable"
+                  @test-provider="testProvider"
+                  @delete-provider="deleteProvider"
+                  @add-model-provider="addModelProvider"
+                />
+                =======
+                <ProviderModelsPanel
+                  :entries="filteredMergedModelEntries"
+                  :available-count="availableModels.length"
+                  v-model:model-search="modelSearch"
+                  :loading-models="loadingModels"
+                  :is-source-modified="isSourceModified"
+                  :supports-image-input="supportsImageInput"
+                  :supports-audio-input="supportsAudioInput"
+                  :supports-tool-call="supportsToolCall"
+                  :supports-reasoning="supportsReasoning"
+                  :format-context-limit="formatContextLimit"
+                  :testing-providers="testingProviders"
+                  :tm="tm"
+                  @fetch-models="fetchAvailableModels"
+                  @open-manual-model="openManualModelDialog"
+                  @open-provider-edit="openProviderEdit"
+                  @toggle-provider-enable="toggleProviderEnable"
+                  @test-provider="testProvider"
+                  @delete-provider="deleteProvider"
+                  @add-model-provider="addModelProvider"
+                />
+                >>>>>>> origin
               </div>
             </div>
             <div
@@ -292,6 +338,7 @@ const {
   resolveSourceIcon,
   getSourceDisplayName,
   supportsImageInput,
+  supportsAudioInput,
   supportsToolCall,
   supportsReasoning,
   formatContextLimit,

dashboard/src/components/chat/ProviderModelMenu.vue

@@ -1,95 +1,67 @@
 <template>
-  <v-menu
-    v-model="menuOpen"
-    :close-on-content-click="false"
-    location="top"
-    @update:model-value="handleMenuToggle"
-  >
-    <template #activator="{ props: menuProps }">
-      <v-chip
-        v-bind="menuProps"
-        class="text-none provider-chip"
-        variant="tonal"
-        :size="chipSize"
-      >
-        <v-icon start size="14"> mdi-creation </v-icon>
-        <span v-if="selectedProviderId">
-          {{ selectedProviderId }}
-        </span>
-        <span v-else>Model</span>
-      </v-chip>
-    </template>
-    <v-card class="provider-menu-card" min-width="280" max-width="400">
-      <v-card-text class="pa-2">
-        <v-text-field
-          v-model="searchQuery"
-          placeholder="Search..."
-          hide-details
-          variant="plain"
-          flat
-          density="compact"
-          prepend-inner-icon="mdi-magnify"
-          class="ml-2 mb-2 mr-2"
-          clearable
-        />
-        <v-list density="compact" nav class="provider-menu-list">
-          <v-list-item
-            v-for="provider in filteredProviders"
-            :key="provider.id"
-            :active="selectedProviderId === provider.id"
-            rounded="lg"
-            class="provider-menu-item"
-            @click="selectProvider(provider)"
-          >
-            <v-list-item-title class="text-body-2">
-              {{ provider.id }}
-            </v-list-item-title>
-            <v-list-item-subtitle class="provider-subtitle">
-              <span class="model-name">{{ provider.model }}</span>
-              <span class="meta-icons">
-                <v-tooltip
-                  v-if="supportsImageInput(provider)"
-                  text="支持图像输入"
-                  location="top"
-                >
-                  <template #activator="{ props: tipProps }">
-                    <v-icon v-bind="tipProps" size="12" color="grey"
-                      >mdi-eye-outline</v-icon
-                    >
-                  </template>
-                </v-tooltip>
-                <v-tooltip
-                  v-if="supportsToolCall(provider)"
-                  text="支持工具调用"
-                  location="top"
-                >
-                  <template #activator="{ props: tipProps }">
-                    <v-icon v-bind="tipProps" size="12" color="grey"
-                      >mdi-wrench</v-icon
-                    >
-                  </template>
-                </v-tooltip>
-                <v-tooltip
-                  v-if="supportsReasoning(provider)"
-                  text="支持推理"
-                  location="top"
-                >
-                  <template #activator="{ props: tipProps }">
-                    <v-icon v-bind="tipProps" size="12" color="grey"
-                      >mdi-brain</v-icon
-                    >
-                  </template>
-                </v-tooltip>
-              </span>
-            </v-list-item-subtitle>
-          </v-list-item>
-        </v-list>
-        <div v-if="providerConfigs.length === 0" class="empty-hint">
-          No available models
-        </div>
-      </v-card-text>
-    </v-card>
-  </v-menu>
+
+    <v-menu v-model="menuOpen" :close-on-content-click="false" location="top" @update:model-value="handleMenuToggle">
+        <template v-slot:activator="{ props: menuProps }">
+            <v-chip v-bind="menuProps" class="text-none provider-chip" variant="tonal" :size="chipSize">
+                <v-icon start size="14">mdi-creation</v-icon>
+                <span v-if="selectedProviderId">
+                    {{ selectedProviderId }}
+                </span>
+                <span v-else>Model</span>
+            </v-chip>
+        </template>
+        <v-card class="provider-menu-card" min-width="280" max-width="400">
+            <v-card-text class="pa-2">
+                <v-text-field
+                    v-model="searchQuery"
+                    placeholder="Search..."
+                    hide-details
+                    variant="plain"
+                    flat
+                    density="compact"
+                    prepend-inner-icon="mdi-magnify"
+                    class="ml-2 mb-2 mr-2"
+                    clearable
+                />
+                <v-list density="compact" nav class="provider-menu-list">
+                    <v-list-item v-for="provider in filteredProviders" :key="provider.id"
+                        :active="selectedProviderId === provider.id" @click="selectProvider(provider)" rounded="lg"
+                        class="provider-menu-item">
+                        <v-list-item-title class="text-body-2">{{ provider.id }}</v-list-item-title>
+                        <v-list-item-subtitle class="provider-subtitle">
+                            <span class="model-name">{{ provider.model }}</span>
+                            <span class="meta-icons">
+                                <v-tooltip text="支持图像输入" location="top" v-if="supportsImageInput(provider)">
+                                    <template v-slot:activator="{ props: tipProps }">
+                                        <v-icon v-bind="tipProps" size="12" color="grey">mdi-eye-outline</v-icon>
+                                    </template>
+                                </v-tooltip>
+                                <v-tooltip text="支持音频输入" location="top" v-if="supportsAudioInput(provider)">
+                                    <template v-slot:activator="{ props: tipProps }">
+                                        <v-icon v-bind="tipProps" size="12" color="grey">mdi-music-note-outline</v-icon>
+                                    </template>
+                                </v-tooltip>
+                                <v-tooltip text="支持工具调用" location="top" v-if="supportsToolCall(provider)">
+                                    <template v-slot:activator="{ props: tipProps }">
+                                        <v-icon v-bind="tipProps" size="12" color="grey">mdi-wrench</v-icon>
+                                    </template>
+                                </v-tooltip>
+                                <v-tooltip text="支持推理" location="top" v-if="supportsReasoning(provider)">
+                                    <template v-slot:activator="{ props: tipProps }">
+                                        <v-icon v-bind="tipProps" size="12" color="grey">mdi-brain</v-icon>
+                                    </template>
+                                </v-tooltip>
+                            </span>
+                        </v-list-item-subtitle>
+                    </v-list-item>
+                </v-list>
+                <div v-if="providerConfigs.length === 0" class="empty-hint">
+                    No available models
+                </div>
+            </v-card-text>
+        </v-card>
+    </v-menu>
+
 </template>
 
 <script setup lang="ts">
@@ -173,6 +145,11 @@ function supportsImageInput(provider: ProviderConfig): boolean {
   return inputs.includes("image");
 }
 
+function supportsAudioInput(provider: ProviderConfig): boolean {
+    const inputs = provider.model_metadata?.modalities?.input || [];
+    return inputs.includes('audio');
+}
+
 function supportsToolCall(provider: ProviderConfig): boolean {
   return Boolean(provider.model_metadata?.tool_call);
 }

dashboard/src/components/provider/AddNewProvider.vue

@@ -5,23 +5,23 @@
         <v-tabs v-model="activeProviderTab" grow>
           <v-tab value="agent_runner" class="font-weight-medium px-3">
             <v-icon start> mdi-cogs </v-icon>
-            {{ tm("dialogs.addProvider.tabs.agentRunner") }}
+            {{ tm('dialogs.addProvider.tabs.agentRunner') }}
           </v-tab>
           <v-tab value="speech_to_text" class="font-weight-medium px-3">
             <v-icon start> mdi-microphone-message </v-icon>
-            {{ tm("dialogs.addProvider.tabs.speechToText") }}
+            {{ tm('dialogs.addProvider.tabs.speechToText') }}
           </v-tab>
           <v-tab value="text_to_speech" class="font-weight-medium px-3">
             <v-icon start> mdi-volume-high </v-icon>
-            {{ tm("dialogs.addProvider.tabs.textToSpeech") }}
+            {{ tm('dialogs.addProvider.tabs.textToSpeech') }}
           </v-tab>
           <v-tab value="embedding" class="font-weight-medium px-3">
             <v-icon start> mdi-code-json </v-icon>
-            {{ tm("dialogs.addProvider.tabs.embedding") }}
+            {{ tm('dialogs.addProvider.tabs.embedding') }}
           </v-tab>
           <v-tab value="rerank" class="font-weight-medium px-3">
             <v-icon start> mdi-compare-vertical </v-icon>
-            {{ tm("dialogs.addProvider.tabs.rerank") }}
+            {{ tm('dialogs.addProvider.tabs.rerank') }}
           </v-tab>
         </v-tabs>
 
@@ -81,7 +81,7 @@
                 cols="12"
               >
                 <v-alert type="info" variant="tonal">
-                  {{ tm("dialogs.addProvider.noTemplates") }}
+                  {{ tm('dialogs.addProvider.noTemplates') }}
                 </v-alert>
               </v-col>
             </v-row>
@@ -91,7 +91,7 @@
       <v-card-actions>
         <v-spacer />
         <v-btn text @click="closeDialog">
-          {{ tm("dialogs.config.cancel") }}
+          {{ tm('dialogs.config.cancel') }}
         </v-btn>
       </v-card-actions>
     </v-card>
@@ -102,6 +102,8 @@
 import { useModuleI18n } from "@/i18n/composables";
 import { getProviderIcon, getProviderDescription } from "@/utils/providerUtils";
 
+const AVAILABLE_PROVIDER_TABS = ['agent_runner', 'speech_to_text', 'text_to_speech', 'embedding', 'rerank'];
+
 export default {
   name: "AddNewProvider",
   props: {
@@ -113,6 +115,10 @@ export default {
       type: Object,
       default: () => ({}),
     },
+    currentProviderType: {
+      type: String,
+      default: 'agent_runner'
+    }
   },
   emits: ["update:show", "select-template"],
   setup() {
@@ -121,7 +127,7 @@ export default {
   },
   data() {
     return {
-      activeProviderTab: "chat_completion",
+      activeProviderTab: "agent_runner",
     };
   },
   computed: {
@@ -134,7 +140,25 @@ export default {
       },
     },
   },
+  watch: {
+    show(value) {
+      if (value) {
+        this.syncActiveProviderTab();
+      }
+    },
+    currentProviderType() {
+      if (this.showDialog) {
+        this.syncActiveProviderTab();
+      }
+    }
+  },
   methods: {
+    syncActiveProviderTab() {
+      this.activeProviderTab = AVAILABLE_PROVIDER_TABS.includes(this.currentProviderType)
+        ? this.currentProviderType
+        : 'agent_runner';
+    },
+
     closeDialog() {
       this.showDialog = false;
     },

dashboard/src/components/provider/ProviderModelsPanel.vue

@@ -80,6 +80,13 @@
                   >
                     mdi-eye-outline
                   </v-icon>
+                  <v-icon
+                    v-if="supportsAudioInput(entry.metadata)"
+                    size="14"
+                    color="grey"
+                  >
+                    mdi-music-note-outline
+                  </v-icon>
                   <v-icon
                     v-if="supportsToolCall(entry.metadata)"
                     size="14"
@@ -183,6 +190,13 @@
                   >
                     mdi-eye-outline
                   </v-icon>
+                  <v-icon
+                    v-if="supportsAudioInput(entry.metadata)"
+                    size="14"
+                    color="grey"
+                  >
+                    mdi-music-note-outline
+                  </v-icon>
                   <v-icon
                     v-if="supportsToolCall(entry.metadata)"
                     size="14"
@@ -259,6 +273,10 @@ const props = defineProps({
     type: Function,
     required: true,
   },
+  supportsAudioInput: {
+    type: Function,
+    required: true
+  },
   supportsToolCall: {
     type: Function,
     required: true,

dashboard/src/components/provider/ProviderSourcesPanel.vue

@@ -251,7 +251,7 @@ const emitDeleteSource = (source) => emit("delete-provider-source", source);
 }
 
 .provider-sources-mobile {
-  padding: 14px 18px 0;
+  padding: 16px;
 }
 
 .provider-sources-list-wrap {

dashboard/src/components/shared/AstrBotConfig.vue

@@ -593,10 +593,21 @@ function hasVisibleItemsAfter(items, currentIndex) {
     padding: 8px 0;
   }
 
-  .property-info,
+  .property-info {
+    padding: 4px 4px;
+  }
+
+  .property-info :deep(.v-list-item) {
+    padding-inline: 0;
+  }
+
   .type-indicator,
   .config-input {
     padding: 4px;
   }
+
+  .config-divider {
+    display: none;
+  }
 }
 </style>

dashboard/src/components/shared/AstrBotConfigV4.vue

@@ -603,14 +603,17 @@ function getSpecialSubtype(value) {
     padding: 8px 0;
   }
 
-  .property-info,
   .type-indicator {
     padding: 4px 8px;
   }
 
   .config-input {
-    padding-left: 24px;
-    padding-right: 24px;
+    padding-left: 16px;
+    padding-right: 16px;
+  }
+
+  .config-divider {
+    display: none;
   }
 }
 </style>

dashboard/src/components/shared/ConfigItemRenderer.vue

@@ -108,7 +108,7 @@
         itemMeta?.options &&
         itemMeta?.render_type === 'checkbox'
       "
-      class="d-flex flex-wrap gap-20"
+      class="checkbox-group d-flex flex-wrap gap-20"
     >
       <v-checkbox
         v-for="(option, optionIndex) in itemMeta.options"
@@ -116,8 +116,9 @@
         :model-value="modelValue"
         :label="getLabel(itemMeta, optionIndex, option)"
         :value="option"
-        class="mr-2"
+        class="config-checkbox"
         color="primary"
+        density="compact"
         hide-details
         @update:model-value="emitUpdate"
       />
@@ -400,8 +401,29 @@ function getSpecialSubtype(value) {
   background-color: rgba(0, 0, 0, 0.5);
 }
 
-.gap-20 {
-  gap: 20px;
+.checkbox-group {
+  gap: 6px 12px;
+}
+
+.config-checkbox {
+  margin-right: 0;
+}
+
+.config-checkbox :deep(.v-selection-control) {
+  min-height: 28px;
+}
+
+.config-checkbox :deep(.v-selection-control__wrapper) {
+  width: 18px;
+  height: 18px;
+}
+
+.config-checkbox :deep(.v-icon) {
+  font-size: 18px;
+}
+
+.config-checkbox :deep(.v-label) {
+  font-size: 0.9rem;
 }
 
 ::v-deep(.v-field__input) {

dashboard/src/composables/useProviderSources.ts

@@ -363,6 +363,11 @@ export function useProviderSources(options: UseProviderSourcesOptions) {
     return inputs.includes("image");
   }
 
+  function supportsAudioInput(meta: any) {
+    const inputs = meta?.modalities?.input || [];
+    return inputs.includes("audio");
+  }
+
   function supportsToolCall(meta: any) {
     return Boolean(meta?.tool_call);
   }
@@ -650,12 +655,15 @@ export function useProviderSources(options: UseProviderSourcesOptions) {
     let modalities: string[];
 
     if (!metadata) {
-      modalities = ["text", "image", "tool_use"];
+      modalities = ["text", "image", "audio", "tool_use"];
     } else {
       modalities = ["text"];
       if (supportsImageInput(metadata)) {
         modalities.push("image");
       }
+      if (supportsAudioInput(metadata)) {
+        modalities.push("audio");
+      }
       if (supportsToolCall(metadata)) {
         modalities.push("tool_use");
       }
@@ -817,6 +825,7 @@ export function useProviderSources(options: UseProviderSourcesOptions) {
     getSourceDisplayName,
     getModelMetadata,
     supportsImageInput,
+    supportsAudioInput,
     supportsToolCall,
     supportsReasoning,
     formatContextLimit,

dashboard/src/composables/useRecording.ts

@@ -1,78 +1,96 @@
-import { ref } from "vue";
-import axios from "@/utils/request";
+import { ref } from 'vue';
+import axios from '@/utils/request';
 
 export function useRecording() {
-  const isRecording = ref(false);
-  const audioChunks = ref<Blob[]>([]);
-  const mediaRecorder = ref<MediaRecorder | null>(null);
+    const isRecording = ref(false);
+    const audioChunks = ref<Blob[]>([]);
+    const mediaRecorder = ref<MediaRecorder | null>(null);
 
-  async function startRecording(onStart?: (label: string) => void) {
-    try {
-      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
-      mediaRecorder.value = new MediaRecorder(stream);
-
-      mediaRecorder.value.ondataavailable = (event) => {
-        audioChunks.value.push(event.data);
-      };
-
-      mediaRecorder.value.start();
-      isRecording.value = true;
-
-      if (onStart) {
-        onStart("录音中...");
-      }
-    } catch (error) {
-      console.error("Failed to start recording:", error);
+    function getRecordingMimeType(): string {
+        const chunkType = audioChunks.value.find(chunk => chunk.type)?.type;
+        return chunkType || mediaRecorder.value?.mimeType || 'audio/webm';
     }
-  }
 
-  async function stopRecording(
-    onStop?: (label: string) => void,
-  ): Promise<string> {
-    return new Promise((resolve, reject) => {
-      if (!mediaRecorder.value) {
-        reject("No media recorder");
-        return;
-      }
-
-      isRecording.value = false;
-      if (onStop) {
-        onStop("聊天输入框");
-      }
-
-      mediaRecorder.value.stop();
-      mediaRecorder.value.onstop = async () => {
-        const audioBlob = new Blob(audioChunks.value, { type: "audio/wav" });
-        audioChunks.value = [];
-
-        mediaRecorder.value?.stream
-          .getTracks()
-          .forEach((track) => track.stop());
-
-        const formData = new FormData();
-        formData.append("file", audioBlob);
+    function getRecordingFilename(mimeType: string): string {
+        const extensionMap: Record<string, string> = {
+            'audio/webm': 'webm',
+            'audio/webm;codecs=opus': 'webm',
+            'audio/ogg': 'ogg',
+            'audio/ogg;codecs=opus': 'ogg',
+            'audio/mp4': 'm4a',
+            'audio/mpeg': 'mp3',
+            'audio/wav': 'wav'
+        };
+        const normalizedMimeType = mimeType.toLowerCase();
+        const extension = extensionMap[normalizedMimeType] || normalizedMimeType.split('/')[1]?.split(';')[0] || 'webm';
+        return `${crypto.randomUUID()}.${extension}`;
+    }
 
+    async function startRecording(onStart?: (label: string) => void) {
         try {
-          const response = await axios.post("/api/chat/post_file", formData, {
-            headers: {
-              "Content-Type": "multipart/form-data",
-            },
-          });
-
-          const audio = response.data.data.filename;
-          console.info("Audio uploaded:", audio);
-          resolve(audio);
-        } catch (err) {
-          console.error("Error uploading audio:", err);
-          reject(err);
+            const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+            mediaRecorder.value = new MediaRecorder(stream);
+            
+            mediaRecorder.value.ondataavailable = (event) => {
+                audioChunks.value.push(event.data);
+            };
+            
+            mediaRecorder.value.start();
+            isRecording.value = true;
+            
+            if (onStart) {
+                onStart('录音中...');
+            }
+        } catch (error) {
+            console.error('Failed to start recording:', error);
         }
-      };
-    });
-  }
+    }
 
-  return {
-    isRecording,
-    startRecording,
-    stopRecording,
-  };
+    async function stopRecording(onStop?: (label: string) => void): Promise<string> {
+        return new Promise((resolve, reject) => {
+            if (!mediaRecorder.value) {
+                reject('No media recorder');
+                return;
+            }
+
+            isRecording.value = false;
+            if (onStop) {
+                onStop('聊天输入框');
+            }
+
+            mediaRecorder.value.stop();
+            mediaRecorder.value.onstop = async () => {
+                const mimeType = getRecordingMimeType();
+                const audioBlob = new Blob(audioChunks.value, { type: mimeType });
+                const filename = getRecordingFilename(mimeType);
+                audioChunks.value = [];
+
+                mediaRecorder.value?.stream.getTracks().forEach(track => track.stop());
+
+                const formData = new FormData();
+                formData.append('file', audioBlob, filename);
+
+                try {
+                    const response = await axios.post('/api/chat/post_file', formData, {
+                        headers: {
+                            'Content-Type': 'multipart/form-data'
+                        }
+                    });
+
+                    const attachmentId = response.data.data.attachment_id;
+                    console.log('Audio uploaded:', attachmentId);
+                    resolve(attachmentId);
+                } catch (err) {
+                    console.error('Error uploading audio:', err);
+                    reject(err);
+                }
+            };
+        });
+    }
+
+    return {
+        isRecording,
+        startRecording,
+        stopRecording
+    };
 }

dashboard/src/i18n/locales/en-US/features/config-metadata.json

@@ -1087,7 +1087,11 @@
       },
       "rerank_api_base": {
         "description": "Rerank Model API Base URL",
-        "hint": "AstrBot appends /v1/rerank to the request URL."
+        "hint": "The full request URL is formed by combining the Base URL and a path suffix (defaults to /v1/rerank)."
+      },
+      "rerank_api_suffix": {
+        "description": "API URL path suffix",
+        "hint": "Path appended to base_url, e.g. /v1/rerank. Leave empty to disable auto-append."
       },
       "rerank_api_key": {
         "description": "API Key",
@@ -1104,6 +1108,24 @@
         "description": "Custom rerank task description",
         "hint": "Only effective for qwen3-rerank models. Recommended to write in English."
       },
+      "nvidia_rerank_api_base": {
+          "description": "API Base URL"
+      },
+      "nvidia_rerank_api_key": {
+          "description": "API Key"
+      },
+      "nvidia_rerank_model": {
+          "description": "Rerank Model Name",
+          "hint": "Please refer to the NVIDIA Docs for the model name."
+      },
+      "nvidia_rerank_model_endpoint": {
+          "description": "Custom Model Endpoint",
+          "hint": "Custom URL suffix endpoint, defaults to /reranking."
+      },
+      "nvidia_rerank_truncate": {
+          "description": "Text Truncation Strategy",
+          "hint": "Whether to truncate the input to fit the model's maximum context length when the input text is too long."
+      },
       "launch_model_if_not_running": {
         "description": "Auto-start model if not running",
         "hint": "If the model is not running in Xinference, attempt to start it automatically. Recommended to disable in production."
@@ -1114,6 +1136,7 @@
         "labels": [
           "Text",
           "Image",
+          "Audio",
           "Tool use"
         ]
       },
@@ -1606,4 +1629,4 @@
     "helpMiddle": "or",
     "helpSuffix": "."
   }
-}
+}

dashboard/src/i18n/locales/ru-RU/features/config-metadata.json

@@ -1088,7 +1088,11 @@
             },
             "rerank_api_base": {
                 "description": "Base URL API модели Rerank",
-                "hint": "AstrBot добавляет /v1/rerank к URL запроса."
+                "hint": "Полный URL запроса формируется путём добавления суффикса к Base URL (по умолчанию /v1/rerank)."
+            },
+            "rerank_api_suffix": {
+                "description": "Суффикс пути API",
+                "hint": "Суффикс пути, добавляемый к base_url, например /v1/rerank. Оставьте пустым, чтобы не добавлять."
             },
             "rerank_api_key": {
                 "description": "API Key",
@@ -1105,6 +1109,24 @@
                 "description": "Описание задачи для Rerank",
                 "hint": "Эффективно только для моделей qwen3-rerank. Рекомендуется писать на английском."
             },
+            "nvidia_rerank_api_base": {
+                "description": "Базовый URL API"
+            },
+            "nvidia_rerank_api_key": {
+                "description": "API-ключ"
+            },
+            "nvidia_rerank_model": {
+                "description": "Название модели Rerank",
+                "hint": "Укажите название модели в соответствии с документацией NVIDIA."
+            },
+            "nvidia_rerank_model_endpoint": {
+                "description": "Пользовательский endpoint модели",
+                "hint": "Пользовательский суффикс URL endpoint, по умолчанию /reranking."
+            },
+            "nvidia_rerank_truncate": {
+                "description": "Стратегия усечения текста",
+                "hint": "Определяет, следует ли усекать входной текст, если он слишком длинный и не помещается в максимальную длину контекста модели."
+            },
             "launch_model_if_not_running": {
                 "description": "Автозапуск модели",
                 "hint": "Если модель не запущена в Xinference, попытаться запустить её автоматически. Рекомендуется отключать в продакшене."
@@ -1115,6 +1137,7 @@
                 "labels": [
                     "Текст",
                     "Изображение",
+                    "Аудио",
                     "Инструменты"
                 ]
             },

dashboard/src/i18n/locales/zh-CN/features/config-metadata.json

@@ -1085,7 +1085,11 @@
       },
       "rerank_api_base": {
         "description": "重排序模型 API Base URL",
-        "hint": "AstrBot 会在请求时在末尾加上 /v1/rerank。"
+        "hint": "最终请求路径由 Base URL 和路径后缀拼接而成（默认为 /v1/rerank）。"
+      },
+      "rerank_api_suffix": {
+        "description": "API URL 路径后缀",
+        "hint": "追加到 base_url 后的路径后缀，如 /v1/rerank。留空则不追加。"
       },
       "rerank_api_key": {
         "description": "API Key",
@@ -1102,6 +1106,24 @@
         "description": "自定义排序任务类型说明",
         "hint": "仅在使用 qwen3-rerank 模型时生效。建议使用英文撰写。"
       },
+      "nvidia_rerank_api_base": {
+          "description": "API Base URL"
+      },
+      "nvidia_rerank_api_key": {
+          "description": "API Key"
+      },
+      "nvidia_rerank_model": {
+          "description": "重排序模型名称",
+          "hint": "请参照NVIDIA Docs中模型名称填写。"
+      },
+      "nvidia_rerank_model_endpoint": {
+          "description": "自定义模型端点",
+          "hint": "自定义URL末尾端点，默认为 /reranking"
+      },
+      "nvidia_rerank_truncate": {
+          "description": "文本截断策略",
+          "hint": "当输入文本过长时，是否截断输入以适应模型的最大上下文长度。"
+      },
       "launch_model_if_not_running": {
         "description": "模型未运行时自动启动",
         "hint": "如果模型当前未在 Xinference 服务中运行，是否尝试自动启动它。在生产环境中建议关闭。"
@@ -1112,6 +1134,7 @@
         "labels": [
           "文本",
           "图像",
+          "音频",
           "工具使用"
         ]
       },

dashboard/src/utils/platformUtils.js

@@ -63,6 +63,9 @@ export function getPlatformIcon(name) {
   } else if (name === "line") {
     return new URL("@/assets/images/platform_logos/line.png", import.meta.url)
       .href;
+  } else if (name === "matrix") {
+    return new URL("@/assets/images/platform_logos/matrix.svg", import.meta.url)
+      .href;
   }
 }
 
@@ -92,6 +95,7 @@ export function getTutorialLink(platformType) {
     satori: "https://docs.astrbot.app/platform/satori/llonebot.html",
     misskey: "https://docs.astrbot.app/platform/misskey.html",
     line: "https://docs.astrbot.app/platform/line.html",
+    matrix: "https://docs.astrbot.app/platform/matrix.html",
   };
   return tutorialMap[platformType] || "https://docs.astrbot.app";
 }
@@ -134,6 +138,7 @@ export function getPlatformDisplayName(platformId) {
     vocechat: "vocechat (VoceChat)",
     satori: "satori (Satori)",
     line: "line (LINE)",
+    matrix: "matrix (Matrix)",
   };
   return displayNameMap[platformId] || platformId;
 }

dashboard/src/utils/providerUtils.js

@@ -63,6 +63,14 @@ export function getProviderIcon(type) {
       "https://cdn.jsdelivr.net/npm/@lobehub/icons-static-svg@latest/icons/openrouter.svg",
     tokenpony: "https://tokenpony.cn/tokenpony-web/logo.png",
     compshare: "https://compshare.cn/favicon.ico",
+    xinference:
+      "https://cdn.jsdelivr.net/npm/@lobehub/icons-static-svg@latest/icons/xinference-color.svg",
+    bailian:
+      "https://cdn.jsdelivr.net/npm/@lobehub/icons-static-svg@latest/icons/bailian-color.svg",
+    volcengine:
+      "https://cdn.jsdelivr.net/npm/@lobehub/icons-static-svg@latest/icons/volcengine-color.svg",
+    longcat:
+      "https://cdn.jsdelivr.net/npm/@lobehub/icons-static-svg@latest/icons/longcat-color.svg",
   };
   return icons[type] || "";
 }

dashboard/src/views/ProviderPage.vue

@@ -141,6 +141,7 @@
                         :loading-models="loadingModels"
                         :is-source-modified="isSourceModified"
                         :supports-image-input="supportsImageInput"
+                        :supports-audio-input="supportsAudioInput"
                         :supports-tool-call="supportsToolCall"
                         :supports-reasoning="supportsReasoning"
                         :format-context-limit="formatContextLimit"
@@ -470,6 +471,7 @@ const {
   resolveSourceIcon,
   getSourceDisplayName,
   supportsImageInput,
+  supportsAudioInput,
   supportsToolCall,
   supportsReasoning,
   formatContextLimit,

docs/zh/platform/matrix.md

@@ -3,6 +3,27 @@
 > [!TIP]
 > 该平台适配器由社区([stevessr](https://github.com/stevessr)) 维护。如果您觉得有帮助，请支持开发者，给该仓库点一个 Star。❤️
 
+## 部署 Matrix 服务器
+
+Matrix 是一个 IM 协议，有着丰富的服务端实现。
+
+请在 [Matrix Server](https://matrix.org/ecosystem/servers/)查看可用的服务端。
+
+
+
+## 支持的基本消息类型
+
+
+| 消息类型     | 是否支持接收 | 是否支持发送 | 备注                                           |
+| ------------ | ------------ | ------------ | ---------------------------------------------- |
+| 文本         | 是           | 是           |                                                |
+| 图片*        | 是           | 是           |                                                |
+| 语音*        | 是           | 是           |                                                |
+| 视频*        | 是           | 是           |                                                |
+| 文件*        | 是           | 是           |                                                |
+| 投票         | 是           | 否           |                                                |
+*: 会持久化到本地，插件会按配置清理，在发送前会进行上传操作，超过服务器允许大小的上传将会失败
+
 ## 安装 astrbot_plugin_matrix_adapter 插件
 
 进入 AstrBot WebUI 的插件市场，搜索 `astrbot_plugin_matrix_adapter`，点击安装。
@@ -13,7 +34,11 @@
 
 ## 配置
 
-请参考该仓库的 [README.md](https://github.com/stevessr/astrbot_plugin_matrix_adapter?tab=readme-ov-file#astrbot-matrix-adapter-%E6%8F%92%E4%BB%B6) 进行配置。
+- **`matrix_homeserver` (必填)`**: 你的 matrix 服务器实例的完整URL地址,支持域名委托自动探测。例如官方实例`https://matrix.org`
+- **`matrix_user_id`**: 你的 matrix 完整用户名。如 `@username:homeserver.com`
+- **`matrix_auth_method` (必填)** : 你的登陆方式，可选`password`,`token`,`oauth2`,`qr`推荐使用`password`或`oauth2/qr`模式 (oauth2/qr 模式下请确保用于认证/扫码的设备回调可以访问到 astrbot 配置的公开地址)
+
+更多请参考该仓库的 [README.md](https://github.com/stevessr/astrbot_plugin_matrix_adapter?tab=readme-ov-file#astrbot-matrix-adapter-%E6%8F%92%E4%BB%B6) 进行配置。
 
 ## 问题提交
 

tests/test_openai_source.py

@@ -2,6 +2,7 @@ from types import SimpleNamespace
 
 import pytest
 from openai.types.chat.chat_completion import ChatCompletion
+from openai.types.chat.chat_completion_chunk import ChatCompletionChunk
 from PIL import Image as PILImage
 
 from astrbot.core.exceptions import EmptyModelOutputError
@@ -1197,6 +1198,93 @@ async def test_parse_openai_completion_raises_empty_model_output_error():
         await provider.terminate()
 
 
+@pytest.mark.asyncio
+async def test_query_stream_extracts_usage_from_empty_choices_chunk(monkeypatch):
+    provider = _make_provider()
+    try:
+        chunks = [
+            ChatCompletionChunk.model_validate(
+                {
+                    "id": "chatcmpl-stream",
+                    "object": "chat.completion.chunk",
+                    "created": 0,
+                    "model": "gpt-4o-mini",
+                    "choices": [
+                        {
+                            "index": 0,
+                            "delta": {
+                                "role": "assistant",
+                                "content": "ok",
+                            },
+                            "finish_reason": None,
+                        }
+                    ],
+                }
+            ),
+            ChatCompletionChunk.model_validate(
+                {
+                    "id": "chatcmpl-stream",
+                    "object": "chat.completion.chunk",
+                    "created": 0,
+                    "model": "gpt-4o-mini",
+                    "choices": [
+                        {
+                            "index": 0,
+                            "delta": {},
+                            "finish_reason": "stop",
+                        }
+                    ],
+                }
+            ),
+            ChatCompletionChunk.model_validate(
+                {
+                    "id": "chatcmpl-stream",
+                    "object": "chat.completion.chunk",
+                    "created": 0,
+                    "model": "gpt-4o-mini",
+                    "choices": [],
+                    "usage": {
+                        "prompt_tokens": 2550,
+                        "completion_tokens": 125,
+                        "total_tokens": 2675,
+                        "prompt_tokens_details": {
+                            "cached_tokens": 2488,
+                        },
+                    },
+                }
+            ),
+        ]
+
+        async def fake_stream():
+            for chunk in chunks:
+                yield chunk
+
+        async def fake_create(**kwargs):
+            return fake_stream()
+
+        monkeypatch.setattr(provider.client.chat.completions, "create", fake_create)
+
+        responses = [
+            response
+            async for response in provider._query_stream(
+                payloads={
+                    "model": "gpt-4o-mini",
+                    "messages": [{"role": "user", "content": "hello"}],
+                },
+                tools=None,
+            )
+        ]
+
+        final_response = responses[-1]
+        assert final_response.completion_text == "ok"
+        assert final_response.usage is not None
+        assert final_response.usage.input_other == 62
+        assert final_response.usage.input_cached == 2488
+        assert final_response.usage.output == 125
+    finally:
+        await provider.terminate()
+
+
 @pytest.mark.asyncio
 async def test_query_filters_empty_assistant_message_without_tool_calls(monkeypatch):
     """Test that empty assistant messages without tool_calls are filtered out."""

tests/unit/test_astr_main_agent.py

@@ -674,7 +674,7 @@ class TestModalitiesFix:
 
         module._modalities_fix(mock_provider, req)
 
-        assert "[图片]" in req.prompt
+        assert "[Image]" in req.prompt
         assert req.image_urls == []
 
     def test_modalities_fix_tool_not_supported(self, mock_provider):