From 480dffb51bab4b691198cb2b15409cd91e77bf62 Mon Sep 17 00:00:00 2001 From: Soulter <905617992@qq.com> Date: Fri, 10 Jan 2025 21:48:15 +0800 Subject: [PATCH 1/5] =?UTF-8?q?feat:=20=E5=88=9D=E6=AD=A5=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0=20webchat=20=E9=A1=B5=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- astrbot/core/__init__.py | 3 + astrbot/core/db/__init__.py | 26 +- astrbot/core/db/po.py | 13 +- astrbot/core/db/sqlite.py | 66 +++- astrbot/core/db/sqlite_init.sql | 8 + .../core/pipeline/whitelist_check/stage.py | 4 + astrbot/core/platform/manager.py | 5 +- .../sources/webchat/webchat_adapter.py | 88 ++++++ .../platform/sources/webchat/webchat_event.py | 31 ++ astrbot/core/utils/io.py | 16 +- astrbot/dashboard/routes/__init__.py | 4 +- astrbot/dashboard/routes/chat.py | 129 ++++++++ astrbot/dashboard/server.py | 8 +- dashboard/package-lock.json | 17 ++ dashboard/package.json | 1 + .../full/vertical-sidebar/sidebarItem.ts | 5 + dashboard/src/router/MainRoutes.ts | 5 + dashboard/src/views/ChatPage.vue | 285 ++++++++++++++++++ 18 files changed, 704 insertions(+), 10 deletions(-) create mode 100644 astrbot/core/platform/sources/webchat/webchat_adapter.py create mode 100644 astrbot/core/platform/sources/webchat/webchat_event.py create mode 100644 astrbot/dashboard/routes/chat.py create mode 100644 dashboard/src/views/ChatPage.vue diff --git a/astrbot/core/__init__.py b/astrbot/core/__init__.py index f11165f03..0ef8e039d 100644 --- a/astrbot/core/__init__.py +++ b/astrbot/core/__init__.py @@ -1,4 +1,5 @@ import os +import asyncio from .log import LogManager, LogBroker from astrbot.core.utils.t2i.renderer import HtmlRenderer from astrbot.core.utils.shared_preferences import SharedPreferences @@ -19,4 +20,6 @@ if os.environ.get('TESTING', ""): db_helper = SQLiteDatabase(DB_PATH) sp = SharedPreferences() # 简单的偏好设置存储 pip_installer = PipInstaller(astrbot_config.get('pip_install_arg', '')) +web_chat_queue = asyncio.Queue() +web_chat_back_queue = asyncio.Queue() WEBUI_SK = "Advanced_System_for_Text_Response_and_Bot_Operations_Tool" diff --git a/astrbot/core/db/__init__.py b/astrbot/core/db/__init__.py index d993cd6bc..424c1539f 100644 --- a/astrbot/core/db/__init__.py +++ b/astrbot/core/db/__init__.py @@ -1,7 +1,7 @@ import abc from dataclasses import dataclass from typing import List -from astrbot.core.db.po import Stats, LLMHistory, ATRIVision +from astrbot.core.db.po import Stats, LLMHistory, ATRIVision, WebChatConversation @dataclass class BaseDatabase(abc.ABC): @@ -76,4 +76,28 @@ class BaseDatabase(abc.ABC): @abc.abstractmethod def get_atri_vision_data_by_path_or_id(self, url_or_path: str, id: str) -> ATRIVision: '''通过 url 或 path 获取 ATRI 视觉数据''' + raise NotImplementedError + + @abc.abstractmethod + def get_webchat_conversation_by_user_id(self, user_id: str, cid: str) -> WebChatConversation: + '''通过 user_id 和 cid 获取 WebChatConversation''' + raise NotImplementedError + + @abc.abstractmethod + def webchat_new_conversation(self, user_id: str, cid: str): + '''新建 WebChatConversation''' + raise NotImplementedError + + @abc.abstractmethod + def get_webchat_conversations(self, user_id: str) -> List[WebChatConversation]: + raise NotImplementedError + + @abc.abstractmethod + def update_webchat_conversation(self, user_id: str, cid: str, history: str): + '''更新 WebChatConversation''' + raise NotImplementedError + + @abc.abstractmethod + def delete_webchat_conversation(self, user_id: str, cid: str): + '''删除 WebChatConversation''' raise NotImplementedError \ No newline at end of file diff --git a/astrbot/core/db/po.py b/astrbot/core/db/po.py index 0e6b2e92c..72235f4e4 100644 --- a/astrbot/core/db/po.py +++ b/astrbot/core/db/po.py @@ -51,4 +51,15 @@ class ATRIVision(): platform_name: str session_id: str sender_nickname: str - timestamp: int = -1 \ No newline at end of file + timestamp: int = -1 + + + +@dataclass +class WebChatConversation(): + user_id: str + cid: str + history: str = "" + created_at: int = 0 + updated_at: int = 0 + \ No newline at end of file diff --git a/astrbot/core/db/sqlite.py b/astrbot/core/db/sqlite.py index 3daf54436..94a5a23e4 100644 --- a/astrbot/core/db/sqlite.py +++ b/astrbot/core/db/sqlite.py @@ -5,7 +5,8 @@ from astrbot.core.db.po import ( Platform, Stats, LLMHistory, - ATRIVision + ATRIVision, + WebChatConversation ) from . import BaseDatabase from typing import Tuple @@ -199,6 +200,69 @@ class SQLiteDatabase(BaseDatabase): c.close() return Stats(platform, [], []) + + + def get_webchat_conversation_by_user_id(self, user_id: str, cid: str) -> WebChatConversation: + try: + c = self.conn.cursor() + except sqlite3.ProgrammingError: + c = self._get_conn(self.db_path).cursor() + + c.execute( + ''' + SELECT * FROM webchat_conversation WHERE user_id = ? AND cid = ? + ''', (user_id, cid) + ) + + res = c.fetchone() + c.close() + return WebChatConversation(*res) + + def webchat_new_conversation(self, user_id: str, cid: str): + history = "[]" + updated_at = int(time.time()) + created_at = updated_at + self._exec_sql( + ''' + INSERT INTO webchat_conversation(user_id, cid, history, updated_at, created_at) VALUES (?, ?, ?, ?, ?) + ''', (user_id, cid, history, updated_at, created_at) + ) + + def get_webchat_conversations(self, user_id: str) -> Tuple: + try: + c = self.conn.cursor() + except sqlite3.ProgrammingError: + c = self._get_conn(self.db_path).cursor() + + c.execute( + ''' + SELECT cid, created_at, updated_at FROM webchat_conversation WHERE user_id = ? ORDER BY updated_at DESC + ''', (user_id,) + ) + + res = c.fetchall() + c.close() + conversations = [] + for row in res: + cid = row[0] + created_at = row[1] + updated_at = row[2] + conversations.append(WebChatConversation("", cid, '[]', created_at, updated_at)) + return conversations + + def update_webchat_conversation(self, user_id: str, cid: str, history: str): + self._exec_sql( + ''' + UPDATE webchat_conversation SET history = ? WHERE user_id = ? AND cid = ? + ''', (history, user_id, cid) + ) + + def delete_webchat_conversation(self, user_id: str, cid: str): + self._exec_sql( + ''' + DELETE FROM webchat_conversation WHERE user_id = ? AND cid = ? + ''', (user_id, cid) + ) def insert_atri_vision_data(self, vision: ATRIVision): diff --git a/astrbot/core/db/sqlite_init.sql b/astrbot/core/db/sqlite_init.sql index 2cd7e77b3..e58f8bad9 100644 --- a/astrbot/core/db/sqlite_init.sql +++ b/astrbot/core/db/sqlite_init.sql @@ -35,4 +35,12 @@ CREATE TABLE IF NOT EXISTS atri_vision( session_id VARCHAR(32), sender_nickname VARCHAR(32), timestamp INTEGER +); + +CREATE TABLE IF NOT EXISTS webchat_conversation( + user_id TEXT, + cid TEXT, + history TEXT, + created_at INTEGER, + updated_at INTEGER ); \ No newline at end of file diff --git a/astrbot/core/pipeline/whitelist_check/stage.py b/astrbot/core/pipeline/whitelist_check/stage.py index 6a4e7097e..1a4952fd0 100644 --- a/astrbot/core/pipeline/whitelist_check/stage.py +++ b/astrbot/core/pipeline/whitelist_check/stage.py @@ -20,6 +20,10 @@ class WhitelistCheckStage(Stage): if not self.enable_whitelist_check: return + if event.get_platform_name() == 'webchat': + # WebChat 豁免 + return + # 检查是否在白名单 if self.wl_ignore_admin_on_group: if event.role == 'admin' and event.get_message_type() == MessageType.GROUP_MESSAGE: diff --git a/astrbot/core/platform/manager.py b/astrbot/core/platform/manager.py index e9e361e29..0308d5746 100644 --- a/astrbot/core/platform/manager.py +++ b/astrbot/core/platform/manager.py @@ -4,7 +4,7 @@ from typing import List from asyncio import Queue from .register import platform_cls_map from astrbot.core import logger - +from .sources.webchat.webchat_adapter import WebChatAdapter class PlatformManager(): def __init__(self, config: AstrBotConfig, event_queue: Queue): @@ -25,6 +25,7 @@ class PlatformManager(): from .sources.qqofficial.qqofficial_platform_adapter import QQOfficialPlatformAdapter # noqa: F401 case "vchat": from .sources.vchat.vchat_platform_adapter import VChatPlatformAdapter # noqa: F401 + async def initialize(self): for platform in self.platforms_config: @@ -37,6 +38,8 @@ class PlatformManager(): logger.info(f"尝试实例化 {platform['type']}({platform['id']}) 平台适配器 ...") inst = cls_type(platform, self.settings, self.event_queue) self.platform_insts.append(inst) + + self.platform_insts.append(WebChatAdapter({}, self.settings, self.event_queue)) def get_insts(self): return self.platform_insts \ No newline at end of file diff --git a/astrbot/core/platform/sources/webchat/webchat_adapter.py b/astrbot/core/platform/sources/webchat/webchat_adapter.py new file mode 100644 index 000000000..d78bfeadf --- /dev/null +++ b/astrbot/core/platform/sources/webchat/webchat_adapter.py @@ -0,0 +1,88 @@ +import time +import asyncio +import uuid +from typing import Awaitable, Any +from astrbot.api.platform import Platform, AstrBotMessage, MessageMember, MessageType, PlatformMetadata +from astrbot.api.event import MessageChain +from astrbot.api.message_components import * # noqa: F403 +from astrbot.api import logger +from astrbot.core import web_chat_queue, web_chat_back_queue +from .webchat_event import WebChatMessageEvent +from astrbot.core.platform.astr_message_event import MessageSesion +from ...register import register_platform_adapter + +class QueueListener: + def __init__(self, queue: asyncio.Queue, callback: callable) -> None: + self.queue = queue + self.callback = callback + + async def run(self): + while True: + data = await self.queue.get() + await self.callback(data) + +@register_platform_adapter("webchat", "webchat") +class WebChatAdapter(Platform): + def __init__(self, platform_config: dict, platform_settings: dict, event_queue: asyncio.Queue) -> None: + super().__init__(event_queue) + + self.config = platform_config + self.settings = platform_settings + self.unique_session = platform_settings['unique_session'] + + self.metadata = PlatformMetadata( + "webchat", + "webchat", + ) + + async def send_by_session(self, session: MessageSesion, message_chain: MessageChain): + plain = "" + for comp in message_chain.chain: + if isinstance(comp, Plain): + plain += comp.text + web_chat_back_queue.put_nowait(plain) + + await super().send_by_session(session, message_chain) + + async def convert_message(self, data: tuple) -> AstrBotMessage: + username, cid, message = data + + + abm = AstrBotMessage() + abm.self_id = "webchat" + abm.tag = "webchat" + abm.sender = MessageMember(username, username) + + abm.type = MessageType.FRIEND_MESSAGE + + abm.session_id = f"webchat!{username}!{cid}" + + abm.message_id = str(uuid.uuid4()) + abm.message = [Plain(message)] + message_str = message + abm.timestamp = int(time.time()) + abm.message_str = message_str + abm.raw_message = data + return abm + + def run(self) -> Awaitable[Any]: + async def callback(data: tuple): + abm = await self.convert_message(data) + await self.handle_msg(abm) + + bot = QueueListener(web_chat_queue, callback) + return bot.run() + + def meta(self) -> PlatformMetadata: + return self.metadata + + async def handle_msg(self, message: AstrBotMessage): + + message_event = WebChatMessageEvent( + message_str=message.message_str, + message_obj=message, + platform_meta=self.meta(), + session_id=message.session_id + ) + + self.commit_event(message_event) \ No newline at end of file diff --git a/astrbot/core/platform/sources/webchat/webchat_event.py b/astrbot/core/platform/sources/webchat/webchat_event.py new file mode 100644 index 000000000..c988724be --- /dev/null +++ b/astrbot/core/platform/sources/webchat/webchat_event.py @@ -0,0 +1,31 @@ +import os +import uuid +from astrbot.api.event import AstrMessageEvent, MessageChain +from astrbot.api.message_components import Plain, Image +from astrbot.core.utils.io import file_to_base64, download_image_by_url +from astrbot.core import web_chat_back_queue + +class WebChatMessageEvent(AstrMessageEvent): + def __init__(self, message_str, message_obj, platform_meta, session_id): + super().__init__(message_str, message_obj, platform_meta, session_id) + self.imgs_dir = "data/webchat/imgs" + os.makedirs(self.imgs_dir, exist_ok=True) + + async def send(self, message: MessageChain): + for comp in message.chain: + if isinstance(comp, Plain): + await web_chat_back_queue.put(comp.text) + elif isinstance(comp, Image): + # save image to local + filename = str(uuid.uuid4()) + ".jpg" + path = os.path.join(self.imgs_dir, filename) + if comp.file and comp.file.startswith("file:///"): + ph = comp.file[8:] + with open(path, "wb") as f: + with open(ph, "rb") as f2: + f.write(f2.read()) + elif comp.file and comp.file.startswith("http"): + await download_image_by_url(comp.file, path=path) + await web_chat_back_queue.put(f"[IMAGE]{filename}") + await web_chat_back_queue.put(None) + await super().send(message) \ No newline at end of file diff --git a/astrbot/core/utils/io.py b/astrbot/core/utils/io.py index 07cc0bd64..a20b3f27a 100644 --- a/astrbot/core/utils/io.py +++ b/astrbot/core/utils/io.py @@ -65,7 +65,7 @@ def save_temp_img(img: Image) -> str: f.write(img) return p -async def download_image_by_url(url: str, post: bool = False, post_data: dict = None) -> str: +async def download_image_by_url(url: str, post: bool = False, post_data: dict = None, path = None) -> str: ''' 下载图片, 返回 path ''' @@ -73,10 +73,20 @@ async def download_image_by_url(url: str, post: bool = False, post_data: dict = async with aiohttp.ClientSession() as session: if post: async with session.post(url, json=post_data) as resp: - return save_temp_img(await resp.read()) + if not path: + return save_temp_img(await resp.read()) + else: + with open(path, "wb") as f: + f.write(await resp.read()) + return path else: async with session.get(url) as resp: - return save_temp_img(await resp.read()) + if not path: + return save_temp_img(await resp.read()) + else: + with open(path, "wb") as f: + f.write(await resp.read()) + return path except aiohttp.client_exceptions.ClientConnectorSSLError: # 关闭SSL验证 ssl_context = ssl.create_default_context() diff --git a/astrbot/dashboard/routes/__init__.py b/astrbot/dashboard/routes/__init__.py index 5149b4bae..b1dee8bed 100644 --- a/astrbot/dashboard/routes/__init__.py +++ b/astrbot/dashboard/routes/__init__.py @@ -5,6 +5,7 @@ from .update import UpdateRoute from .stat import StatRoute from .log import LogRoute from .static_file import StaticFileRoute +from .chat import ChatRoute __all__ = [ @@ -14,6 +15,7 @@ __all__ = [ "UpdateRoute", "StatRoute", "LogRoute", - "StaticFileRoute" + "StaticFileRoute", + "ChatRoute", ] diff --git a/astrbot/dashboard/routes/chat.py b/astrbot/dashboard/routes/chat.py new file mode 100644 index 000000000..2e44db351 --- /dev/null +++ b/astrbot/dashboard/routes/chat.py @@ -0,0 +1,129 @@ +import uuid +import json +import os +from .route import Route, Response, RouteContext +from astrbot.core import web_chat_queue, web_chat_back_queue +from quart import request, Response as QuartResponse, g +from astrbot.core.db import BaseDatabase +import asyncio + +class ChatRoute(Route): + def __init__(self, context: RouteContext, db: BaseDatabase) -> None: + super().__init__(context) + self.routes = { + '/chat/send': ('POST', self.chat), + '/chat/new_conversation': ('GET', self.new_conversation), + '/chat/conversations': ('GET', self.get_conversations), + '/chat/get_conversation': ('GET', self.get_conversation), + '/chat/delete_conversation': ('GET', self.delete_conversation), + '/chat/get_file': ('GET', self.get_file) + } + self.db = db + self.register_routes() + self.imgs_dir = "data/webchat/imgs" + + async def get_file(self): + filename = request.args.get('filename') + if not filename: + return Response().error("Missing key: filename").__dict__ + + try: + with open(os.path.join(self.imgs_dir, filename), "rb") as f: + return QuartResponse(f.read(), mimetype="image/jpeg") + except FileNotFoundError: + return Response().error("File not found").__dict__ + + async def chat(self): + username = g.get('username', 'guest') + + post_data = await request.json + if 'message' not in post_data: + return Response().error("Missing key: message").__dict__ + + if 'conversation_id' not in post_data: + return Response().error("Missing key: conversation_id").__dict__ + + message = post_data['message'] + conversation_id = post_data['conversation_id'] + if not message: + return Response().error("Message is empty").__dict__ + if not conversation_id: + return Response().error("conversation_id is empty").__dict__ + + await web_chat_queue.put((username, conversation_id, message)) + + async def stream(): + ret = [] + while True: + result = await web_chat_back_queue.get() + + if result is None: + break + + ret.append(result) + + yield result + '\n' + + await asyncio.sleep(0.5) + + conversation = self.db.get_webchat_conversation_by_user_id(username, conversation_id) + try: + history = json.loads(conversation['history']) + except BaseException: + history = [] + history.append({ + 'type': 'user', + 'message': message + }) + # history.append({ + # 'type': 'bot', + # 'message': ret + # }) + for r in ret: + history.append({ + 'type': 'bot', + 'message': r + }) + self.db.update_webchat_conversation(username, conversation_id, history=json.dumps(history)) + + return QuartResponse( + stream(), + mimetype="text/event-stream", + headers={ + "Content-Type": "text/event-stream", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "Access-Control-Allow-Origin": "*" # 如果是跨域请求 + } + ) + + async def delete_conversation(self): + username = g.get('username', 'guest') + conversation_id = request.args.get('conversation_id') + if not conversation_id: + return Response().error("Missing key: conversation_id").__dict__ + + self.db.delete_webchat_conversation(username, conversation_id) + return Response().ok().__dict__ + + async def new_conversation(self): + username = g.get('username', 'guest') + conversation_id = str(uuid.uuid4()) + self.db.webchat_new_conversation(username, conversation_id) + return Response().ok(data={ + 'conversation_id': conversation_id + }).__dict__ + + async def get_conversations(self): + username = g.get('username', 'guest') + conversations = self.db.get_webchat_conversations(username) + return Response().ok(data=conversations).__dict__ + + async def get_conversation(self): + username = g.get('username', 'guest') + conversation_id = request.args.get('conversation_id') + if not conversation_id: + return Response().error("Missing key: conversation_id").__dict__ + + conversation = self.db.get_webchat_conversation_by_user_id(username, conversation_id) + return Response().ok(data=conversation).__dict__ \ No newline at end of file diff --git a/astrbot/dashboard/server.py b/astrbot/dashboard/server.py index 2b5f76067..97013b423 100644 --- a/astrbot/dashboard/server.py +++ b/astrbot/dashboard/server.py @@ -2,7 +2,7 @@ import logging import jwt import asyncio import os -from quart import Quart, request, jsonify +from quart import Quart, request, jsonify, g from quart.logging import default_handler from astrbot.core.core_lifecycle import AstrBotCoreLifecycle from .routes import * @@ -31,12 +31,15 @@ class AstrBotDashboard(): self.lr = LogRoute(self.context, core_lifecycle.log_broker) self.sfr = StaticFileRoute(self.context) self.ar = AuthRoute(self.context) + self.chat_route = ChatRoute(self.context, db) async def auth_middleware(self): if not request.path.startswith("/api"): return if request.path == "/api/auth/login": return + if request.path == "/api/chat/get_file": + return # claim jwt token = request.headers.get("Authorization") if not token: @@ -46,7 +49,8 @@ class AstrBotDashboard(): if token.startswith("Bearer "): token = token[7:] try: - jwt.decode(token, WEBUI_SK, algorithms=["HS256"]) + payload = jwt.decode(token, WEBUI_SK, algorithms=["HS256"]) + g.username = payload["username"] except jwt.ExpiredSignatureError: r = jsonify(Response().error("Token 过期").__dict__) r.status_code = 401 diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index 3b33a7e28..7a1478cae 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -18,6 +18,7 @@ "date-fns": "2.30.0", "js-md5": "^0.8.3", "lodash": "4.17.21", + "marked": "^15.0.6", "pinia": "2.1.6", "remixicon": "3.5.0", "vee-validate": "4.11.3", @@ -3702,6 +3703,17 @@ "markdown-it": "bin/markdown-it.js" } }, + "node_modules/marked": { + "version": "15.0.6", + "resolved": "https://registry.npmjs.org/marked/-/marked-15.0.6.tgz", + "integrity": "sha512-Y07CUOE+HQXbVDCGl3LXggqJDbXDP2pArc2C1N1RRMN0ONiShoSsIInMd5Gsxupe7fKLpgimTV+HOJ9r7bA+pg==", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 18" + } + }, "node_modules/mdurl": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", @@ -8460,6 +8472,11 @@ "uc.micro": "^1.0.5" } }, + "marked": { + "version": "15.0.6", + "resolved": "https://registry.npmjs.org/marked/-/marked-15.0.6.tgz", + "integrity": "sha512-Y07CUOE+HQXbVDCGl3LXggqJDbXDP2pArc2C1N1RRMN0ONiShoSsIInMd5Gsxupe7fKLpgimTV+HOJ9r7bA+pg==" + }, "mdurl": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/mdurl/-/mdurl-1.0.1.tgz", diff --git a/dashboard/package.json b/dashboard/package.json index 54fc44daa..2888d4415 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -23,6 +23,7 @@ "date-fns": "2.30.0", "js-md5": "^0.8.3", "lodash": "4.17.21", + "marked": "^15.0.6", "pinia": "2.1.6", "remixicon": "3.5.0", "vee-validate": "4.11.3", diff --git a/dashboard/src/layouts/full/vertical-sidebar/sidebarItem.ts b/dashboard/src/layouts/full/vertical-sidebar/sidebarItem.ts index 7d4766bd3..4761ae2a8 100644 --- a/dashboard/src/layouts/full/vertical-sidebar/sidebarItem.ts +++ b/dashboard/src/layouts/full/vertical-sidebar/sidebarItem.ts @@ -30,6 +30,11 @@ const sidebarItem: menu[] = [ icon: 'mdi-puzzle', to: '/extension' }, + { + title: '聊天', + icon: 'mdi-chat', + to: '/chat' + }, { title: '控制台', icon: 'mdi-console', diff --git a/dashboard/src/router/MainRoutes.ts b/dashboard/src/router/MainRoutes.ts index 32c82696a..766dd3df5 100644 --- a/dashboard/src/router/MainRoutes.ts +++ b/dashboard/src/router/MainRoutes.ts @@ -36,6 +36,11 @@ const MainRoutes = { name: 'Project ATRI', path: '/project-atri', component: () => import('@/views/ATRIProject.vue') + }, + { + name: 'Chat', + path: '/chat', + component: () => import('@/views/ChatPage.vue') } ] }; diff --git a/dashboard/src/views/ChatPage.vue b/dashboard/src/views/ChatPage.vue new file mode 100644 index 000000000..0a60e4c23 --- /dev/null +++ b/dashboard/src/views/ChatPage.vue @@ -0,0 +1,285 @@ + + + + + + \ No newline at end of file From d4f32673ab4c034a3080f56cbc5eeed78fe2944e Mon Sep 17 00:00:00 2001 From: Soulter <905617992@qq.com> Date: Fri, 10 Jan 2025 22:08:43 +0800 Subject: [PATCH 2/5] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=8C=81=E4=B9=85?= =?UTF-8?q?=E5=8C=96=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- astrbot/dashboard/routes/chat.py | 5 +++-- dashboard/src/views/ChatPage.vue | 9 ++++++--- stt.py | 33 ++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 stt.py diff --git a/astrbot/dashboard/routes/chat.py b/astrbot/dashboard/routes/chat.py index 2e44db351..735673976 100644 --- a/astrbot/dashboard/routes/chat.py +++ b/astrbot/dashboard/routes/chat.py @@ -68,8 +68,9 @@ class ChatRoute(Route): conversation = self.db.get_webchat_conversation_by_user_id(username, conversation_id) try: - history = json.loads(conversation['history']) - except BaseException: + history = json.loads(conversation.history) + except BaseException as e: + print(e) history = [] history.append({ 'type': 'user', diff --git a/dashboard/src/views/ChatPage.vue b/dashboard/src/views/ChatPage.vue index 0a60e4c23..fcf7f92a8 100644 --- a/dashboard/src/views/ChatPage.vue +++ b/dashboard/src/views/ChatPage.vue @@ -14,12 +14,13 @@ marked.setOptions({
-
+
- 创建对话 - 删除对话 + + 创建对话 + @@ -36,6 +37,8 @@ marked.setOptions({ + + 删除此对话
diff --git a/stt.py b/stt.py new file mode 100644 index 000000000..9eee69b8a --- /dev/null +++ b/stt.py @@ -0,0 +1,33 @@ +import torch +from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline +from datasets import load_dataset + + +device = "cuda:0" if torch.cuda.is_available() else "cpu" +torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32 + +model_id = "openai/whisper-large-v3" + +model = AutoModelForSpeechSeq2Seq.from_pretrained( + model_id, torch_dtype=torch_dtype, low_cpu_mem_usage=True +) +model.to(device) + +processor = AutoProcessor.from_pretrained(model_id) + +pipe = pipeline( + "automatic-speech-recognition", + model=model, + tokenizer=processor.tokenizer, + feature_extractor=processor.feature_extractor, + chunk_length_s=30, + batch_size=16, # batch size for inference - set based on your device + torch_dtype=torch_dtype, + device=device, +) + +dataset = load_dataset("distil-whisper/librispeech_long", "clean", split="validation") +sample = dataset[0]["audio"] + +result = pipe(sample) +print(result["text"]) From 5fc63aeaf17aad302b8281011a95230c950171e6 Mon Sep 17 00:00:00 2001 From: Soulter <905617992@qq.com> Date: Fri, 10 Jan 2025 22:45:14 +0800 Subject: [PATCH 3/5] perf: ui --- dashboard/src/views/ChatPage.vue | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/dashboard/src/views/ChatPage.vue b/dashboard/src/views/ChatPage.vue index fcf7f92a8..8362d15f4 100644 --- a/dashboard/src/views/ChatPage.vue +++ b/dashboard/src/views/ChatPage.vue @@ -43,9 +43,17 @@ marked.setOptions({
- Hello, I'm - AstrBot ⭐ + style="height: 100%; display: flex; justify-content: center; align-items: center; flex-direction: column;"> +
+ Hello, I'm + AstrBot ⭐ +
+
+ 输入 + /help + 获取帮助 😊 +
+
Date: Sat, 11 Jan 2025 00:06:19 +0800 Subject: [PATCH 4/5] =?UTF-8?q?perf:=20webchat=E6=94=AF=E6=8C=81=E4=BC=A0?= =?UTF-8?q?=E5=9B=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sources/webchat/webchat_adapter.py | 22 ++- .../core/provider/sources/openai_source.py | 2 + astrbot/dashboard/routes/chat.py | 43 ++++-- dashboard/src/views/ChatPage.vue | 146 +++++++++++++----- 4 files changed, 162 insertions(+), 51 deletions(-) diff --git a/astrbot/core/platform/sources/webchat/webchat_adapter.py b/astrbot/core/platform/sources/webchat/webchat_adapter.py index d78bfeadf..87e9d5f7b 100644 --- a/astrbot/core/platform/sources/webchat/webchat_adapter.py +++ b/astrbot/core/platform/sources/webchat/webchat_adapter.py @@ -1,10 +1,11 @@ import time import asyncio import uuid +import os from typing import Awaitable, Any from astrbot.api.platform import Platform, AstrBotMessage, MessageMember, MessageType, PlatformMetadata from astrbot.api.event import MessageChain -from astrbot.api.message_components import * # noqa: F403 +from astrbot.api.message_components import Plain, Image # noqa: F403 from astrbot.api import logger from astrbot.core import web_chat_queue, web_chat_back_queue from .webchat_event import WebChatMessageEvent @@ -29,6 +30,7 @@ class WebChatAdapter(Platform): self.config = platform_config self.settings = platform_settings self.unique_session = platform_settings['unique_session'] + self.imgs_dir = "data/webchat/imgs" self.metadata = PlatformMetadata( "webchat", @@ -45,7 +47,7 @@ class WebChatAdapter(Platform): await super().send_by_session(session, message_chain) async def convert_message(self, data: tuple) -> AstrBotMessage: - username, cid, message = data + username, cid, payload = data abm = AstrBotMessage() @@ -58,8 +60,20 @@ class WebChatAdapter(Platform): abm.session_id = f"webchat!{username}!{cid}" abm.message_id = str(uuid.uuid4()) - abm.message = [Plain(message)] - message_str = message + abm.message = [] + + if payload['message']: + abm.message.append(Plain(payload['message'])) + if payload['image_url']: + if isinstance(payload['image_url'], list): + for img in payload['image_url']: + abm.message.append(Image.fromFileSystem(os.path.join(self.imgs_dir, img))) + else: + abm.message.append(Image.fromFileSystem(os.path.join(self.imgs_dir, payload['image_url']))) + + logger.debug(f"WebChatAdapter: {abm.message}") + + message_str = payload['message'] abm.timestamp = int(time.time()) abm.message_str = message_str abm.raw_message = data diff --git a/astrbot/core/provider/sources/openai_source.py b/astrbot/core/provider/sources/openai_source.py index b0c9f0a58..7e601e958 100644 --- a/astrbot/core/provider/sources/openai_source.py +++ b/astrbot/core/provider/sources/openai_source.py @@ -209,6 +209,8 @@ class ProviderOpenAIOfficial(Provider): image_path = await download_image_by_url(image_url) image_data = await self.encode_image_bs64(image_path) else: + if image_url.startswith("file:///"): + image_url = image_url.replace("file:///", "") image_data = await self.encode_image_bs64(image_url) user_content["content"].append({"type": "image_url", "image_url": {"url": image_data}}) return user_content diff --git a/astrbot/dashboard/routes/chat.py b/astrbot/dashboard/routes/chat.py index 735673976..cef7e563f 100644 --- a/astrbot/dashboard/routes/chat.py +++ b/astrbot/dashboard/routes/chat.py @@ -16,7 +16,8 @@ class ChatRoute(Route): '/chat/conversations': ('GET', self.get_conversations), '/chat/get_conversation': ('GET', self.get_conversation), '/chat/delete_conversation': ('GET', self.delete_conversation), - '/chat/get_file': ('GET', self.get_file) + '/chat/get_file': ('GET', self.get_file), + '/chat/post_image': ('POST', self.post_image) } self.db = db self.register_routes() @@ -32,25 +33,43 @@ class ChatRoute(Route): return QuartResponse(f.read(), mimetype="image/jpeg") except FileNotFoundError: return Response().error("File not found").__dict__ + + async def post_image(self): + post_data = await request.files + if 'file' not in post_data: + return Response().error("Missing key: file").__dict__ + + file = post_data['file'] + filename = str(uuid.uuid4()) + ".jpg" + path = os.path.join(self.imgs_dir, filename) + await file.save(path) + + return Response().ok(data={ + 'filename': filename + }).__dict__ async def chat(self): username = g.get('username', 'guest') post_data = await request.json - if 'message' not in post_data: - return Response().error("Missing key: message").__dict__ + if 'message' not in post_data and 'image_url' not in post_data: + return Response().error("Missing key: message or image_url").__dict__ if 'conversation_id' not in post_data: return Response().error("Missing key: conversation_id").__dict__ message = post_data['message'] conversation_id = post_data['conversation_id'] - if not message: - return Response().error("Message is empty").__dict__ + image_url = post_data.get('image_url') + if not message and not image_url: + return Response().error("Message and image_url are empty").__dict__ if not conversation_id: return Response().error("conversation_id is empty").__dict__ - await web_chat_queue.put((username, conversation_id, message)) + await web_chat_queue.put((username, conversation_id, { + 'message': message, + 'image_url': image_url # list + })) async def stream(): ret = [] @@ -72,14 +91,14 @@ class ChatRoute(Route): except BaseException as e: print(e) history = [] - history.append({ + + new_his = { 'type': 'user', 'message': message - }) - # history.append({ - # 'type': 'bot', - # 'message': ret - # }) + } + if image_url: + new_his['image_url'] = image_url + history.append(new_his) for r in ret: history.append({ 'type': 'bot', diff --git a/dashboard/src/views/ChatPage.vue b/dashboard/src/views/ChatPage.vue index 8362d15f4..f4cb9b0e5 100644 --- a/dashboard/src/views/ChatPage.vue +++ b/dashboard/src/views/ChatPage.vue @@ -16,32 +16,27 @@ marked.setOptions({
- + 创建对话 - - - - - 新对话 - {{ formatDate(item.updated_at) }} + + 创建对话 - + + + + 新对话 + {{ formatDate(item.updated_at) }} + + - 删除此对话 + 删除此对话
-
+
@@ -49,11 +44,12 @@ marked.setOptions({ AstrBot ⭐
- 输入 - /help - 获取帮助 😊 + 输入 + /help + 获取帮助 😊
- +
{{ msg.message }} +
+
+ +
+
@@ -72,19 +75,38 @@ marked.setOptions({
-
-
- -