opensource/bisheng
3
0
Fork 0
mirror of https://github.com/dataelement/bisheng.git synced 2026-10-28 03:40:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge zhyd

Browse Source
This commit is contained in:
yaojin
2024-09-26 16:38:09 +08:00
parent 58429c79bf a807490ee5
commit b7d0a720d9
44 changed files with 2251 additions and 726 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.drone.yml
+68 -42
View File
@@ -15,37 +15,13 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行
https_proxy:
from_secret: PROXY
commands:
- git clone https://github.com/dataelement/bisheng.git .
- git clone https://github.com/dataelement/bisheng-enterprise.git .
- git checkout $DRONE_COMMIT
- name: package # 流水线名称
pull: if-not-exists
image: python:3.10-slim # 定义创建容器的Docker镜像
volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置
- name: bisheng-cache
path: /app/build # 将应用打包好的Jar和执行脚本挂载出来
environment:
RELEASE_VERSION: 99.99.99
NEXUS_USER:
from_secret: NEXUS_USER
NEXUS_PASSWORD:
from_secret: NEXUS_PASSWORD
REPO:
from_secret: PY_NEXUS
commands: # 定义在Docker容器中执行的shell命令
- pip install Cython
- pip install wheel
- pip install twine
- pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
- cd ./src/bisheng-langchain
- python setup.py bdist_wheel
- twine upload --verbose -u $NEXUS_USER -p $NEXUS_PASSWORD --repository-url $REPO dist/*.whl
- name: set poetry
pull: if-not-exists
image: golang
environment:
RELEASE_VERSION: 99.99.99
NEXUS_PUBLIC:
from_secret: NEXUS_PUBLIC
NEXUS_PUBLIC_PASSWORD:
@@ -63,9 +39,7 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行
- echo $REPO
- REPO2=$(echo $REPO | sed 's/http:\\/\\///g')
- sed '/apt-get/ s|$| '"$PROXY"'|' Dockerfile
- sed -i 's/^bisheng_langchain.*/bisheng_langchain = "'$RELEASE_VERSION'"/g' pyproject.toml
- sed -i '6i\RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple' Dockerfile
- sed -i '23i\RUN poetry source add --priority=supplemental foo http://'$NEXUS_PUBLIC':'$NEXUS_PUBLIC_PASSWORD'@'$REPO2'simple' Dockerfile
- sed -i '23i\RUN poetry source add --priority=primary qh https://pypi.tuna.tsinghua.edu.cn/simple' Dockerfile
- cat Dockerfile
@@ -73,6 +47,12 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行
pull: if-not-exists
image: docker:24.0.6
privileged: true
environment:
http_proxy:
from_secret: PROXY
https_proxy:
from_secret: PROXY
no_proxy: 192.168.106.8
volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置
- name: apt-cache
path: /var/cache/apt/archives # 将应用打包好的Jar和执行脚本挂载出来
@@ -171,20 +151,66 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行
when: # 成功
status:
- success
trigger:
branch:
- release
# - feat/*
event:
- push
ref: # git 分支
- refs/tags/*
event: # 事 件
- tag
- name: publish_nexus
pull: if-not-exists
image: docker:24.0.6
environment:
version: ${DRONE_TAG}
USERNAME:
from_secret: upload_name
docker_username:
from_secret: DOCKER_USER
docker_password:
from_secret: DOCKER_PASSWORD
commands:
- apk add --no-cache curl
- apk add --no-cache tar
- apk add --no-cache wget
- docker login -u $docker_username -p $docker_password 192.168.106.8:6082
- docker pull 192.168.106.8:6082/project/bisheng-frontend:$version
- docker pull 192.168.106.8:6082/project/bisheng-backend:$version
- docker image tag 192.168.106.8:6082/project/bisheng-frontend:$version dataelement/bisheng-frontend:$version
- docker image tag 192.168.106.8:6082/project/bisheng-backend:$version dataelement/bisheng-backend:$version
- docker save -o bisheng-frontend-$version.tar dataelement/bisheng-frontend:$version
- docker save -o bisheng-backend-$version.tar dataelement/bisheng-backend:$version
- docker rmi dataelement/frontend:$version -f
- docker rmi dataelement/backend:$version -f
- mkdir -p deployment/docker-images
- cp -r docker/* deployment/
- mv bisheng-fronteend-$version.tar bisheng-backend-$version.tar deployment/docker-images
- mv deployment bisheng-$version
- tar czf bisheng-$version.tar.gz bisheng-$version
- curl -k --progress-bar -o /dev/null --user $USERNAME --upload-file bisheng-$version.tar.gz http://192.168.106.244:5081/repository/raw-hosted/product/bisheng-ent/bisheng-$version.tar.gz
volumes:
- name: dockersock
path: /var/run
when:
status:
- success
ref: # git 分支
- refs/tags/*
event: # 事 件
- tag
volumes:
- name: bisheng-cache
host:
path: /opt/drone/data/bisheng/
- name: apt-cache
host:
path: /opt/drone/data/bisheng/apt/
- name: socket
host:
path: /var/run/docker.sock
- name: bisheng-cache
host:
path: /opt/drone/data/bisheng/
- name: apt-cache
host:
path: /opt/drone/data/bisheng/apt/
- name: maven_local_repo
host:
path: /data/drone-data/m2
- name: data
host:
path: /data/drone-data/package/
- name: dockersock
host:
path: /var/run/
.github/workflows/ci.yml
+34 -82
View File
@@ -1,4 +1,4 @@
name: CI
name: release
on:
push:
@@ -7,53 +7,29 @@ on:
- "v*"
env:
DOCKERHUB_REPO: dataelement/
DOCKERHUB_REPO: project/
PY_NEXUS: 110.16.193.170:50083
DOCKER_NEXUS: 110.16.193.170:50080
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build_bisheng_langchain:
build:
runs-on: ubuntu-latest
#if: startsWith(github.event.ref, 'refs/tags')
steps:
# deploy
- name: checkout
uses: actions/checkout@v2
- name: Get version
id: get_version
run: |
echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
- name: Set Environment Variable
run: echo "RELEASE_VERSION=${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_ENV
uses: actions/checkout@v2
# 构建 bisheng_langchain
- name: Set python version 3.8
# 构建 bisheng-langchain
- name: Set python version 3.10
uses: actions/setup-python@v1
with:
python-version: 3.8
- name: Build PyPi bisheng-langchain and push
id: pypi_build_bisheng_langchain
run: |
pip install Cython
pip install wheel
pip install twine
cd ./src/bisheng-langchain
python setup.py bdist_wheel
set +e
twine upload dist/* -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASSWORD }} --repository pypi
set -e
build_bisheng:
needs: build_bisheng_langchain
runs-on: ubuntu-latest
# if: startsWith(github.event.ref, 'refs/tags')
steps:
- name: checkout
uses: actions/checkout@v2
python-version: "3.10"
- name: Get version
id: get_version
@@ -63,45 +39,39 @@ jobs:
- name: Set Environment Variable
run: echo "RELEASE_VERSION=${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_ENV
# 登录 docker hub
- name: Login to DockerHub
uses: docker/login-action@v1
# 发布到 私有仓库
- name: set insecure registry
run: |
echo "{ \"insecure-registries\": [\"http://${{ env.DOCKER_NEXUS }}\"] }" | sudo tee /etc/docker/daemon.json
sudo service docker restart
# - name: Set up QEMU
# uses: docker/setup-qemu-action@v1
- name: Login Nexus Container Registry
uses: docker/login-action@v2
with:
# GitHub Repo => Settings => Secrets 增加 docker hub 登录密钥信息
# DOCKERHUB_USERNAME 是 docker hub 账号名.
# DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建.
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
registry: http://${{ env.DOCKER_NEXUS }}/
username: ${{ secrets.NEXUS_USER }}
password: ${{ secrets.NEXUS_PASSWORD }}
# - name: Login to DockerHub
# uses: docker/login-action@v1
# with:
# registry: https://cr.dataelem.com/
# username: ${{ secrets.CR_DOCKERHUB_USERNAME }}
# password: ${{ secrets.CR_DOCKERHUB_TOKEN }}
# 构建 backend 并推送到 Docker hub
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: install poetry
# 替换poetry编译为私有服务
- name: replace self-host repo
uses: snok/install-poetry@v1
with:
installer-parallel: true
- name: build lock
run: |
run: |
cd ./src/backend
pip install bisheng_langchain==$RELEASE_VERSION
poetry lock
poetry source add --priority=supplemental foo http://${{ secrets.NEXUS_PUBLIC }}:${{ secrets.NEXUS_PUBLIC_PASSWORD }}@${{ env.PY_NEXUS }}/repository/pypi-group/simple
# 安装 nltk
pip install nltk
python -c "import nltk; nltk.download('punkt'); nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger'); nltk.download('averaged_perceptron_tagger_eng'); "
mv /home/runner/nltk_data ./
cd ../../
# 构建 backend 并推送到 Docker hub
- name: Build backend and push
id: docker_build_backend
uses: docker/build-push-action@v2
@@ -111,14 +81,12 @@ jobs:
# 是否 docker push
push: true
# docker build arg, 注入 APP_NAME/APP_VERSION
platforms: linux/amd64,linux/arm64
build-args: |
APP_NAME="bisheng-backend"
APP_VERSION=${{ steps.get_version.outputs.VERSION }}
# 生成两个 docker tag: ${APP_VERSION} 和 latest
tags: |
${{ env.DOCKERHUB_REPO }}bisheng-backend:latest
${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}
${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}
# 构建 Docker frontend 并推送到 Docker hub
- name: Build frontend and push
id: docker_build_frontend
@@ -129,26 +97,10 @@ jobs:
# 是否 docker push
push: true
# docker build arg, 注入 APP_NAME/APP_VERSION
platforms: linux/amd64,linux/arm64
build-args: |
APP_NAME="bisheng-frontend"
APP_VERSION=${{ steps.get_version.outputs.VERSION }}
# 生成两个 docker tag: ${APP_VERSION} 和 latest
tags: |
${{ env.DOCKERHUB_REPO }}bisheng-frontend:latest
${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}
- name: Process git message
id: process_message
run: |
value=$(echo "${{ github.event.head_commit.message }}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/%0A/g')
value=$(echo "${value}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\r/%0A/g')
echo "message=${value}" >> $GITHUB_ENV
shell: bash
${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}
- name: notify feishu
uses: fjogeleit/http-request-action@v1
with:
url: ${{ secrets.FEISHU_WEBHOOK }}
method: 'POST'
data: '{"msg_type":"post","content":{"post":{"zh_cn":{"title": "${{ steps.get_version.outputs.VERSION }}发布成功", "content": [[{"tag":"text","text":"发布功能："},{"tag":"text","text":"${{ env.message }}"}]]}}}}'
.github/workflows/release.yml.bak
-150
View File
@@ -1,150 +0,0 @@
# name: release
# on:
# push:
# # Sequence of patterns matched against refs/tags
# branches:
# - "release"
# env:
# DOCKERHUB_REPO: dataelement/
# PY_NEXUS: 110.16.193.170:50083
# DOCKER_NEXUS: 110.16.193.170:50080
# concurrency:
# group: ${{ github.workflow }}-${{ github.ref }}
# cancel-in-progress: true
# jobs:
# build:
# runs-on: ubuntu-latest
# #if: startsWith(github.event.ref, 'refs/tags')
# steps:
# # deploy
# - name: checkout
# uses: actions/checkout@v2
# - name: Process git message
# id: process_message
# run: |
# value=$(echo "${{ github.event.head_commit.message }}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/%0A/g' )
# echo "message=${value}" >> $GITHUB_ENV
# shell: bash
# - name: notify feishu
# uses: fjogeleit/http-request-action@v1
# with:
# url: ' https://open.feishu.cn/open-apis/bot/v2/hook/2cfe0d8d-647c-4408-9f39-c59134035c4b'
# method: 'POST'
# data: '{"msg_type":"post","content":{"post":{"zh_cn":{"title": "${{github.event.pusher.name}}提交代码，开始编译", "content": [[{"tag":"text","text":"发布功能："},{"tag":"text","text":"${{ env.message }}"}]]}}}}'
# - name: Set Environment Variable
# run: echo "RELEASE_VERSION=99.99.99" >> $GITHUB_ENV
# - name: Set python version 3.8
# uses: actions/setup-python@v1
# with:
# python-version: 3.8
# - name: Build PyPi bisheng-langchain and push
# id: pypi_build_bisheng_langchain
# run: |
# pip install Cython
# pip install wheel
# pip install twine
# cd ./src/bisheng-langchain
# python setup.py bdist_wheel
# repo="http://${{ env.PY_NEXUS }}/repository/pypi-hosted/"
# twine upload --verbose -u ${{ secrets.NEXUS_USER }} -p ${{ secrets.NEXUS_PASSWORD }} --repository-url $repo dist/*.whl
# cd ../../
# - name: Set up QEMU
# uses: docker/setup-qemu-action@v1
# - name: set up Docker Buildx
# uses: docker/setup-buildx-action@v3
# # 发布到 私有仓库
# - name: set insecure registry
# run: |
# echo "{ \"insecure-registries\": [\"http://${{ env.DOCKER_NEXUS }}\"] }" | sudo tee /etc/docker/daemon.json
# sudo service docker restart
# # - name: Set up QEMU
# # uses: docker/setup-qemu-action@v1
# - name: Login Nexus Container Registry
# uses: docker/login-action@v2
# with:
# registry: http://${{ env.DOCKER_NEXUS }}/
# username: ${{ secrets.NEXUS_USER }}
# password: ${{ secrets.NEXUS_PASSWORD }}
# # 替换poetry编译为私有服务
# - name: replace self-host repo
# uses: snok/install-poetry@v1
# with:
# installer-parallel: true
# - name: build lock
# run: |
# cd ./src/backend
# sed -i 's/^bisheng_langchain.*/bisheng_langchain = "'$RELEASE_VERSION'"/g' pyproject.toml
# poetry source add --priority=supplemental foo http://${{ secrets.NEXUS_PUBLIC }}:${{ secrets.NEXUS_PUBLIC_PASSWORD }}@${{ env.PY_NEXUS }}/repository/pypi-group/simple
# poetry lock
# cd ../../
# # 构建 backend 并推送到 Docker hub
# - name: Build backend and push
# id: docker_build_backend
# uses: docker/build-push-action@v2
# with:
# # backend 的context目录
# context: "./src/backend/"
# # 是否 docker push
# push: true
# platforms: linux/amd64,linux/arm64
# # docker build arg, 注入 APP_NAME/APP_VERSION
# build-args: |
# APP_NAME="bisheng-backend"
# APP_VERSION="release"
# # 生成两个 docker tag: ${APP_VERSION} 和 latest
# tags: |
# ${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-backend:release
# # 构建 Docker frontend 并推送到 Docker hub
# - name: Build frontend and push
# id: docker_build_frontend
# uses: docker/build-push-action@v2
# with:
# # frontend 的context目录
# context: "./src/frontend/"
# # 是否 docker push
# push: true
# platforms: linux/amd64,linux/arm64
# # docker build arg, 注入 APP_NAME/APP_VERSION
# build-args: |
# APP_NAME="bisheng-frontend"
# APP_VERSION="release"
# # 生成两个 docker tag: ${APP_VERSION} 和 latest
# tags: |
# ${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-frontend:release
# # deploy
# - name: notify feishu
# uses: fjogeleit/http-request-action@v1
# with:
# url: ' https://open.feishu.cn/open-apis/bot/v2/hook/2cfe0d8d-647c-4408-9f39-c59134035c4b'
# method: 'POST'
# data: '{"msg_type":"text","content":{"text":"release 编译成功, 准备部署"}}'
# - name: Deploy Stage
# uses: fjogeleit/http-request-action@v1
# with:
# timeout: 200000
# url: 'https://bisheng.dataelem.com/deploy/cgi-bin/deploy_script.py'
# method: 'GET'
# - name: notify feishu
# uses: fjogeleit/http-request-action@v1
# with:
# url: ' https://open.feishu.cn/open-apis/bot/v2/hook/2cfe0d8d-647c-4408-9f39-c59134035c4b'
# method: 'POST'
# data: '{"msg_type":"text","content":{"text":"release 部署成功"}}'
.github/workflows/test.yml
+100
View File
@@ -0,0 +1,100 @@
name: test_build
on:
push:
# Sequence of patterns matched against refs/tags
branches:
- "develop/*"
env:
DOCKERHUB_REPO: project/
PY_NEXUS: 110.16.193.170:50083
DOCKER_NEXUS: 110.16.193.170:50080
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build:
runs-on: ubuntu-latest
#if: startsWith(github.event.ref, 'refs/tags')
steps:
- name: checkout
uses: actions/checkout@v2
- name: Get version
id: get_version
run: |
echo ::set-output name=VERSION::${GITHUB_REF/refs\/heads\/develop\//}
echo $GITHUB_REF
echo $VERSION
# 构建 bisheng-langchain
- name: Set python version 3.8
uses: actions/setup-python@v1
with:
python-version: 3.8
# 发布到 私有仓库
- name: set insecure registry
run: |
echo "{ \"insecure-registries\": [\"http://${{ env.DOCKER_NEXUS }}\"] }" | sudo tee /etc/docker/daemon.json
sudo service docker restart
# - name: Set up QEMU
# uses: docker/setup-qemu-action@v1
- name: Login Nexus Container Registry
uses: docker/login-action@v2
with:
registry: http://${{ env.DOCKER_NEXUS }}/
username: ${{ secrets.NEXUS_USER }}
password: ${{ secrets.NEXUS_PASSWORD }}
# 替换poetry编译为私有服务
- name: replace self-host repo
uses: snok/install-poetry@v1
with:
installer-parallel: true
- name: build lock
run: |
cd ./src/backend
poetry source add --priority=supplemental foo http://${{ secrets.NEXUS_PUBLIC }}:${{ secrets.NEXUS_PUBLIC_PASSWORD }}@${{ env.PY_NEXUS }}/repository/pypi-group/simple
poetry lock
cd ../../
# 构建 backend 并推送到 Docker hub
- name: Build backend and push
id: docker_build_backend
uses: docker/build-push-action@v2
with:
# backend 的context目录
context: "./src/backend/"
# 是否 docker push
push: true
# docker build arg, 注入 APP_NAME/APP_VERSION
build-args: |
APP_NAME="bisheng-backend"
APP_VERSION=${{ steps.get_version.outputs.VERSION }}
# 生成两个 docker tag: ${APP_VERSION} 和 latest
tags: |
${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }}
# 构建 Docker frontend 并推送到 Docker hub
- name: Build frontend and push
id: docker_build_frontend
uses: docker/build-push-action@v2
with:
# frontend 的context目录
context: "./src/frontend/"
# 是否 docker push
push: true
# docker build arg, 注入 APP_NAME/APP_VERSION
build-args: |
APP_NAME="bisheng-frontend"
APP_VERSION=${{ steps.get_version.outputs.VERSION }}
# 生成两个 docker tag: ${APP_VERSION} 和 latest
tags: |
${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }}
src/backend/bisheng/api/services/dataset_service.py
+74
View File
@@ -0,0 +1,74 @@
from typing import Dict, List, Optional
from bisheng.api.services.base import BaseService
from bisheng.api.v1.schema.dataset_param import CreateDatasetParam
from bisheng.database.models.dataset import Dataset, DatasetCreate, DatasetDao, DatasetRead
from bisheng.database.models.user import UserDao
from bisheng.utils.minio_client import MinioClient
from fastapi import HTTPException
class DatasetService(BaseService):
@classmethod
def build_dataset_list(cls,
page: int,
limit: int,
keyword: Optional[str] = None) -> List[Dict]:
"""补全list 数据"""
dataset_list = DatasetDao.filter_dataset_by_ids(dataset_ids=[],
keyword=keyword,
page=page,
limit=limit)
count_filter = []
if keyword:
count_filter.append(Dataset.name.like('%{}%'.format(keyword)))
total_count = DatasetDao.get_count_by_filter(count_filter)
user_ids = [one.user_id for one in dataset_list]
user_list = UserDao.get_user_by_ids(user_ids)
user_dict = {one.user_id: one for one in user_list}
res = [DatasetRead.validate(one) for one in dataset_list]
for one in res:
one.user_name = user_dict[one.user_id].user_name
if one.object_name:
one.url = MinioClient().get_share_link(one.object_name)
return res, total_count
@classmethod
def create_dataset(cls, user_id: int, data: CreateDatasetParam):
"""创建数据集"""
dataset_insert = DatasetCreate.validate(data)
dataset_insert.user_id = user_id
isExist = DatasetDao.get_dataset_by_name(data.name)
if isExist:
raise ValueError('数据集名称已存在')
dataset = DatasetDao.insert(dataset_insert)
# 处理文件
object_name = f'/dataset/{dataset.id}/{dataset.name}'
if data.file_url:
# MinioClient().upload_minio()
dataset.object_name = object_name
if data.qa_list:
for qa in data.qa_list:
qa.dataset_id = dataset.id
# QADao.insert(qa)
dataset = DatasetDao.update(dataset)
return dataset
@classmethod
def delete_dataset(cls, dataset_id: int):
dataset = DatasetDao.get_dataset_by_id(dataset_id)
if not dataset:
raise HTTPException(status_code=404, detail='Dataset not found')
# 处理minio
object_name = dataset.object_name
if object_name:
minio_client = MinioClient()
minio_client.delete_minio(object_name)
DatasetDao.delete(dataset)
return True
src/backend/bisheng/api/services/finetune_file.py
+38 -7
View File
@@ -3,6 +3,7 @@ import uuid
from typing import Any, List
from bisheng.api.errcode.finetune import TrainFileNotExistError
from bisheng.api.v1.schema.base_schema import PageList
from bisheng.api.v1.schemas import UnifiedResponseModel, resp_200
from bisheng.database.models.preset_train import PresetTrain, PresetTrainDao
from bisheng.utils.logger import logger
@@ -15,7 +16,8 @@ class FinetuneFileService(BaseModel):
""" 训练任务 文件管理 """
@classmethod
def upload_file(cls, files: List[UploadFile], is_preset: bool, user: Any) -> UnifiedResponseModel:
def upload_file(cls, files: List[UploadFile], is_preset: bool,
user: Any) -> UnifiedResponseModel:
if len(files) == 0:
return TrainFileNotExistError.return_resp()
@@ -28,6 +30,25 @@ class FinetuneFileService(BaseModel):
PresetTrainDao.insert_batch(file_list)
return resp_200(data=file_list)
@classmethod
def upload_preset_file(cls, name: str, type: int, file_path: str,
user: Any) -> UnifiedResponseModel:
# 将训练文件上传到minio
file_root = cls.get_upload_file_root(False)
file_id = uuid.uuid4().hex
file_ext = os.path.basename(file_path).split('.')[-1]
object_name = f'{file_root}/{file_id}.{file_ext}'
MinioClient().upload_minio(object_name, file_path)
# 将预置数据存入数据库
file_info = PresetTrain(id=file_id,
name=name,
url=object_name,
type=type,
user_id=user.get('user_id'),
user_name=user.get('user_name'))
PresetTrainDao.insert_batch([file_info])
return resp_200(data=file_info)
@classmethod
def get_upload_file_root(cls, is_preset: bool) -> str:
if is_preset:
@@ -36,22 +57,32 @@ class FinetuneFileService(BaseModel):
return 'finetune/train_file/personal'
@classmethod
def upload_file_to_minio(cls, files: List[UploadFile], file_root: str, user: Any) -> List[PresetTrain]:
def upload_file_to_minio(cls, files: List[UploadFile], file_root: str,
user: Any) -> List[PresetTrain]:
minio_client = MinioClient()
ret = []
for file in files:
file_id = uuid.uuid4().hex
file_ext = os.path.basename(file.filename).split('.')[-1]
file_info = PresetTrain(id=file_id, name=file.filename,
file_info = PresetTrain(id=file_id,
name=file.filename,
url=f'{file_root}/{file_id}.{file_ext}',
user_id=user.get('user_id'), user_name=user.get('user_name'))
minio_client.upload_minio_file(file_info.url, file.file, length=file.size, content_type=file.content_type)
user_id=user.get('user_id'),
user_name=user.get('user_name'))
minio_client.upload_minio_file(file_info.url,
file.file,
length=file.size,
content_type=file.content_type)
ret.append(file_info)
return ret
@classmethod
def get_preset_file(cls) -> List[PresetTrain]:
return PresetTrainDao.find_all()
def get_preset_file(cls,
keyword: str = None,
page_size: int = None,
page_num: int = None) -> List[PresetTrain]:
list_res, total_count = PresetTrainDao.search_name(keyword, page_size, page_num)
return PageList(list=list_res, total=total_count)
@classmethod
def delete_preset_file(cls, file_id: uuid.UUID, user: Any) -> UnifiedResponseModel:
src/backend/bisheng/api/services/knowledge_imp.py
+302 -3
View File
@@ -5,6 +5,27 @@ from typing import Any, Dict, List
import requests
from loguru import logger
from pymilvus import Collection
from typing import Any, Dict, List, Optional
from uuid import uuid4
import requests
from bisheng.cache.utils import file_download
from bisheng.database.base import session_getter
from bisheng.database.models.knowledge import Knowledge, KnowledgeCreate, KnowledgeDao
from bisheng.database.models.knowledge_file import (KnowledgeFile, KnowledgeFileDao, QAKnoweldgeDao,
QAKnowledge, QAKnowledgeUpsert)
from bisheng.interface.embeddings.custom import FakeEmbedding
from bisheng.interface.importing.utils import import_by_type, import_vectorstore
from bisheng.interface.initialize.loading import instantiate_llm, instantiate_vectorstore
from bisheng.settings import settings
from bisheng.utils import minio_client
from bisheng.utils.embedding import decide_embeddings
from bisheng.utils.minio_client import MinioClient
from bisheng_langchain.document_loaders import ElemUnstructuredLoader
from bisheng_langchain.rag.extract_info import extract_title
from bisheng_langchain.text_splitter import ElemCharacterTextSplitter
from bisheng_langchain.vectorstores import ElasticKeywordsSearch
from fastapi import HTTPException
from langchain.embeddings.base import Embeddings
from langchain.schema.document import Document
from langchain.text_splitter import CharacterTextSplitter
@@ -31,6 +52,10 @@ from bisheng.utils.minio_client import MinioClient
from bisheng_langchain.document_loaders import ElemUnstructuredLoader
from bisheng_langchain.rag.extract_info import extract_title
from bisheng_langchain.text_splitter import ElemCharacterTextSplitter
from loguru import logger
from pymilvus import Collection
from sqlalchemy import delete, func, or_
from sqlmodel import select
filetype_load_map = {
'txt': TextLoader,
@@ -171,14 +196,19 @@ def decide_vectorstores(collection_name: str, vector_store: str,
# 无相关配置
return None
param = {'embedding': embedding}
if vector_store == 'ElasticKeywordsSearch':
param = {'index_name': collection_name, 'embedding': embedding}
param['index_name'] = collection_name
if isinstance(vector_config['ssl_verify'], str):
vector_config['ssl_verify'] = eval(vector_config['ssl_verify'])
else:
param = {'collection_name': collection_name, 'embedding': embedding}
elif vector_store == 'Milvus':
param['collection_name'] = collection_name
vector_config.pop('partition_suffix', '')
vector_config.pop('is_partition', '')
else:
# 适配其他的vector
collection_name = vector_config.pop('collection_or_index_name', '')
vector_config[collection_name] = collection_name
param.update(vector_config)
class_obj = import_vectorstore(vector_store)
@@ -531,3 +561,272 @@ def delete_es(index_name: str):
if esvectore_client:
res = esvectore_client.client.indices.delete(index=index_name, ignore=[400, 404])
logger.info(f'act=delete_es index={index_name} res={res}')
def QA_save_knowledge(db_knowledge: Knowledge, QA: QAKnowledge):
"""使用text 导入knowledge"""
questions = QA.questions
answer = json.loads(QA.answers)[0]
extra = {}
if QA.extra_meta:
extra = json.loads(QA.extra_meta) or {}
extra.update({'answer': answer})
docs = [Document(page_content=question, metadata=extra) for question in questions]
try:
embeddings = decide_embeddings(db_knowledge.model)
vectore_config_dict: dict = settings.get_knowledge().get('vectorstores')
if not vectore_config_dict:
raise Exception('向量数据库必须配置')
vectore_client_list = [
decide_vectorstores(db_knowledge.index_name or db_knowledge.collection_name, db,
embeddings) if db == 'ElasticKeywordsSearch' else
decide_vectorstores(db_knowledge.collection_name, db, embeddings)
for db in vectore_config_dict.keys()
]
logger.info('vector_init_conn_done col={} dbs={}', db_knowledge.collection_name,
vectore_config_dict.keys())
except Exception as e:
logger.exception(e)
try:
# 统一document
metadata = [{
'file_id': QA.id,
'knowledge_id': f'{db_knowledge.id}',
'page': doc.metadata.pop('page', 1),
'source': doc.metadata.pop('source', ''),
'bbox': doc.metadata.pop('bbox', ''),
'extra': json.dumps(doc.metadata, ensure_ascii=False)
} for doc in docs]
# 向量存储
for vectore_client in vectore_client_list:
vectore_client.add_texts(texts=[t.page_content for t in docs], metadatas=metadata)
QA.status = 1
with session_getter() as session:
session.add(QA)
session.commit()
session.refresh(QA)
except Exception as e:
logger.error(e)
setattr(QA, 'status', 0)
setattr(QA, 'remark', str(e)[:500])
with session_getter() as session:
session.add(QA)
session.commit()
session.refresh(QA)
return QA
def add_qa(db_knowledge: Knowledge, data: QAKnowledgeUpsert) -> QAKnowledge:
"""使用text 导入QAknowledge"""
if db_knowledge.type != 1:
raise Exception('knowledge type error')
try:
# 相似问统一插入
questions = data.questions
if questions:
if data.id:
qa_db = QAKnoweldgeDao.get_qa_knowledge_by_primary_id(data.id)
qa_db.questions = questions
qa_db.answers = data.answers
qa = QAKnoweldgeDao.update(qa_db)
# 需要先删除再插入
delete_vector_data(db_knowledge, [data.id])
else:
qa = QAKnoweldgeDao.insert_qa(data)
# 对question进行embedding，然后录入知识库
qa = QA_save_knowledge(db_knowledge, qa)
return qa
except Exception as e:
logger.exception(e)
raise e
def qa_status_change(qa_id: int, target_status: int):
"""QA 状态切换"""
qa_db = QAKnoweldgeDao.get_qa_knowledge_by_primary_id(qa_id)
if qa_db.status == target_status:
logger.info('qa status is same, skip')
return
db_knowledge = KnowledgeDao.query_by_id(qa_db.knowledge_id)
if target_status == 0:
delete_vector_data(db_knowledge, [qa_id])
qa_db.status = target_status
QAKnoweldgeDao.update(qa_db)
else:
qa_db.status = target_status
QAKnoweldgeDao.update(qa_db)
QA_save_knowledge(db_knowledge, qa_db)
return qa_db
def list_qa_by_knowledge_id(knowledge_id: int,
page_size: int = 10,
page_num: int = 1,
question: Optional[str] = None,
answer: Optional[str] = None,
keyword: Optional[str] = None,
status: Optional[int] = None) -> List[QAKnowledge]:
"""获取知识库下的所有qa"""
if not knowledge_id:
return []
count_sql = select(func.count(QAKnowledge.id)).where(QAKnowledge.knowledge_id == knowledge_id)
list_sql = select(QAKnowledge).where(QAKnowledge.knowledge_id == knowledge_id)
if status:
count_sql = count_sql.where(QAKnowledge.status == status)
list_sql = list_sql.where(QAKnowledge.status == status)
if question:
count_sql = count_sql.where(QAKnowledge.questions.like(f'%{question}%'))
list_sql = list_sql.where(QAKnowledge.questions.like(f'%{question}%'))
if answer:
count_sql = count_sql.where(QAKnowledge.answers.like(f'%{answer}%'))
list_sql = list_sql.where(QAKnowledge.answers.like(f'%{answer}%'))
if keyword:
count_sql = count_sql.where(
or_(QAKnowledge.questions.like(f'%{keyword}%'),
QAKnowledge.answers.like(f'%{keyword}%')))
list_sql = list_sql.where(
or_(QAKnowledge.answers.like(f'%{keyword}%'),
QAKnowledge.questions.like(f'%{keyword}%')))
list_sql = list_sql.order_by(QAKnowledge.update_time.desc()).limit(page_size).offset(
(page_num - 1) * page_size)
count = QAKnoweldgeDao.total_count(count_sql)
list_qa = QAKnoweldgeDao.query_by_condition(list_sql)
return list_qa, count
def delete_vector_data(knowledge: Knowledge, file_ids: List[int]):
"""删除向量数据, 想做一个通用的，可以对接langchain的vectorDB"""
# embeddings = FakeEmbedding()
# vectore_config_dict: dict = settings.get_knowledge().get('vectorstores')
# if not vectore_config_dict:
# raise Exception('向量数据库必须配置')
# elastic_index = knowledge.index_name or knowledge.collection_name
# vectore_client_list = [
# decide_vectorstores(elastic_index, db, embeddings) if db == 'ElasticKeywordsSearch' else
# decide_vectorstores(knowledge.collection_name, db, embeddings)
# for db in vectore_config_dict.keys()
# ]
# logger.info('vector_init_conn_done col={} dbs={}', knowledge.collection_name,
# vectore_config_dict.keys())
# for vectore_client in vectore_client_list:
# 查询vector primary key
embeddings = FakeEmbedding()
collection_name = knowledge.collection_name
# 处理vectordb
vectore_client = decide_vectorstores(collection_name, 'Milvus', embeddings)
try:
if isinstance(vectore_client.col, Collection):
pk = vectore_client.col.query(expr=f'file_id in {file_ids}',
output_fields=['pk'],
timeout=10)
else:
pk = []
except Exception:
# 重试一次
logger.error('timeout_except')
vectore_client.close_connection(vectore_client.alias)
vectore_client = decide_vectorstores(collection_name, 'Milvus', embeddings)
pk = vectore_client.col.query(expr=f'file_id in {file_ids}',
output_fields=['pk'],
timeout=10)
logger.info('query_milvus pk={}', pk)
if pk:
res = vectore_client.col.delete(f"pk in {[p['pk'] for p in pk]}", timeout=10)
logger.info(f'act=delete_vector file_id={file_ids} res={res}')
vectore_client.close_connection(vectore_client.alias)
# elastic
index_name = knowledge.index_name or collection_name
esvectore_client = decide_vectorstores(index_name, 'ElasticKeywordsSearch', embeddings)
if esvectore_client:
res = esvectore_client.client.delete_by_query(
index=index_name, body={'query': {
'terms': {
'metadata.file_id': file_ids
}
}})
logger.info(f'act=delete_es res={res}')
return True
def recommend_question(question: str, answer: str, number: int = 3) -> List[str]:
from langchain.chains.llm import LLMChain
from langchain_core.prompts.prompt import PromptTemplate
prompt = """- Role: 问题生成专家
- Background: 用户希望通过人工智能模型根据给定的问题和答案生成相似的问题，以便于扩展知识库或用于教育和测试目的。
- Profile: 你是一位专业的数据分析师和语言模型专家，擅长从现有数据中提取模式，并生成新的相关问题。
- Constrains: 确保生成的问题在语义上与原始问题相似，同时保持多样性，避免重复。
- Workflow:
1. 分析用户输入的问题和答案，提取关键词和主题。
2. 根据提取的关键词和主题创建相似问题。
3. 验证生成的问题与原始问题在语义上的相似性，并确保多样性。
- Examples:
问题："法国的首都是哪里？"
答案："巴黎"
生成3个相似问题：
- "法国的首都叫什么名字？"
- "哪个城市是法国的首都？"
- "巴黎是哪个国家的首都？"
请使用json 返回
{{"question": 生成的问题列表}}
以下是用户提供的问题和答案：
问题：{question}
答案：{answer}
你生成的{number}个相似问题：
"""
keyword_conf = settings.get_default_llm() or {}
if keyword_conf:
keyword_conf['temperature'] = 0.7
keyword_conf['cache'] = False
node_type = keyword_conf.pop('type', 'HostQwenChat') # 兼容旧配置
class_object = import_by_type(_type='llms', name=node_type)
llm = instantiate_llm(node_type, class_object, keyword_conf)
llm_chain = LLMChain(llm=llm, prompt=PromptTemplate.from_template(prompt))
gen_question = llm_chain.predict(question=question, answer=answer, number=number)
try:
code_ret = extract_code_blocks(gen_question)
if code_ret:
question_dict = json.loads(code_ret[0])
return question_dict['question']
else:
logger.info('md_code_extract_error {}', gen_question)
return []
except Exception as exc:
logger.error('recommend_question json.loads error:{}', gen_question)
raise ValueError(gen_question) from exc
else:
logger.info('llm_chain is None recommend_over')
return '默认大模型未配置'
def extract_code_blocks(markdown_code_block: str):
# 定义正则表达式模式
pattern = r'```\w*\s*(.*?)```'
# 使用 re.DOTALL 使 . 能够匹配换行符
matches = re.findall(pattern, markdown_code_block, re.DOTALL)
# 去除每段代码块两端的空白字符
return [match.strip() for match in matches]
src/backend/bisheng/api/services/user_service.py
+66 -11
View File
@@ -1,15 +1,13 @@
import json
import functools
import json
from base64 import b64decode
from typing import List
import rsa
from fastapi import HTTPException, Depends, Request
from fastapi_jwt_auth import AuthJWT
from bisheng.api.JWT import ACCESS_TOKEN_EXPIRE_TIME
from bisheng.api.errcode.base import UnAuthorizedError
from bisheng.api.errcode.user import UserLoginOfflineError, UserNameAlreadyExistError, UserNeedGroupAndRoleError
from bisheng.api.errcode.user import (UserLoginOfflineError, UserNameAlreadyExistError,
UserNeedGroupAndRoleError)
from bisheng.api.JWT import ACCESS_TOKEN_EXPIRE_TIME
from bisheng.api.utils import md5_hash
from bisheng.api.v1.schemas import CreateUserReq
from bisheng.cache.redis import redis_client
@@ -22,7 +20,9 @@ from bisheng.database.models.user import User, UserDao
from bisheng.database.models.user_group import UserGroupDao
from bisheng.database.models.user_role import UserRoleDao
from bisheng.settings import settings
from bisheng.utils.constants import USER_CURRENT_SESSION, RSA_KEY
from bisheng.utils.constants import RSA_KEY, USER_CURRENT_SESSION
from fastapi import Depends, HTTPException, Request
from fastapi_jwt_auth import AuthJWT
class UserPayload:
@@ -140,7 +140,7 @@ def sso_login():
def gen_user_role(db_user: User):
# 查询用户的角色列表
db_user_role = UserRoleDao.get_user_roles(db_user.user_id)
role = ""
role = ''
role_ids = []
for user_role in db_user_role:
if user_role.role_id == 1:
@@ -148,7 +148,7 @@ def gen_user_role(db_user: User):
role = 'admin'
else:
role_ids.append(user_role.role_id)
if role != "admin":
if role != 'admin':
# 判断是否是用户组管理员
db_user_groups = UserGroupDao.get_user_admin_group(db_user.user_id)
if len(db_user_groups) > 0:
@@ -169,7 +169,8 @@ def gen_user_jwt(db_user: User):
# 生成JWT令牌
payload = {'user_name': db_user.user_name, 'user_id': db_user.user_id, 'role': role}
# Create the tokens and passing to set_access_cookies or set_refresh_cookies
access_token = AuthJWT().create_access_token(subject=json.dumps(payload), expires_time=ACCESS_TOKEN_EXPIRE_TIME)
access_token = AuthJWT().create_access_token(subject=json.dumps(payload),
expires_time=ACCESS_TOKEN_EXPIRE_TIME)
refresh_token = AuthJWT().create_refresh_token(subject=db_user.user_name)
@@ -177,6 +178,60 @@ def gen_user_jwt(db_user: User):
return access_token, refresh_token, role, web_menu
def get_knowledge_list_by_access(role_id: int, name: str, page_num: int, page_size: int):
count_filter = []
if name:
count_filter.append(Knowledge.name.like('%{}%'.format(name)))
db_role_access = KnowledgeDao.get_knowledge_by_access(role_id, page_num, page_size)
total_count = KnowledgeDao.get_count_by_filter(count_filter)
# 补充用户名
user_ids = [access[0].user_id for access in db_role_access]
db_users = UserDao.get_user_by_ids(user_ids)
user_dict = {user.user_id: user.user_name for user in db_users}
return {
'data': [
KnowledgeRead.validate({
'name': access[0].name,
'user_name': user_dict.get(access[0].user_id),
'user_id': access[0].user_id,
'update_time': access[0].update_time,
'id': access[0].id
}) for access in db_role_access
],
'total':
total_count
}
def get_flow_list_by_access(role_id: int, name: str, page_num: int, page_size: int):
count_filter = []
if name:
count_filter.append(Flow.name.like('%{}%'.format(name)))
db_role_access = FlowDao.get_flow_by_access(role_id, name, page_num, page_size)
total_count = FlowDao.get_count_by_filters(count_filter)
# 补充用户名
user_ids = [access[0].user_id for access in db_role_access]
db_users = UserDao.get_user_by_ids(user_ids)
user_dict = {user.user_id: user.user_name for user in db_users}
return {
'data': [
FlowRead.validate({
'name': access[0].name,
'user_name': user_dict.get(access[0].user_id),
'user_id': access[0].user_id,
'update_time': access[0].update_time,
'id': access[0].id
}) for access in db_role_access
],
'total':
total_count
}
def get_assistant_list_by_access(role_id: int, name: str, page_num: int, page_size: int):
count_filter = []
if name:
@@ -198,7 +253,7 @@ def get_assistant_list_by_access(role_id: int, name: str, page_num: int, page_si
'id': access[0].id
} for access in db_role_access],
'total':
total_count
total_count
}
src/backend/bisheng/api/utils.py
+21 -16
View File
@@ -5,17 +5,16 @@ from pathlib import Path
from typing import Dict, List
import aiohttp
from fastapi import Request, WebSocket
from fastapi_jwt_auth import AuthJWT
from platformdirs import user_cache_dir
from sqlalchemy import delete
from sqlmodel import select
from bisheng.api.v1.schemas import StreamData
from bisheng.database.base import session_getter
from bisheng.database.models.variable_value import Variable
from bisheng.graph.graph.base import Graph
from bisheng.utils.logger import logger
from fastapi import Request, WebSocket
from fastapi_jwt_auth import AuthJWT
from platformdirs import user_cache_dir
from sqlalchemy import delete
from sqlmodel import select
API_WORDS = ['api', 'key', 'token']
@@ -97,7 +96,8 @@ async def build_flow(graph_data: dict,
}
yield str(StreamData(event='log', data=log_dict))
# # 如果存在文件，当前不操作文件，避免重复操作
if not process_file and vertex.base_type == 'documentloaders':
if not process_file and (vertex.base_type == 'documentloaders'
or vertex.base_type == 'input_output'):
template_dict = {
key: value
for key, value in vertex.data['node']['template'].items()
@@ -175,7 +175,8 @@ async def build_flow_no_yield(graph_data: dict,
for vertex in sorted_vertices:
try:
# 如果存在文件，当前不操作文件，避免重复操作
if not process_file and vertex.base_type == 'documentloaders':
if not process_file and (vertex.base_type == 'documentloaders'
or vertex.base_type == 'input_output'):
template_dict = {
key: value
for key, value in vertex.data['node']['template'].items()
@@ -192,15 +193,19 @@ async def build_flow_no_yield(graph_data: dict,
if vertex.base_type == 'vectorstores':
# 注入user_name
vertex.params['user_name'] = kwargs.get('user_name') if kwargs else ''
if vertex.vertex_type not in ["MilvusWithPermissionCheck", "ElasticsearchWithPermissionCheck"]:
if vertex.vertex_type not in [
'MilvusWithPermissionCheck', 'ElasticsearchWithPermissionCheck'
]:
# 知识库通过参数传参
if 'collection_name' in kwargs and 'collection_name' in vertex.params:
vertex.params['collection_name'] = kwargs['collection_name']
if 'collection_name' in kwargs and 'index_name' in vertex.params:
vertex.params['index_name'] = kwargs['collection_name']
if 'collection_name' in vertex.params and not vertex.params.get('collection_name'):
vertex.params['collection_name'] = f'tmp_{flow_id}_{chat_id if chat_id else 1}'
if 'collection_name' in vertex.params and not vertex.params.get(
'collection_name'):
vertex.params[
'collection_name'] = f'tmp_{flow_id}_{chat_id if chat_id else 1}'
logger.info(f"rename_vector_col col={vertex.params['collection_name']}")
if process_file:
# L1 清除Milvus历史记录
@@ -397,15 +402,15 @@ def parse_gpus(gpu_str: str) -> List[Dict]:
'gpu_util')[0]
res.append({
'gpu_uuid':
gpu_uuid_elem.firstChild.data,
gpu_uuid_elem.firstChild.data,
'gpu_id':
gpu_id_elem.firstChild.data,
gpu_id_elem.firstChild.data,
'gpu_total_mem':
'%.2f G' % (float(gpu_total_mem.firstChild.data.split(' ')[0]) / 1024),
'%.2f G' % (float(gpu_total_mem.firstChild.data.split(' ')[0]) / 1024),
'gpu_used_mem':
'%.2f G' % (float(free_mem.firstChild.data.split(' ')[0]) / 1024),
'%.2f G' % (float(free_mem.firstChild.data.split(' ')[0]) / 1024),
'gpu_utility':
round(float(gpu_utility_elem.firstChild.data.split(' ')[0]) / 100, 2)
round(float(gpu_utility_elem.firstChild.data.split(' ')[0]) / 100, 2)
})
return res
src/backend/bisheng/api/v1/callback.py
+14 -4
View File
@@ -20,7 +20,12 @@ from langchain.schema.messages import BaseMessage
class AsyncStreamingLLMCallbackHandler(AsyncCallbackHandler):
"""Callback handler for streaming LLM responses."""
def __init__(self, websocket: WebSocket, flow_id: str, chat_id: str, user_id: int = None, **kwargs: Any):
def __init__(self,
websocket: WebSocket,
flow_id: str,
chat_id: str,
user_id: int = None,
**kwargs: Any):
self.websocket = websocket
self.flow_id = flow_id
self.chat_id = chat_id
@@ -108,7 +113,7 @@ class AsyncStreamingLLMCallbackHandler(AsyncCallbackHandler):
result = output
# Create a formatted message.
intermediate_steps = f'{observation_prefix}{result}'
intermediate_steps = f'{observation_prefix}{result[:100]}'
# Create a ChatResponse instance.
resp = ChatResponse(type='stream',
@@ -242,7 +247,12 @@ class AsyncStreamingLLMCallbackHandler(AsyncCallbackHandler):
class StreamingLLMCallbackHandler(BaseCallbackHandler):
"""Callback handler for streaming LLM responses."""
def __init__(self, websocket: WebSocket, flow_id: str, chat_id: str, user_id: int = None, **kwargs: Any):
def __init__(self,
websocket: WebSocket,
flow_id: str,
chat_id: str,
user_id: int = None,
**kwargs: Any):
self.websocket = websocket
self.flow_id = flow_id
self.chat_id = chat_id
@@ -538,7 +548,7 @@ class AsyncGptsDebugCallbackHandler(AsyncGptsLLMCallbackHandler):
is_bot=1,
message=json.dumps(output_info),
intermediate_steps=f'{input_info["steps"]}\n\nTool output:\n\n Error: ' +
str(error),
str(error),
category=tool_category,
type='end',
flow_id=self.flow_id,
src/backend/bisheng/api/v1/chat.py
+143 -36
View File
@@ -1,16 +1,9 @@
import json
import time
import math
from typing import List, Optional
from uuid import UUID
from fastapi import APIRouter, HTTPException, Query, WebSocket, WebSocketException, status, Body, Request
from fastapi.params import Depends
from fastapi.responses import StreamingResponse
from fastapi_jwt_auth import AuthJWT
from sqlalchemy import func
from sqlmodel import select
from bisheng.api.errcode.base import UnAuthorizedError
from bisheng.api.services.assistant import AssistantService
from bisheng.api.services.audit_log import AuditLogService
from bisheng.api.services.base import BaseService
@@ -19,19 +12,31 @@ from bisheng.api.services.flow import FlowService
from bisheng.api.services.knowledge_imp import delete_es, delete_vector
from bisheng.api.services.user_service import UserPayload, get_login_user
from bisheng.api.utils import build_flow, build_input_keys_response, get_request_ip
from bisheng.api.v1.schemas import (BuildStatus, BuiltResponse, ChatInput, ChatList,
FlowGptsOnlineList, InitResponse, StreamData,
UnifiedResponseModel, resp_200, AddChatMessages)
from bisheng.api.v1.schema.base_schema import PageList
from bisheng.api.v1.schema.chat_schema import AppChatList
from bisheng.api.v1.schemas import (AddChatMessages, BuildStatus, BuiltResponse, ChatInput,
ChatList, FlowGptsOnlineList, InitResponse, StreamData,
UnifiedResponseModel, resp_200)
from bisheng.cache.redis import redis_client
from bisheng.chat.manager import ChatManager
from bisheng.database.base import session_getter
from bisheng.database.models.assistant import AssistantDao, AssistantStatus
from bisheng.database.models.flow import Flow, FlowDao, FlowStatus
from bisheng.database.models.flow_version import FlowVersionDao
from bisheng.database.models.message import ChatMessage, ChatMessageDao, ChatMessageRead
from bisheng.database.models.group_resource import GroupResourceDao, ResourceTypeEnum
from bisheng.database.models.message import ChatMessage, ChatMessageDao, ChatMessageRead, MessageDao
from bisheng.database.models.user import UserDao
from bisheng.database.models.user_group import UserGroupDao
from bisheng.graph.graph.base import Graph
from bisheng.utils.logger import logger
from bisheng.utils.util import get_cache_key
from fastapi import (APIRouter, Body, HTTPException, Query, Request, WebSocket, WebSocketException,
status)
from fastapi.params import Depends
from fastapi.responses import StreamingResponse
from fastapi_jwt_auth import AuthJWT
from sqlalchemy import func
from sqlmodel import select
router = APIRouter(tags=['Chat'])
chat_manager = ChatManager()
@@ -39,6 +44,75 @@ flow_data_store = redis_client
expire = 600 # reids 60s 过期
@router.get('/chat/app/list',
response_model=UnifiedResponseModel[PageList[AppChatList]],
status_code=200)
def get_app_chat_list(*,
keyword: Optional[str] = None,
page_num: Optional[int] = 1,
page_size: Optional[int] = 20,
login_user: UserPayload = Depends(get_login_user)):
"""通过消息表进行聊天App统计，全量表查询，"""
"""性能问题后续优化"""
group_flow_ids = []
if not login_user.is_admin():
# 判断下是否是用户组管理员
user_groups = UserGroupDao.get_user_admin_group(login_user.user_id)
if not user_groups:
raise UnAuthorizedError.http_exception()
user_group_ids = [user_group.group_id for user_group in user_groups]
# 获取分组下的所有资源ID
resources = GroupResourceDao.get_groups_resource(
user_group_ids, resource_types=[ResourceTypeEnum.FLOW, ResourceTypeEnum.ASSISTANT])
group_flow_ids = [one.third_id for one in resources]
if not group_flow_ids:
return resp_200(PageList(list=[], total=0))
flow_ids, user_ids = [], []
if keyword:
flows = FlowDao.get_flow_list_by_name(name=keyword)
assistants = AssistantDao.get_all_assistants(name=keyword, page=0, limit=0)
users = UserDao.search_user_by_name(user_name=keyword)
if flows:
flow_ids = [flow.id for flow in flows]
if assistants:
flow_ids = flow_ids.extend([assistant.id for assistant in assistants])
if user_ids:
user_ids = [user.user_id for user in users]
# 检索内容为空
if not flow_ids and not user_ids:
return resp_200(PageList(list=[], total=0))
if group_flow_ids:
if flow_ids:
flow_ids = list(set(flow_ids) & set(group_flow_ids))
else:
flow_ids = group_flow_ids
res, count = MessageDao.app_list_group_by_chat_id(page_size=page_size,
page_num=page_num,
flow_ids=flow_ids,
user_ids=user_ids)
# 补齐中文
user_ids = [one.get('user_id') for one in res]
flow_ids = [one.get('flow_id') for one in res]
user_list = UserDao.get_user_by_ids(user_ids)
flow_list = FlowDao.get_flow_by_ids(flow_ids)
assistant_list = AssistantDao.get_assistants_by_ids(flow_ids)
user_map = {user.user_id: user.user_name for user in user_list}
flow_map = {flow.id: flow.name for flow in flow_list}
assistant_map = {assistant.id: assistant.name for assistant in assistant_list}
flow_map.update(assistant_map)
res_obj = PageList(list=[
AppChatList(user_name=user_map.get(one['user_id'], one['user_id']),
flow_name=flow_map.get(one['flow_id'], one['flow_id']),
**one) for one in res
],
total=count)
return resp_200(res_obj)
@router.get('/chat/history',
response_model=UnifiedResponseModel[List[ChatMessageRead]],
status_code=200)
@@ -51,8 +125,7 @@ def get_chatmessage(*,
if not chat_id or not flow_id:
return {'code': 500, 'message': 'chat_id 和 flow_id 必传参数'}
where = select(ChatMessage).where(ChatMessage.flow_id == flow_id,
ChatMessage.chat_id == chat_id,
ChatMessage.user_id == login_user.user_id)
ChatMessage.chat_id == chat_id)
if id:
where = where.where(ChatMessage.id < int(id))
with session_getter() as session:
@@ -81,7 +154,8 @@ def del_chat_id(*,
else:
assistant_info = AssistantDao.get_one_assistant(message.flow_id)
if assistant_info:
AuditLogService.delete_chat_assistant(login_user, get_request_ip(request), assistant_info)
AuditLogService.delete_chat_assistant(login_user, get_request_ip(request),
assistant_info)
return resp_200(message='删除成功')
@@ -98,10 +172,20 @@ def add_chat_messages(*,
chat_id = data.chat_id
if not chat_id or not flow_id:
raise HTTPException(status_code=500, detail='chat_id 和 flow_id 必传参数')
human_message = ChatMessage(flow_id=flow_id.hex, chat_id=chat_id, user_id=login_user.user_id, is_bot=False,
message=data.human_message, type='human', category='question')
bot_message = ChatMessage(flow_id=flow_id.hex, chat_id=chat_id, user_id=login_user.user_id, is_bot=True,
message=data.answer_message, type='bot', category='answer')
human_message = ChatMessage(flow_id=flow_id.hex,
chat_id=chat_id,
user_id=login_user.user_id,
is_bot=False,
message=data.human_message,
type='human',
category='question')
bot_message = ChatMessage(flow_id=flow_id.hex,
chat_id=chat_id,
user_id=login_user.user_id,
is_bot=True,
message=data.answer_message,
type='bot',
category='answer')
ChatMessageDao.insert_batch([human_message, bot_message])
# 写审计日志, 判断是否是新建会话
@@ -115,7 +199,8 @@ def add_chat_messages(*,
else:
assistant_info = AssistantDao.get_one_assistant(flow_id)
if assistant_info:
AuditLogService.create_chat_assistant(login_user, get_request_ip(request), flow_id.hex)
AuditLogService.create_chat_assistant(login_user, get_request_ip(request),
flow_id.hex)
return resp_200(message='添加成功')
@@ -126,7 +211,9 @@ def update_chat_message(*,
message: str = Body(embed=True),
login_user: UserPayload = Depends(get_login_user)):
""" 更新一条消息的内容 安全检查使用"""
logger.info(f"update_chat_message message_id={message_id} message={message} login_user={login_user.user_name}")
logger.info(
f'update_chat_message message_id={message_id} message={message} login_user={login_user.user_name}'
)
chat_message = ChatMessageDao.get_message_by_id(message_id)
if not chat_message:
return resp_200(message='消息不存在')
@@ -165,6 +252,15 @@ def like_response(*, data: ChatInput, login_user: UserPayload = Depends(get_logi
return resp_200(message='操作成功')
@router.post('/chat/copied', status_code=200)
def copied_message(*,
message_id: int = Body(embed=True),
login_user: UserPayload = Depends(get_login_user)):
""" 上传复制message的数据 """
ChatMessageDao.update_message_copied(message_id, 1)
return resp_200(message='操作成功')
@router.post('/chat/comment', status_code=200)
def comment_resp(*, data: ChatInput, login_user: UserPayload = Depends(get_login_user)):
comment_answer(data.message_id, data.comment)
@@ -179,9 +275,9 @@ def get_chatlist_list(*,
smt = (select(ChatMessage.flow_id, ChatMessage.chat_id,
func.min(ChatMessage.create_time).label('create_time'),
func.max(ChatMessage.update_time).label('update_time')).where(
ChatMessage.user_id == login_user.user_id).group_by(
ChatMessage.flow_id,
ChatMessage.chat_id).order_by(func.max(ChatMessage.update_time).desc()))
ChatMessage.user_id == login_user.user_id).group_by(
ChatMessage.flow_id,
ChatMessage.chat_id).order_by(func.max(ChatMessage.update_time).desc()))
with session_getter() as session:
db_message = session.exec(smt).all()
@@ -237,18 +333,29 @@ def get_chatlist_list(*,
def get_online_chat(*,
keyword: Optional[str] = None,
tag_id: Optional[int] = None,
page: Optional[int] = 0,
limit: Optional[int] = 0,
page: Optional[int] = 1,
limit: Optional[int] = 10,
user: UserPayload = Depends(get_login_user)):
# 由于是获取助手和技能两个表，需要将page修改下
if page and limit:
search_page = math.ceil(page / 2)
else:
search_page = 1
limit = 10
res = []
all_assistant = AssistantService.get_assistant(user, keyword, AssistantStatus.ONLINE.value,
tag_id, page=search_page, limit=limit)
all_assistant = AssistantService.get_assistant(user,
keyword,
AssistantStatus.ONLINE.value,
tag_id,
page=search_page,
limit=limit)
all_assistant = all_assistant.data.get('data')
flows = FlowService.get_all_flows(user, keyword, FlowStatus.ONLINE.value,
tag_id=tag_id, page=search_page, page_size=limit)
flows = FlowService.get_all_flows(user,
keyword,
FlowStatus.ONLINE.value,
tag_id=tag_id,
page=search_page,
page_size=limit)
flows = flows.data.get('data')
for one in all_assistant:
res.append(
@@ -263,12 +370,12 @@ def get_online_chat(*,
# 获取用户可见的所有已上线的技能
for one in flows:
res.append(
FlowGptsOnlineList(id=one["id"],
name=one["name"],
desc=one["description"],
logo=one["logo"],
create_time=one["create_time"],
update_time=one["update_time"],
FlowGptsOnlineList(id=one['id'],
name=one['name'],
desc=one['description'],
logo=one['logo'],
create_time=one['create_time'],
update_time=one['update_time'],
flow_type='flow'))
res.sort(key=lambda x: x.update_time, reverse=True)
if page and limit:
src/backend/bisheng/api/v1/finetune.py
+60 -43
View File
@@ -1,4 +1,5 @@
import json
import tempfile
from typing import List, Optional
from uuid import UUID
@@ -9,23 +10,28 @@ from bisheng.api.services.finetune import FinetuneService
from bisheng.api.services.finetune_file import FinetuneFileService
from bisheng.api.services.user_service import get_login_user, UserPayload
from bisheng.api.v1.schemas import FinetuneCreateReq, UnifiedResponseModel, resp_200
from bisheng.cache.utils import file_download
from bisheng.database.models.finetune import Finetune, FinetuneChangeModelName, FinetuneList
from bisheng.database.models.model_deploy import ModelDeploy
from bisheng.database.models.knowledge import KnowledgeDao
from bisheng.database.models.knowledge_file import QAKnoweldgeDao
from bisheng.database.models.preset_train import PresetTrain
from bisheng.utils.minio_client import MinioClient
from fastapi import APIRouter, Body, Depends, File, Query, UploadFile
from fastapi_jwt_auth import AuthJWT
from loguru import logger
router = APIRouter(prefix='/finetune', tags=['Finetune'], dependencies=[Depends(get_login_user)])
# create finetune job
@router.post('/job', response_model=UnifiedResponseModel[Finetune])
async def create_job(*,
finetune: FinetuneCreateReq,
Authorize: AuthJWT = Depends()):
async def create_job(*, finetune: FinetuneCreateReq, Authorize: AuthJWT = Depends()):
# get login user
Authorize.jwt_required()
current_user = json.loads(Authorize.get_jwt_subject())
finetune = Finetune(**finetune.dict(exclude={'method'}), method=finetune.method.value,
finetune = Finetune(**finetune.dict(exclude={'method'}),
method=finetune.method.value,
user_id=current_user.get('user_id'),
user_name=current_user.get('user_name'))
return FinetuneService.create_job(finetune)
@@ -33,9 +39,7 @@ async def create_job(*,
# 删除训练任务
@router.delete('/job', response_model=UnifiedResponseModel)
async def delete_job(*,
job_id: UUID,
Authorize: AuthJWT = Depends()):
async def delete_job(*, job_id: UUID, Authorize: AuthJWT = Depends()):
# get login user
Authorize.jwt_required()
current_user = json.loads(Authorize.get_jwt_subject())
@@ -44,9 +48,7 @@ async def delete_job(*,
# 中止训练任务
@router.post('/job/cancel', response_model=UnifiedResponseModel)
async def cancel_job(*,
job_id: UUID,
Authorize: AuthJWT = Depends()):
async def cancel_job(*, job_id: UUID, Authorize: AuthJWT = Depends()):
# get login user
Authorize.jwt_required()
current_user = json.loads(Authorize.get_jwt_subject())
@@ -55,9 +57,7 @@ async def cancel_job(*,
# 发布训练任务
@router.post('/job/publish', response_model=UnifiedResponseModel)
async def publish_job(*,
job_id: UUID,
Authorize: AuthJWT = Depends()):
async def publish_job(*, job_id: UUID, Authorize: AuthJWT = Depends()):
# get login user
Authorize.jwt_required()
current_user = json.loads(Authorize.get_jwt_subject())
@@ -65,9 +65,7 @@ async def publish_job(*,
@router.post('/job/publish/cancel', response_model=UnifiedResponseModel)
async def cancel_publish_job(*,
job_id: UUID,
Authorize: AuthJWT = Depends()):
async def cancel_publish_job(*, job_id: UUID, Authorize: AuthJWT = Depends()):
# get login user
Authorize.jwt_required()
current_user = json.loads(Authorize.get_jwt_subject())
@@ -78,8 +76,10 @@ async def cancel_publish_job(*,
@router.get('/job', response_model=UnifiedResponseModel[List[Finetune]])
async def get_job(*,
server: str = Query(default=None, description='关联的RT服务名字'),
status: str = Query(default='', title='多个以英文逗号,分隔',
description='训练任务的状态，1: 训练中 2: 训练失败 3: 任务中止 4: 训练成功 5: 发布完成'),
status: str = Query(
default='',
title='多个以英文逗号,分隔',
description='训练任务的状态，1: 训练中 2: 训练失败 3: 任务中止 4: 训练成功 5: 发布完成'),
model_name: Optional[str] = Query(default='', description='模型名称,模糊搜索'),
page: Optional[int] = Query(default=1, description='页码'),
limit: Optional[int] = Query(default=10, description='每页条数'),
@@ -89,24 +89,24 @@ async def get_job(*,
status_list = []
if status.strip():
status_list = [int(one) for one in status.strip().split(',')]
req_data = FinetuneList(server_name=server, status=status_list, model_name=model_name, page=page, limit=limit)
req_data = FinetuneList(server_name=server,
status=status_list,
model_name=model_name,
page=page,
limit=limit)
return FinetuneService.get_all_job(req_data)
# 获取任务最新详细信息，此接口会同步查询SFT-backend侧将任务状态更新到最新
@router.get('/job/info', response_model=UnifiedResponseModel[Finetune])
async def get_job_info(*,
job_id: UUID,
Authorize: AuthJWT = Depends()):
async def get_job_info(*, job_id: UUID, Authorize: AuthJWT = Depends()):
# get login user
Authorize.jwt_required()
return FinetuneService.get_job_info(job_id)
@router.patch('/job/model', response_model=UnifiedResponseModel)
async def update_job(*,
req_data: FinetuneChangeModelName,
Authorize: AuthJWT = Depends()):
async def update_job(*, req_data: FinetuneChangeModelName, Authorize: AuthJWT = Depends()):
# get login user
Authorize.jwt_required()
current_user = json.loads(Authorize.get_jwt_subject())
@@ -126,27 +126,50 @@ async def upload_file(*,
@router.post('/job/file/preset', response_model=UnifiedResponseModel[List[PresetTrain]])
async def upload_preset_file(*,
files: list[UploadFile] = File(description='预置训练文件列表'),
files: Optional[str] = Body(default=None, description='预置训练文件列表'),
name: Optional[str] = Body(description='数据集名字'),
qa_list: Optional[list[int]] = Body(default=None,
description='QA知识库'),
Authorize: AuthJWT = Depends()):
Authorize.jwt_required()
current_user = json.loads(Authorize.get_jwt_subject())
return FinetuneFileService.upload_file(files, True, current_user)
if files:
filepath, file_name = file_download(files)
logger.info(f'dataset upload_file_name: {file_name}')
return FinetuneFileService.upload_preset_file(name, 0, filepath, current_user)
elif qa_list:
# 将qa数据按照finetune格式，进行文件存储
qa_knowledge_db = KnowledgeDao.get_list_by_ids(qa_list)
qa_knowledge_db_ids = [qa_knowledge.id for qa_knowledge in qa_knowledge_db]
qa_db_list = QAKnoweldgeDao.get_qa_knowledge_by_knowledge_ids(qa_knowledge_db_ids)
qa_list = []
for qa in qa_db_list:
qa_list.extend([{
'instruction': question,
'input': '',
'output': json.loads(qa.answers)[0]
} for question in qa.questions])
with tempfile.NamedTemporaryFile(mode='w+', suffix='.json') as filepath:
json.dump(qa_list, filepath, ensure_ascii=False, indent=2)
filepath.seek(0)
return FinetuneFileService.upload_preset_file(name, 1, filepath.name, current_user)
# 获取预置训练文件列表
@router.get('/job/file/preset', response_model=UnifiedResponseModel[List[PresetTrain]])
async def get_preset_file(*, Authorize: AuthJWT = Depends()):
async def get_preset_file(*,
page_size: Optional[int] = None,
page_num: Optional[int] = None,
keyword: Optional[str] = None,
Authorize: AuthJWT = Depends()):
# get login user
Authorize.jwt_required()
ret = FinetuneFileService.get_preset_file()
ret = FinetuneFileService.get_preset_file(keyword, page_size, page_num)
return resp_200(ret)
@router.delete('/job/file/preset', response_model=UnifiedResponseModel)
async def delete_preset_file(*,
file_id: UUID,
Authorize: AuthJWT = Depends()):
async def delete_preset_file(*, file_id: UUID, Authorize: AuthJWT = Depends()):
# get login user
Authorize.jwt_required()
current_user = json.loads(Authorize.get_jwt_subject())
@@ -154,20 +177,15 @@ async def delete_preset_file(*,
@router.get('/job/file/download', response_model=UnifiedResponseModel)
async def get_download_url(*,
file_url: str,
Authorize: AuthJWT = Depends()):
async def get_download_url(*, file_url: str, Authorize: AuthJWT = Depends()):
Authorize.jwt_required()
minio_client = MinioClient()
download_url = minio_client.get_share_link(file_url)
return resp_200(data={
'url': download_url
})
return resp_200(data={'url': download_url})
@router.get('/server/filters', response_model=UnifiedResponseModel)
async def get_server_filters(*,
Authorize: AuthJWT = Depends()):
async def get_server_filters(*, Authorize: AuthJWT = Depends()):
Authorize.jwt_required()
return FinetuneService.get_server_filters()
@@ -183,8 +201,7 @@ async def get_model_list(request: Request,
@router.get('/gpu', response_model=UnifiedResponseModel)
async def get_gpu_info(*,
Authorize: AuthJWT = Depends()):
async def get_gpu_info(*, Authorize: AuthJWT = Depends()):
# get login user
Authorize.jwt_required()
return FinetuneService.get_gpu_info()
src/backend/bisheng/api/v1/knowledge.py
+223 -3
View File
@@ -1,4 +1,8 @@
import urllib.parse
import json
import os
import re
import time
from typing import List, Optional
from fastapi import APIRouter, BackgroundTasks, Depends, File, HTTPException, UploadFile, Request, Body, Query
@@ -12,6 +16,36 @@ from bisheng.cache.utils import save_uploaded_file
from bisheng.database.models.knowledge import KnowledgeCreate, KnowledgeRead, KnowledgeUpdate, KnowledgeDao
from bisheng.database.models.knowledge_file import KnowledgeFile, KnowledgeFileDao
from bisheng.utils.logger import logger
from bisheng.api.errcode.base import UnAuthorizedError
from bisheng.api.services import knowledge_imp
from bisheng.api.services.audit_log import AuditLogService
from bisheng.api.services.knowledge_imp import (add_qa, addEmbedding, decide_vectorstores,
delete_knowledge_file_vectors, retry_files)
from bisheng.api.services.user_service import UserPayload, get_login_user
from bisheng.api.utils import get_request_ip
from bisheng.api.v1.schemas import UnifiedResponseModel, UploadFileResponse, resp_200, resp_500
from bisheng.cache.utils import file_download, save_uploaded_file
from bisheng.database.base import session_getter
from bisheng.database.models.group_resource import GroupResource, GroupResourceDao, ResourceTypeEnum
from bisheng.database.models.knowledge import (Knowledge, KnowledgeCreate, KnowledgeDao,
KnowledgeRead, KnowledgeTypeEnum)
from bisheng.database.models.knowledge_file import (KnowledgeFile, KnowledgeFileDao,
KnowledgeFileRead, QAKnoweldgeDao,
QAKnowledgeUpsert)
from bisheng.database.models.role_access import AccessType, RoleAccess
from bisheng.database.models.user import User, UserDao
from bisheng.database.models.user_group import UserGroupDao
from bisheng.interface.embeddings.custom import FakeEmbedding
from bisheng.settings import settings
from bisheng.utils.logger import logger
from bisheng.utils.minio_client import MinioClient
from bisheng_langchain.vectorstores import ElasticKeywordsSearch
from fastapi import (APIRouter, BackgroundTasks, Body, Depends, File, HTTPException, Request,
UploadFile)
from fastapi.encoders import jsonable_encoder
from pymilvus import Collection
from sqlalchemy import delete, func, or_
from sqlmodel import select
# build router
router = APIRouter(prefix='/knowledge', tags=['Knowledge'])
@@ -85,7 +119,7 @@ def create_knowledge(*,
return resp_200(db_knowledge)
@router.get('/', status_code=200)
@router.get('', status_code=200)
def get_knowledge(*,
request: Request,
login_user: UserPayload = Depends(get_login_user),
@@ -138,6 +172,56 @@ def get_filelist(*,
})
@router.get('/qa/list/{qa_knowledge_id}', status_code=200)
def get_QA_list(*,
qa_knowledge_id: int,
page_size: int = 10,
page_num: int = 1,
question: Optional[str] = None,
answer: Optional[str] = None,
keyword: Optional[str] = None,
status: Optional[int] = None,
login_user: UserPayload = Depends(get_login_user)):
""" 获取知识库文件信息. """
# 查询当前知识库，是否有写入权限
with session_getter() as session:
db_knowledge: Knowledge = session.get(Knowledge, qa_knowledge_id)
if not db_knowledge:
raise HTTPException(status_code=500, detail='当前知识库不可用，返回上级目录')
if not login_user.access_check(db_knowledge.user_id, str(qa_knowledge_id),
AccessType.KNOWLEDGE):
return UnAuthorizedError.return_resp()
if db_knowledge.type == KnowledgeTypeEnum.NORMAL.value:
return HTTPException(status_code=500, detail='知识库为普通知识库')
if keyword:
question = keyword
qa_list, total_count = knowledge_imp.list_qa_by_knowledge_id(qa_knowledge_id, page_size,
page_num, question, answer,
status)
user_list = UserDao.get_user_by_ids([qa.user_id for qa in qa_list])
user_map = {
user.user_id: user.user_name
for user in user_list
}
data = [jsonable_encoder(qa) for qa in qa_list]
for qa in data:
qa['questions'] = qa['questions'][0]
qa['answers'] = json.loads(qa['answers'])[0]
qa['user_name'] = user_map.get(qa['user_id'], qa['user_id'])
return resp_200({
'data': data,
'total': total_count,
'writeable':
login_user.access_check(db_knowledge.user_id, str(qa_knowledge_id),
AccessType.KNOWLEDGE_WRITE)
})
@router.post('/retry', status_code=200)
def retry(*, request: Request, login_user: UserPayload = Depends(get_login_user),
background_tasks: BackgroundTasks,
@@ -147,15 +231,74 @@ def retry(*, request: Request, login_user: UserPayload = Depends(get_login_user)
return resp_200()
@router.delete('/{knowledge_id}', status_code=200)
def delete_knowledge(*,
request: Request,
knowledge_id: int,
login_user: UserPayload = Depends(get_login_user)):
""" 删除知识库信息. """
with session_getter() as session:
knowledge = session.get(Knowledge, knowledge_id)
if not knowledge:
raise HTTPException(status_code=404, detail='knowledge not found')
if not login_user.access_check(knowledge.user_id, str(knowledge_id),
AccessType.KNOWLEDGE_WRITE):
raise HTTPException(status_code=404, detail='没有权限执行操作')
# 处理knowledgefile
with session_getter() as session:
session.exec(delete(KnowledgeFile).where(KnowledgeFile.knowledge_id == knowledge_id))
session.commit()
# 处理vector
embeddings = FakeEmbedding()
vectore_client = decide_vectorstores(knowledge.collection_name, 'Milvus', embeddings)
if isinstance(vectore_client.col, Collection):
logger.info(f'delete_vectore col={knowledge.collection_name}')
if knowledge.collection_name.startswith('col'):
vectore_client.col.drop()
else:
pk = vectore_client.col.query(expr=f'knowledge_id=="{knowledge.id}"',
output_fields=['pk'])
vectore_client.col.delete(f"pk in {[p['pk'] for p in pk]}")
# 判断milvus 是否还有entity
if vectore_client.col.is_empty:
vectore_client.col.drop()
# 处理 es
# elastic
esvectore_client: 'ElasticKeywordsSearch' = decide_vectorstores(knowledge.index_name,
'ElasticKeywordsSearch',
embeddings)
if esvectore_client:
index_name = knowledge.index_name or knowledge.collection_name # 兼容老版本
res = esvectore_client.client.indices.delete(index=index_name, ignore=[400, 404])
logger.info(f'act=delete_es index={index_name} res={res}')
with session_getter() as session:
session.delete(knowledge)
session.commit()
delete_knowledge_hook(request, knowledge, login_user)
return resp_200(message='删除成功')
def delete_knowledge_hook(request: Request, knowledge: Knowledge, login_user: UserPayload):
logger.info(f'delete_knowledge_hook id={knowledge.id}, user: {login_user.user_id}')
# 删除知识库的审计日志
AuditLogService.delete_knowledge(login_user, get_request_ip(request), knowledge)
GroupResourceDao.delete_group_resource_by_third_id(str(knowledge.id),
ResourceTypeEnum.KNOWLEDGE)
@router.delete('/file/{file_id}', status_code=200)
def delete_knowledge_file(*,
request: Request,
file_id: int,
login_user: UserPayload = Depends(get_login_user)):
""" 删除知识文件信息 """
KnowledgeService.delete_knowledge_file(request, login_user, [file_id])
return resp_200(message='删除成功')
@@ -212,3 +355,80 @@ async def get_file_bbox(request: Request, login_user: UserPayload = Depends(get_
file_id: int = Query(description='文件唯一ID')):
res = KnowledgeService.get_file_bbox(request, login_user, file_id)
return resp_200(data=res)
@router.post('/qa/add', status_code=200)
def qa_add(*, QACreate: QAKnowledgeUpsert, login_user: UserPayload = Depends(get_login_user)):
""" 增加知识库信息. """
QACreate.user_id = login_user.user_id
db_knowledge = KnowledgeDao.query_by_id(QACreate.knowledge_id)
if db_knowledge.type != KnowledgeTypeEnum.QA.value:
raise HTTPException(status_code=404, detail='知识库类型错误')
add_qa(db_knowledge=db_knowledge, data=QACreate)
return resp_200()
@router.post('/qa/status_switch', status_code=200)
def qa_status_switch(*,
status: int = Body(embed=True),
id: int = Body(embed=True),
login_user: UserPayload = Depends(get_login_user)):
""" 修改知识库信息. """
return resp_200(knowledge_imp.qa_status_change(id, status))
@router.get('/qa/detail', status_code=200)
def qa_list(*, id: int, login_user: UserPayload = Depends(get_login_user)):
""" 增加知识库信息. """
qa_knowledge = QAKnoweldgeDao.get_qa_knowledge_by_primary_id(id)
qa_knowledge.answers = json.loads(qa_knowledge.answers)[0]
return resp_200(data=qa_knowledge)
@router.post('/qa/append', status_code=200)
def qa_append(
*,
ids: list[int] = Body(..., embed=True),
question: str = Body(..., embed=True),
login_user: UserPayload = Depends(get_login_user),
):
""" 增加知识库信息. """
QA_list = QAKnoweldgeDao.select_list(ids)
knowledge = KnowledgeDao.query_by_id(QA_list[0].knowledge_id)
for qa in QA_list:
qa.questions.append(question)
knowledge_imp.add_qa(knowledge, qa)
return resp_200()
@router.delete('/qa/delete', status_code=200)
def qa_delete(*,
ids: list[int] = Body(embed=True),
login_user: UserPayload = Depends(get_login_user)):
""" 删除知识文件信息 """
knowledge_dbs = QAKnoweldgeDao.select_list(ids)
knowledge = KnowledgeDao.query_by_id(knowledge_dbs[0].knowledge_id)
if not login_user.access_check(knowledge.user_id, str(knowledge.id),
AccessType.KNOWLEDGE_WRITE):
raise HTTPException(status_code=404, detail='没有权限执行操作')
if knowledge.type == KnowledgeTypeEnum.NORMAL.value:
return HTTPException(status_code=500, detail='知识库类型错误')
QAKnoweldgeDao.delete_batch(ids)
# knowledge_imp.delete_vector_data(knowledge.id, ids)
return resp_200()
@router.post('/qa/auto_question')
def qa_auto_question(
*,
number: int = Body(default=3, embed=True),
ori_question: str = Body(default='', embed=True),
answer: str = Body(default='', embed=True),
):
"""通过大模型自动生成问题"""
questions = knowledge_imp.recommend_question(ori_question, number=number, answer=answer)
return resp_200(data={'questions': questions})
src/backend/bisheng/api/v1/schema/base_schema.py
+11
View File
@@ -0,0 +1,11 @@
from typing import Generic, List, TypeVar
from pydantic import BaseModel
# 创建泛型变量
DataT = TypeVar('DataT')
class PageList(Generic[DataT], BaseModel):
list: List[DataT]
total: int
src/backend/bisheng/api/v1/schema/chat_schema.py
+23
View File
@@ -0,0 +1,23 @@
from datetime import datetime
from typing import List, Optional
from uuid import UUID
from pydantic import BaseModel
class AppChatList(BaseModel):
flow_name: str
user_name: str
user_id: int
chat_id: str
flow_id: UUID
create_time: datetime
like_count: int
dislike_count: int
copied_count: int
class APIAddQAParam(BaseModel):
question: str
answer: List[str]
relative_questions: Optional[List[str]] = []
src/backend/bisheng/api/v2/assistant.py
+40 -31
View File
@@ -4,12 +4,6 @@ import uuid
from typing import Optional
from uuid import UUID
from fastapi import APIRouter, Request, HTTPException, WebSocket, WebSocketException
from fastapi import status as http_status
from fastapi.responses import StreamingResponse, ORJSONResponse
from langchain_core.messages import HumanMessage, AIMessage
from loguru import logger
from bisheng.api.services.assistant import AssistantService
from bisheng.api.services.assistant_agent import AssistantAgent
from bisheng.api.utils import get_request_ip
@@ -17,20 +11,28 @@ from bisheng.api.v1.chat import chat_manager
from bisheng.api.v1.schemas import OpenAIChatCompletionResp, OpenAIChatCompletionReq, UnifiedResponseModel, \
AssistantInfo, OpenAIChoice
from bisheng.api.v2.utils import get_default_operator
from bisheng.api.v1.schemas import (AssistantInfo, OpenAIChatCompletionReq,
OpenAIChatCompletionResp, OpenAIChoice, UnifiedResponseModel)
from bisheng.chat.types import WorkType
from bisheng.settings import settings
from fastapi import APIRouter, HTTPException, Query, Request, WebSocket, WebSocketException
from fastapi import status as http_status
from fastapi.responses import ORJSONResponse, StreamingResponse
from langchain_core.messages import AIMessage, HumanMessage
from loguru import logger
router = APIRouter(prefix='/assistant', tags=['OpenAPI', 'Assistant'])
@router.post('/chat/completions', response_model=OpenAIChatCompletionResp)
async def assistant_chat_completions(request: Request,
req_data: OpenAIChatCompletionReq):
async def assistant_chat_completions(request: Request, req_data: OpenAIChatCompletionReq):
"""
兼容openai接口格式，所有的错误必须返回非http200的状态码
和助手进行聊天
"""
logger.info(f'act=assistant_chat_completions assistant_id={req_data.model}, ip={get_request_ip(request)}')
logger.info(
f'act=assistant_chat_completions assistant_id={req_data.model}, ip={get_request_ip(request)}'
)
try:
# 获取系统配置里配置的默认用户信息
login_user = get_default_operator()
@@ -39,7 +41,9 @@ async def assistant_chat_completions(request: Request,
# 查找助手信息
res = AssistantService.get_assistant_info(UUID(req_data.model), login_user)
if res.status_code != 200:
return ORJSONResponse(status_code=500, content=res.status_message, media_type='application/json')
return ORJSONResponse(status_code=500,
content=res.status_message,
media_type='application/json')
assistant_info = res.data
# 覆盖温度设置
@@ -73,13 +77,10 @@ async def assistant_chat_completions(request: Request,
object='chat.completion',
created=int(time.time()),
model=req_data.model,
choices=[OpenAIChoice(
index=0,
message={
'role': 'assistant',
'content': answer
}
)],
choices=[OpenAIChoice(index=0, message={
'role': 'assistant',
'content': answer
})],
)
# 非流式直接返回结果
@@ -88,14 +89,7 @@ async def assistant_chat_completions(request: Request,
# 流式返回最终结果, 兼容openai格式处理
openai_resp.object = 'chat.completion.chunk'
openai_resp.choices = [
OpenAIChoice(
index=0,
delta={
'content': answer
}
)
]
openai_resp.choices = [OpenAIChoice(index=0, delta={'content': answer})]
async def _event_stream():
# todo：zgq 后续优化成真正的流式输出，目前是出现最终答案之后直接流式返回的
@@ -123,11 +117,26 @@ async def get_assistant_info(request: Request, assistant_id: UUID):
return AssistantService.get_assistant_info(assistant_id, login_user)
@router.get('/list', status_code=200)
def get_assistant_list(request: Request,
name: str = Query(default=None, description='助手名称，模糊匹配, 包含描述的模糊匹配'),
tag_id: int = Query(default=None, description='标签ID'),
page: Optional[int] = Query(default=1, gt=0, description='页码'),
limit: Optional[int] = Query(default=10, gt=0, description='每页条数'),
status: Optional[int] = Query(default=None, description='是否上线状态'),
user_id: int = None):
"""
公开的获取技能信息的接口
"""
logger.info(f'public_get_list ip: {request.client.host} user_id:{user_id}')
user_id = user_id if user_id else settings.get_from_db('default_operator').get('user')
login_user = UserPayload(**{'user_id': user_id, 'role': ''})
return AssistantService.get_assistant(login_user, name, status, tag_id, page, limit)
@router.websocket('/chat/{assistant_id}')
async def chat(*,
websocket: WebSocket,
assistant_id: str,
chat_id: Optional[str] = None):
async def chat(*, websocket: WebSocket, assistant_id: str, chat_id: Optional[str] = None):
"""
助手的ws免登录接口
"""
@@ -135,8 +144,8 @@ async def chat(*,
login_user = get_default_operator()
try:
request = websocket
await chat_manager.dispatch_client(request, assistant_id, chat_id, login_user, WorkType.GPTS,
websocket)
await chat_manager.dispatch_client(request, assistant_id, chat_id, login_user,
WorkType.GPTS, websocket)
except WebSocketException as exc:
logger.error(f'Websocket exception: {str(exc)}')
await websocket.close(code=http_status.WS_1011_INTERNAL_ERROR, reason=str(exc))
src/backend/bisheng/api/v2/chat.py
+28 -4
View File
@@ -1,19 +1,20 @@
import json
from typing import Optional
from uuid import uuid4
from typing import List, Optional
from uuid import UUID, uuid4
from bisheng.api.services.chat_imp import comment_answer
from bisheng.api.services.utils import set_flow_knowledge_id
from bisheng.api.v1.schemas import ChatInput, resp_200
from bisheng.api.v2.schema.message import SyncMessage
from bisheng.cache.redis import redis_client
from bisheng.chat.manager import ChatManager
from bisheng.database.base import session_getter
from bisheng.database.models.flow import Flow
from bisheng.database.models.message import ChatMessage
from bisheng.database.models.message import ChatMessage, ChatMessageDao
from bisheng.processing.process import process_tweaks
from bisheng.settings import settings
from bisheng.utils.logger import logger
from fastapi import APIRouter, WebSocket, status
from fastapi import APIRouter, Body, WebSocket, status
router = APIRouter(prefix='/chat', tags=['OpenAPI', 'Chat'])
chat_manager = ChatManager()
@@ -97,3 +98,26 @@ def solve_response(*, data: dict):
def comment(*, data: ChatInput):
comment_answer(data.message_id, data.comment)
return resp_200()
@router.post('/sync/messages', status_code=200)
def sync_message(*,
flow_id: UUID = Body(embed=True),
message_list: List[SyncMessage] = Body(embed=True),
user_id: int = Body(default=None, embed=True)):
user_id = user_id if user_id else settings.get_from_db('default_operator').get('user')
batch_message = [
ChatMessage(is_bot=message.is_send,
source=0,
message=message.message,
extra=json.dumps(message.extra),
type='answer',
category='answer',
flow_id=flow_id,
user_id=user_id,
chat_id=flow_id.hex) for message in message_list
]
ChatMessageDao.insert_batch(batch_message)
return resp_200()
src/backend/bisheng/api/v2/filelib.py
+140
View File
@@ -2,8 +2,31 @@ import hashlib
import json
import os
from typing import Dict, List, Optional
from urllib.parse import unquote
from fastapi import APIRouter, BackgroundTasks, File, Form, HTTPException, UploadFile, Request
from bisheng.api.services import knowledge_imp
from bisheng.api.services.knowledge_imp import (addEmbedding, create_knowledge, decide_vectorstores,
delete_es, delete_knowledge_by,
delete_knowledge_file_vectors, delete_vector,
text_knowledge)
from bisheng.api.v1.schemas import ChunkInput, UnifiedResponseModel, resp_200, resp_500
from bisheng.api.v2.schema.filelib import APIAddQAParam, APIAppendQAParam, QueryQAParam
from bisheng.cache.utils import save_download_file
from bisheng.database.base import session_getter
from bisheng.database.models.knowledge import (Knowledge, KnowledgeCreate, KnowledgeDao,
KnowledgeRead, KnowledgeTypeEnum, KnowledgeUpdate)
from bisheng.database.models.knowledge_file import (KnowledgeFile, KnowledgeFileDao,
KnowledgeFileRead, QAKnoweldgeDao, QAKnowledge,
QAKnowledgeUpsert)
from bisheng.database.models.message import ChatMessageDao
from bisheng.database.models.role_access import AccessType, RoleAccess
from bisheng.database.models.user import User
from bisheng.interface.embeddings.custom import FakeEmbedding
from bisheng.settings import settings
from bisheng.utils.logger import logger
from bisheng.utils.minio_client import MinioClient
from fastapi import APIRouter, BackgroundTasks, Body, File, Form, HTTPException, UploadFile
from fastapi.encoders import jsonable_encoder
from sqlalchemy import func
from sqlmodel import select
@@ -262,3 +285,120 @@ def dump_vector_knowledge(collection_name: str, expr: str = None, store: str = '
def download_statistic_file(file_path: str):
file_name = os.path.basename(file_path)
return FileResponse(file_path, filename=file_name)
@router.post('/add_qa')
def add_qa(*,
knowledge_id: int = Body(embed=True),
data: List[APIAddQAParam] = Body(embed=True),
user_id: Optional[int] = Body(default=None, embed=True)):
user_id = user_id if user_id else settings.get_from_db('default_operator').get('user')
knowledge = KnowledgeDao.query_by_id(knowledge_id)
logger.info('add_qa_data knowledge_id={} size={}', knowledge_id, len(data))
res = []
for item in data:
qa_insert = QAKnowledgeUpsert(knowledge_id=knowledge_id,
questions=[item.question],
answers=item.answer,
user_id=user_id,
extra_meta=json.dumps(item.extra),
source=3)
res.append(knowledge_imp.add_qa(knowledge, qa_insert))
return resp_200(res)
@router.post('/add_relative_qa', response_model=List[QAKnowledge])
def append_qa(*,
knowledge_id: int = Body(embed=True),
data: APIAppendQAParam = Body(embed=True),
user_id: Optional[int] = Body(default=None, embed=True)):
user_id = user_id if user_id else settings.get_from_db('default_operator').get('user')
knowledge = KnowledgeDao.query_by_id(knowledge_id)
qa_db = QAKnoweldgeDao.get_qa_knowledge_by_primary_id(data.id)
if not qa_db:
return HTTPException(404, detail='qa 对没有找到')
qa_insert = QAKnowledgeUpsert.validate(knowledge)
qa_insert.questions.extend(data.relative_questions)
return resp_200(knowledge_imp.add_qa(knowledge, qa_insert))
@router.delete('/qa/{qa_id}', status_code=200)
def delete_qa_data(*, qa_id: int, question: Optional[str] = None):
""" 删除qa 问题对信息 """
qa = QAKnoweldgeDao.get_qa_knowledge_by_primary_id(qa_id)
if not qa:
raise HTTPException(status_code=404, detail='qa 不存在')
if question:
qa.questions = [q for q in qa.questions if q != question]
QAKnoweldgeDao.update(qa)
else:
QAKnoweldgeDao.delete_batch([qa_id])
try:
knowledge = KnowledgeDao.query_by_id(qa.knowledge_id)
knowledge_imp.delete_vector_data(knowledge, file_ids=[qa_id])
if question:
knowledge_imp.QA_save_knowledge(knowledge, qa)
return resp_200()
except Exception as e:
return resp_500(message=f'error e={str(e)}')
@router.post('/update_qa', status_code=200)
def update_qa(
*,
id: int = Body(embed=True),
question: Optional[str] = Body(default=None, embed=True),
original_question: Optional[str] = Body(default=None, embed=True),
answer: Optional[List[str]] = Body(default=None, embed=True),
):
""" 删除qa 问题对信息 """
qa = QAKnoweldgeDao.get_qa_knowledge_by_primary_id(id)
if not qa:
raise HTTPException(status_code=404, detail='qa 不存在')
if original_question:
qa.questions = [q if q != question else question for q in qa.questions]
else:
qa.questions = [question]
if answer:
qa.answers = json.dumps(answer, ensure_ascii=False)
QAKnoweldgeDao.update(qa)
try:
knowledge = KnowledgeDao.query_by_id(qa.knowledge_id)
if question:
knowledge_imp.delete_vector_data(knowledge, file_ids=[id])
knowledge_imp.QA_save_knowledge(knowledge, qa)
return resp_200()
except Exception as e:
return resp_500(message=f'error e={str(e)}')
@router.get('/detail_qa', status_code=200)
def detail_qa(*, id: int):
""" 获取问题对信息 """
qa = QAKnoweldgeDao.get_qa_knowledge_by_primary_id(id)
return resp_200(qa)
@router.post('/query_qa', status_code=200)
def query_qa(QueryQAParam: QueryQAParam):
""" 删除qa 问题对信息 """
sources = [1, 2] # 3 是api倒入的
qa_list = QAKnoweldgeDao.query_by_condition_v1(source=sources,
create_start=QueryQAParam.timeRange[0],
create_end=QueryQAParam.timeRange[1])
if qa_list:
for q in qa_list:
q.answers = json.loads(q.answers)
return resp_200(qa_list)
src/backend/bisheng/api/v2/flow.py
+26 -7
View File
@@ -1,24 +1,21 @@
from uuid import UUID
from loguru import logger
from fastapi import APIRouter, Request, HTTPException
from bisheng.api.services.flow import FlowService
from bisheng.api.services.user_service import UserPayload
from bisheng.api.v1.schemas import resp_200
from bisheng.api.v2.assistant import get_default_operator
from bisheng.database.models.flow import FlowDao
from bisheng.settings import settings
from fastapi import APIRouter, HTTPException, Query, Request
from loguru import logger
router = APIRouter(prefix='/flows', tags=['OpenAPI', 'FlowV2'])
@router.get("/{flow_id}", status_code=200)
@router.get('/{flow_id}', status_code=200)
def get_flow(request: Request, flow_id: UUID):
"""
公开的获取技能信息的接口
"""
logger.info(f"public_get_flow ip: {request.client.host} flow_id:{flow_id}")
logger.info(f'public_get_flow ip: {request.client.host} flow_id:{flow_id}')
# 判断下配置是否打开
if not settings.get_from_db("default_operator").get("enable_guest_access"):
raise HTTPException(status_code=403, detail="无权限访问")
@@ -30,3 +27,25 @@ def get_flow(request: Request, flow_id: UUID):
})
return FlowService.get_one_flow(login_user, flow_id.hex)
@router.get('', status_code=200)
def get_flow_list(request: Request,
name: str = Query(default=None, description='根据name查找数据库，包含描述的模糊搜索'),
tag_id: int = Query(default=None, description='标签ID'),
page_size: int = Query(default=10, description='每页数量'),
page_num: int = Query(default=1, description='页数'),
status: int = None,
user_id: int = None):
"""
公开的获取技能信息的接口
"""
logger.info(f'public_get_flow_list ip: {request.client.host} user_id={user_id}')
user_id = user_id if user_id else settings.get_from_db('default_operator').get('user')
login_user = UserPayload(**{'user_id': user_id, 'role': ''})
try:
return FlowService.get_all_flows(login_user, name, status, tag_id, page_num, page_size)
except Exception as e:
logger.error(e)
raise HTTPException(status_code=500, detail='获取技能列表失败')
src/backend/bisheng/api/v2/schema/filelib.py
+18
View File
@@ -0,0 +1,18 @@
from typing import Dict, List, Optional
from pydantic import BaseModel
class APIAddQAParam(BaseModel):
question: str
answer: List[str]
extra: Optional[Dict] = {}
class APIAppendQAParam(BaseModel):
relative_questions: List[str] = []
id: str = None
class QueryQAParam(BaseModel):
timeRange: List[str]
src/backend/bisheng/api/v2/schema/message.py
+8
View File
@@ -0,0 +1,8 @@
from pydantic import BaseModel
class SyncMessage(BaseModel):
is_send: bool
message: str
create_time: str
extra: dict
src/backend/bisheng/chat/handlers.py
+12 -10
View File
@@ -3,6 +3,7 @@ import time
from queue import Queue
from typing import Dict
from bisheng.api.utils import build_input_keys_response
from bisheng.api.v1.schemas import ChatMessage, ChatResponse
from bisheng.chat.manager import ChatManager
from bisheng.chat.utils import judge_source, process_graph, process_source_document
@@ -286,7 +287,6 @@ class Handler:
async def process_file(self, session: ChatManager, client_id: str, chat_id: str, payload: dict,
user_id: int):
file_name = payload['inputs']
file_name.pop('id', '')
batch_question = payload['inputs']['questions']
# 如果L3
file = ChatMessage(is_bot=False, message=file_name, type='end', user_id=user_id)
@@ -305,12 +305,13 @@ class Handler:
input_key = 'input'
input_dict = {}
else:
input_key = langchain_object.input_keys[0]
input_key = list(build_input_keys_response(langchain_object,
{})['input_keys'].keys())[0]
input_dict = {k: '' for k in langchain_object.input_keys}
batch_question = ['start'] if not batch_question else batch_question # 确保点击确定，会执行LLM
report = ''
logger.info(f'process_file batch_question={batch_question}')
logger.info(f'process_file batch_question={batch_question} input_key={input_key}')
for question in batch_question:
if not question:
continue
@@ -332,13 +333,14 @@ class Handler:
await session.send_json(client_id, chat_id, response_step)
report = f"""{report}### {question} \n {result} \n """
start_resp.category = 'report'
await session.send_json(client_id, chat_id, start_resp)
response = ChatResponse(type='end',
intermediate_steps=report,
category='report',
user_id=user_id)
await session.send_json(client_id, chat_id, response)
if len(batch_question) > 1:
start_resp.category = 'report'
await session.send_json(client_id, chat_id, start_resp)
response = ChatResponse(type='end',
intermediate_steps=report,
category='report',
user_id=user_id)
await session.send_json(client_id, chat_id, response)
close_resp = ChatResponse(type='close', category='system', user_id=user_id)
await session.send_json(client_id, chat_id, close_resp)
src/backend/bisheng/chat/manager.py
+68 -32
View File
@@ -4,12 +4,9 @@ import json
import time
import uuid
from collections import defaultdict
from queue import Queue
from typing import Any, Dict, List
from uuid import UUID
from queue import Queue
from loguru import logger
from fastapi import WebSocket, WebSocketDisconnect, status, Request
from bisheng.api.services.audit_log import AuditLogService
from bisheng.api.services.user_service import UserPayload
@@ -25,10 +22,13 @@ from bisheng.database.base import session_getter
from bisheng.database.models.flow import Flow
from bisheng.database.models.message import ChatMessageDao
from bisheng.database.models.user import User, UserDao
from bisheng.graph.utils import cut_graph_bynode, find_next_node
from bisheng.processing.process import process_tweaks
from bisheng.utils.threadpool import ThreadPoolManager, thread_pool
from bisheng.utils.util import get_cache_key
from bisheng_langchain.input_output.output import Report
from fastapi import Request, WebSocket, WebSocketDisconnect, status
from loguru import logger
class ChatHistory(Subject):
@@ -38,10 +38,10 @@ class ChatHistory(Subject):
self.history: Dict[str, List[ChatMessage]] = defaultdict(list)
def add_message(
self,
client_id: str,
chat_id: str,
message: ChatMessage,
self,
client_id: str,
chat_id: str,
message: ChatMessage,
):
"""Add a message to the chat history."""
t1 = time.time()
@@ -187,14 +187,15 @@ class ChatManager:
if 'after sending' in str(exc):
logger.error(exc)
async def dispatch_client(self,
request: Request, # 原始请求体
client_id: str,
chat_id: str,
login_user: UserPayload,
work_type: WorkType,
websocket: WebSocket,
graph_data: dict = None):
async def dispatch_client(
self,
request: Request, # 原始请求体
client_id: str,
chat_id: str,
login_user: UserPayload,
work_type: WorkType,
websocket: WebSocket,
graph_data: dict = None):
client_key = uuid.uuid4().hex
chat_client = ChatClient(request,
client_key,
@@ -243,17 +244,17 @@ class ChatManager:
self.clear_client(client_key)
async def handle_websocket(
self,
flow_id: str,
chat_id: str,
websocket: WebSocket,
user_id: int,
gragh_data: dict = None,
self,
flow_id: str,
chat_id: str,
websocket: WebSocket,
user_id: int,
gragh_data: dict = None,
):
# 建立连接，并存储映射，兼容不复用ws 场景
key_list = set([get_cache_key(flow_id, chat_id)])
await self.connect(flow_id, chat_id, websocket)
autogen_pool = ThreadPoolManager(max_workers=1, thread_name_prefix='autogen')
# autogen_pool = ThreadPoolManager(max_workers=1, thread_name_prefix='autogen')
context_dict = {
get_cache_key(flow_id, chat_id): {
'status': 'init',
@@ -407,10 +408,11 @@ class ChatManager:
if len(res) <= 1: # 说明是新建会话
websocket = self.active_connections[key]
login_user = UserPayload(**{
"user_id": user_id,
"user_name": UserDao.get_user(user_id).user_name,
'user_id': user_id,
'user_name': UserDao.get_user(user_id).user_name,
})
AuditLogService.create_chat_flow(login_user, get_request_ip(websocket), flow_id)
AuditLogService.create_chat_flow(login_user, get_request_ip(websocket),
flow_id)
start_resp.type = 'start'
# should input data
@@ -440,7 +442,7 @@ class ChatManager:
if payload and self.in_memory_cache.get(langchain_obj_key):
action, over = await self.preper_action(flow_id, chat_id, langchain_obj_key, payload,
start_resp, step_resp)
logger.info(
logger.debug(
f"processing_message message={payload.get('inputs')} action={action} over={over}")
if not over:
# task_service: 'TaskService' = get_task_service()
@@ -461,9 +463,13 @@ class ChatManager:
if isinstance(self.in_memory_cache.get(langchain_obj_key), AutoGenChain):
# autogen chain
logger.info(f'autogen_submit {langchain_obj_key}')
autogen_pool.submit(key, Handler(stream_queue=self.stream_queue[key]).dispatch_task, **params)
autogen_pool.submit(key,
Handler(stream_queue=self.stream_queue[key]).dispatch_task,
**params)
else:
thread_pool.submit(key, Handler(stream_queue=self.stream_queue[key]).dispatch_task, **params)
thread_pool.submit(key,
Handler(stream_queue=self.stream_queue[key]).dispatch_task,
**params)
status_ = 'init'
context.update({'status': status_})
context.update({'payload': {}}) # clean message
@@ -489,7 +495,33 @@ class ChatManager:
or 'file_path' in payload['inputs']):
node_data = payload['inputs'].get('data', '') or [payload['inputs']]
graph_data = self.refresh_graph_data(graph_data, node_data)
self.set_cache(langchain_obj_key, None) # rebuild object
# 上传文件就重新解压，有点粗, 只有document loader 需要
node_loader = False
for nod in node_data:
if any('Loader' in x['id'] for x in find_next_node(graph_data, nod['id'])):
node_loader = True
break
if node_loader:
self.set_cache(langchain_obj_key, None) # rebuild object
else:
# 能力，只考虑一个file input的情况
node_id = find_next_node(graph_data, nod['id'])[0]['id']
nodes, edges = cut_graph_bynode(graph_data, node_id)
chat_history = payload.get('chatHistory') or [{
'isSend': False,
'message': '请问想买什么'
}]
last_msg = [m['message'] for m in chat_history if not m['isSend']][-1]
graph_data_input = {'edges': edges, 'nodes': nodes}
graph_data_input = process_tweaks(graph_data_input,
tweaks={node_id: {
'question': last_msg
}})
obj = await build_flow_no_yield(graph_data_input, {}, process_file=True)
obj.abuild()
question = [await x.get_result() for x in obj.get_input_nodes()][-1]
self.set_cache(langchain_obj_key + '_question', [question])
has_file = any(['InputFile' in nd.get('id', '') for nd in node_data])
has_variable = any(['VariableNode' in nd.get('id', '') for nd in node_data])
if has_file:
@@ -590,8 +622,12 @@ class ChatManager:
logger.info(f'init_langchain build_end timecost={time.time() - start_time}')
question = []
for node in graph.vertices:
if node.vertex_type == 'InputNode':
question.extend(await node.get_result())
if node.vertex_type in {'InputNode', 'AudioInputNode', 'FileInputNode'}:
question_parse = await node.get_result()
if isinstance(question_parse, list):
question.extend(question_parse)
else:
question.append(question_parse)
self.set_cache(key_node + '_question', question)
input_nodes = graph.get_input_nodes()
src/backend/bisheng/custom/customs.py
+2 -2
View File
@@ -23,9 +23,9 @@ CUSTOM_NODES = {
},
'memories': {
'PostgresChatMessageHistory':
frontend_node.memories.PostgresChatMessageHistoryFrontendNode(),
frontend_node.memories.PostgresChatMessageHistoryFrontendNode(),
'MongoDBChatMessageHistory':
frontend_node.memories.MongoDBChatMessageHistoryFrontendNode(),
frontend_node.memories.MongoDBChatMessageHistoryFrontendNode(),
},
'chains': {
'SeriesCharacterChain': frontend_node.chains.SeriesCharacterChainNode(),
src/backend/bisheng/database/models/assistant.py
+2 -2
View File
@@ -160,8 +160,8 @@ class AssistantDao(Assistant):
return session.exec(statement).all()
@classmethod
def get_all_assistants(cls, name: str, page: int, limit: int, assistant_ids: List[UUID] = None, status: int = None) \
-> (List[Assistant], int):
def get_all_assistants(cls, name: str, page: int, limit: int, assistant_ids: List[UUID] = None,
status: int = None) -> (List[Assistant], int):
with session_getter() as session:
statement = select(Assistant).where(Assistant.is_delete == 0)
count_statement = session.query(func.count(
src/backend/bisheng/database/models/dataset.py
+93
View File
@@ -0,0 +1,93 @@
from datetime import datetime
from typing import Any, List, Optional
from bisheng.database.base import session_getter
from bisheng.database.models.base import SQLModelSerializable
from sqlalchemy import Column, DateTime, delete, text
from sqlmodel import Field, select
class DatasetBase(SQLModelSerializable):
user_id: Optional[int] = Field(index=True, description='创建用户id')
name: str = Field(index=True, description='数据集名称')
type: str = Field(index=False, default=0, description='预留字段')
description: Optional[str] = Field(index=False, description='数据集描述')
object_name: Optional[str] = Field(index=False, description='数据集S3名称')
create_time: Optional[datetime] = Field(
sa_column=Column(DateTime, nullable=False, server_default=text('CURRENT_TIMESTAMP')))
update_time: Optional[datetime] = Field(
sa_column=Column(DateTime,
nullable=True,
server_default=text('CURRENT_TIMESTAMP'),
onupdate=text('CURRENT_TIMESTAMP')))
class Dataset(DatasetBase, table=True):
id: Optional[int] = Field(default=None, primary_key=True)
class DatasetCreate(DatasetBase):
pass
class DatasetRead(DatasetBase):
user_name: Optional[str]
url: Optional[str]
class DatasetUpdate(DatasetBase):
pass
class DatasetDao(DatasetBase):
@classmethod
def filter_dataset_by_ids(cls,
dataset_ids: List[int],
keyword: str = None,
page: int = 0,
limit: int = 0) -> List[Dataset]:
with session_getter() as session:
query = select(Dataset)
if dataset_ids:
query = query.where(Dataset.id.in_(dataset_ids))
if keyword:
query = query.where(Dataset.name.like(f'%{keyword}%'))
if page and limit:
query = query.order_by(Dataset.update_time.desc()).offset(
(page - 1) * limit).limit(limit)
return session.exec(query).all()
@classmethod
def get_count_by_filter(cls, filters: List[Any]) -> int:
with session_getter() as session:
return session.scalar(select(Dataset.id).where(*filters))
@classmethod
def insert(cls, dataset: DatasetCreate):
with session_getter() as session:
db_insert = Dataset.validate(dataset)
session.add(db_insert)
session.commit()
session.refresh(db_insert)
return db_insert
@classmethod
def get_dataset_by_name(cls, name: str):
with session_getter() as session:
return session.exec(select(Dataset).where(Dataset.name == name)).all()
@classmethod
def update(cls, data: Dataset):
with session_getter() as session:
session.add(data)
session.commit()
session.refresh(data)
return data
@classmethod
def delete(cls, dataset_id: int):
with session_getter() as session:
session.exec(delete(Dataset).where(Dataset.id == dataset_id))
session.commit()
src/backend/bisheng/database/models/flow.py
+44 -23
View File
@@ -95,7 +95,10 @@ class FlowDao(FlowBase):
with session_getter() as session:
session.add(flow_info)
# 创建一个默认的版本
flow_version = FlowVersion(name="v0", is_current=1, data=flow_info.data, flow_id=flow_info.id.hex,
flow_version = FlowVersion(name='v0',
is_current=1,
data=flow_info.data,
flow_id=flow_info.id.hex,
user_id=flow_info.user_id)
session.add(flow_version)
session.commit()
@@ -108,7 +111,8 @@ class FlowDao(FlowBase):
with session_getter() as session:
session.delete(flow_info)
# 删除对应的版本信息
update_statement = update(FlowVersion).where(FlowVersion.flow_id == flow_info.id.hex).values(is_delete=1)
update_statement = update(FlowVersion).where(
FlowVersion.flow_id == flow_info.id.hex).values(is_delete=1)
session.exec(update_statement)
session.commit()
return flow_info
@@ -139,6 +143,12 @@ class FlowDao(FlowBase):
statement = select(Flow).where(Flow.user_id == user_id, Flow.name == name)
return session.exec(statement).first()
@classmethod
def get_flow_list_by_name(cls, name: str) -> List[Flow]:
with session_getter() as session:
statement = select(Flow).where(Flow.name.like('%{}%'.format(name)))
return session.exec(statement).all()
@classmethod
def get_flow_by_access(cls, role_id: int, name: str, page_size: int,
page_num: int) -> List[Tuple[Flow, RoleAccess]]:
@@ -176,7 +186,8 @@ class FlowDao(FlowBase):
elif not extra_ids:
statement = statement.where(Flow.user_id == user_id)
if name:
statement = statement.where(or_(Flow.name.like(f'%{name}%'), Flow.description.like(f'%{name}%')))
statement = statement.where(
or_(Flow.name.like(f'%{name}%'), Flow.description.like(f'%{name}%')))
if status is not None:
statement = statement.where(Flow.status == status)
if flow_ids:
@@ -189,17 +200,22 @@ class FlowDao(FlowBase):
return [Flow.model_validate(f) for f in flows_partial]
@classmethod
def count_flows(cls, user_id: Optional[int], extra_ids: Union[List[str], str], name: str,
status: Optional[int] = None, flow_ids: List[str] = None) -> int:
def count_flows(cls,
user_id: Optional[int],
extra_ids: Union[List[str], str],
name: str,
status: Optional[int] = None,
flow_ids: List[str] = None) -> int:
with session_getter() as session:
count_statement = session.query(func.count(Flow.id))
if extra_ids and isinstance(extra_ids, List):
count_statement = count_statement.filter(or_(Flow.id.in_(extra_ids), Flow.user_id == user_id))
count_statement = count_statement.filter(
or_(Flow.id.in_(extra_ids), Flow.user_id == user_id))
elif not extra_ids:
count_statement = count_statement.filter(Flow.user_id == user_id)
if name:
count_statement = count_statement.filter(or_(Flow.name.like(f'%{name}%'),
Flow.description.like(f'%{name}%')))
count_statement = count_statement.filter(
or_(Flow.name.like(f'%{name}%'), Flow.description.like(f'%{name}%')))
if flow_ids:
count_statement = count_statement.filter(Flow.id.in_(flow_ids))
if status is not None:
@@ -210,17 +226,22 @@ class FlowDao(FlowBase):
def get_all_online_flows(cls, keyword: str = None, flow_ids: List[str] = None) -> List[Flow]:
with session_getter() as session:
statement = select(Flow.id, Flow.user_id, Flow.name, Flow.status, Flow.create_time,
Flow.logo, Flow.update_time, Flow.description, Flow.guide_word).where(
Flow.status == FlowStatus.ONLINE.value)
Flow.logo, Flow.update_time, Flow.description,
Flow.guide_word).where(Flow.status == FlowStatus.ONLINE.value)
if flow_ids:
statement = statement.where(Flow.id.in_(flow_ids))
if keyword:
statement = statement.where(or_(Flow.name.like(f'%{keyword}%'), Flow.description.like(f'%{keyword}%')))
statement = statement.where(
or_(Flow.name.like(f'%{keyword}%'), Flow.description.like(f'%{keyword}%')))
result = session.exec(statement).mappings().all()
return [Flow.model_validate(f) for f in result]
@classmethod
def get_user_access_online_flows(cls, user_id: int, page: int = 0, limit: int = 0, keyword: str = None,
def get_user_access_online_flows(cls,
user_id: int,
page: int = 0,
limit: int = 0,
keyword: str = None,
flow_ids: List[str] = None) -> List[Flow]:
user_role = UserRoleDao.get_user_roles(user_id)
flow_id_extra = []
@@ -233,7 +254,12 @@ class FlowDao(FlowBase):
role_access = RoleAccessDao.get_role_access(role_ids, AccessType.FLOW)
if role_access:
flow_id_extra = [access.third_id for access in role_access]
return FlowDao.get_flows(user_id, flow_id_extra, keyword, FlowStatus.ONLINE.value, flow_ids=flow_ids, page=page,
return FlowDao.get_flows(user_id,
flow_id_extra,
keyword,
FlowStatus.ONLINE.value,
flow_ids=flow_ids,
page=page,
limit=limit)
@classmethod
@@ -243,21 +269,16 @@ class FlowDao(FlowBase):
通过技能ID过滤技能列表，只返回简略信息，不包含data
"""
statement = select(Flow.id, Flow.user_id, Flow.name, Flow.status, Flow.create_time,
Flow.update_time, Flow.description,
Flow.guide_word)
Flow.update_time, Flow.description, Flow.guide_word)
count_statement = select(func.count(Flow.id))
if flow_ids:
statement = statement.where(Flow.id.in_(flow_ids))
count_statement = count_statement.where(Flow.id.in_(flow_ids))
if keyword:
statement = statement.where(or_(
Flow.name.like(f'%{keyword}%'),
Flow.description.like(f'%{keyword}%')
))
count_statement = count_statement.where(or_(
Flow.name.like(f'%{keyword}%'),
Flow.description.like(f'%{keyword}%')
))
statement = statement.where(
or_(Flow.name.like(f'%{keyword}%'), Flow.description.like(f'%{keyword}%')))
count_statement = count_statement.where(
or_(Flow.name.like(f'%{keyword}%'), Flow.description.like(f'%{keyword}%')))
if page and limit:
statement = statement.offset((page - 1) * limit).limit(limit)
statement = statement.order_by(Flow.update_time.desc())
src/backend/bisheng/database/models/group_resource.py
+17
View File
@@ -79,6 +79,23 @@ class GroupResourceDao(GroupResourceBase):
statement = statement.offset(page_size * (page_num - 1)).limit(page_size)
return session.exec(statement).all()
@classmethod
def get_groups_resource(cls,
group_ids: List[int],
resource_types: List[ResourceTypeEnum] = None,
name: str = None,
page_size: int = None,
page_num: int = None) -> list[GroupResource]:
with session_getter() as session:
statement = select(GroupResource).where(GroupResource.group_id.in_(group_ids))
if resource_types:
statement = statement.where(GroupResource.type.in_([r.value for r in resource_types]))
if name:
statement = statement.where(GroupResource.third_id.like(f'%{name}%'))
if page_num and page_size:
statement = statement.offset(page_size * (page_num - 1)).limit(page_size)
return session.exec(statement).all()
@classmethod
def get_resource_group(cls, resource_type: ResourceTypeEnum, third_id: str) -> list[GroupResource]:
"""
src/backend/bisheng/database/models/knowledge.py
+10 -1
View File
@@ -2,6 +2,8 @@ from datetime import datetime
from typing import Any, List, Optional, Union
from sqlmodel.sql.expression import Select, SelectOfScalar
from enum import Enum
from typing import Any, List, Optional, Tuple
from bisheng.database.base import session_getter
from bisheng.database.models.base import SQLModelSerializable
@@ -14,9 +16,16 @@ from sqlalchemy import Column, DateTime, func, text, delete, update
from sqlmodel import Field, or_, select
class KnowledgeTypeEnum(Enum):
QA = 1
NORMAL = 0
class KnowledgeBase(SQLModelSerializable):
user_id: Optional[int] = Field(index=True)
name: str = Field(index=True, min_length=1, max_length=30, description="知识库名, 最少一个字符，最多30个字符")
name: str = Field(index=True)
type: str = Field(index=False, default=0, description='0 为普通知识库，1 为QA知识库')
description: Optional[str] = Field(index=True)
model: Optional[str] = Field(index=False)
collection_name: Optional[str] = Field(index=False)
@@ -81,7 +90,7 @@ class KnowledgeDao(KnowledgeBase):
return session.get(Knowledge, knowledge_id)
@classmethod
def get_list_by_ids(cls, ids: List[int]):
def get_list_by_ids(cls, ids: List[int]) -> List[Knowledge]:
with session_getter() as session:
return session.exec(select(Knowledge).where(Knowledge.id.in_(ids))).all()
src/backend/bisheng/database/models/knowledge_file.py
+125 -1
View File
@@ -1,10 +1,13 @@
import json
from datetime import datetime
from enum import Enum
from typing import List, Optional
from bisheng.database.base import session_getter
from bisheng.database.models.base import SQLModelSerializable
from sqlalchemy import Column, DateTime, String, text
# if TYPE_CHECKING:
from pydantic import validator
from sqlalchemy import JSON, Column, DateTime, String, text
from sqlmodel import Field, delete, func, select
@@ -40,10 +43,54 @@ class KnowledgeFileBase(SQLModelSerializable):
onupdate=text('CURRENT_TIMESTAMP')))
class QAKnowledgeBase(SQLModelSerializable):
user_id: Optional[int] = Field(index=True)
knowledge_id: int = Field(index=True)
questions: List[str] = Field(index=False)
answers: str = Field(index=False)
source: Optional[int] = Field(index=False, description='0: 未知 1: 手动；2: 审计, 3: api')
status: Optional[int] = Field(index=False, description='1: 解析中；2: 解析成功；3: 解析失败')
extra_meta: Optional[str] = Field(index=False)
remark: Optional[str] = Field(sa_column=Column(String(length=512)))
create_time: Optional[datetime] = Field(
sa_column=Column(DateTime, nullable=False, server_default=text('CURRENT_TIMESTAMP')))
update_time: Optional[datetime] = Field(
sa_column=Column(DateTime,
nullable=False,
server_default=text('CURRENT_TIMESTAMP'),
onupdate=text('CURRENT_TIMESTAMP')))
@validator('questions')
def validate_json(v):
# dict_keys(['description', 'name', 'id', 'data'])
if not v:
return v
if not isinstance(v, List):
raise ValueError('question must be a valid JSON')
return v
@validator('answers')
def validate_answer(v):
# dict_keys(['description', 'name', 'id', 'data'])
if not v:
return v
if isinstance(v, List):
return json.dumps(v, ensure_ascii=False)
return v
class KnowledgeFile(KnowledgeFileBase, table=True):
id: Optional[int] = Field(default=None, primary_key=True)
class QAKnowledge(QAKnowledgeBase, table=True):
id: Optional[int] = Field(default=None, primary_key=True)
questions: Optional[List[str]] = Field(default=None, sa_column=Column(JSON))
answers: Optional[str] = Field(default=None, sa_column=Column(String(length=2048)))
class KnowledgeFileRead(KnowledgeFileBase):
id: int
@@ -52,6 +99,12 @@ class KnowledgeFileCreate(KnowledgeFileBase):
pass
class QAKnowledgeUpsert(QAKnowledgeBase):
"""支持修改"""
id: Optional[int]
answers: Optional[List[str]]
class KnowledgeFileDao(KnowledgeFileBase):
@classmethod
@@ -142,3 +195,74 @@ class KnowledgeFileDao(KnowledgeFileBase):
statement = statement.where(KnowledgeFile.status == status)
with session_getter() as session:
return session.scalar(statement)
class QAKnoweldgeDao(QAKnowledgeBase):
@classmethod
def get_qa_knowledge_by_knowledge_id(cls, knowledge_id: int):
with session_getter() as session:
return session.exec(
select(QAKnowledge).where(QAKnowledge.knowledge_id == knowledge_id)).all()
@classmethod
def get_qa_knowledge_by_knowledge_ids(cls, knowledge_ids: List[int]) -> List[QAKnowledge]:
with session_getter() as session:
return session.exec(
select(QAKnowledge).where(QAKnowledge.knowledge_id.in_(knowledge_ids))).all()
@classmethod
def get_qa_knowledge_by_primary_id(cls, qa_id: int) -> QAKnowledge:
with session_getter() as session:
return session.exec(select(QAKnowledge).where(QAKnowledge.id == qa_id)).first()
@classmethod
def update(cls, qa_knowledge: QAKnowledge):
if qa_knowledge.id is None:
raise ValueError('id不能为空')
with session_getter() as session:
session.add(QAKnowledge.validate(qa_knowledge))
session.commit()
session.refresh(qa_knowledge)
return qa_knowledge
@classmethod
def delete_batch(cls, qa_ids: List[int]) -> bool:
with session_getter() as session:
session.exec(delete(QAKnowledge).where(QAKnowledge.id.in_(qa_ids)))
session.commit()
return True
@classmethod
def select_list(cls, ids: List[int]) -> List[QAKnowledge]:
with session_getter() as session:
QAKnowledges = session.exec(select(QAKnowledge).where(QAKnowledge.id.in_(ids))).all()
if not QAKnowledges:
raise ValueError('知识库不存在')
return QAKnowledges
@classmethod
def insert_qa(cls, qa_knowledge: QAKnowledgeUpsert):
with session_getter() as session:
qa = QAKnowledge.validate(qa_knowledge)
session.add(qa)
session.commit()
session.refresh(qa)
return qa
@classmethod
def total_count(cls, sql):
with session_getter() as session:
return session.scalar(sql)
@classmethod
def query_by_condition(cls, sql):
with session_getter() as session:
return session.exec(sql).all()
@classmethod
def query_by_condition_v1(cls, source: List[int], create_start: str, create_end: str):
with session_getter() as session:
sql = select(QAKnowledge).where(QAKnowledge.source.in_(source)).where(
QAKnowledge.create_time.between(create_start, create_end))
return session.exec(sql).all()
src/backend/bisheng/database/models/message.py
+55 -5
View File
@@ -1,12 +1,12 @@
from datetime import datetime
from typing import Dict, Optional, List
from typing import Dict, List, Optional, Tuple
from uuid import UUID
from bisheng.database.base import session_getter
from bisheng.database.models.base import SQLModelSerializable
from loguru import logger
from pydantic import BaseModel
from sqlalchemy import JSON, Column, DateTime, String, Text, func, text, update
from sqlalchemy import JSON, Column, DateTime, String, Text, case, func, text, update
from sqlmodel import Field, delete, select
@@ -22,6 +22,7 @@ class MessageBase(SQLModelSerializable):
user_id: Optional[str] = Field(index=True, description='用户id')
liked: Optional[int] = Field(index=False, default=0, description='用户是否喜欢 0未评价/1 喜欢/2 不喜欢')
solved: Optional[int] = Field(index=False, default=0, description='用户是否喜欢 0未评价/1 解决/2 未解决')
copied: Optional[int] = Field(index=False, default=0, description='用户是否复制 0：未复制 1：已复制')
sender: Optional[str] = Field(index=False, default='', description='autogen 的发送方')
receiver: Optional[Dict] = Field(index=False, default=None, description='autogen 的发送方')
intermediate_steps: Optional[str] = Field(sa_column=Column(Text), description='过程日志')
@@ -74,6 +75,48 @@ class MessageDao(MessageBase):
with session_getter() as session:
return session.scalar(base_condition)
@classmethod
def app_list_group_by_chat_id(
cls,
page_size: int,
page_num: int,
flow_ids: Optional[list[str]],
user_ids: Optional[list[int]],
) -> Tuple[List[Dict], int]:
with session_getter() as session:
count_stat = select(func.count(func.distinct(ChatMessage.chat_id)))
sql = select(ChatMessage.chat_id, ChatMessage.user_id, ChatMessage.flow_id,
func.max(ChatMessage.create_time).label('create_time'),
func.sum(case((ChatMessage.liked == 1, 1), else_=0)),
func.sum(case((ChatMessage.liked == 2, 1), else_=0)),
func.sum(case((ChatMessage.copied == 1, 1), else_=0)), )
if flow_ids:
count_stat = count_stat.where(ChatMessage.flow_id.in_(flow_ids))
sql = sql.where(ChatMessage.flow_id.in_(flow_ids))
if user_ids:
count_stat = count_stat.where(ChatMessage.user_id.in_(user_ids))
sql = sql.where(ChatMessage.user_id.in_(user_ids))
sql = sql.group_by(ChatMessage.chat_id, ChatMessage.user_id,
ChatMessage.flow_id).order_by(
func.max(ChatMessage.create_time).desc()).offset(
page_size * (page_num - 1)).limit(page_size)
res_list = session.exec(sql).all()
total_count = session.scalar(count_stat)
dict_res = [{
'chat_id': chat_id,
'user_id': user_id,
'flow_id': flow_id,
'like_count': like_num,
'dislike_count': dislike_num,
'copied_count': copied_num,
'create_time': create_time
} for chat_id, user_id, flow_id, create_time, like_num, dislike_num, copied_num in res_list]
logger.info(res_list)
return dict_res, total_count
class ChatMessageDao(MessageBase):
@@ -92,7 +135,8 @@ class ChatMessageDao(MessageBase):
"""
获取每个会话最近的一次消息内容
"""
statement = select(ChatMessage.chat_id, func.max(ChatMessage.id)).where(ChatMessage.chat_id.in_(chat_ids))
statement = select(ChatMessage.chat_id,
func.max(ChatMessage.id)).where(ChatMessage.chat_id.in_(chat_ids))
if category:
statement = statement.where(ChatMessage.category == category)
statement = statement.group_by(ChatMessage.chat_id)
@@ -163,8 +207,7 @@ class ChatMessageDao(MessageBase):
@classmethod
def update_message(cls, message_id: int, user_id: int, message: str):
with session_getter() as session:
statement = update(ChatMessage).where(
ChatMessage.id == message_id).where(
statement = update(ChatMessage).where(ChatMessage.id == message_id).where(
ChatMessage.user_id == user_id).values(message=message)
session.exec(statement)
session.commit()
@@ -176,3 +219,10 @@ class ChatMessageDao(MessageBase):
session.commit()
session.refresh(message)
return message
@classmethod
def update_message_copied(cls, message_id: int, copied: int):
with session_getter() as session:
statement = update(ChatMessage).where(ChatMessage.id == message_id).values(copied=copied)
session.exec(statement)
session.commit()
src/backend/bisheng/database/models/preset_train.py
+24 -3
View File
@@ -1,9 +1,10 @@
from datetime import datetime
from typing import List, Optional
from typing import List, Optional, Tuple
from uuid import UUID, uuid4
from bisheng.database.base import session_getter
from bisheng.database.models.base import SQLModelSerializable
from sqlalchemy import func
from sqlmodel import Column, DateTime, Field, select, text
@@ -14,10 +15,13 @@ class PresetTrainBase(SQLModelSerializable):
name: str = Field(default='', index=True, description='上传的文件名字')
user_id: str = Field(default='', index=True, description='创建人ID')
user_name: str = Field(default='', index=True, description='创建人姓名')
type: int = Field(default=0, index=True, description='0 文件 1 QA')
create_time: Optional[datetime] = Field(sa_column=Column(
DateTime, nullable=False, index=True, server_default=text('CURRENT_TIMESTAMP')))
update_time: Optional[datetime] = Field(sa_column=Column(
DateTime, nullable=False, server_default=text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP')))
update_time: Optional[datetime] = Field(
sa_column=Column(DateTime,
nullable=False,
server_default=text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP')))
class PresetTrain(PresetTrainBase, table=True):
@@ -54,3 +58,20 @@ class PresetTrainDao(PresetTrainBase):
with session_getter() as session:
statement = select(PresetTrain)
return session.exec(statement).all()
@classmethod
def search_name(cls,
keyword: str = None,
page_size: int = None,
page_num: int = None) -> Tuple[List[PresetTrain], int]:
with session_getter() as session:
statement = select(PresetTrain)
count = select(func.count(PresetTrain.id))
if keyword:
statement = statement.where(PresetTrain.name.like('%{}%'.format(keyword)))
count = count.where(PresetTrain.name.like('%{}%'.format(keyword)))
if page_num and page_size:
statement = statement.offset((page_num - 1) * page_size).limit(page_size)
statement = statement.order_by(PresetTrain.create_time.desc())
return session.exec(statement).all(), session.scalar(count)
src/backend/bisheng/database/models/user.py
+21 -12
View File
@@ -1,15 +1,14 @@
from datetime import datetime
from typing import List, Optional
from pydantic import validator
from sqlalchemy import Column, DateTime, text, func
from sqlmodel import Field, select
from bisheng.database.base import session_getter
from bisheng.database.models.base import SQLModelSerializable
from bisheng.database.models.role import DefaultRole, AdminRole
from bisheng.database.models.role import AdminRole, DefaultRole
from bisheng.database.models.user_group import UserGroup
from bisheng.database.models.user_role import UserRole
from pydantic import validator
from sqlalchemy import Column, DateTime, func, text
from sqlmodel import Field, select
class UserBase(SQLModelSerializable):
@@ -38,10 +37,9 @@ class UserBase(SQLModelSerializable):
class User(UserBase, table=True):
user_id: Optional[int] = Field(default=None, primary_key=True)
password: str = Field(index=False)
password_update_time: Optional[datetime] = Field(sa_column=Column(DateTime,
nullable=False,
server_default=text('CURRENT_TIMESTAMP')),
description="密码最近的修改时间")
password_update_time: Optional[datetime] = Field(sa_column=Column(
DateTime, nullable=False, server_default=text('CURRENT_TIMESTAMP')),
description='密码最近的修改时间')
class UserRead(UserBase):
@@ -65,7 +63,7 @@ class UserLogin(UserBase):
class UserCreate(UserBase):
password: Optional[str] = Field(default="")
password: Optional[str] = Field(default='')
captcha_key: Optional[str]
captcha: Optional[str]
@@ -104,7 +102,11 @@ class UserDao(UserBase):
return user
@classmethod
def filter_users(cls, user_ids: List[int], keyword: str = None, page: int = 0, limit: int = 0) -> (List[User], int):
def filter_users(cls,
user_ids: List[int],
keyword: str = None,
page: int = 0,
limit: int = 0) -> (List[User], int):
statement = select(User)
count_statement = select(func.count(User.user_id))
if user_ids:
@@ -125,6 +127,12 @@ class UserDao(UserBase):
statement = select(User).where(User.user_name == user_name)
return session.exec(statement).first()
@classmethod
def search_user_by_name(cls, user_name: str) -> List[User] | None:
with session_getter() as session:
statement = select(User).where(User.user_name.like('%{}%'.format(user_name)))
return session.exec(statement).all()
@classmethod
def create_user(cls, db_user: User) -> User:
with session_getter() as session:
@@ -164,7 +172,8 @@ class UserDao(UserBase):
return user
@classmethod
def add_user_with_groups_and_roles(cls, user: User, group_ids: List[int], role_ids: List[int]) -> User:
def add_user_with_groups_and_roles(cls, user: User, group_ids: List[int],
role_ids: List[int]) -> User:
with session_getter() as session:
session.add(user)
session.flush()
src/backend/bisheng/graph/utils.py
+39 -1
View File
@@ -243,7 +243,8 @@ def update_source_handle(new_edge, g_nodes, g_edges):
dict: The updated edge with the new source handle.
"""
last_node = copy.deepcopy(find_last_node(g_nodes, g_edges))
new_edge['sourceHandle'] = new_edge['sourceHandle'].replace(new_edge['source'], last_node['id'])
new_edge['sourceHandle'] = new_edge['sourceHandle'].replace(new_edge['source'],
last_node['id'])
new_edge['source'] = last_node['id']
return new_edge
@@ -273,3 +274,40 @@ def get_updated_edges(base_flow, g_nodes, g_edges, group_node_id):
if edge['target'] == group_node_id or edge['source'] == group_node_id:
updated_edges.append(new_edge)
return updated_edges
def find_next_node(graph_data: Dict, node_id: str) -> List[Dict]:
"""
Finds the next node in the graph data based on the given node id.
"""
nodes = graph_data.get('nodes', [])
edges = graph_data.get('edges', [])
edges_ = [e['target'] for e in edges if e['source'] == node_id]
return [n for n in nodes if n['id'] in edges_]
def cut_graph_bynode(graph_data: Dict, node_id: str) -> List[Dict]:
"""
通过node_id 找到和node相关的所有依赖节点。
"""
nodes = graph_data.get('nodes', [])
edges = graph_data.get('edges', [])
nodes_new_list = []
edges_new_list = []
iflast = True
for e in edges:
if e['target'] == node_id:
iflast = False
node_list, edge_list = cut_graph_bynode(graph_data, e['source'])
nodes_new_list.extend(node_list)
edges_new_list.extend(edge_list)
nodes_new_list.append([n for n in nodes if n['id'] == node_id][0])
edges_new_list.append(e)
if iflast:
for node in nodes:
if node['id'] == node_id:
return [node], []
return nodes_new_list, edges_new_list
src/backend/bisheng/interface/initialize/loading.py
+29 -21
View File
@@ -25,6 +25,7 @@ from bisheng.interface.wrappers.base import wrapper_creator
from bisheng.settings import settings
from bisheng.utils import validate
from bisheng.utils.constants import NODE_ID_DICT, PRESET_QUESTION
from bisheng.utils.embedding import decide_embeddings
from bisheng_langchain.vectorstores import VectorStoreFilterRetriever
from langchain.agents import agent as agent_module
from langchain.agents.agent import AgentExecutor
@@ -39,8 +40,6 @@ from loguru import logger
from pydantic import SecretStr, ValidationError, create_model
from pydantic.fields import FieldInfo
from bisheng.utils.embedding import decide_embeddings
if TYPE_CHECKING:
from bisheng import CustomComponent
@@ -211,6 +210,13 @@ def instantiate_input_output(node_type, class_object, params, id_dict):
return [file_path, file_name2 if file_name2 else file_path[1]]
else:
return ''
if 'file_path' in params:
file_path = params['file_path']
if not file_path:
return ''
if isinstance(file_path, list):
params['file_path'] = file_path[0]
return class_object(**params).text()
@@ -474,33 +480,35 @@ def instantiate_vectorstore(node_type: str, class_object: Type[VectorStore], par
# 过滤掉用户没有权限的知识库
# TODO zgq 后续统一技能执行流程后将和业务有关的逻辑都迁移到初始化技能对象之前
if node_type == "MilvusWithPermissionCheck" or node_type == "ElasticsearchWithPermissionCheck":
col_name = "collection_name"
if node_type == "ElasticsearchWithPermissionCheck":
col_name = "index_name"
if node_type == 'MilvusWithPermissionCheck' or node_type == 'ElasticsearchWithPermissionCheck':
col_name = 'collection_name'
if node_type == 'ElasticsearchWithPermissionCheck':
col_name = 'index_name'
# 获取执行用户 有权限查看的知识库列表
knowledge_ids = [one["key"] for one in params[col_name]]
knowledge_ids = [one['key'] for one in params[col_name]]
knowledge_list = KnowledgeDao.judge_knowledge_permission(user_name, knowledge_ids)
logger.debug(f"{node_type} after filter, get knowledge_list: {knowledge_list}")
logger.debug(f'{node_type} after filter, get knowledge_list: {knowledge_list}')
if not knowledge_list:
logger.warning(f"{node_type}: after filter, get zero knowledge")
logger.warning(f'{node_type}: after filter, get zero knowledge')
# 没有任何知识库的话，提供假的embedding和空的collection_name
if node_type == "MilvusWithPermissionCheck":
if node_type == 'MilvusWithPermissionCheck':
params[col_name] = []
params["partition_keys"] = []
params['partition_keys'] = []
for knowledge in knowledge_list:
params[col_name].append(knowledge.collection_name)
if knowledge.collection_name.startswith("partition"):
params["partition_keys"].append(knowledge.id)
if knowledge.collection_name.startswith('partition'):
params['partition_keys'].append(knowledge.id)
else:
params["partition_keys"].append(None)
params["embedding"] = decide_embeddings(knowledge_list[0].model) if knowledge_list else FakeEmbedding()
params['partition_keys'].append(None)
params['embedding'] = decide_embeddings(
knowledge_list[0].model) if knowledge_list else FakeEmbedding()
else:
params[col_name] = [knowledge.index_name or knowledge.collection_name
for knowledge in knowledge_list]
params[col_name] = [
knowledge.index_name or knowledge.collection_name for knowledge in knowledge_list
]
if initializer := vecstore_initializer.get(class_object.__name__):
vecstore = initializer(class_object, params, search_kwargs)
@@ -565,8 +573,8 @@ def instantiate_documentloader(class_object: Type[BaseLoader], params: Dict):
def instantiate_textsplitter(
class_object,
params: Dict,
class_object,
params: Dict,
):
try:
documents = params.pop('documents')
@@ -577,7 +585,7 @@ def instantiate_textsplitter(
'Try changing the chunk_size of the Text Splitter.') from exc
if ('separator_type' in params
and params['separator_type'] == 'Text') or 'separator_type' not in params:
and params['separator_type'] == 'Text') or 'separator_type' not in params:
params.pop('separator_type', None)
# separators might come in as an escaped string like \\n
# so we need to convert it to a string
@@ -608,7 +616,7 @@ def replace_zero_shot_prompt_with_prompt_template(nodes):
# Build Prompt Template
tools = [
tool for tool in nodes if tool['type'] != 'chatOutputNode'
and 'Tool' in tool['data']['node']['base_classes']
and 'Tool' in tool['data']['node']['base_classes']
]
node['data'] = build_prompt_template(prompt=node['data'], tools=tools)
break
src/backend/bisheng/interface/vector_store/custom.py
+142 -93
View File
@@ -1,20 +1,19 @@
from abc import ABC
from typing import Optional, Any, List, Tuple, Dict, TYPE_CHECKING, Iterable, Callable
from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Tuple
import jieba
from bisheng_langchain.vectorstores.elastic_keywords_search import DEFAULT_PROMPT
from bisheng_langchain.vectorstores.milvus import DEFAULT_MILVUS_CONNECTION
from langchain.chains.llm import LLMChain
from langchain.docstore.document import Document
from langchain.embeddings.base import Embeddings
from langchain.utils import get_from_dict_or_env
from langchain.vectorstores.base import VectorStore
from langchain_community.vectorstores.milvus import Milvus as MilvusLangchain
from langchain_core.language_models import BaseLLM
from langchain_core.prompts import PromptTemplate
from langchain.utils import get_from_dict_or_env
from langchain.embeddings.base import Embeddings
from langchain_community.vectorstores.milvus import Milvus as MilvusLangchain, Milvus
from langchain.docstore.document import Document
from langchain.vectorstores.base import VectorStore
from loguru import logger
from bisheng_langchain.vectorstores.milvus import DEFAULT_MILVUS_CONNECTION
from bisheng_langchain.vectorstores.elastic_keywords_search import DEFAULT_PROMPT
if TYPE_CHECKING:
from elasticsearch import Elasticsearch # noqa: F401
@@ -49,16 +48,65 @@ class MilvusWithPermissionCheck(MilvusLangchain):
# Default search params when one is not provided.
self.default_search_params = {
'IVF_FLAT': {'metric_type': 'L2', 'params': {'nprobe': 10}},
'IVF_SQ8': {'metric_type': 'L2', 'params': {'nprobe': 10}},
'IVF_PQ': {'metric_type': 'L2', 'params': {'nprobe': 10}},
'HNSW': {'metric_type': 'L2', 'params': {'ef': 100}},
'RHNSW_FLAT': {'metric_type': 'L2', 'params': {'ef': 10}},
'RHNSW_SQ': {'metric_type': 'L2', 'params': {'ef': 10}},
'RHNSW_PQ': {'metric_type': 'L2', 'params': {'ef': 10}},
'IVF_HNSW': {'metric_type': 'L2', 'params': {'nprobe': 10, 'ef': 10}},
'ANNOY': {'metric_type': 'L2', 'params': {'search_k': 10}},
'AUTOINDEX': {'metric_type': 'L2', 'params': {}},
'IVF_FLAT': {
'metric_type': 'L2',
'params': {
'nprobe': 10
}
},
'IVF_SQ8': {
'metric_type': 'L2',
'params': {
'nprobe': 10
}
},
'IVF_PQ': {
'metric_type': 'L2',
'params': {
'nprobe': 10
}
},
'HNSW': {
'metric_type': 'L2',
'params': {
'ef': 100
}
},
'RHNSW_FLAT': {
'metric_type': 'L2',
'params': {
'ef': 10
}
},
'RHNSW_SQ': {
'metric_type': 'L2',
'params': {
'ef': 10
}
},
'RHNSW_PQ': {
'metric_type': 'L2',
'params': {
'ef': 10
}
},
'IVF_HNSW': {
'metric_type': 'L2',
'params': {
'nprobe': 10,
'ef': 10
}
},
'ANNOY': {
'metric_type': 'L2',
'params': {
'search_k': 10
}
},
'AUTOINDEX': {
'metric_type': 'L2',
'params': {}
},
}
self.embedding_func = embedding_function
@@ -106,7 +154,7 @@ class MilvusWithPermissionCheck(MilvusLangchain):
raise e
if not self.col:
logger.warning("No collection found, please confirm user have knowledge access")
logger.warning('No collection found, please confirm user have knowledge access')
# raise ValueError("No collection found, please confirm collection name correctly.")
# Initialize the vector store
self._init()
@@ -115,13 +163,14 @@ class MilvusWithPermissionCheck(MilvusLangchain):
from pymilvus import connections
connections.remove_connection(using)
def _init(self,
embeddings: Optional[list] = None,
metadatas: Optional[list[dict]] = None,
partition_names: Optional[list] = None,
replica_number: int = 1,
timeout: Optional[float] = None,
) -> None:
def _init(
self,
embeddings: Optional[list] = None,
metadatas: Optional[list[dict]] = None,
partition_names: Optional[list] = None,
replica_number: int = 1,
timeout: Optional[float] = None,
) -> None:
self._extract_fields(col_index=0)
self._create_search_params()
self._load()
@@ -175,18 +224,18 @@ class MilvusWithPermissionCheck(MilvusLangchain):
@classmethod
def from_texts(
cls,
texts: List[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
collection_name: list[str] = None,
connection_args: dict[str, Any] = DEFAULT_MILVUS_CONNECTION,
consistency_level: str = "Session",
index_params: Optional[dict] = None,
search_params: Optional[dict] = None,
drop_old: bool = False,
no_embedding: bool = False,
**kwargs: Any,
cls,
texts: List[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
collection_name: list[str] = None,
connection_args: dict[str, Any] = DEFAULT_MILVUS_CONNECTION,
consistency_level: str = 'Session',
index_params: Optional[dict] = None,
search_params: Optional[dict] = None,
drop_old: bool = False,
no_embedding: bool = False,
**kwargs: Any,
):
"""
no insert data into milvus, only search from milvus
@@ -204,13 +253,13 @@ class MilvusWithPermissionCheck(MilvusLangchain):
return vector_db
def similarity_search(
self,
query: str,
k: int = 4,
param: Optional[dict] = None,
expr: Optional[str] = None,
timeout: Optional[int] = None,
**kwargs: Any,
self,
query: str,
k: int = 4,
param: Optional[dict] = None,
expr: Optional[str] = None,
timeout: Optional[int] = None,
**kwargs: Any,
) -> List[Document]:
"""Perform a similarity search against the query string.
@@ -236,13 +285,13 @@ class MilvusWithPermissionCheck(MilvusLangchain):
return [doc for doc, _ in res]
def similarity_search_by_vector(
self,
embedding: List[float],
k: int = 4,
param: Optional[dict] = None,
expr: Optional[str] = None,
timeout: Optional[int] = None,
**kwargs: Any,
self,
embedding: List[float],
k: int = 4,
param: Optional[dict] = None,
expr: Optional[str] = None,
timeout: Optional[int] = None,
**kwargs: Any,
) -> List[Document]:
"""Perform a similarity search against the query string.
@@ -268,13 +317,13 @@ class MilvusWithPermissionCheck(MilvusLangchain):
return [doc for doc, _ in res]
def similarity_search_with_score(
self,
query: str,
k: int = 4,
param: Optional[dict] = None,
expr: Optional[str] = None,
timeout: Optional[int] = None,
**kwargs: Any,
self,
query: str,
k: int = 4,
param: Optional[dict] = None,
expr: Optional[str] = None,
timeout: Optional[int] = None,
**kwargs: Any,
) -> List[Tuple[Document, float]]:
"""Perform a search on a query string and return results with score.
@@ -314,13 +363,13 @@ class MilvusWithPermissionCheck(MilvusLangchain):
return res
def similarity_search_with_score_by_vector(
self,
embedding: List[float],
k: int = 4,
param: Optional[dict] = None,
expr: Optional[str] = None,
timeout: Optional[int] = None,
**kwargs: Any,
self,
embedding: List[float],
k: int = 4,
param: Optional[dict] = None,
expr: Optional[str] = None,
timeout: Optional[int] = None,
**kwargs: Any,
) -> List[Tuple[Document, float]]:
"""Perform a search on a query string and return results with score.
@@ -355,7 +404,7 @@ class MilvusWithPermissionCheck(MilvusLangchain):
output_fields = self.fields[:]
output_fields.remove(self._vector_field)
finally_k = kwargs.pop("k", k)
finally_k = kwargs.pop('k', k)
ret = []
@@ -408,13 +457,13 @@ class ElasticsearchWithPermissionCheck(VectorStore, ABC):
"""
def __init__(
self,
elasticsearch_url: str,
index_name: List[str],
drop_old: Optional[bool] = False,
*,
ssl_verify: Optional[Dict[str, Any]] = None,
llm_chain: Optional[LLMChain] = None,
self,
elasticsearch_url: str,
index_name: List[str],
drop_old: Optional[bool] = False,
*,
ssl_verify: Optional[Dict[str, Any]] = None,
llm_chain: Optional[LLMChain] = None,
):
"""Initialize with necessary components."""
try:
@@ -477,10 +526,11 @@ class ElasticsearchWithPermissionCheck(VectorStore, ABC):
keywords = eval(keywords_str)
if not isinstance(keywords, list):
raise ValueError('Keywords extracted by llm is not list.')
except Exception as e:
except Exception:
keywords = jieba.analyse.extract_tags(query, topK=10, withWeight=False)
else:
keywords = jieba.analyse.extract_tags(query, topK=10, withWeight=False)
keywords = keywords or [query]
logger.debug(f'finally search keywords: {keywords}')
match_query = {'bool': {must_or_should: []}}
for key in keywords:
@@ -491,13 +541,12 @@ class ElasticsearchWithPermissionCheck(VectorStore, ABC):
response = self.client_search(self.client, one_index_name, match_query, size=k)
hits = [hit for hit in response['hits']['hits']]
for hit in hits:
ret.append((
Document(page_content=hit['_source']['text'], metadata=hit['_source']['metadata']),
hit['_score']
))
logger.debug(f'ElasticsearchWithPermissionCheck Search {one_index_name} results: {hits}')
ret.append((Document(page_content=hit['_source']['text'],
metadata=hit['_source']['metadata']), hit['_score']))
logger.debug(
f'ElasticsearchWithPermissionCheck Search {one_index_name} results: {hits}')
logger.debug(f'ElasticsearchWithPermissionCheck Search all results: {len(ret)}')
finally_k = kwargs.pop("finally_k", k)
finally_k = kwargs.pop('finally_k', k)
ret.sort(key=lambda x: x[1], reverse=True)
ret = ret[:finally_k]
logger.debug(f'ElasticsearchWithPermissionCheck Search finally results: {len(ret)}')
@@ -515,17 +564,17 @@ class ElasticsearchWithPermissionCheck(VectorStore, ABC):
@classmethod
def from_texts(
cls,
texts: List[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
ids: Optional[List[str]] = None,
index_name: Optional[List[str]] = None,
refresh_indices: bool = True,
llm: Optional[BaseLLM] = None,
prompt: Optional[PromptTemplate] = DEFAULT_PROMPT,
drop_old: Optional[bool] = False,
**kwargs: Any,
cls,
texts: List[str],
embedding: Embeddings,
metadatas: Optional[List[dict]] = None,
ids: Optional[List[str]] = None,
index_name: Optional[List[str]] = None,
refresh_indices: bool = True,
llm: Optional[BaseLLM] = None,
prompt: Optional[PromptTemplate] = DEFAULT_PROMPT,
drop_old: Optional[bool] = False,
**kwargs: Any,
):
"""Construct ElasticKeywordsSearch wrapper from raw documents.
src/backend/bisheng/main.py
+11 -7
View File
@@ -21,12 +21,15 @@ from fastapi_jwt_auth.exceptions import AuthJWTException
from loguru import logger
def handle_http_exception(req: Request, exc: HTTPException) -> ORJSONResponse:
msg = {
'status_code': exc.status_code,
'status_message': exc.detail['error'] if isinstance(exc.detail, dict) else exc.detail
}
logger.error(f'{req.method} {req.url} {exc.status_code} {exc.detail}')
def handle_http_exception(req: Request, exc: Exception) -> ORJSONResponse:
if isinstance(exc, HTTPException):
msg = {
'status_code': exc.status_code,
'status_message': exc.detail['error'] if isinstance(exc.detail, dict) else exc.detail
}
else:
msg = {'status_code': 500, 'status_message': str(exc)}
logger.error(f'{req.method} {req.url} {str(exc)}')
return ORJSONResponse(content=msg)
@@ -38,7 +41,8 @@ def handle_request_validation_error(req: Request, exc: RequestValidationError) -
_EXCEPTION_HANDLERS = {
HTTPException: handle_http_exception,
RequestValidationError: handle_request_validation_error
RequestValidationError: handle_request_validation_error,
Exception: handle_http_exception
}
src/backend/bisheng/template/frontend_node/input_output.py
+45 -8
View File
@@ -5,10 +5,30 @@ from bisheng.template.frontend_node.base import FrontendNode
from bisheng.template.template.base import Template
def build_file_field(suffixes: list,
fileTypes: list,
name: str = 'file_path',
fieldType='fileNode') -> TemplateField:
"""Build a template field for a document loader."""
return TemplateField(
field_type=fieldType,
required=True,
show=True,
name=name,
value='',
suffixes=suffixes,
fileTypes=fileTypes,
)
class InputOutputNode(FrontendNode):
name: str = 'InputOutputNode'
base_classes: list[str] = ['input', 'output']
def add_extra_base_classes(self) -> None:
if self.name in {'AudioInputNode', 'FileInputNode'}:
self.base_classes.append('input')
def add_extra_fields(self) -> None:
pass
@@ -36,6 +56,25 @@ class InputOutputNode(FrontendNode):
field.show = True
field.field_type = 'variable'
field.required = True
if name == 'AudioInputNode':
if field.name == 'file_path':
field.show = True
field.field_type = 'fileNode'
field.required = True
field.suffixes = ['.mp3']
field.fileTypes = ['mp3']
elif field.name == 'openai_proxy':
field.show = True
if name == 'FileInputNode':
if field.name == 'file_path':
field.show = True
field.field_type = 'fileNode'
field.required = True
field.suffixes = ['.jpg', '.png', '.jpeg']
field.fileTypes = ['jpg', 'png', 'jpeg']
elif field.name == 'openai_proxy':
field.show = True
class InputNode(FrontendNode):
@@ -72,14 +111,12 @@ class InputFileNode(FrontendNode):
name='file_path',
value='',
),
TemplateField(
field_type='str',
show=True,
name='file_type',
placeholder='提示上传文件类型',
display_name='Name',
info='Tips for which file should upload'
),
TemplateField(field_type='str',
show=True,
name='file_type',
placeholder='提示上传文件类型',
display_name='Name',
info='Tips for which file should upload'),
],
)
description: str = """输入节点，用来自动对接输入"""
src/backend/bisheng/utils/constants.py
+2 -9
View File
@@ -8,15 +8,8 @@ OPENAI_MODELS = [
'text-ada-001',
]
CHAT_OPENAI_MODELS = [
'gpt-3.5-turbo-0613',
'gpt-3.5-turbo',
'gpt-3.5-turbo-16k-0613',
'gpt-3.5-turbo-16k',
'gpt-4-0613',
'gpt-4-32k-0613',
'gpt-4',
'gpt-4-32k',
'gpt-4-1106-preview',
'gpt-3.5-turbo-0613', 'gpt-3.5-turbo', 'gpt-3.5-turbo-16k-0613', 'gpt-3.5-turbo-16k',
'gpt-4-0613', 'gpt-4-32k-0613', 'gpt-4', 'gpt-4-32k', 'gpt-4-1106-preview', 'gpt-4o'
]
ANTHROPIC_MODELS = [
src/backend/test/milvus_trans.py
+11 -10
View File
@@ -154,18 +154,19 @@ import requests
def milvus_clean():
milvus_cli = MilvusClient(uri='http://192.168.106.116:19530')
milvus_cli = MilvusClient(uri='http://192.168.106.109:19530')
collection = milvus_cli.list_collections()
for col in collection:
if col.startswith('rag'):
if col.startswith('tmp'):
print(col)
collection_col = Collection(col, using=milvus_cli._using)
# milvus_cli.drop_collection(col)
try:
collection_col.release(timeout=1)
except Exception:
continue
# collection_col = Collection(col, using=milvus_cli._using)
milvus_cli.drop_collection(col)
# try:
# collection_col.release(timeout=1)
# except Exception:
# continue
def elastic_clean():
@@ -184,6 +185,6 @@ def elastic_clean():
print(x)
# milvus_clean()
elastic_clean()
milvus_clean()
# elastic_clean()
# milvus_trans()
src/bisheng-langchain/bisheng_langchain/chains/qa_generation/base_v2.py
+14 -33
View File
@@ -134,8 +134,6 @@ class TrainsetGenerator:
chunk_size: int = 1024,
seed: int = 42,
prompt: Optional[ChatPromptTemplate] = SEED_QUESTION_CHAT_PROMPT,
filter_lowquality_context: bool = False,
filter_lowquality_question: bool = False,
answer_prompt: Optional[HumanMessagePromptTemplate] = ANSWER_FORMULATE,
) -> None:
self.generator_llm = generator_llm
@@ -154,8 +152,6 @@ class TrainsetGenerator:
self.threshold = 5.0
self.rng = default_rng(seed)
self.prompt = prompt
self.filter_lowquality_context = filter_lowquality_context
self.filter_lowquality_question = filter_lowquality_question
if answer_prompt is None:
answer_prompt = ANSWER_FORMULATE
self.answer_prompt = answer_prompt
@@ -167,8 +163,6 @@ class TrainsetGenerator:
chunk_size: int = 512,
trainset_distribution: dict = DEFAULT_TRAIN_DISTRIBUTION,
prompt: Optional[ChatPromptTemplate] = SEED_QUESTION_CHAT_PROMPT,
filter_lowquality_context: bool = False,
filter_lowquality_question: bool = False,
answer_prompt: Optional[PromptTemplate] = ANSWER_FORMULATE,
):
generator_llm = llm
@@ -179,8 +173,6 @@ class TrainsetGenerator:
chunk_size=chunk_size,
trainset_distribution=trainset_distribution,
prompt=prompt,
filter_lowquality_context=filter_lowquality_context,
filter_lowquality_question=filter_lowquality_question,
answer_prompt=answer_prompt,
)
@@ -324,17 +316,14 @@ class TrainsetGenerator:
)
text_chunk = " ".join([node.get_content() for node in nodes])
if self.filter_lowquality_context:
score = self._filter_context(text_chunk)
if not score:
continue
score = self._filter_context(text_chunk)
if not score:
continue
seed_question = self._seed_question(text_chunk)
question = seed_question
if self.filter_lowquality_question:
is_valid_question = self._filter_question(question)
else:
is_valid_question = True
# is_valid_question = self._filter_question(question)
is_valid_question = True
if is_valid_question:
context = [text_chunk] * len(question.split("\n"))
is_conv = len(context) > 1
@@ -372,8 +361,6 @@ class QAGenerationChainV2(Chain):
llm: BaseLanguageModel,
k: Optional[int] = None,
chunk_size: int = 512,
filter_lowquality_context: bool = False,
filter_lowquality_question: bool = False,
question_prompt: Optional[ChatPromptTemplate] = SEED_QUESTION_CHAT_PROMPT,
answer_prompt: Optional[HumanMessagePromptTemplate] = ANSWER_FORMULATE,
**kwargs: Any,
@@ -390,14 +377,8 @@ class QAGenerationChainV2(Chain):
Returns:
a QAGenerationChain class
"""
generator = TrainsetGenerator.from_default(
llm,
chunk_size=chunk_size,
prompt=question_prompt,
answer_prompt=answer_prompt,
filter_lowquality_context=filter_lowquality_context,
filter_lowquality_question=filter_lowquality_question
)
generator = TrainsetGenerator.from_default(llm, chunk_size=chunk_size, prompt=question_prompt,
answer_prompt=answer_prompt)
return cls(documents=documents, generator=generator, k=k, **kwargs)
@property
@@ -424,14 +405,14 @@ class QAGenerationChainV2(Chain):
dataset = self.generator.generate(documents=self.documents, train_size=self.k)
df = dataset.to_pandas()
qa_pairs = df.to_dict("records")
qa = []
qa = ''
for pair in qa_pairs:
qa.append({
"question": pair["question"],
"answer": pair["ground_truth"][0],
"context": pair["ground_truth_context"][0],
})
qa = f'```json\n{json.dumps(qa, ensure_ascii=False, indent=4)}\n```'
qa += json.dumps(
{
"question": pair["question"],
"answer": pair["ground_truth"][0],
"context": pair["ground_truth_context"][0],
}, ensure_ascii=False)
return {self.output_key: qa}
async def _acall(
src/bisheng-langchain/bisheng_langchain/rag/init_retrievers/keyword_retriever.py
+1 -1
View File
@@ -16,7 +16,7 @@ from langchain.text_splitter import TextSplitter
class KeywordRetriever(BaseRetriever):
keyword_store: VectorStore
keyword_store: ElasticKeywordsSearch
text_splitter: TextSplitter
search_type: str = 'similarity'
search_kwargs: dict = Field(default_factory=dict)
src/bisheng-langchain/bisheng_langchain/vectorstores/elastic_keywords_search.py
+45 -1
View File
@@ -13,6 +13,7 @@ from langchain.llms.base import BaseLLM
from langchain.prompts.prompt import PromptTemplate
from langchain.utils import get_from_dict_or_env
from langchain.vectorstores.base import VectorStore
from loguru import logger
if TYPE_CHECKING:
from elasticsearch import Elasticsearch # noqa: F401
@@ -326,6 +327,49 @@ class ElasticKeywordsSearch(VectorStore, ABC):
response = client.search(index=index_name, body={'query': script_query, 'size': size})
return response
def delete(self, **kwargs: Any) -> None:
def delete_index(self, **kwargs: Any) -> None:
# TODO: Check if this can be done in bulk
self.client.indices.delete(index=self.index_name)
def delete(
self,
ids: Optional[List[str]] = None,
refresh_indices: Optional[bool] = True,
**kwargs: Any,
) -> Optional[bool]:
"""Delete documents from the Elasticsearch index.
Args:
ids: List of ids of documents to delete.
refresh_indices: Whether to refresh the index
after deleting documents. Defaults to True.
"""
try:
from elasticsearch.helpers import BulkIndexError, bulk
except ImportError:
raise ImportError('Could not import elasticsearch python package. '
'Please install it with `pip install elasticsearch`.')
body = []
if ids is None:
raise ValueError('ids must be provided.')
for _id in ids:
body.append({'_op_type': 'delete', '_index': self.index_name, '_id': _id})
if len(body) > 0:
try:
bulk(self.client, body, refresh=refresh_indices, ignore_status=404)
logger.debug(f'Deleted {len(body)} texts from index')
return True
except BulkIndexError as e:
logger.error(f'Error deleting texts: {e}')
firstError = e.errors[0].get('index', {}).get('error', {})
logger.error(f"First error reason: {firstError.get('reason')}")
raise e
else:
logger.debug('No texts to delete from index')
return False
src/bisheng-langchain/tests/chains/test_qa_generation.py
+2 -12
View File
@@ -32,18 +32,8 @@ def generator():
unstructured_api_url='https://bisheng.dataelem.com/api/v1/etl4llm/predict',
)
documents = loader.load()
k = 5
chunk_size = 512
filter_lowquality_context = False
filter_lowquality_question = False
# qa_generator = QAGenerationChain.from_llm(documents, llm, k=k, chunk_size=chunk_size, filter_lowquality_context=filter_lowquality_context, filter_lowquality_question=filter_lowquality_question)
qa_generator = QAGenerationChainV2.from_llm(documents,
llm,
k=k,
chunk_size=chunk_size,
filter_lowquality_context=filter_lowquality_context,
filter_lowquality_question=filter_lowquality_question
)
# qa_generator = QAGenerationChain.from_llm(documents, llm, k=5)
qa_generator = QAGenerationChainV2.from_llm(documents, llm)
inputs = {'begin': '开始'}
response = qa_generator(inputs)
question_answers = response['questions']