diff --git a/.drone.yml b/.drone.yml index b6ace209e..17f132a78 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,37 +15,13 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行 https_proxy: from_secret: PROXY commands: - - git clone https://github.com/dataelement/bisheng.git . + - git clone https://github.com/dataelement/bisheng-enterprise.git . - git checkout $DRONE_COMMIT - - name: package # 流水线名称 - pull: if-not-exists - image: python:3.10-slim # 定义创建容器的Docker镜像 - volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置 - - name: bisheng-cache - path: /app/build # 将应用打包好的Jar和执行脚本挂载出来 - environment: - RELEASE_VERSION: 99.99.99 - NEXUS_USER: - from_secret: NEXUS_USER - NEXUS_PASSWORD: - from_secret: NEXUS_PASSWORD - REPO: - from_secret: PY_NEXUS - commands: # 定义在Docker容器中执行的shell命令 - - pip install Cython - - pip install wheel - - pip install twine - - pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple - - cd ./src/bisheng-langchain - - python setup.py bdist_wheel - - twine upload --verbose -u $NEXUS_USER -p $NEXUS_PASSWORD --repository-url $REPO dist/*.whl - - name: set poetry pull: if-not-exists image: golang environment: - RELEASE_VERSION: 99.99.99 NEXUS_PUBLIC: from_secret: NEXUS_PUBLIC NEXUS_PUBLIC_PASSWORD: @@ -63,9 +39,7 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行 - echo $REPO - REPO2=$(echo $REPO | sed 's/http:\\/\\///g') - sed '/apt-get/ s|$| '"$PROXY"'|' Dockerfile - - sed -i 's/^bisheng_langchain.*/bisheng_langchain = "'$RELEASE_VERSION'"/g' pyproject.toml - sed -i '6i\RUN pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple' Dockerfile - - sed -i '23i\RUN poetry source add --priority=supplemental foo http://'$NEXUS_PUBLIC':'$NEXUS_PUBLIC_PASSWORD'@'$REPO2'simple' Dockerfile - sed -i '23i\RUN poetry source add --priority=primary qh https://pypi.tuna.tsinghua.edu.cn/simple' Dockerfile - cat Dockerfile @@ -73,6 +47,12 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行 pull: if-not-exists image: docker:24.0.6 privileged: true + environment: + http_proxy: + from_secret: PROXY + https_proxy: + from_secret: PROXY + no_proxy: 192.168.106.8 volumes: # 将容器内目录挂载到宿主机，仓库需要开启Trusted设置 - name: apt-cache path: /var/cache/apt/archives # 将应用打包好的Jar和执行脚本挂载出来 @@ -171,20 +151,66 @@ steps: # 定义流水线执行步骤，这些步骤将顺序执行 when: # 成功 status: - success -trigger: - branch: - - release - # - feat/* - event: - - push + ref: # git 分支 + - refs/tags/* + event: # 事 件 + - tag + + - name: publish_nexus + pull: if-not-exists + image: docker:24.0.6 + environment: + version: ${DRONE_TAG} + USERNAME: + from_secret: upload_name + docker_username: + from_secret: DOCKER_USER + docker_password: + from_secret: DOCKER_PASSWORD + commands: + - apk add --no-cache curl + - apk add --no-cache tar + - apk add --no-cache wget + - docker login -u $docker_username -p $docker_password 192.168.106.8:6082 + - docker pull 192.168.106.8:6082/project/bisheng-frontend:$version + - docker pull 192.168.106.8:6082/project/bisheng-backend:$version + - docker image tag 192.168.106.8:6082/project/bisheng-frontend:$version dataelement/bisheng-frontend:$version + - docker image tag 192.168.106.8:6082/project/bisheng-backend:$version dataelement/bisheng-backend:$version + - docker save -o bisheng-frontend-$version.tar dataelement/bisheng-frontend:$version + - docker save -o bisheng-backend-$version.tar dataelement/bisheng-backend:$version + - docker rmi dataelement/frontend:$version -f + - docker rmi dataelement/backend:$version -f + - mkdir -p deployment/docker-images + - cp -r docker/* deployment/ + - mv bisheng-fronteend-$version.tar bisheng-backend-$version.tar deployment/docker-images + - mv deployment bisheng-$version + - tar czf bisheng-$version.tar.gz bisheng-$version + - curl -k --progress-bar -o /dev/null --user $USERNAME --upload-file bisheng-$version.tar.gz http://192.168.106.244:5081/repository/raw-hosted/product/bisheng-ent/bisheng-$version.tar.gz + volumes: + - name: dockersock + path: /var/run + when: + status: + - success + ref: # git 分支 + - refs/tags/* + event: # 事 件 + - tag + volumes: -- name: bisheng-cache - host: - path: /opt/drone/data/bisheng/ -- name: apt-cache - host: - path: /opt/drone/data/bisheng/apt/ -- name: socket - host: - path: /var/run/docker.sock \ No newline at end of file + - name: bisheng-cache + host: + path: /opt/drone/data/bisheng/ + - name: apt-cache + host: + path: /opt/drone/data/bisheng/apt/ + - name: maven_local_repo + host: + path: /data/drone-data/m2 + - name: data + host: + path: /data/drone-data/package/ + - name: dockersock + host: + path: /var/run/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c8a346a7..d1cb2604c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: CI +name: release on: push: @@ -7,53 +7,29 @@ on: - "v*" env: - DOCKERHUB_REPO: dataelement/ - + DOCKERHUB_REPO: project/ + PY_NEXUS: 110.16.193.170:50083 + DOCKER_NEXUS: 110.16.193.170:50080 + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: - build_bisheng_langchain: + build: runs-on: ubuntu-latest #if: startsWith(github.event.ref, 'refs/tags') steps: + # deploy - name: checkout - uses: actions/checkout@v2 - - - name: Get version - id: get_version - run: | - echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} - - - name: Set Environment Variable - run: echo "RELEASE_VERSION=${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_ENV + uses: actions/checkout@v2 - # 构建 bisheng_langchain - - name: Set python version 3.8 + + # 构建 bisheng-langchain + - name: Set python version 3.10 uses: actions/setup-python@v1 with: - python-version: 3.8 - - - name: Build PyPi bisheng-langchain and push - id: pypi_build_bisheng_langchain - run: | - pip install Cython - pip install wheel - pip install twine - cd ./src/bisheng-langchain - python setup.py bdist_wheel - set +e - twine upload dist/* -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASSWORD }} --repository pypi - set -e - - build_bisheng: - needs: build_bisheng_langchain - runs-on: ubuntu-latest - # if: startsWith(github.event.ref, 'refs/tags') - steps: - - name: checkout - uses: actions/checkout@v2 + python-version: "3.10" - name: Get version id: get_version @@ -63,45 +39,39 @@ jobs: - name: Set Environment Variable run: echo "RELEASE_VERSION=${{ steps.get_version.outputs.VERSION }}" >> $GITHUB_ENV - # 登录 docker hub - - name: Login to DockerHub - uses: docker/login-action@v1 + # 发布到 私有仓库 + - name: set insecure registry + run: | + echo "{ \"insecure-registries\": [\"http://${{ env.DOCKER_NEXUS }}\"] }" | sudo tee /etc/docker/daemon.json + sudo service docker restart + + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v1 + + - name: Login Nexus Container Registry + uses: docker/login-action@v2 with: - # GitHub Repo => Settings => Secrets 增加 docker hub 登录密钥信息 - # DOCKERHUB_USERNAME 是 docker hub 账号名. - # DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建. - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + registry: http://${{ env.DOCKER_NEXUS }}/ + username: ${{ secrets.NEXUS_USER }} + password: ${{ secrets.NEXUS_PASSWORD }} - # - name: Login to DockerHub - # uses: docker/login-action@v1 - # with: - # registry: https://cr.dataelem.com/ - # username: ${{ secrets.CR_DOCKERHUB_USERNAME }} - # password: ${{ secrets.CR_DOCKERHUB_TOKEN }} - - # 构建 backend 并推送到 Docker hub - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: install poetry + # 替换poetry编译为私有服务 + - name: replace self-host repo uses: snok/install-poetry@v1 with: installer-parallel: true + - name: build lock - run: | + run: | cd ./src/backend - pip install bisheng_langchain==$RELEASE_VERSION - poetry lock + poetry source add --priority=supplemental foo http://${{ secrets.NEXUS_PUBLIC }}:${{ secrets.NEXUS_PUBLIC_PASSWORD }}@${{ env.PY_NEXUS }}/repository/pypi-group/simple # 安装 nltk pip install nltk python -c "import nltk; nltk.download('punkt'); nltk.download('punkt_tab'); nltk.download('averaged_perceptron_tagger'); nltk.download('averaged_perceptron_tagger_eng'); " mv /home/runner/nltk_data ./ cd ../../ + # 构建 backend 并推送到 Docker hub - name: Build backend and push id: docker_build_backend uses: docker/build-push-action@v2 @@ -111,14 +81,12 @@ jobs: # 是否 docker push push: true # docker build arg, 注入 APP_NAME/APP_VERSION - platforms: linux/amd64,linux/arm64 build-args: | APP_NAME="bisheng-backend" APP_VERSION=${{ steps.get_version.outputs.VERSION }} # 生成两个 docker tag: ${APP_VERSION} 和 latest tags: | - ${{ env.DOCKERHUB_REPO }}bisheng-backend:latest - ${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }} + ${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }} # 构建 Docker frontend 并推送到 Docker hub - name: Build frontend and push id: docker_build_frontend @@ -129,26 +97,10 @@ jobs: # 是否 docker push push: true # docker build arg, 注入 APP_NAME/APP_VERSION - platforms: linux/amd64,linux/arm64 build-args: | APP_NAME="bisheng-frontend" APP_VERSION=${{ steps.get_version.outputs.VERSION }} # 生成两个 docker tag: ${APP_VERSION} 和 latest tags: | - ${{ env.DOCKERHUB_REPO }}bisheng-frontend:latest - ${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }} - - - name: Process git message - id: process_message - run: | - value=$(echo "${{ github.event.head_commit.message }}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/%0A/g') - value=$(echo "${value}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\r/%0A/g') - echo "message=${value}" >> $GITHUB_ENV - shell: bash + ${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }} - - name: notify feishu - uses: fjogeleit/http-request-action@v1 - with: - url: ${{ secrets.FEISHU_WEBHOOK }} - method: 'POST' - data: '{"msg_type":"post","content":{"post":{"zh_cn":{"title": "${{ steps.get_version.outputs.VERSION }}发布成功", "content": [[{"tag":"text","text":"发布功能："},{"tag":"text","text":"${{ env.message }}"}]]}}}}' diff --git a/.github/workflows/release.yml.bak b/.github/workflows/release.yml.bak deleted file mode 100644 index 5f39f0ff2..000000000 --- a/.github/workflows/release.yml.bak +++ /dev/null @@ -1,150 +0,0 @@ -# name: release - -# on: -# push: -# # Sequence of patterns matched against refs/tags -# branches: -# - "release" - -# env: -# DOCKERHUB_REPO: dataelement/ -# PY_NEXUS: 110.16.193.170:50083 -# DOCKER_NEXUS: 110.16.193.170:50080 - -# concurrency: -# group: ${{ github.workflow }}-${{ github.ref }} -# cancel-in-progress: true - -# jobs: -# build: -# runs-on: ubuntu-latest -# #if: startsWith(github.event.ref, 'refs/tags') -# steps: -# # deploy -# - name: checkout -# uses: actions/checkout@v2 - -# - name: Process git message -# id: process_message -# run: | -# value=$(echo "${{ github.event.head_commit.message }}" | sed -e ':a' -e 'N' -e '$!ba' -e 's/\n/%0A/g' ) -# echo "message=${value}" >> $GITHUB_ENV -# shell: bash - -# - name: notify feishu -# uses: fjogeleit/http-request-action@v1 -# with: -# url: ' https://open.feishu.cn/open-apis/bot/v2/hook/2cfe0d8d-647c-4408-9f39-c59134035c4b' -# method: 'POST' -# data: '{"msg_type":"post","content":{"post":{"zh_cn":{"title": "${{github.event.pusher.name}}提交代码，开始编译", "content": [[{"tag":"text","text":"发布功能："},{"tag":"text","text":"${{ env.message }}"}]]}}}}' - -# - name: Set Environment Variable -# run: echo "RELEASE_VERSION=99.99.99" >> $GITHUB_ENV - -# - name: Set python version 3.8 -# uses: actions/setup-python@v1 -# with: -# python-version: 3.8 - -# - name: Build PyPi bisheng-langchain and push -# id: pypi_build_bisheng_langchain -# run: | -# pip install Cython -# pip install wheel -# pip install twine -# cd ./src/bisheng-langchain -# python setup.py bdist_wheel -# repo="http://${{ env.PY_NEXUS }}/repository/pypi-hosted/" -# twine upload --verbose -u ${{ secrets.NEXUS_USER }} -p ${{ secrets.NEXUS_PASSWORD }} --repository-url $repo dist/*.whl -# cd ../../ - -# - name: Set up QEMU -# uses: docker/setup-qemu-action@v1 - -# - name: set up Docker Buildx -# uses: docker/setup-buildx-action@v3 - -# # 发布到 私有仓库 -# - name: set insecure registry -# run: | -# echo "{ \"insecure-registries\": [\"http://${{ env.DOCKER_NEXUS }}\"] }" | sudo tee /etc/docker/daemon.json -# sudo service docker restart - -# # - name: Set up QEMU -# # uses: docker/setup-qemu-action@v1 -# - name: Login Nexus Container Registry -# uses: docker/login-action@v2 -# with: -# registry: http://${{ env.DOCKER_NEXUS }}/ -# username: ${{ secrets.NEXUS_USER }} -# password: ${{ secrets.NEXUS_PASSWORD }} - -# # 替换poetry编译为私有服务 -# - name: replace self-host repo -# uses: snok/install-poetry@v1 -# with: -# installer-parallel: true - -# - name: build lock -# run: | -# cd ./src/backend -# sed -i 's/^bisheng_langchain.*/bisheng_langchain = "'$RELEASE_VERSION'"/g' pyproject.toml -# poetry source add --priority=supplemental foo http://${{ secrets.NEXUS_PUBLIC }}:${{ secrets.NEXUS_PUBLIC_PASSWORD }}@${{ env.PY_NEXUS }}/repository/pypi-group/simple -# poetry lock -# cd ../../ - -# # 构建 backend 并推送到 Docker hub -# - name: Build backend and push -# id: docker_build_backend -# uses: docker/build-push-action@v2 -# with: -# # backend 的context目录 -# context: "./src/backend/" -# # 是否 docker push -# push: true -# platforms: linux/amd64,linux/arm64 -# # docker build arg, 注入 APP_NAME/APP_VERSION -# build-args: | -# APP_NAME="bisheng-backend" -# APP_VERSION="release" -# # 生成两个 docker tag: ${APP_VERSION} 和 latest -# tags: | -# ${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-backend:release -# # 构建 Docker frontend 并推送到 Docker hub -# - name: Build frontend and push -# id: docker_build_frontend -# uses: docker/build-push-action@v2 -# with: -# # frontend 的context目录 -# context: "./src/frontend/" -# # 是否 docker push -# push: true -# platforms: linux/amd64,linux/arm64 -# # docker build arg, 注入 APP_NAME/APP_VERSION -# build-args: | -# APP_NAME="bisheng-frontend" -# APP_VERSION="release" -# # 生成两个 docker tag: ${APP_VERSION} 和 latest -# tags: | -# ${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-frontend:release -# # deploy -# - name: notify feishu -# uses: fjogeleit/http-request-action@v1 -# with: -# url: ' https://open.feishu.cn/open-apis/bot/v2/hook/2cfe0d8d-647c-4408-9f39-c59134035c4b' -# method: 'POST' -# data: '{"msg_type":"text","content":{"text":"release 编译成功, 准备部署"}}' - -# - name: Deploy Stage -# uses: fjogeleit/http-request-action@v1 -# with: -# timeout: 200000 -# url: 'https://bisheng.dataelem.com/deploy/cgi-bin/deploy_script.py' -# method: 'GET' - -# - name: notify feishu -# uses: fjogeleit/http-request-action@v1 -# with: -# url: ' https://open.feishu.cn/open-apis/bot/v2/hook/2cfe0d8d-647c-4408-9f39-c59134035c4b' -# method: 'POST' -# data: '{"msg_type":"text","content":{"text":"release 部署成功"}}' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 000000000..28f37a697 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,100 @@ +name: test_build + +on: + push: + # Sequence of patterns matched against refs/tags + branches: + - "develop/*" + +env: + DOCKERHUB_REPO: project/ + PY_NEXUS: 110.16.193.170:50083 + DOCKER_NEXUS: 110.16.193.170:50080 + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + #if: startsWith(github.event.ref, 'refs/tags') + steps: + - name: checkout + uses: actions/checkout@v2 + + - name: Get version + id: get_version + run: | + echo ::set-output name=VERSION::${GITHUB_REF/refs\/heads\/develop\//} + echo $GITHUB_REF + echo $VERSION + + # 构建 bisheng-langchain + - name: Set python version 3.8 + uses: actions/setup-python@v1 + with: + python-version: 3.8 + + # 发布到 私有仓库 + - name: set insecure registry + run: | + echo "{ \"insecure-registries\": [\"http://${{ env.DOCKER_NEXUS }}\"] }" | sudo tee /etc/docker/daemon.json + sudo service docker restart + + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v1 + + - name: Login Nexus Container Registry + uses: docker/login-action@v2 + with: + registry: http://${{ env.DOCKER_NEXUS }}/ + username: ${{ secrets.NEXUS_USER }} + password: ${{ secrets.NEXUS_PASSWORD }} + + # 替换poetry编译为私有服务 + - name: replace self-host repo + uses: snok/install-poetry@v1 + with: + installer-parallel: true + + - name: build lock + run: | + cd ./src/backend + poetry source add --priority=supplemental foo http://${{ secrets.NEXUS_PUBLIC }}:${{ secrets.NEXUS_PUBLIC_PASSWORD }}@${{ env.PY_NEXUS }}/repository/pypi-group/simple + poetry lock + cd ../../ + + # 构建 backend 并推送到 Docker hub + - name: Build backend and push + id: docker_build_backend + uses: docker/build-push-action@v2 + with: + # backend 的context目录 + context: "./src/backend/" + # 是否 docker push + push: true + # docker build arg, 注入 APP_NAME/APP_VERSION + build-args: | + APP_NAME="bisheng-backend" + APP_VERSION=${{ steps.get_version.outputs.VERSION }} + # 生成两个 docker tag: ${APP_VERSION} 和 latest + tags: | + ${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-backend:${{ steps.get_version.outputs.VERSION }} + # 构建 Docker frontend 并推送到 Docker hub + - name: Build frontend and push + id: docker_build_frontend + uses: docker/build-push-action@v2 + with: + # frontend 的context目录 + context: "./src/frontend/" + # 是否 docker push + push: true + # docker build arg, 注入 APP_NAME/APP_VERSION + build-args: | + APP_NAME="bisheng-frontend" + APP_VERSION=${{ steps.get_version.outputs.VERSION }} + # 生成两个 docker tag: ${APP_VERSION} 和 latest + tags: | + ${{ env.DOCKER_NEXUS }}/${{ env.DOCKERHUB_REPO }}bisheng-frontend:${{ steps.get_version.outputs.VERSION }} + \ No newline at end of file diff --git a/src/backend/bisheng/api/services/dataset_service.py b/src/backend/bisheng/api/services/dataset_service.py new file mode 100644 index 000000000..d2528c672 --- /dev/null +++ b/src/backend/bisheng/api/services/dataset_service.py @@ -0,0 +1,74 @@ +from typing import Dict, List, Optional + +from bisheng.api.services.base import BaseService +from bisheng.api.v1.schema.dataset_param import CreateDatasetParam +from bisheng.database.models.dataset import Dataset, DatasetCreate, DatasetDao, DatasetRead +from bisheng.database.models.user import UserDao +from bisheng.utils.minio_client import MinioClient +from fastapi import HTTPException + + +class DatasetService(BaseService): + + @classmethod + def build_dataset_list(cls, + page: int, + limit: int, + keyword: Optional[str] = None) -> List[Dict]: + """补全list 数据""" + + dataset_list = DatasetDao.filter_dataset_by_ids(dataset_ids=[], + keyword=keyword, + page=page, + limit=limit) + count_filter = [] + if keyword: + count_filter.append(Dataset.name.like('%{}%'.format(keyword))) + total_count = DatasetDao.get_count_by_filter(count_filter) + + user_ids = [one.user_id for one in dataset_list] + user_list = UserDao.get_user_by_ids(user_ids) + user_dict = {one.user_id: one for one in user_list} + res = [DatasetRead.validate(one) for one in dataset_list] + for one in res: + one.user_name = user_dict[one.user_id].user_name + if one.object_name: + one.url = MinioClient().get_share_link(one.object_name) + + return res, total_count + + @classmethod + def create_dataset(cls, user_id: int, data: CreateDatasetParam): + """创建数据集""" + dataset_insert = DatasetCreate.validate(data) + dataset_insert.user_id = user_id + isExist = DatasetDao.get_dataset_by_name(data.name) + if isExist: + raise ValueError('数据集名称已存在') + dataset = DatasetDao.insert(dataset_insert) + # 处理文件 + object_name = f'/dataset/{dataset.id}/{dataset.name}' + if data.file_url: + + # MinioClient().upload_minio() + dataset.object_name = object_name + if data.qa_list: + for qa in data.qa_list: + qa.dataset_id = dataset.id + # QADao.insert(qa) + + dataset = DatasetDao.update(dataset) + return dataset + + @classmethod + def delete_dataset(cls, dataset_id: int): + dataset = DatasetDao.get_dataset_by_id(dataset_id) + if not dataset: + raise HTTPException(status_code=404, detail='Dataset not found') + # 处理minio + object_name = dataset.object_name + if object_name: + minio_client = MinioClient() + minio_client.delete_minio(object_name) + DatasetDao.delete(dataset) + return True diff --git a/src/backend/bisheng/api/services/finetune_file.py b/src/backend/bisheng/api/services/finetune_file.py index 3a9ff8e69..d0de0b002 100644 --- a/src/backend/bisheng/api/services/finetune_file.py +++ b/src/backend/bisheng/api/services/finetune_file.py @@ -3,6 +3,7 @@ import uuid from typing import Any, List from bisheng.api.errcode.finetune import TrainFileNotExistError +from bisheng.api.v1.schema.base_schema import PageList from bisheng.api.v1.schemas import UnifiedResponseModel, resp_200 from bisheng.database.models.preset_train import PresetTrain, PresetTrainDao from bisheng.utils.logger import logger @@ -15,7 +16,8 @@ class FinetuneFileService(BaseModel): """ 训练任务 文件管理 """ @classmethod - def upload_file(cls, files: List[UploadFile], is_preset: bool, user: Any) -> UnifiedResponseModel: + def upload_file(cls, files: List[UploadFile], is_preset: bool, + user: Any) -> UnifiedResponseModel: if len(files) == 0: return TrainFileNotExistError.return_resp() @@ -28,6 +30,25 @@ class FinetuneFileService(BaseModel): PresetTrainDao.insert_batch(file_list) return resp_200(data=file_list) + @classmethod + def upload_preset_file(cls, name: str, type: int, file_path: str, + user: Any) -> UnifiedResponseModel: + # 将训练文件上传到minio + file_root = cls.get_upload_file_root(False) + file_id = uuid.uuid4().hex + file_ext = os.path.basename(file_path).split('.')[-1] + object_name = f'{file_root}/{file_id}.{file_ext}' + MinioClient().upload_minio(object_name, file_path) + # 将预置数据存入数据库 + file_info = PresetTrain(id=file_id, + name=name, + url=object_name, + type=type, + user_id=user.get('user_id'), + user_name=user.get('user_name')) + PresetTrainDao.insert_batch([file_info]) + return resp_200(data=file_info) + @classmethod def get_upload_file_root(cls, is_preset: bool) -> str: if is_preset: @@ -36,22 +57,32 @@ class FinetuneFileService(BaseModel): return 'finetune/train_file/personal' @classmethod - def upload_file_to_minio(cls, files: List[UploadFile], file_root: str, user: Any) -> List[PresetTrain]: + def upload_file_to_minio(cls, files: List[UploadFile], file_root: str, + user: Any) -> List[PresetTrain]: minio_client = MinioClient() ret = [] for file in files: file_id = uuid.uuid4().hex file_ext = os.path.basename(file.filename).split('.')[-1] - file_info = PresetTrain(id=file_id, name=file.filename, + file_info = PresetTrain(id=file_id, + name=file.filename, url=f'{file_root}/{file_id}.{file_ext}', - user_id=user.get('user_id'), user_name=user.get('user_name')) - minio_client.upload_minio_file(file_info.url, file.file, length=file.size, content_type=file.content_type) + user_id=user.get('user_id'), + user_name=user.get('user_name')) + minio_client.upload_minio_file(file_info.url, + file.file, + length=file.size, + content_type=file.content_type) ret.append(file_info) return ret @classmethod - def get_preset_file(cls) -> List[PresetTrain]: - return PresetTrainDao.find_all() + def get_preset_file(cls, + keyword: str = None, + page_size: int = None, + page_num: int = None) -> List[PresetTrain]: + list_res, total_count = PresetTrainDao.search_name(keyword, page_size, page_num) + return PageList(list=list_res, total=total_count) @classmethod def delete_preset_file(cls, file_id: uuid.UUID, user: Any) -> UnifiedResponseModel: diff --git a/src/backend/bisheng/api/services/knowledge_imp.py b/src/backend/bisheng/api/services/knowledge_imp.py index fa51d31e4..267fb9e4d 100644 --- a/src/backend/bisheng/api/services/knowledge_imp.py +++ b/src/backend/bisheng/api/services/knowledge_imp.py @@ -5,6 +5,27 @@ from typing import Any, Dict, List import requests from loguru import logger from pymilvus import Collection +from typing import Any, Dict, List, Optional +from uuid import uuid4 + +import requests +from bisheng.cache.utils import file_download +from bisheng.database.base import session_getter +from bisheng.database.models.knowledge import Knowledge, KnowledgeCreate, KnowledgeDao +from bisheng.database.models.knowledge_file import (KnowledgeFile, KnowledgeFileDao, QAKnoweldgeDao, + QAKnowledge, QAKnowledgeUpsert) +from bisheng.interface.embeddings.custom import FakeEmbedding +from bisheng.interface.importing.utils import import_by_type, import_vectorstore +from bisheng.interface.initialize.loading import instantiate_llm, instantiate_vectorstore +from bisheng.settings import settings +from bisheng.utils import minio_client +from bisheng.utils.embedding import decide_embeddings +from bisheng.utils.minio_client import MinioClient +from bisheng_langchain.document_loaders import ElemUnstructuredLoader +from bisheng_langchain.rag.extract_info import extract_title +from bisheng_langchain.text_splitter import ElemCharacterTextSplitter +from bisheng_langchain.vectorstores import ElasticKeywordsSearch +from fastapi import HTTPException from langchain.embeddings.base import Embeddings from langchain.schema.document import Document from langchain.text_splitter import CharacterTextSplitter @@ -31,6 +52,10 @@ from bisheng.utils.minio_client import MinioClient from bisheng_langchain.document_loaders import ElemUnstructuredLoader from bisheng_langchain.rag.extract_info import extract_title from bisheng_langchain.text_splitter import ElemCharacterTextSplitter +from loguru import logger +from pymilvus import Collection +from sqlalchemy import delete, func, or_ +from sqlmodel import select filetype_load_map = { 'txt': TextLoader, @@ -171,14 +196,19 @@ def decide_vectorstores(collection_name: str, vector_store: str, # 无相关配置 return None + param = {'embedding': embedding} if vector_store == 'ElasticKeywordsSearch': - param = {'index_name': collection_name, 'embedding': embedding} + param['index_name'] = collection_name if isinstance(vector_config['ssl_verify'], str): vector_config['ssl_verify'] = eval(vector_config['ssl_verify']) - else: - param = {'collection_name': collection_name, 'embedding': embedding} + elif vector_store == 'Milvus': + param['collection_name'] = collection_name vector_config.pop('partition_suffix', '') vector_config.pop('is_partition', '') + else: + # 适配其他的vector + collection_name = vector_config.pop('collection_or_index_name', '') + vector_config[collection_name] = collection_name param.update(vector_config) class_obj = import_vectorstore(vector_store) @@ -531,3 +561,272 @@ def delete_es(index_name: str): if esvectore_client: res = esvectore_client.client.indices.delete(index=index_name, ignore=[400, 404]) logger.info(f'act=delete_es index={index_name} res={res}') + + +def QA_save_knowledge(db_knowledge: Knowledge, QA: QAKnowledge): + """使用text 导入knowledge""" + + questions = QA.questions + answer = json.loads(QA.answers)[0] + extra = {} + if QA.extra_meta: + extra = json.loads(QA.extra_meta) or {} + extra.update({'answer': answer}) + docs = [Document(page_content=question, metadata=extra) for question in questions] + try: + embeddings = decide_embeddings(db_knowledge.model) + vectore_config_dict: dict = settings.get_knowledge().get('vectorstores') + if not vectore_config_dict: + raise Exception('向量数据库必须配置') + vectore_client_list = [ + decide_vectorstores(db_knowledge.index_name or db_knowledge.collection_name, db, + embeddings) if db == 'ElasticKeywordsSearch' else + decide_vectorstores(db_knowledge.collection_name, db, embeddings) + for db in vectore_config_dict.keys() + ] + logger.info('vector_init_conn_done col={} dbs={}', db_knowledge.collection_name, + vectore_config_dict.keys()) + except Exception as e: + logger.exception(e) + + try: + # 统一document + metadata = [{ + 'file_id': QA.id, + 'knowledge_id': f'{db_knowledge.id}', + 'page': doc.metadata.pop('page', 1), + 'source': doc.metadata.pop('source', ''), + 'bbox': doc.metadata.pop('bbox', ''), + 'extra': json.dumps(doc.metadata, ensure_ascii=False) + } for doc in docs] + + # 向量存储 + for vectore_client in vectore_client_list: + vectore_client.add_texts(texts=[t.page_content for t in docs], metadatas=metadata) + + QA.status = 1 + with session_getter() as session: + session.add(QA) + session.commit() + session.refresh(QA) + except Exception as e: + logger.error(e) + setattr(QA, 'status', 0) + setattr(QA, 'remark', str(e)[:500]) + with session_getter() as session: + session.add(QA) + session.commit() + session.refresh(QA) + + return QA + + +def add_qa(db_knowledge: Knowledge, data: QAKnowledgeUpsert) -> QAKnowledge: + """使用text 导入QAknowledge""" + if db_knowledge.type != 1: + raise Exception('knowledge type error') + try: + # 相似问统一插入 + questions = data.questions + if questions: + if data.id: + qa_db = QAKnoweldgeDao.get_qa_knowledge_by_primary_id(data.id) + qa_db.questions = questions + qa_db.answers = data.answers + qa = QAKnoweldgeDao.update(qa_db) + # 需要先删除再插入 + delete_vector_data(db_knowledge, [data.id]) + else: + qa = QAKnoweldgeDao.insert_qa(data) + + # 对question进行embedding，然后录入知识库 + qa = QA_save_knowledge(db_knowledge, qa) + return qa + except Exception as e: + logger.exception(e) + raise e + + +def qa_status_change(qa_id: int, target_status: int): + """QA 状态切换""" + qa_db = QAKnoweldgeDao.get_qa_knowledge_by_primary_id(qa_id) + + if qa_db.status == target_status: + logger.info('qa status is same, skip') + return + + db_knowledge = KnowledgeDao.query_by_id(qa_db.knowledge_id) + if target_status == 0: + delete_vector_data(db_knowledge, [qa_id]) + qa_db.status = target_status + QAKnoweldgeDao.update(qa_db) + else: + qa_db.status = target_status + QAKnoweldgeDao.update(qa_db) + QA_save_knowledge(db_knowledge, qa_db) + return qa_db + + +def list_qa_by_knowledge_id(knowledge_id: int, + page_size: int = 10, + page_num: int = 1, + question: Optional[str] = None, + answer: Optional[str] = None, + keyword: Optional[str] = None, + status: Optional[int] = None) -> List[QAKnowledge]: + """获取知识库下的所有qa""" + if not knowledge_id: + return [] + + count_sql = select(func.count(QAKnowledge.id)).where(QAKnowledge.knowledge_id == knowledge_id) + list_sql = select(QAKnowledge).where(QAKnowledge.knowledge_id == knowledge_id) + + if status: + count_sql = count_sql.where(QAKnowledge.status == status) + list_sql = list_sql.where(QAKnowledge.status == status) + + if question: + count_sql = count_sql.where(QAKnowledge.questions.like(f'%{question}%')) + list_sql = list_sql.where(QAKnowledge.questions.like(f'%{question}%')) + + if answer: + count_sql = count_sql.where(QAKnowledge.answers.like(f'%{answer}%')) + list_sql = list_sql.where(QAKnowledge.answers.like(f'%{answer}%')) + + if keyword: + count_sql = count_sql.where( + or_(QAKnowledge.questions.like(f'%{keyword}%'), + QAKnowledge.answers.like(f'%{keyword}%'))) + list_sql = list_sql.where( + or_(QAKnowledge.answers.like(f'%{keyword}%'), + QAKnowledge.questions.like(f'%{keyword}%'))) + + list_sql = list_sql.order_by(QAKnowledge.update_time.desc()).limit(page_size).offset( + (page_num - 1) * page_size) + count = QAKnoweldgeDao.total_count(count_sql) + list_qa = QAKnoweldgeDao.query_by_condition(list_sql) + + return list_qa, count + + +def delete_vector_data(knowledge: Knowledge, file_ids: List[int]): + """删除向量数据, 想做一个通用的，可以对接langchain的vectorDB""" + # embeddings = FakeEmbedding() + # vectore_config_dict: dict = settings.get_knowledge().get('vectorstores') + # if not vectore_config_dict: + # raise Exception('向量数据库必须配置') + # elastic_index = knowledge.index_name or knowledge.collection_name + # vectore_client_list = [ + # decide_vectorstores(elastic_index, db, embeddings) if db == 'ElasticKeywordsSearch' else + # decide_vectorstores(knowledge.collection_name, db, embeddings) + # for db in vectore_config_dict.keys() + # ] + # logger.info('vector_init_conn_done col={} dbs={}', knowledge.collection_name, + # vectore_config_dict.keys()) + + # for vectore_client in vectore_client_list: + # 查询vector primary key + embeddings = FakeEmbedding() + collection_name = knowledge.collection_name + # 处理vectordb + vectore_client = decide_vectorstores(collection_name, 'Milvus', embeddings) + try: + if isinstance(vectore_client.col, Collection): + pk = vectore_client.col.query(expr=f'file_id in {file_ids}', + output_fields=['pk'], + timeout=10) + else: + pk = [] + except Exception: + # 重试一次 + logger.error('timeout_except') + vectore_client.close_connection(vectore_client.alias) + vectore_client = decide_vectorstores(collection_name, 'Milvus', embeddings) + pk = vectore_client.col.query(expr=f'file_id in {file_ids}', + output_fields=['pk'], + timeout=10) + logger.info('query_milvus pk={}', pk) + if pk: + res = vectore_client.col.delete(f"pk in {[p['pk'] for p in pk]}", timeout=10) + logger.info(f'act=delete_vector file_id={file_ids} res={res}') + vectore_client.close_connection(vectore_client.alias) + + # elastic + index_name = knowledge.index_name or collection_name + esvectore_client = decide_vectorstores(index_name, 'ElasticKeywordsSearch', embeddings) + + if esvectore_client: + res = esvectore_client.client.delete_by_query( + index=index_name, body={'query': { + 'terms': { + 'metadata.file_id': file_ids + } + }}) + logger.info(f'act=delete_es res={res}') + return True + + +def recommend_question(question: str, answer: str, number: int = 3) -> List[str]: + from langchain.chains.llm import LLMChain + from langchain_core.prompts.prompt import PromptTemplate + prompt = """- Role: 问题生成专家 + - Background: 用户希望通过人工智能模型根据给定的问题和答案生成相似的问题，以便于扩展知识库或用于教育和测试目的。 + - Profile: 你是一位专业的数据分析师和语言模型专家，擅长从现有数据中提取模式，并生成新的相关问题。 + - Constrains: 确保生成的问题在语义上与原始问题相似，同时保持多样性，避免重复。 + - Workflow: + 1. 分析用户输入的问题和答案，提取关键词和主题。 + 2. 根据提取的关键词和主题创建相似问题。 + 3. 验证生成的问题与原始问题在语义上的相似性，并确保多样性。 + - Examples: + 问题："法国的首都是哪里？" + 答案："巴黎" + 生成3个相似问题： + - "法国的首都叫什么名字？" + - "哪个城市是法国的首都？" + - "巴黎是哪个国家的首都？" + + 请使用json 返回 + {{"question": 生成的问题列表}} + + 以下是用户提供的问题和答案： + 问题：{question} + 答案：{answer} + + 你生成的{number}个相似问题： + """ + keyword_conf = settings.get_default_llm() or {} + if keyword_conf: + keyword_conf['temperature'] = 0.7 + keyword_conf['cache'] = False + node_type = keyword_conf.pop('type', 'HostQwenChat') # 兼容旧配置 + class_object = import_by_type(_type='llms', name=node_type) + llm = instantiate_llm(node_type, class_object, keyword_conf) + + llm_chain = LLMChain(llm=llm, prompt=PromptTemplate.from_template(prompt)) + gen_question = llm_chain.predict(question=question, answer=answer, number=number) + + try: + code_ret = extract_code_blocks(gen_question) + if code_ret: + question_dict = json.loads(code_ret[0]) + return question_dict['question'] + else: + logger.info('md_code_extract_error {}', gen_question) + return [] + except Exception as exc: + logger.error('recommend_question json.loads error:{}', gen_question) + raise ValueError(gen_question) from exc + else: + logger.info('llm_chain is None recommend_over') + return '默认大模型未配置' + + +def extract_code_blocks(markdown_code_block: str): + # 定义正则表达式模式 + pattern = r'```\w*\s*(.*?)```' + + # 使用 re.DOTALL 使 . 能够匹配换行符 + matches = re.findall(pattern, markdown_code_block, re.DOTALL) + + # 去除每段代码块两端的空白字符 + return [match.strip() for match in matches] diff --git a/src/backend/bisheng/api/services/user_service.py b/src/backend/bisheng/api/services/user_service.py index 91065c501..1c77eb4bd 100644 --- a/src/backend/bisheng/api/services/user_service.py +++ b/src/backend/bisheng/api/services/user_service.py @@ -1,15 +1,13 @@ -import json import functools +import json from base64 import b64decode from typing import List import rsa -from fastapi import HTTPException, Depends, Request -from fastapi_jwt_auth import AuthJWT - -from bisheng.api.JWT import ACCESS_TOKEN_EXPIRE_TIME from bisheng.api.errcode.base import UnAuthorizedError -from bisheng.api.errcode.user import UserLoginOfflineError, UserNameAlreadyExistError, UserNeedGroupAndRoleError +from bisheng.api.errcode.user import (UserLoginOfflineError, UserNameAlreadyExistError, + UserNeedGroupAndRoleError) +from bisheng.api.JWT import ACCESS_TOKEN_EXPIRE_TIME from bisheng.api.utils import md5_hash from bisheng.api.v1.schemas import CreateUserReq from bisheng.cache.redis import redis_client @@ -22,7 +20,9 @@ from bisheng.database.models.user import User, UserDao from bisheng.database.models.user_group import UserGroupDao from bisheng.database.models.user_role import UserRoleDao from bisheng.settings import settings -from bisheng.utils.constants import USER_CURRENT_SESSION, RSA_KEY +from bisheng.utils.constants import RSA_KEY, USER_CURRENT_SESSION +from fastapi import Depends, HTTPException, Request +from fastapi_jwt_auth import AuthJWT class UserPayload: @@ -140,7 +140,7 @@ def sso_login(): def gen_user_role(db_user: User): # 查询用户的角色列表 db_user_role = UserRoleDao.get_user_roles(db_user.user_id) - role = "" + role = '' role_ids = [] for user_role in db_user_role: if user_role.role_id == 1: @@ -148,7 +148,7 @@ def gen_user_role(db_user: User): role = 'admin' else: role_ids.append(user_role.role_id) - if role != "admin": + if role != 'admin': # 判断是否是用户组管理员 db_user_groups = UserGroupDao.get_user_admin_group(db_user.user_id) if len(db_user_groups) > 0: @@ -169,7 +169,8 @@ def gen_user_jwt(db_user: User): # 生成JWT令牌 payload = {'user_name': db_user.user_name, 'user_id': db_user.user_id, 'role': role} # Create the tokens and passing to set_access_cookies or set_refresh_cookies - access_token = AuthJWT().create_access_token(subject=json.dumps(payload), expires_time=ACCESS_TOKEN_EXPIRE_TIME) + access_token = AuthJWT().create_access_token(subject=json.dumps(payload), + expires_time=ACCESS_TOKEN_EXPIRE_TIME) refresh_token = AuthJWT().create_refresh_token(subject=db_user.user_name) @@ -177,6 +178,60 @@ def gen_user_jwt(db_user: User): return access_token, refresh_token, role, web_menu +def get_knowledge_list_by_access(role_id: int, name: str, page_num: int, page_size: int): + count_filter = [] + if name: + count_filter.append(Knowledge.name.like('%{}%'.format(name))) + + db_role_access = KnowledgeDao.get_knowledge_by_access(role_id, page_num, page_size) + total_count = KnowledgeDao.get_count_by_filter(count_filter) + # 补充用户名 + user_ids = [access[0].user_id for access in db_role_access] + db_users = UserDao.get_user_by_ids(user_ids) + user_dict = {user.user_id: user.user_name for user in db_users} + + return { + 'data': [ + KnowledgeRead.validate({ + 'name': access[0].name, + 'user_name': user_dict.get(access[0].user_id), + 'user_id': access[0].user_id, + 'update_time': access[0].update_time, + 'id': access[0].id + }) for access in db_role_access + ], + 'total': + total_count + } + + +def get_flow_list_by_access(role_id: int, name: str, page_num: int, page_size: int): + count_filter = [] + if name: + count_filter.append(Flow.name.like('%{}%'.format(name))) + + db_role_access = FlowDao.get_flow_by_access(role_id, name, page_num, page_size) + total_count = FlowDao.get_count_by_filters(count_filter) + # 补充用户名 + user_ids = [access[0].user_id for access in db_role_access] + db_users = UserDao.get_user_by_ids(user_ids) + user_dict = {user.user_id: user.user_name for user in db_users} + + return { + 'data': [ + FlowRead.validate({ + 'name': access[0].name, + 'user_name': user_dict.get(access[0].user_id), + 'user_id': access[0].user_id, + 'update_time': access[0].update_time, + 'id': access[0].id + }) for access in db_role_access + ], + 'total': + total_count + } + + def get_assistant_list_by_access(role_id: int, name: str, page_num: int, page_size: int): count_filter = [] if name: @@ -198,7 +253,7 @@ def get_assistant_list_by_access(role_id: int, name: str, page_num: int, page_si 'id': access[0].id } for access in db_role_access], 'total': - total_count + total_count } diff --git a/src/backend/bisheng/api/utils.py b/src/backend/bisheng/api/utils.py index 87ecdabc2..0889ca40d 100644 --- a/src/backend/bisheng/api/utils.py +++ b/src/backend/bisheng/api/utils.py @@ -5,17 +5,16 @@ from pathlib import Path from typing import Dict, List import aiohttp -from fastapi import Request, WebSocket -from fastapi_jwt_auth import AuthJWT -from platformdirs import user_cache_dir -from sqlalchemy import delete -from sqlmodel import select - from bisheng.api.v1.schemas import StreamData from bisheng.database.base import session_getter from bisheng.database.models.variable_value import Variable from bisheng.graph.graph.base import Graph from bisheng.utils.logger import logger +from fastapi import Request, WebSocket +from fastapi_jwt_auth import AuthJWT +from platformdirs import user_cache_dir +from sqlalchemy import delete +from sqlmodel import select API_WORDS = ['api', 'key', 'token'] @@ -97,7 +96,8 @@ async def build_flow(graph_data: dict, } yield str(StreamData(event='log', data=log_dict)) # # 如果存在文件，当前不操作文件，避免重复操作 - if not process_file and vertex.base_type == 'documentloaders': + if not process_file and (vertex.base_type == 'documentloaders' + or vertex.base_type == 'input_output'): template_dict = { key: value for key, value in vertex.data['node']['template'].items() @@ -175,7 +175,8 @@ async def build_flow_no_yield(graph_data: dict, for vertex in sorted_vertices: try: # 如果存在文件，当前不操作文件，避免重复操作 - if not process_file and vertex.base_type == 'documentloaders': + if not process_file and (vertex.base_type == 'documentloaders' + or vertex.base_type == 'input_output'): template_dict = { key: value for key, value in vertex.data['node']['template'].items() @@ -192,15 +193,19 @@ async def build_flow_no_yield(graph_data: dict, if vertex.base_type == 'vectorstores': # 注入user_name vertex.params['user_name'] = kwargs.get('user_name') if kwargs else '' - if vertex.vertex_type not in ["MilvusWithPermissionCheck", "ElasticsearchWithPermissionCheck"]: + if vertex.vertex_type not in [ + 'MilvusWithPermissionCheck', 'ElasticsearchWithPermissionCheck' + ]: # 知识库通过参数传参 if 'collection_name' in kwargs and 'collection_name' in vertex.params: vertex.params['collection_name'] = kwargs['collection_name'] if 'collection_name' in kwargs and 'index_name' in vertex.params: vertex.params['index_name'] = kwargs['collection_name'] - if 'collection_name' in vertex.params and not vertex.params.get('collection_name'): - vertex.params['collection_name'] = f'tmp_{flow_id}_{chat_id if chat_id else 1}' + if 'collection_name' in vertex.params and not vertex.params.get( + 'collection_name'): + vertex.params[ + 'collection_name'] = f'tmp_{flow_id}_{chat_id if chat_id else 1}' logger.info(f"rename_vector_col col={vertex.params['collection_name']}") if process_file: # L1 清除Milvus历史记录 @@ -397,15 +402,15 @@ def parse_gpus(gpu_str: str) -> List[Dict]: 'gpu_util')[0] res.append({ 'gpu_uuid': - gpu_uuid_elem.firstChild.data, + gpu_uuid_elem.firstChild.data, 'gpu_id': - gpu_id_elem.firstChild.data, + gpu_id_elem.firstChild.data, 'gpu_total_mem': - '%.2f G' % (float(gpu_total_mem.firstChild.data.split(' ')[0]) / 1024), + '%.2f G' % (float(gpu_total_mem.firstChild.data.split(' ')[0]) / 1024), 'gpu_used_mem': - '%.2f G' % (float(free_mem.firstChild.data.split(' ')[0]) / 1024), + '%.2f G' % (float(free_mem.firstChild.data.split(' ')[0]) / 1024), 'gpu_utility': - round(float(gpu_utility_elem.firstChild.data.split(' ')[0]) / 100, 2) + round(float(gpu_utility_elem.firstChild.data.split(' ')[0]) / 100, 2) }) return res diff --git a/src/backend/bisheng/api/v1/callback.py b/src/backend/bisheng/api/v1/callback.py index 9e1417ed5..06fe51d2a 100644 --- a/src/backend/bisheng/api/v1/callback.py +++ b/src/backend/bisheng/api/v1/callback.py @@ -20,7 +20,12 @@ from langchain.schema.messages import BaseMessage class AsyncStreamingLLMCallbackHandler(AsyncCallbackHandler): """Callback handler for streaming LLM responses.""" - def __init__(self, websocket: WebSocket, flow_id: str, chat_id: str, user_id: int = None, **kwargs: Any): + def __init__(self, + websocket: WebSocket, + flow_id: str, + chat_id: str, + user_id: int = None, + **kwargs: Any): self.websocket = websocket self.flow_id = flow_id self.chat_id = chat_id @@ -108,7 +113,7 @@ class AsyncStreamingLLMCallbackHandler(AsyncCallbackHandler): result = output # Create a formatted message. - intermediate_steps = f'{observation_prefix}{result}' + intermediate_steps = f'{observation_prefix}{result[:100]}' # Create a ChatResponse instance. resp = ChatResponse(type='stream', @@ -242,7 +247,12 @@ class AsyncStreamingLLMCallbackHandler(AsyncCallbackHandler): class StreamingLLMCallbackHandler(BaseCallbackHandler): """Callback handler for streaming LLM responses.""" - def __init__(self, websocket: WebSocket, flow_id: str, chat_id: str, user_id: int = None, **kwargs: Any): + def __init__(self, + websocket: WebSocket, + flow_id: str, + chat_id: str, + user_id: int = None, + **kwargs: Any): self.websocket = websocket self.flow_id = flow_id self.chat_id = chat_id @@ -538,7 +548,7 @@ class AsyncGptsDebugCallbackHandler(AsyncGptsLLMCallbackHandler): is_bot=1, message=json.dumps(output_info), intermediate_steps=f'{input_info["steps"]}\n\nTool output:\n\n Error: ' + - str(error), + str(error), category=tool_category, type='end', flow_id=self.flow_id, diff --git a/src/backend/bisheng/api/v1/chat.py b/src/backend/bisheng/api/v1/chat.py index df1acd63c..c2a349b01 100644 --- a/src/backend/bisheng/api/v1/chat.py +++ b/src/backend/bisheng/api/v1/chat.py @@ -1,16 +1,9 @@ import json -import time import math from typing import List, Optional from uuid import UUID -from fastapi import APIRouter, HTTPException, Query, WebSocket, WebSocketException, status, Body, Request -from fastapi.params import Depends -from fastapi.responses import StreamingResponse -from fastapi_jwt_auth import AuthJWT -from sqlalchemy import func -from sqlmodel import select - +from bisheng.api.errcode.base import UnAuthorizedError from bisheng.api.services.assistant import AssistantService from bisheng.api.services.audit_log import AuditLogService from bisheng.api.services.base import BaseService @@ -19,19 +12,31 @@ from bisheng.api.services.flow import FlowService from bisheng.api.services.knowledge_imp import delete_es, delete_vector from bisheng.api.services.user_service import UserPayload, get_login_user from bisheng.api.utils import build_flow, build_input_keys_response, get_request_ip -from bisheng.api.v1.schemas import (BuildStatus, BuiltResponse, ChatInput, ChatList, - FlowGptsOnlineList, InitResponse, StreamData, - UnifiedResponseModel, resp_200, AddChatMessages) +from bisheng.api.v1.schema.base_schema import PageList +from bisheng.api.v1.schema.chat_schema import AppChatList +from bisheng.api.v1.schemas import (AddChatMessages, BuildStatus, BuiltResponse, ChatInput, + ChatList, FlowGptsOnlineList, InitResponse, StreamData, + UnifiedResponseModel, resp_200) from bisheng.cache.redis import redis_client from bisheng.chat.manager import ChatManager from bisheng.database.base import session_getter from bisheng.database.models.assistant import AssistantDao, AssistantStatus from bisheng.database.models.flow import Flow, FlowDao, FlowStatus from bisheng.database.models.flow_version import FlowVersionDao -from bisheng.database.models.message import ChatMessage, ChatMessageDao, ChatMessageRead +from bisheng.database.models.group_resource import GroupResourceDao, ResourceTypeEnum +from bisheng.database.models.message import ChatMessage, ChatMessageDao, ChatMessageRead, MessageDao +from bisheng.database.models.user import UserDao +from bisheng.database.models.user_group import UserGroupDao from bisheng.graph.graph.base import Graph from bisheng.utils.logger import logger from bisheng.utils.util import get_cache_key +from fastapi import (APIRouter, Body, HTTPException, Query, Request, WebSocket, WebSocketException, + status) +from fastapi.params import Depends +from fastapi.responses import StreamingResponse +from fastapi_jwt_auth import AuthJWT +from sqlalchemy import func +from sqlmodel import select router = APIRouter(tags=['Chat']) chat_manager = ChatManager() @@ -39,6 +44,75 @@ flow_data_store = redis_client expire = 600 # reids 60s 过期 +@router.get('/chat/app/list', + response_model=UnifiedResponseModel[PageList[AppChatList]], + status_code=200) +def get_app_chat_list(*, + keyword: Optional[str] = None, + page_num: Optional[int] = 1, + page_size: Optional[int] = 20, + login_user: UserPayload = Depends(get_login_user)): + """通过消息表进行聊天App统计，全量表查询，""" + """性能问题后续优化""" + + group_flow_ids = [] + if not login_user.is_admin(): + # 判断下是否是用户组管理员 + user_groups = UserGroupDao.get_user_admin_group(login_user.user_id) + if not user_groups: + raise UnAuthorizedError.http_exception() + user_group_ids = [user_group.group_id for user_group in user_groups] + # 获取分组下的所有资源ID + resources = GroupResourceDao.get_groups_resource( + user_group_ids, resource_types=[ResourceTypeEnum.FLOW, ResourceTypeEnum.ASSISTANT]) + group_flow_ids = [one.third_id for one in resources] + if not group_flow_ids: + return resp_200(PageList(list=[], total=0)) + + flow_ids, user_ids = [], [] + if keyword: + flows = FlowDao.get_flow_list_by_name(name=keyword) + assistants = AssistantDao.get_all_assistants(name=keyword, page=0, limit=0) + users = UserDao.search_user_by_name(user_name=keyword) + if flows: + flow_ids = [flow.id for flow in flows] + if assistants: + flow_ids = flow_ids.extend([assistant.id for assistant in assistants]) + if user_ids: + user_ids = [user.user_id for user in users] + # 检索内容为空 + if not flow_ids and not user_ids: + return resp_200(PageList(list=[], total=0)) + if group_flow_ids: + if flow_ids: + flow_ids = list(set(flow_ids) & set(group_flow_ids)) + else: + flow_ids = group_flow_ids + + res, count = MessageDao.app_list_group_by_chat_id(page_size=page_size, + page_num=page_num, + flow_ids=flow_ids, + user_ids=user_ids) + # 补齐中文 + user_ids = [one.get('user_id') for one in res] + flow_ids = [one.get('flow_id') for one in res] + user_list = UserDao.get_user_by_ids(user_ids) + flow_list = FlowDao.get_flow_by_ids(flow_ids) + assistant_list = AssistantDao.get_assistants_by_ids(flow_ids) + user_map = {user.user_id: user.user_name for user in user_list} + flow_map = {flow.id: flow.name for flow in flow_list} + assistant_map = {assistant.id: assistant.name for assistant in assistant_list} + + flow_map.update(assistant_map) + res_obj = PageList(list=[ + AppChatList(user_name=user_map.get(one['user_id'], one['user_id']), + flow_name=flow_map.get(one['flow_id'], one['flow_id']), + **one) for one in res + ], + total=count) + return resp_200(res_obj) + + @router.get('/chat/history', response_model=UnifiedResponseModel[List[ChatMessageRead]], status_code=200) @@ -51,8 +125,7 @@ def get_chatmessage(*, if not chat_id or not flow_id: return {'code': 500, 'message': 'chat_id 和 flow_id 必传参数'} where = select(ChatMessage).where(ChatMessage.flow_id == flow_id, - ChatMessage.chat_id == chat_id, - ChatMessage.user_id == login_user.user_id) + ChatMessage.chat_id == chat_id) if id: where = where.where(ChatMessage.id < int(id)) with session_getter() as session: @@ -81,7 +154,8 @@ def del_chat_id(*, else: assistant_info = AssistantDao.get_one_assistant(message.flow_id) if assistant_info: - AuditLogService.delete_chat_assistant(login_user, get_request_ip(request), assistant_info) + AuditLogService.delete_chat_assistant(login_user, get_request_ip(request), + assistant_info) return resp_200(message='删除成功') @@ -98,10 +172,20 @@ def add_chat_messages(*, chat_id = data.chat_id if not chat_id or not flow_id: raise HTTPException(status_code=500, detail='chat_id 和 flow_id 必传参数') - human_message = ChatMessage(flow_id=flow_id.hex, chat_id=chat_id, user_id=login_user.user_id, is_bot=False, - message=data.human_message, type='human', category='question') - bot_message = ChatMessage(flow_id=flow_id.hex, chat_id=chat_id, user_id=login_user.user_id, is_bot=True, - message=data.answer_message, type='bot', category='answer') + human_message = ChatMessage(flow_id=flow_id.hex, + chat_id=chat_id, + user_id=login_user.user_id, + is_bot=False, + message=data.human_message, + type='human', + category='question') + bot_message = ChatMessage(flow_id=flow_id.hex, + chat_id=chat_id, + user_id=login_user.user_id, + is_bot=True, + message=data.answer_message, + type='bot', + category='answer') ChatMessageDao.insert_batch([human_message, bot_message]) # 写审计日志, 判断是否是新建会话 @@ -115,7 +199,8 @@ def add_chat_messages(*, else: assistant_info = AssistantDao.get_one_assistant(flow_id) if assistant_info: - AuditLogService.create_chat_assistant(login_user, get_request_ip(request), flow_id.hex) + AuditLogService.create_chat_assistant(login_user, get_request_ip(request), + flow_id.hex) return resp_200(message='添加成功') @@ -126,7 +211,9 @@ def update_chat_message(*, message: str = Body(embed=True), login_user: UserPayload = Depends(get_login_user)): """ 更新一条消息的内容 安全检查使用""" - logger.info(f"update_chat_message message_id={message_id} message={message} login_user={login_user.user_name}") + logger.info( + f'update_chat_message message_id={message_id} message={message} login_user={login_user.user_name}' + ) chat_message = ChatMessageDao.get_message_by_id(message_id) if not chat_message: return resp_200(message='消息不存在') @@ -165,6 +252,15 @@ def like_response(*, data: ChatInput, login_user: UserPayload = Depends(get_logi return resp_200(message='操作成功') +@router.post('/chat/copied', status_code=200) +def copied_message(*, + message_id: int = Body(embed=True), + login_user: UserPayload = Depends(get_login_user)): + """ 上传复制message的数据 """ + ChatMessageDao.update_message_copied(message_id, 1) + return resp_200(message='操作成功') + + @router.post('/chat/comment', status_code=200) def comment_resp(*, data: ChatInput, login_user: UserPayload = Depends(get_login_user)): comment_answer(data.message_id, data.comment) @@ -179,9 +275,9 @@ def get_chatlist_list(*, smt = (select(ChatMessage.flow_id, ChatMessage.chat_id, func.min(ChatMessage.create_time).label('create_time'), func.max(ChatMessage.update_time).label('update_time')).where( - ChatMessage.user_id == login_user.user_id).group_by( - ChatMessage.flow_id, - ChatMessage.chat_id).order_by(func.max(ChatMessage.update_time).desc())) + ChatMessage.user_id == login_user.user_id).group_by( + ChatMessage.flow_id, + ChatMessage.chat_id).order_by(func.max(ChatMessage.update_time).desc())) with session_getter() as session: db_message = session.exec(smt).all() @@ -237,18 +333,29 @@ def get_chatlist_list(*, def get_online_chat(*, keyword: Optional[str] = None, tag_id: Optional[int] = None, - page: Optional[int] = 0, - limit: Optional[int] = 0, + page: Optional[int] = 1, + limit: Optional[int] = 10, user: UserPayload = Depends(get_login_user)): # 由于是获取助手和技能两个表，需要将page修改下 if page and limit: search_page = math.ceil(page / 2) + else: + search_page = 1 + limit = 10 res = [] - all_assistant = AssistantService.get_assistant(user, keyword, AssistantStatus.ONLINE.value, - tag_id, page=search_page, limit=limit) + all_assistant = AssistantService.get_assistant(user, + keyword, + AssistantStatus.ONLINE.value, + tag_id, + page=search_page, + limit=limit) all_assistant = all_assistant.data.get('data') - flows = FlowService.get_all_flows(user, keyword, FlowStatus.ONLINE.value, - tag_id=tag_id, page=search_page, page_size=limit) + flows = FlowService.get_all_flows(user, + keyword, + FlowStatus.ONLINE.value, + tag_id=tag_id, + page=search_page, + page_size=limit) flows = flows.data.get('data') for one in all_assistant: res.append( @@ -263,12 +370,12 @@ def get_online_chat(*, # 获取用户可见的所有已上线的技能 for one in flows: res.append( - FlowGptsOnlineList(id=one["id"], - name=one["name"], - desc=one["description"], - logo=one["logo"], - create_time=one["create_time"], - update_time=one["update_time"], + FlowGptsOnlineList(id=one['id'], + name=one['name'], + desc=one['description'], + logo=one['logo'], + create_time=one['create_time'], + update_time=one['update_time'], flow_type='flow')) res.sort(key=lambda x: x.update_time, reverse=True) if page and limit: diff --git a/src/backend/bisheng/api/v1/finetune.py b/src/backend/bisheng/api/v1/finetune.py index 37076aed2..84eb3e83d 100644 --- a/src/backend/bisheng/api/v1/finetune.py +++ b/src/backend/bisheng/api/v1/finetune.py @@ -1,4 +1,5 @@ import json +import tempfile from typing import List, Optional from uuid import UUID @@ -9,23 +10,28 @@ from bisheng.api.services.finetune import FinetuneService from bisheng.api.services.finetune_file import FinetuneFileService from bisheng.api.services.user_service import get_login_user, UserPayload from bisheng.api.v1.schemas import FinetuneCreateReq, UnifiedResponseModel, resp_200 +from bisheng.cache.utils import file_download from bisheng.database.models.finetune import Finetune, FinetuneChangeModelName, FinetuneList from bisheng.database.models.model_deploy import ModelDeploy +from bisheng.database.models.knowledge import KnowledgeDao +from bisheng.database.models.knowledge_file import QAKnoweldgeDao from bisheng.database.models.preset_train import PresetTrain from bisheng.utils.minio_client import MinioClient +from fastapi import APIRouter, Body, Depends, File, Query, UploadFile +from fastapi_jwt_auth import AuthJWT +from loguru import logger router = APIRouter(prefix='/finetune', tags=['Finetune'], dependencies=[Depends(get_login_user)]) # create finetune job @router.post('/job', response_model=UnifiedResponseModel[Finetune]) -async def create_job(*, - finetune: FinetuneCreateReq, - Authorize: AuthJWT = Depends()): +async def create_job(*, finetune: FinetuneCreateReq, Authorize: AuthJWT = Depends()): # get login user Authorize.jwt_required() current_user = json.loads(Authorize.get_jwt_subject()) - finetune = Finetune(**finetune.dict(exclude={'method'}), method=finetune.method.value, + finetune = Finetune(**finetune.dict(exclude={'method'}), + method=finetune.method.value, user_id=current_user.get('user_id'), user_name=current_user.get('user_name')) return FinetuneService.create_job(finetune) @@ -33,9 +39,7 @@ async def create_job(*, # 删除训练任务 @router.delete('/job', response_model=UnifiedResponseModel) -async def delete_job(*, - job_id: UUID, - Authorize: AuthJWT = Depends()): +async def delete_job(*, job_id: UUID, Authorize: AuthJWT = Depends()): # get login user Authorize.jwt_required() current_user = json.loads(Authorize.get_jwt_subject()) @@ -44,9 +48,7 @@ async def delete_job(*, # 中止训练任务 @router.post('/job/cancel', response_model=UnifiedResponseModel) -async def cancel_job(*, - job_id: UUID, - Authorize: AuthJWT = Depends()): +async def cancel_job(*, job_id: UUID, Authorize: AuthJWT = Depends()): # get login user Authorize.jwt_required() current_user = json.loads(Authorize.get_jwt_subject()) @@ -55,9 +57,7 @@ async def cancel_job(*, # 发布训练任务 @router.post('/job/publish', response_model=UnifiedResponseModel) -async def publish_job(*, - job_id: UUID, - Authorize: AuthJWT = Depends()): +async def publish_job(*, job_id: UUID, Authorize: AuthJWT = Depends()): # get login user Authorize.jwt_required() current_user = json.loads(Authorize.get_jwt_subject()) @@ -65,9 +65,7 @@ async def publish_job(*, @router.post('/job/publish/cancel', response_model=UnifiedResponseModel) -async def cancel_publish_job(*, - job_id: UUID, - Authorize: AuthJWT = Depends()): +async def cancel_publish_job(*, job_id: UUID, Authorize: AuthJWT = Depends()): # get login user Authorize.jwt_required() current_user = json.loads(Authorize.get_jwt_subject()) @@ -78,8 +76,10 @@ async def cancel_publish_job(*, @router.get('/job', response_model=UnifiedResponseModel[List[Finetune]]) async def get_job(*, server: str = Query(default=None, description='关联的RT服务名字'), - status: str = Query(default='', title='多个以英文逗号,分隔', - description='训练任务的状态，1: 训练中 2: 训练失败 3: 任务中止 4: 训练成功 5: 发布完成'), + status: str = Query( + default='', + title='多个以英文逗号,分隔', + description='训练任务的状态，1: 训练中 2: 训练失败 3: 任务中止 4: 训练成功 5: 发布完成'), model_name: Optional[str] = Query(default='', description='模型名称,模糊搜索'), page: Optional[int] = Query(default=1, description='页码'), limit: Optional[int] = Query(default=10, description='每页条数'), @@ -89,24 +89,24 @@ async def get_job(*, status_list = [] if status.strip(): status_list = [int(one) for one in status.strip().split(',')] - req_data = FinetuneList(server_name=server, status=status_list, model_name=model_name, page=page, limit=limit) + req_data = FinetuneList(server_name=server, + status=status_list, + model_name=model_name, + page=page, + limit=limit) return FinetuneService.get_all_job(req_data) # 获取任务最新详细信息，此接口会同步查询SFT-backend侧将任务状态更新到最新 @router.get('/job/info', response_model=UnifiedResponseModel[Finetune]) -async def get_job_info(*, - job_id: UUID, - Authorize: AuthJWT = Depends()): +async def get_job_info(*, job_id: UUID, Authorize: AuthJWT = Depends()): # get login user Authorize.jwt_required() return FinetuneService.get_job_info(job_id) @router.patch('/job/model', response_model=UnifiedResponseModel) -async def update_job(*, - req_data: FinetuneChangeModelName, - Authorize: AuthJWT = Depends()): +async def update_job(*, req_data: FinetuneChangeModelName, Authorize: AuthJWT = Depends()): # get login user Authorize.jwt_required() current_user = json.loads(Authorize.get_jwt_subject()) @@ -126,27 +126,50 @@ async def upload_file(*, @router.post('/job/file/preset', response_model=UnifiedResponseModel[List[PresetTrain]]) async def upload_preset_file(*, - files: list[UploadFile] = File(description='预置训练文件列表'), + files: Optional[str] = Body(default=None, description='预置训练文件列表'), + name: Optional[str] = Body(description='数据集名字'), + qa_list: Optional[list[int]] = Body(default=None, + description='QA知识库'), Authorize: AuthJWT = Depends()): Authorize.jwt_required() current_user = json.loads(Authorize.get_jwt_subject()) - - return FinetuneFileService.upload_file(files, True, current_user) + if files: + filepath, file_name = file_download(files) + logger.info(f'dataset upload_file_name: {file_name}') + return FinetuneFileService.upload_preset_file(name, 0, filepath, current_user) + elif qa_list: + # 将qa数据按照finetune格式，进行文件存储 + qa_knowledge_db = KnowledgeDao.get_list_by_ids(qa_list) + qa_knowledge_db_ids = [qa_knowledge.id for qa_knowledge in qa_knowledge_db] + qa_db_list = QAKnoweldgeDao.get_qa_knowledge_by_knowledge_ids(qa_knowledge_db_ids) + qa_list = [] + for qa in qa_db_list: + qa_list.extend([{ + 'instruction': question, + 'input': '', + 'output': json.loads(qa.answers)[0] + } for question in qa.questions]) + with tempfile.NamedTemporaryFile(mode='w+', suffix='.json') as filepath: + json.dump(qa_list, filepath, ensure_ascii=False, indent=2) + filepath.seek(0) + return FinetuneFileService.upload_preset_file(name, 1, filepath.name, current_user) # 获取预置训练文件列表 @router.get('/job/file/preset', response_model=UnifiedResponseModel[List[PresetTrain]]) -async def get_preset_file(*, Authorize: AuthJWT = Depends()): +async def get_preset_file(*, + page_size: Optional[int] = None, + page_num: Optional[int] = None, + keyword: Optional[str] = None, + Authorize: AuthJWT = Depends()): # get login user Authorize.jwt_required() - ret = FinetuneFileService.get_preset_file() + ret = FinetuneFileService.get_preset_file(keyword, page_size, page_num) return resp_200(ret) @router.delete('/job/file/preset', response_model=UnifiedResponseModel) -async def delete_preset_file(*, - file_id: UUID, - Authorize: AuthJWT = Depends()): +async def delete_preset_file(*, file_id: UUID, Authorize: AuthJWT = Depends()): # get login user Authorize.jwt_required() current_user = json.loads(Authorize.get_jwt_subject()) @@ -154,20 +177,15 @@ async def delete_preset_file(*, @router.get('/job/file/download', response_model=UnifiedResponseModel) -async def get_download_url(*, - file_url: str, - Authorize: AuthJWT = Depends()): +async def get_download_url(*, file_url: str, Authorize: AuthJWT = Depends()): Authorize.jwt_required() minio_client = MinioClient() download_url = minio_client.get_share_link(file_url) - return resp_200(data={ - 'url': download_url - }) + return resp_200(data={'url': download_url}) @router.get('/server/filters', response_model=UnifiedResponseModel) -async def get_server_filters(*, - Authorize: AuthJWT = Depends()): +async def get_server_filters(*, Authorize: AuthJWT = Depends()): Authorize.jwt_required() return FinetuneService.get_server_filters() @@ -183,8 +201,7 @@ async def get_model_list(request: Request, @router.get('/gpu', response_model=UnifiedResponseModel) -async def get_gpu_info(*, - Authorize: AuthJWT = Depends()): +async def get_gpu_info(*, Authorize: AuthJWT = Depends()): # get login user Authorize.jwt_required() return FinetuneService.get_gpu_info() diff --git a/src/backend/bisheng/api/v1/knowledge.py b/src/backend/bisheng/api/v1/knowledge.py index c4a496b76..b8476d491 100644 --- a/src/backend/bisheng/api/v1/knowledge.py +++ b/src/backend/bisheng/api/v1/knowledge.py @@ -1,4 +1,8 @@ import urllib.parse +import json +import os +import re +import time from typing import List, Optional from fastapi import APIRouter, BackgroundTasks, Depends, File, HTTPException, UploadFile, Request, Body, Query @@ -12,6 +16,36 @@ from bisheng.cache.utils import save_uploaded_file from bisheng.database.models.knowledge import KnowledgeCreate, KnowledgeRead, KnowledgeUpdate, KnowledgeDao from bisheng.database.models.knowledge_file import KnowledgeFile, KnowledgeFileDao from bisheng.utils.logger import logger +from bisheng.api.errcode.base import UnAuthorizedError +from bisheng.api.services import knowledge_imp +from bisheng.api.services.audit_log import AuditLogService +from bisheng.api.services.knowledge_imp import (add_qa, addEmbedding, decide_vectorstores, + delete_knowledge_file_vectors, retry_files) +from bisheng.api.services.user_service import UserPayload, get_login_user +from bisheng.api.utils import get_request_ip +from bisheng.api.v1.schemas import UnifiedResponseModel, UploadFileResponse, resp_200, resp_500 +from bisheng.cache.utils import file_download, save_uploaded_file +from bisheng.database.base import session_getter +from bisheng.database.models.group_resource import GroupResource, GroupResourceDao, ResourceTypeEnum +from bisheng.database.models.knowledge import (Knowledge, KnowledgeCreate, KnowledgeDao, + KnowledgeRead, KnowledgeTypeEnum) +from bisheng.database.models.knowledge_file import (KnowledgeFile, KnowledgeFileDao, + KnowledgeFileRead, QAKnoweldgeDao, + QAKnowledgeUpsert) +from bisheng.database.models.role_access import AccessType, RoleAccess +from bisheng.database.models.user import User, UserDao +from bisheng.database.models.user_group import UserGroupDao +from bisheng.interface.embeddings.custom import FakeEmbedding +from bisheng.settings import settings +from bisheng.utils.logger import logger +from bisheng.utils.minio_client import MinioClient +from bisheng_langchain.vectorstores import ElasticKeywordsSearch +from fastapi import (APIRouter, BackgroundTasks, Body, Depends, File, HTTPException, Request, + UploadFile) +from fastapi.encoders import jsonable_encoder +from pymilvus import Collection +from sqlalchemy import delete, func, or_ +from sqlmodel import select # build router router = APIRouter(prefix='/knowledge', tags=['Knowledge']) @@ -85,7 +119,7 @@ def create_knowledge(*, return resp_200(db_knowledge) -@router.get('/', status_code=200) +@router.get('', status_code=200) def get_knowledge(*, request: Request, login_user: UserPayload = Depends(get_login_user), @@ -138,6 +172,56 @@ def get_filelist(*, }) +@router.get('/qa/list/{qa_knowledge_id}', status_code=200) +def get_QA_list(*, + qa_knowledge_id: int, + page_size: int = 10, + page_num: int = 1, + question: Optional[str] = None, + answer: Optional[str] = None, + keyword: Optional[str] = None, + status: Optional[int] = None, + login_user: UserPayload = Depends(get_login_user)): + """ 获取知识库文件信息. """ + + # 查询当前知识库，是否有写入权限 + with session_getter() as session: + db_knowledge: Knowledge = session.get(Knowledge, qa_knowledge_id) + if not db_knowledge: + raise HTTPException(status_code=500, detail='当前知识库不可用，返回上级目录') + if not login_user.access_check(db_knowledge.user_id, str(qa_knowledge_id), + AccessType.KNOWLEDGE): + return UnAuthorizedError.return_resp() + + if db_knowledge.type == KnowledgeTypeEnum.NORMAL.value: + return HTTPException(status_code=500, detail='知识库为普通知识库') + + if keyword: + question = keyword + + qa_list, total_count = knowledge_imp.list_qa_by_knowledge_id(qa_knowledge_id, page_size, + page_num, question, answer, + status) + user_list = UserDao.get_user_by_ids([qa.user_id for qa in qa_list]) + user_map = { + user.user_id: user.user_name + for user in user_list + } + data = [jsonable_encoder(qa) for qa in qa_list] + for qa in data: + qa['questions'] = qa['questions'][0] + qa['answers'] = json.loads(qa['answers'])[0] + qa['user_name'] = user_map.get(qa['user_id'], qa['user_id']) + + return resp_200({ + 'data': data, + 'total': total_count, + 'writeable': + login_user.access_check(db_knowledge.user_id, str(qa_knowledge_id), + AccessType.KNOWLEDGE_WRITE) + }) + + @router.post('/retry', status_code=200) def retry(*, request: Request, login_user: UserPayload = Depends(get_login_user), background_tasks: BackgroundTasks, @@ -147,15 +231,74 @@ def retry(*, request: Request, login_user: UserPayload = Depends(get_login_user) return resp_200() +@router.delete('/{knowledge_id}', status_code=200) +def delete_knowledge(*, + request: Request, + knowledge_id: int, + login_user: UserPayload = Depends(get_login_user)): + """ 删除知识库信息. """ + + with session_getter() as session: + knowledge = session.get(Knowledge, knowledge_id) + if not knowledge: + raise HTTPException(status_code=404, detail='knowledge not found') + if not login_user.access_check(knowledge.user_id, str(knowledge_id), + AccessType.KNOWLEDGE_WRITE): + raise HTTPException(status_code=404, detail='没有权限执行操作') + + # 处理knowledgefile + with session_getter() as session: + session.exec(delete(KnowledgeFile).where(KnowledgeFile.knowledge_id == knowledge_id)) + session.commit() + # 处理vector + embeddings = FakeEmbedding() + vectore_client = decide_vectorstores(knowledge.collection_name, 'Milvus', embeddings) + if isinstance(vectore_client.col, Collection): + logger.info(f'delete_vectore col={knowledge.collection_name}') + if knowledge.collection_name.startswith('col'): + vectore_client.col.drop() + else: + pk = vectore_client.col.query(expr=f'knowledge_id=="{knowledge.id}"', + output_fields=['pk']) + vectore_client.col.delete(f"pk in {[p['pk'] for p in pk]}") + # 判断milvus 是否还有entity + if vectore_client.col.is_empty: + vectore_client.col.drop() + + # 处理 es + # elastic + esvectore_client: 'ElasticKeywordsSearch' = decide_vectorstores(knowledge.index_name, + 'ElasticKeywordsSearch', + embeddings) + if esvectore_client: + index_name = knowledge.index_name or knowledge.collection_name # 兼容老版本 + res = esvectore_client.client.indices.delete(index=index_name, ignore=[400, 404]) + logger.info(f'act=delete_es index={index_name} res={res}') + + with session_getter() as session: + session.delete(knowledge) + session.commit() + delete_knowledge_hook(request, knowledge, login_user) + return resp_200(message='删除成功') + + +def delete_knowledge_hook(request: Request, knowledge: Knowledge, login_user: UserPayload): + logger.info(f'delete_knowledge_hook id={knowledge.id}, user: {login_user.user_id}') + + # 删除知识库的审计日志 + AuditLogService.delete_knowledge(login_user, get_request_ip(request), knowledge) + + GroupResourceDao.delete_group_resource_by_third_id(str(knowledge.id), + ResourceTypeEnum.KNOWLEDGE) + + @router.delete('/file/{file_id}', status_code=200) def delete_knowledge_file(*, request: Request, file_id: int, login_user: UserPayload = Depends(get_login_user)): """ 删除知识文件信息 """ - KnowledgeService.delete_knowledge_file(request, login_user, [file_id]) - return resp_200(message='删除成功') @@ -212,3 +355,80 @@ async def get_file_bbox(request: Request, login_user: UserPayload = Depends(get_ file_id: int = Query(description='文件唯一ID')): res = KnowledgeService.get_file_bbox(request, login_user, file_id) return resp_200(data=res) + + +@router.post('/qa/add', status_code=200) +def qa_add(*, QACreate: QAKnowledgeUpsert, login_user: UserPayload = Depends(get_login_user)): + """ 增加知识库信息. """ + QACreate.user_id = login_user.user_id + db_knowledge = KnowledgeDao.query_by_id(QACreate.knowledge_id) + if db_knowledge.type != KnowledgeTypeEnum.QA.value: + raise HTTPException(status_code=404, detail='知识库类型错误') + + add_qa(db_knowledge=db_knowledge, data=QACreate) + return resp_200() + + +@router.post('/qa/status_switch', status_code=200) +def qa_status_switch(*, + status: int = Body(embed=True), + id: int = Body(embed=True), + login_user: UserPayload = Depends(get_login_user)): + """ 修改知识库信息. """ + + return resp_200(knowledge_imp.qa_status_change(id, status)) + + +@router.get('/qa/detail', status_code=200) +def qa_list(*, id: int, login_user: UserPayload = Depends(get_login_user)): + """ 增加知识库信息. """ + qa_knowledge = QAKnoweldgeDao.get_qa_knowledge_by_primary_id(id) + qa_knowledge.answers = json.loads(qa_knowledge.answers)[0] + return resp_200(data=qa_knowledge) + + +@router.post('/qa/append', status_code=200) +def qa_append( + *, + ids: list[int] = Body(..., embed=True), + question: str = Body(..., embed=True), + login_user: UserPayload = Depends(get_login_user), +): + """ 增加知识库信息. """ + QA_list = QAKnoweldgeDao.select_list(ids) + knowledge = KnowledgeDao.query_by_id(QA_list[0].knowledge_id) + for qa in QA_list: + qa.questions.append(question) + knowledge_imp.add_qa(knowledge, qa) + return resp_200() + + +@router.delete('/qa/delete', status_code=200) +def qa_delete(*, + ids: list[int] = Body(embed=True), + login_user: UserPayload = Depends(get_login_user)): + """ 删除知识文件信息 """ + knowledge_dbs = QAKnoweldgeDao.select_list(ids) + knowledge = KnowledgeDao.query_by_id(knowledge_dbs[0].knowledge_id) + if not login_user.access_check(knowledge.user_id, str(knowledge.id), + AccessType.KNOWLEDGE_WRITE): + raise HTTPException(status_code=404, detail='没有权限执行操作') + + if knowledge.type == KnowledgeTypeEnum.NORMAL.value: + return HTTPException(status_code=500, detail='知识库类型错误') + + QAKnoweldgeDao.delete_batch(ids) + # knowledge_imp.delete_vector_data(knowledge.id, ids) + return resp_200() + + +@router.post('/qa/auto_question') +def qa_auto_question( + *, + number: int = Body(default=3, embed=True), + ori_question: str = Body(default='', embed=True), + answer: str = Body(default='', embed=True), +): + """通过大模型自动生成问题""" + questions = knowledge_imp.recommend_question(ori_question, number=number, answer=answer) + return resp_200(data={'questions': questions}) diff --git a/src/backend/bisheng/api/v1/schema/base_schema.py b/src/backend/bisheng/api/v1/schema/base_schema.py new file mode 100644 index 000000000..cb20f602c --- /dev/null +++ b/src/backend/bisheng/api/v1/schema/base_schema.py @@ -0,0 +1,11 @@ +from typing import Generic, List, TypeVar + +from pydantic import BaseModel + +# 创建泛型变量 +DataT = TypeVar('DataT') + + +class PageList(Generic[DataT], BaseModel): + list: List[DataT] + total: int diff --git a/src/backend/bisheng/api/v1/schema/chat_schema.py b/src/backend/bisheng/api/v1/schema/chat_schema.py new file mode 100644 index 000000000..970819e0e --- /dev/null +++ b/src/backend/bisheng/api/v1/schema/chat_schema.py @@ -0,0 +1,23 @@ +from datetime import datetime +from typing import List, Optional +from uuid import UUID + +from pydantic import BaseModel + + +class AppChatList(BaseModel): + flow_name: str + user_name: str + user_id: int + chat_id: str + flow_id: UUID + create_time: datetime + like_count: int + dislike_count: int + copied_count: int + + +class APIAddQAParam(BaseModel): + question: str + answer: List[str] + relative_questions: Optional[List[str]] = [] diff --git a/src/backend/bisheng/api/v2/assistant.py b/src/backend/bisheng/api/v2/assistant.py index 6c797fdc5..e58fcfdcb 100644 --- a/src/backend/bisheng/api/v2/assistant.py +++ b/src/backend/bisheng/api/v2/assistant.py @@ -4,12 +4,6 @@ import uuid from typing import Optional from uuid import UUID -from fastapi import APIRouter, Request, HTTPException, WebSocket, WebSocketException -from fastapi import status as http_status -from fastapi.responses import StreamingResponse, ORJSONResponse -from langchain_core.messages import HumanMessage, AIMessage -from loguru import logger - from bisheng.api.services.assistant import AssistantService from bisheng.api.services.assistant_agent import AssistantAgent from bisheng.api.utils import get_request_ip @@ -17,20 +11,28 @@ from bisheng.api.v1.chat import chat_manager from bisheng.api.v1.schemas import OpenAIChatCompletionResp, OpenAIChatCompletionReq, UnifiedResponseModel, \ AssistantInfo, OpenAIChoice from bisheng.api.v2.utils import get_default_operator +from bisheng.api.v1.schemas import (AssistantInfo, OpenAIChatCompletionReq, + OpenAIChatCompletionResp, OpenAIChoice, UnifiedResponseModel) from bisheng.chat.types import WorkType from bisheng.settings import settings +from fastapi import APIRouter, HTTPException, Query, Request, WebSocket, WebSocketException +from fastapi import status as http_status +from fastapi.responses import ORJSONResponse, StreamingResponse +from langchain_core.messages import AIMessage, HumanMessage +from loguru import logger router = APIRouter(prefix='/assistant', tags=['OpenAPI', 'Assistant']) @router.post('/chat/completions', response_model=OpenAIChatCompletionResp) -async def assistant_chat_completions(request: Request, - req_data: OpenAIChatCompletionReq): +async def assistant_chat_completions(request: Request, req_data: OpenAIChatCompletionReq): """ 兼容openai接口格式，所有的错误必须返回非http200的状态码 和助手进行聊天 """ - logger.info(f'act=assistant_chat_completions assistant_id={req_data.model}, ip={get_request_ip(request)}') + logger.info( + f'act=assistant_chat_completions assistant_id={req_data.model}, ip={get_request_ip(request)}' + ) try: # 获取系统配置里配置的默认用户信息 login_user = get_default_operator() @@ -39,7 +41,9 @@ async def assistant_chat_completions(request: Request, # 查找助手信息 res = AssistantService.get_assistant_info(UUID(req_data.model), login_user) if res.status_code != 200: - return ORJSONResponse(status_code=500, content=res.status_message, media_type='application/json') + return ORJSONResponse(status_code=500, + content=res.status_message, + media_type='application/json') assistant_info = res.data # 覆盖温度设置 @@ -73,13 +77,10 @@ async def assistant_chat_completions(request: Request, object='chat.completion', created=int(time.time()), model=req_data.model, - choices=[OpenAIChoice( - index=0, - message={ - 'role': 'assistant', - 'content': answer - } - )], + choices=[OpenAIChoice(index=0, message={ + 'role': 'assistant', + 'content': answer + })], ) # 非流式直接返回结果 @@ -88,14 +89,7 @@ async def assistant_chat_completions(request: Request, # 流式返回最终结果, 兼容openai格式处理 openai_resp.object = 'chat.completion.chunk' - openai_resp.choices = [ - OpenAIChoice( - index=0, - delta={ - 'content': answer - } - ) - ] + openai_resp.choices = [OpenAIChoice(index=0, delta={'content': answer})] async def _event_stream(): # todo：zgq 后续优化成真正的流式输出，目前是出现最终答案之后直接流式返回的 @@ -123,11 +117,26 @@ async def get_assistant_info(request: Request, assistant_id: UUID): return AssistantService.get_assistant_info(assistant_id, login_user) +@router.get('/list', status_code=200) +def get_assistant_list(request: Request, + name: str = Query(default=None, description='助手名称，模糊匹配, 包含描述的模糊匹配'), + tag_id: int = Query(default=None, description='标签ID'), + page: Optional[int] = Query(default=1, gt=0, description='页码'), + limit: Optional[int] = Query(default=10, gt=0, description='每页条数'), + status: Optional[int] = Query(default=None, description='是否上线状态'), + user_id: int = None): + """ + 公开的获取技能信息的接口 + """ + logger.info(f'public_get_list ip: {request.client.host} user_id:{user_id}') + + user_id = user_id if user_id else settings.get_from_db('default_operator').get('user') + login_user = UserPayload(**{'user_id': user_id, 'role': ''}) + return AssistantService.get_assistant(login_user, name, status, tag_id, page, limit) + + @router.websocket('/chat/{assistant_id}') -async def chat(*, - websocket: WebSocket, - assistant_id: str, - chat_id: Optional[str] = None): +async def chat(*, websocket: WebSocket, assistant_id: str, chat_id: Optional[str] = None): """ 助手的ws免登录接口 """ @@ -135,8 +144,8 @@ async def chat(*, login_user = get_default_operator() try: request = websocket - await chat_manager.dispatch_client(request, assistant_id, chat_id, login_user, WorkType.GPTS, - websocket) + await chat_manager.dispatch_client(request, assistant_id, chat_id, login_user, + WorkType.GPTS, websocket) except WebSocketException as exc: logger.error(f'Websocket exception: {str(exc)}') await websocket.close(code=http_status.WS_1011_INTERNAL_ERROR, reason=str(exc)) diff --git a/src/backend/bisheng/api/v2/chat.py b/src/backend/bisheng/api/v2/chat.py index 6884d9332..28a04b407 100644 --- a/src/backend/bisheng/api/v2/chat.py +++ b/src/backend/bisheng/api/v2/chat.py @@ -1,19 +1,20 @@ import json -from typing import Optional -from uuid import uuid4 +from typing import List, Optional +from uuid import UUID, uuid4 from bisheng.api.services.chat_imp import comment_answer from bisheng.api.services.utils import set_flow_knowledge_id from bisheng.api.v1.schemas import ChatInput, resp_200 +from bisheng.api.v2.schema.message import SyncMessage from bisheng.cache.redis import redis_client from bisheng.chat.manager import ChatManager from bisheng.database.base import session_getter from bisheng.database.models.flow import Flow -from bisheng.database.models.message import ChatMessage +from bisheng.database.models.message import ChatMessage, ChatMessageDao from bisheng.processing.process import process_tweaks from bisheng.settings import settings from bisheng.utils.logger import logger -from fastapi import APIRouter, WebSocket, status +from fastapi import APIRouter, Body, WebSocket, status router = APIRouter(prefix='/chat', tags=['OpenAPI', 'Chat']) chat_manager = ChatManager() @@ -97,3 +98,26 @@ def solve_response(*, data: dict): def comment(*, data: ChatInput): comment_answer(data.message_id, data.comment) return resp_200() + + +@router.post('/sync/messages', status_code=200) +def sync_message(*, + flow_id: UUID = Body(embed=True), + message_list: List[SyncMessage] = Body(embed=True), + user_id: int = Body(default=None, embed=True)): + + user_id = user_id if user_id else settings.get_from_db('default_operator').get('user') + + batch_message = [ + ChatMessage(is_bot=message.is_send, + source=0, + message=message.message, + extra=json.dumps(message.extra), + type='answer', + category='answer', + flow_id=flow_id, + user_id=user_id, + chat_id=flow_id.hex) for message in message_list + ] + ChatMessageDao.insert_batch(batch_message) + return resp_200() diff --git a/src/backend/bisheng/api/v2/filelib.py b/src/backend/bisheng/api/v2/filelib.py index ed2fbd42b..c702b3c62 100644 --- a/src/backend/bisheng/api/v2/filelib.py +++ b/src/backend/bisheng/api/v2/filelib.py @@ -2,8 +2,31 @@ import hashlib import json import os from typing import Dict, List, Optional +from urllib.parse import unquote from fastapi import APIRouter, BackgroundTasks, File, Form, HTTPException, UploadFile, Request +from bisheng.api.services import knowledge_imp +from bisheng.api.services.knowledge_imp import (addEmbedding, create_knowledge, decide_vectorstores, + delete_es, delete_knowledge_by, + delete_knowledge_file_vectors, delete_vector, + text_knowledge) +from bisheng.api.v1.schemas import ChunkInput, UnifiedResponseModel, resp_200, resp_500 +from bisheng.api.v2.schema.filelib import APIAddQAParam, APIAppendQAParam, QueryQAParam +from bisheng.cache.utils import save_download_file +from bisheng.database.base import session_getter +from bisheng.database.models.knowledge import (Knowledge, KnowledgeCreate, KnowledgeDao, + KnowledgeRead, KnowledgeTypeEnum, KnowledgeUpdate) +from bisheng.database.models.knowledge_file import (KnowledgeFile, KnowledgeFileDao, + KnowledgeFileRead, QAKnoweldgeDao, QAKnowledge, + QAKnowledgeUpsert) +from bisheng.database.models.message import ChatMessageDao +from bisheng.database.models.role_access import AccessType, RoleAccess +from bisheng.database.models.user import User +from bisheng.interface.embeddings.custom import FakeEmbedding +from bisheng.settings import settings +from bisheng.utils.logger import logger +from bisheng.utils.minio_client import MinioClient +from fastapi import APIRouter, BackgroundTasks, Body, File, Form, HTTPException, UploadFile from fastapi.encoders import jsonable_encoder from sqlalchemy import func from sqlmodel import select @@ -262,3 +285,120 @@ def dump_vector_knowledge(collection_name: str, expr: str = None, store: str = ' def download_statistic_file(file_path: str): file_name = os.path.basename(file_path) return FileResponse(file_path, filename=file_name) + + +@router.post('/add_qa') +def add_qa(*, + knowledge_id: int = Body(embed=True), + data: List[APIAddQAParam] = Body(embed=True), + user_id: Optional[int] = Body(default=None, embed=True)): + + user_id = user_id if user_id else settings.get_from_db('default_operator').get('user') + knowledge = KnowledgeDao.query_by_id(knowledge_id) + logger.info('add_qa_data knowledge_id={} size={}', knowledge_id, len(data)) + res = [] + for item in data: + qa_insert = QAKnowledgeUpsert(knowledge_id=knowledge_id, + questions=[item.question], + answers=item.answer, + user_id=user_id, + extra_meta=json.dumps(item.extra), + source=3) + + res.append(knowledge_imp.add_qa(knowledge, qa_insert)) + + return resp_200(res) + + +@router.post('/add_relative_qa', response_model=List[QAKnowledge]) +def append_qa(*, + knowledge_id: int = Body(embed=True), + data: APIAppendQAParam = Body(embed=True), + user_id: Optional[int] = Body(default=None, embed=True)): + + user_id = user_id if user_id else settings.get_from_db('default_operator').get('user') + knowledge = KnowledgeDao.query_by_id(knowledge_id) + qa_db = QAKnoweldgeDao.get_qa_knowledge_by_primary_id(data.id) + if not qa_db: + return HTTPException(404, detail='qa 对没有找到') + + qa_insert = QAKnowledgeUpsert.validate(knowledge) + qa_insert.questions.extend(data.relative_questions) + + return resp_200(knowledge_imp.add_qa(knowledge, qa_insert)) + + +@router.delete('/qa/{qa_id}', status_code=200) +def delete_qa_data(*, qa_id: int, question: Optional[str] = None): + """ 删除qa 问题对信息 """ + qa = QAKnoweldgeDao.get_qa_knowledge_by_primary_id(qa_id) + + if not qa: + raise HTTPException(status_code=404, detail='qa 不存在') + + if question: + qa.questions = [q for q in qa.questions if q != question] + QAKnoweldgeDao.update(qa) + else: + QAKnoweldgeDao.delete_batch([qa_id]) + try: + knowledge = KnowledgeDao.query_by_id(qa.knowledge_id) + knowledge_imp.delete_vector_data(knowledge, file_ids=[qa_id]) + if question: + knowledge_imp.QA_save_knowledge(knowledge, qa) + return resp_200() + except Exception as e: + return resp_500(message=f'error e={str(e)}') + + +@router.post('/update_qa', status_code=200) +def update_qa( + *, + id: int = Body(embed=True), + question: Optional[str] = Body(default=None, embed=True), + original_question: Optional[str] = Body(default=None, embed=True), + answer: Optional[List[str]] = Body(default=None, embed=True), +): + """ 删除qa 问题对信息 """ + qa = QAKnoweldgeDao.get_qa_knowledge_by_primary_id(id) + + if not qa: + raise HTTPException(status_code=404, detail='qa 不存在') + + if original_question: + qa.questions = [q if q != question else question for q in qa.questions] + else: + qa.questions = [question] + if answer: + qa.answers = json.dumps(answer, ensure_ascii=False) + QAKnoweldgeDao.update(qa) + + try: + knowledge = KnowledgeDao.query_by_id(qa.knowledge_id) + if question: + knowledge_imp.delete_vector_data(knowledge, file_ids=[id]) + knowledge_imp.QA_save_knowledge(knowledge, qa) + return resp_200() + except Exception as e: + return resp_500(message=f'error e={str(e)}') + + +@router.get('/detail_qa', status_code=200) +def detail_qa(*, id: int): + """ 获取问题对信息 """ + qa = QAKnoweldgeDao.get_qa_knowledge_by_primary_id(id) + return resp_200(qa) + + +@router.post('/query_qa', status_code=200) +def query_qa(QueryQAParam: QueryQAParam): + """ 删除qa 问题对信息 """ + sources = [1, 2] # 3 是api倒入的 + qa_list = QAKnoweldgeDao.query_by_condition_v1(source=sources, + create_start=QueryQAParam.timeRange[0], + create_end=QueryQAParam.timeRange[1]) + if qa_list: + for q in qa_list: + q.answers = json.loads(q.answers) + + return resp_200(qa_list) diff --git a/src/backend/bisheng/api/v2/flow.py b/src/backend/bisheng/api/v2/flow.py index 0213be887..3fafe704f 100644 --- a/src/backend/bisheng/api/v2/flow.py +++ b/src/backend/bisheng/api/v2/flow.py @@ -1,24 +1,21 @@ from uuid import UUID -from loguru import logger - -from fastapi import APIRouter, Request, HTTPException from bisheng.api.services.flow import FlowService from bisheng.api.services.user_service import UserPayload -from bisheng.api.v1.schemas import resp_200 from bisheng.api.v2.assistant import get_default_operator -from bisheng.database.models.flow import FlowDao from bisheng.settings import settings +from fastapi import APIRouter, HTTPException, Query, Request +from loguru import logger router = APIRouter(prefix='/flows', tags=['OpenAPI', 'FlowV2']) -@router.get("/{flow_id}", status_code=200) +@router.get('/{flow_id}', status_code=200) def get_flow(request: Request, flow_id: UUID): """ 公开的获取技能信息的接口 """ - logger.info(f"public_get_flow ip: {request.client.host} flow_id:{flow_id}") + logger.info(f'public_get_flow ip: {request.client.host} flow_id:{flow_id}') # 判断下配置是否打开 if not settings.get_from_db("default_operator").get("enable_guest_access"): raise HTTPException(status_code=403, detail="无权限访问") @@ -30,3 +27,25 @@ def get_flow(request: Request, flow_id: UUID): }) return FlowService.get_one_flow(login_user, flow_id.hex) + + +@router.get('', status_code=200) +def get_flow_list(request: Request, + name: str = Query(default=None, description='根据name查找数据库，包含描述的模糊搜索'), + tag_id: int = Query(default=None, description='标签ID'), + page_size: int = Query(default=10, description='每页数量'), + page_num: int = Query(default=1, description='页数'), + status: int = None, + user_id: int = None): + """ + 公开的获取技能信息的接口 + """ + logger.info(f'public_get_flow_list ip: {request.client.host} user_id={user_id}') + + user_id = user_id if user_id else settings.get_from_db('default_operator').get('user') + login_user = UserPayload(**{'user_id': user_id, 'role': ''}) + try: + return FlowService.get_all_flows(login_user, name, status, tag_id, page_num, page_size) + except Exception as e: + logger.error(e) + raise HTTPException(status_code=500, detail='获取技能列表失败') diff --git a/src/backend/bisheng/api/v2/schema/filelib.py b/src/backend/bisheng/api/v2/schema/filelib.py new file mode 100644 index 000000000..71daf8323 --- /dev/null +++ b/src/backend/bisheng/api/v2/schema/filelib.py @@ -0,0 +1,18 @@ +from typing import Dict, List, Optional + +from pydantic import BaseModel + + +class APIAddQAParam(BaseModel): + question: str + answer: List[str] + extra: Optional[Dict] = {} + + +class APIAppendQAParam(BaseModel): + relative_questions: List[str] = [] + id: str = None + + +class QueryQAParam(BaseModel): + timeRange: List[str] diff --git a/src/backend/bisheng/api/v2/schema/message.py b/src/backend/bisheng/api/v2/schema/message.py new file mode 100644 index 000000000..642e490c3 --- /dev/null +++ b/src/backend/bisheng/api/v2/schema/message.py @@ -0,0 +1,8 @@ +from pydantic import BaseModel + + +class SyncMessage(BaseModel): + is_send: bool + message: str + create_time: str + extra: dict diff --git a/src/backend/bisheng/chat/handlers.py b/src/backend/bisheng/chat/handlers.py index 61b17ebd7..d80c91956 100644 --- a/src/backend/bisheng/chat/handlers.py +++ b/src/backend/bisheng/chat/handlers.py @@ -3,6 +3,7 @@ import time from queue import Queue from typing import Dict +from bisheng.api.utils import build_input_keys_response from bisheng.api.v1.schemas import ChatMessage, ChatResponse from bisheng.chat.manager import ChatManager from bisheng.chat.utils import judge_source, process_graph, process_source_document @@ -286,7 +287,6 @@ class Handler: async def process_file(self, session: ChatManager, client_id: str, chat_id: str, payload: dict, user_id: int): file_name = payload['inputs'] - file_name.pop('id', '') batch_question = payload['inputs']['questions'] # 如果L3 file = ChatMessage(is_bot=False, message=file_name, type='end', user_id=user_id) @@ -305,12 +305,13 @@ class Handler: input_key = 'input' input_dict = {} else: - input_key = langchain_object.input_keys[0] + input_key = list(build_input_keys_response(langchain_object, + {})['input_keys'].keys())[0] input_dict = {k: '' for k in langchain_object.input_keys} batch_question = ['start'] if not batch_question else batch_question # 确保点击确定，会执行LLM report = '' - logger.info(f'process_file batch_question={batch_question}') + logger.info(f'process_file batch_question={batch_question} input_key={input_key}') for question in batch_question: if not question: continue @@ -332,13 +333,14 @@ class Handler: await session.send_json(client_id, chat_id, response_step) report = f"""{report}### {question} \n {result} \n """ - start_resp.category = 'report' - await session.send_json(client_id, chat_id, start_resp) - response = ChatResponse(type='end', - intermediate_steps=report, - category='report', - user_id=user_id) - await session.send_json(client_id, chat_id, response) + if len(batch_question) > 1: + start_resp.category = 'report' + await session.send_json(client_id, chat_id, start_resp) + response = ChatResponse(type='end', + intermediate_steps=report, + category='report', + user_id=user_id) + await session.send_json(client_id, chat_id, response) close_resp = ChatResponse(type='close', category='system', user_id=user_id) await session.send_json(client_id, chat_id, close_resp) diff --git a/src/backend/bisheng/chat/manager.py b/src/backend/bisheng/chat/manager.py index 2d2733439..82365086c 100644 --- a/src/backend/bisheng/chat/manager.py +++ b/src/backend/bisheng/chat/manager.py @@ -4,12 +4,9 @@ import json import time import uuid from collections import defaultdict +from queue import Queue from typing import Any, Dict, List from uuid import UUID -from queue import Queue - -from loguru import logger -from fastapi import WebSocket, WebSocketDisconnect, status, Request from bisheng.api.services.audit_log import AuditLogService from bisheng.api.services.user_service import UserPayload @@ -25,10 +22,13 @@ from bisheng.database.base import session_getter from bisheng.database.models.flow import Flow from bisheng.database.models.message import ChatMessageDao from bisheng.database.models.user import User, UserDao +from bisheng.graph.utils import cut_graph_bynode, find_next_node from bisheng.processing.process import process_tweaks from bisheng.utils.threadpool import ThreadPoolManager, thread_pool from bisheng.utils.util import get_cache_key from bisheng_langchain.input_output.output import Report +from fastapi import Request, WebSocket, WebSocketDisconnect, status +from loguru import logger class ChatHistory(Subject): @@ -38,10 +38,10 @@ class ChatHistory(Subject): self.history: Dict[str, List[ChatMessage]] = defaultdict(list) def add_message( - self, - client_id: str, - chat_id: str, - message: ChatMessage, + self, + client_id: str, + chat_id: str, + message: ChatMessage, ): """Add a message to the chat history.""" t1 = time.time() @@ -187,14 +187,15 @@ class ChatManager: if 'after sending' in str(exc): logger.error(exc) - async def dispatch_client(self, - request: Request, # 原始请求体 - client_id: str, - chat_id: str, - login_user: UserPayload, - work_type: WorkType, - websocket: WebSocket, - graph_data: dict = None): + async def dispatch_client( + self, + request: Request, # 原始请求体 + client_id: str, + chat_id: str, + login_user: UserPayload, + work_type: WorkType, + websocket: WebSocket, + graph_data: dict = None): client_key = uuid.uuid4().hex chat_client = ChatClient(request, client_key, @@ -243,17 +244,17 @@ class ChatManager: self.clear_client(client_key) async def handle_websocket( - self, - flow_id: str, - chat_id: str, - websocket: WebSocket, - user_id: int, - gragh_data: dict = None, + self, + flow_id: str, + chat_id: str, + websocket: WebSocket, + user_id: int, + gragh_data: dict = None, ): # 建立连接，并存储映射，兼容不复用ws 场景 key_list = set([get_cache_key(flow_id, chat_id)]) await self.connect(flow_id, chat_id, websocket) - autogen_pool = ThreadPoolManager(max_workers=1, thread_name_prefix='autogen') + # autogen_pool = ThreadPoolManager(max_workers=1, thread_name_prefix='autogen') context_dict = { get_cache_key(flow_id, chat_id): { 'status': 'init', @@ -407,10 +408,11 @@ class ChatManager: if len(res) <= 1: # 说明是新建会话 websocket = self.active_connections[key] login_user = UserPayload(**{ - "user_id": user_id, - "user_name": UserDao.get_user(user_id).user_name, + 'user_id': user_id, + 'user_name': UserDao.get_user(user_id).user_name, }) - AuditLogService.create_chat_flow(login_user, get_request_ip(websocket), flow_id) + AuditLogService.create_chat_flow(login_user, get_request_ip(websocket), + flow_id) start_resp.type = 'start' # should input data @@ -440,7 +442,7 @@ class ChatManager: if payload and self.in_memory_cache.get(langchain_obj_key): action, over = await self.preper_action(flow_id, chat_id, langchain_obj_key, payload, start_resp, step_resp) - logger.info( + logger.debug( f"processing_message message={payload.get('inputs')} action={action} over={over}") if not over: # task_service: 'TaskService' = get_task_service() @@ -461,9 +463,13 @@ class ChatManager: if isinstance(self.in_memory_cache.get(langchain_obj_key), AutoGenChain): # autogen chain logger.info(f'autogen_submit {langchain_obj_key}') - autogen_pool.submit(key, Handler(stream_queue=self.stream_queue[key]).dispatch_task, **params) + autogen_pool.submit(key, + Handler(stream_queue=self.stream_queue[key]).dispatch_task, + **params) else: - thread_pool.submit(key, Handler(stream_queue=self.stream_queue[key]).dispatch_task, **params) + thread_pool.submit(key, + Handler(stream_queue=self.stream_queue[key]).dispatch_task, + **params) status_ = 'init' context.update({'status': status_}) context.update({'payload': {}}) # clean message @@ -489,7 +495,33 @@ class ChatManager: or 'file_path' in payload['inputs']): node_data = payload['inputs'].get('data', '') or [payload['inputs']] graph_data = self.refresh_graph_data(graph_data, node_data) - self.set_cache(langchain_obj_key, None) # rebuild object + # 上传文件就重新解压，有点粗, 只有document loader 需要 + node_loader = False + + for nod in node_data: + if any('Loader' in x['id'] for x in find_next_node(graph_data, nod['id'])): + node_loader = True + break + if node_loader: + self.set_cache(langchain_obj_key, None) # rebuild object + else: + # 能力，只考虑一个file input的情况 + node_id = find_next_node(graph_data, nod['id'])[0]['id'] + nodes, edges = cut_graph_bynode(graph_data, node_id) + chat_history = payload.get('chatHistory') or [{ + 'isSend': False, + 'message': '请问想买什么' + }] + last_msg = [m['message'] for m in chat_history if not m['isSend']][-1] + graph_data_input = {'edges': edges, 'nodes': nodes} + graph_data_input = process_tweaks(graph_data_input, + tweaks={node_id: { + 'question': last_msg + }}) + obj = await build_flow_no_yield(graph_data_input, {}, process_file=True) + obj.abuild() + question = [await x.get_result() for x in obj.get_input_nodes()][-1] + self.set_cache(langchain_obj_key + '_question', [question]) has_file = any(['InputFile' in nd.get('id', '') for nd in node_data]) has_variable = any(['VariableNode' in nd.get('id', '') for nd in node_data]) if has_file: @@ -590,8 +622,12 @@ class ChatManager: logger.info(f'init_langchain build_end timecost={time.time() - start_time}') question = [] for node in graph.vertices: - if node.vertex_type == 'InputNode': - question.extend(await node.get_result()) + if node.vertex_type in {'InputNode', 'AudioInputNode', 'FileInputNode'}: + question_parse = await node.get_result() + if isinstance(question_parse, list): + question.extend(question_parse) + else: + question.append(question_parse) self.set_cache(key_node + '_question', question) input_nodes = graph.get_input_nodes() diff --git a/src/backend/bisheng/custom/customs.py b/src/backend/bisheng/custom/customs.py index 23acdd828..aa77e7e39 100644 --- a/src/backend/bisheng/custom/customs.py +++ b/src/backend/bisheng/custom/customs.py @@ -23,9 +23,9 @@ CUSTOM_NODES = { }, 'memories': { 'PostgresChatMessageHistory': - frontend_node.memories.PostgresChatMessageHistoryFrontendNode(), + frontend_node.memories.PostgresChatMessageHistoryFrontendNode(), 'MongoDBChatMessageHistory': - frontend_node.memories.MongoDBChatMessageHistoryFrontendNode(), + frontend_node.memories.MongoDBChatMessageHistoryFrontendNode(), }, 'chains': { 'SeriesCharacterChain': frontend_node.chains.SeriesCharacterChainNode(), diff --git a/src/backend/bisheng/database/models/assistant.py b/src/backend/bisheng/database/models/assistant.py index c16f9d9a7..8d38a9e03 100644 --- a/src/backend/bisheng/database/models/assistant.py +++ b/src/backend/bisheng/database/models/assistant.py @@ -160,8 +160,8 @@ class AssistantDao(Assistant): return session.exec(statement).all() @classmethod - def get_all_assistants(cls, name: str, page: int, limit: int, assistant_ids: List[UUID] = None, status: int = None) \ - -> (List[Assistant], int): + def get_all_assistants(cls, name: str, page: int, limit: int, assistant_ids: List[UUID] = None, + status: int = None) -> (List[Assistant], int): with session_getter() as session: statement = select(Assistant).where(Assistant.is_delete == 0) count_statement = session.query(func.count( diff --git a/src/backend/bisheng/database/models/dataset.py b/src/backend/bisheng/database/models/dataset.py new file mode 100644 index 000000000..cf85663a8 --- /dev/null +++ b/src/backend/bisheng/database/models/dataset.py @@ -0,0 +1,93 @@ +from datetime import datetime +from typing import Any, List, Optional + +from bisheng.database.base import session_getter +from bisheng.database.models.base import SQLModelSerializable +from sqlalchemy import Column, DateTime, delete, text +from sqlmodel import Field, select + + +class DatasetBase(SQLModelSerializable): + user_id: Optional[int] = Field(index=True, description='创建用户id') + name: str = Field(index=True, description='数据集名称') + type: str = Field(index=False, default=0, description='预留字段') + description: Optional[str] = Field(index=False, description='数据集描述') + object_name: Optional[str] = Field(index=False, description='数据集S3名称') + create_time: Optional[datetime] = Field( + sa_column=Column(DateTime, nullable=False, server_default=text('CURRENT_TIMESTAMP'))) + update_time: Optional[datetime] = Field( + sa_column=Column(DateTime, + nullable=True, + server_default=text('CURRENT_TIMESTAMP'), + onupdate=text('CURRENT_TIMESTAMP'))) + + +class Dataset(DatasetBase, table=True): + id: Optional[int] = Field(default=None, primary_key=True) + + +class DatasetCreate(DatasetBase): + pass + + +class DatasetRead(DatasetBase): + user_name: Optional[str] + url: Optional[str] + + +class DatasetUpdate(DatasetBase): + pass + + +class DatasetDao(DatasetBase): + + @classmethod + def filter_dataset_by_ids(cls, + dataset_ids: List[int], + keyword: str = None, + page: int = 0, + limit: int = 0) -> List[Dataset]: + with session_getter() as session: + query = select(Dataset) + if dataset_ids: + query = query.where(Dataset.id.in_(dataset_ids)) + if keyword: + query = query.where(Dataset.name.like(f'%{keyword}%')) + if page and limit: + query = query.order_by(Dataset.update_time.desc()).offset( + (page - 1) * limit).limit(limit) + + return session.exec(query).all() + + @classmethod + def get_count_by_filter(cls, filters: List[Any]) -> int: + with session_getter() as session: + return session.scalar(select(Dataset.id).where(*filters)) + + @classmethod + def insert(cls, dataset: DatasetCreate): + with session_getter() as session: + db_insert = Dataset.validate(dataset) + session.add(db_insert) + session.commit() + session.refresh(db_insert) + return db_insert + + @classmethod + def get_dataset_by_name(cls, name: str): + with session_getter() as session: + return session.exec(select(Dataset).where(Dataset.name == name)).all() + + @classmethod + def update(cls, data: Dataset): + with session_getter() as session: + session.add(data) + session.commit() + session.refresh(data) + return data + + @classmethod + def delete(cls, dataset_id: int): + with session_getter() as session: + session.exec(delete(Dataset).where(Dataset.id == dataset_id)) + session.commit() diff --git a/src/backend/bisheng/database/models/flow.py b/src/backend/bisheng/database/models/flow.py index 76d33b74e..b565ca843 100644 --- a/src/backend/bisheng/database/models/flow.py +++ b/src/backend/bisheng/database/models/flow.py @@ -95,7 +95,10 @@ class FlowDao(FlowBase): with session_getter() as session: session.add(flow_info) # 创建一个默认的版本 - flow_version = FlowVersion(name="v0", is_current=1, data=flow_info.data, flow_id=flow_info.id.hex, + flow_version = FlowVersion(name='v0', + is_current=1, + data=flow_info.data, + flow_id=flow_info.id.hex, user_id=flow_info.user_id) session.add(flow_version) session.commit() @@ -108,7 +111,8 @@ class FlowDao(FlowBase): with session_getter() as session: session.delete(flow_info) # 删除对应的版本信息 - update_statement = update(FlowVersion).where(FlowVersion.flow_id == flow_info.id.hex).values(is_delete=1) + update_statement = update(FlowVersion).where( + FlowVersion.flow_id == flow_info.id.hex).values(is_delete=1) session.exec(update_statement) session.commit() return flow_info @@ -139,6 +143,12 @@ class FlowDao(FlowBase): statement = select(Flow).where(Flow.user_id == user_id, Flow.name == name) return session.exec(statement).first() + @classmethod + def get_flow_list_by_name(cls, name: str) -> List[Flow]: + with session_getter() as session: + statement = select(Flow).where(Flow.name.like('%{}%'.format(name))) + return session.exec(statement).all() + @classmethod def get_flow_by_access(cls, role_id: int, name: str, page_size: int, page_num: int) -> List[Tuple[Flow, RoleAccess]]: @@ -176,7 +186,8 @@ class FlowDao(FlowBase): elif not extra_ids: statement = statement.where(Flow.user_id == user_id) if name: - statement = statement.where(or_(Flow.name.like(f'%{name}%'), Flow.description.like(f'%{name}%'))) + statement = statement.where( + or_(Flow.name.like(f'%{name}%'), Flow.description.like(f'%{name}%'))) if status is not None: statement = statement.where(Flow.status == status) if flow_ids: @@ -189,17 +200,22 @@ class FlowDao(FlowBase): return [Flow.model_validate(f) for f in flows_partial] @classmethod - def count_flows(cls, user_id: Optional[int], extra_ids: Union[List[str], str], name: str, - status: Optional[int] = None, flow_ids: List[str] = None) -> int: + def count_flows(cls, + user_id: Optional[int], + extra_ids: Union[List[str], str], + name: str, + status: Optional[int] = None, + flow_ids: List[str] = None) -> int: with session_getter() as session: count_statement = session.query(func.count(Flow.id)) if extra_ids and isinstance(extra_ids, List): - count_statement = count_statement.filter(or_(Flow.id.in_(extra_ids), Flow.user_id == user_id)) + count_statement = count_statement.filter( + or_(Flow.id.in_(extra_ids), Flow.user_id == user_id)) elif not extra_ids: count_statement = count_statement.filter(Flow.user_id == user_id) if name: - count_statement = count_statement.filter(or_(Flow.name.like(f'%{name}%'), - Flow.description.like(f'%{name}%'))) + count_statement = count_statement.filter( + or_(Flow.name.like(f'%{name}%'), Flow.description.like(f'%{name}%'))) if flow_ids: count_statement = count_statement.filter(Flow.id.in_(flow_ids)) if status is not None: @@ -210,17 +226,22 @@ class FlowDao(FlowBase): def get_all_online_flows(cls, keyword: str = None, flow_ids: List[str] = None) -> List[Flow]: with session_getter() as session: statement = select(Flow.id, Flow.user_id, Flow.name, Flow.status, Flow.create_time, - Flow.logo, Flow.update_time, Flow.description, Flow.guide_word).where( - Flow.status == FlowStatus.ONLINE.value) + Flow.logo, Flow.update_time, Flow.description, + Flow.guide_word).where(Flow.status == FlowStatus.ONLINE.value) if flow_ids: statement = statement.where(Flow.id.in_(flow_ids)) if keyword: - statement = statement.where(or_(Flow.name.like(f'%{keyword}%'), Flow.description.like(f'%{keyword}%'))) + statement = statement.where( + or_(Flow.name.like(f'%{keyword}%'), Flow.description.like(f'%{keyword}%'))) result = session.exec(statement).mappings().all() return [Flow.model_validate(f) for f in result] @classmethod - def get_user_access_online_flows(cls, user_id: int, page: int = 0, limit: int = 0, keyword: str = None, + def get_user_access_online_flows(cls, + user_id: int, + page: int = 0, + limit: int = 0, + keyword: str = None, flow_ids: List[str] = None) -> List[Flow]: user_role = UserRoleDao.get_user_roles(user_id) flow_id_extra = [] @@ -233,7 +254,12 @@ class FlowDao(FlowBase): role_access = RoleAccessDao.get_role_access(role_ids, AccessType.FLOW) if role_access: flow_id_extra = [access.third_id for access in role_access] - return FlowDao.get_flows(user_id, flow_id_extra, keyword, FlowStatus.ONLINE.value, flow_ids=flow_ids, page=page, + return FlowDao.get_flows(user_id, + flow_id_extra, + keyword, + FlowStatus.ONLINE.value, + flow_ids=flow_ids, + page=page, limit=limit) @classmethod @@ -243,21 +269,16 @@ class FlowDao(FlowBase): 通过技能ID过滤技能列表，只返回简略信息，不包含data """ statement = select(Flow.id, Flow.user_id, Flow.name, Flow.status, Flow.create_time, - Flow.update_time, Flow.description, - Flow.guide_word) + Flow.update_time, Flow.description, Flow.guide_word) count_statement = select(func.count(Flow.id)) if flow_ids: statement = statement.where(Flow.id.in_(flow_ids)) count_statement = count_statement.where(Flow.id.in_(flow_ids)) if keyword: - statement = statement.where(or_( - Flow.name.like(f'%{keyword}%'), - Flow.description.like(f'%{keyword}%') - )) - count_statement = count_statement.where(or_( - Flow.name.like(f'%{keyword}%'), - Flow.description.like(f'%{keyword}%') - )) + statement = statement.where( + or_(Flow.name.like(f'%{keyword}%'), Flow.description.like(f'%{keyword}%'))) + count_statement = count_statement.where( + or_(Flow.name.like(f'%{keyword}%'), Flow.description.like(f'%{keyword}%'))) if page and limit: statement = statement.offset((page - 1) * limit).limit(limit) statement = statement.order_by(Flow.update_time.desc()) diff --git a/src/backend/bisheng/database/models/group_resource.py b/src/backend/bisheng/database/models/group_resource.py index 118acbc6d..e0abdc49b 100644 --- a/src/backend/bisheng/database/models/group_resource.py +++ b/src/backend/bisheng/database/models/group_resource.py @@ -79,6 +79,23 @@ class GroupResourceDao(GroupResourceBase): statement = statement.offset(page_size * (page_num - 1)).limit(page_size) return session.exec(statement).all() + @classmethod + def get_groups_resource(cls, + group_ids: List[int], + resource_types: List[ResourceTypeEnum] = None, + name: str = None, + page_size: int = None, + page_num: int = None) -> list[GroupResource]: + with session_getter() as session: + statement = select(GroupResource).where(GroupResource.group_id.in_(group_ids)) + if resource_types: + statement = statement.where(GroupResource.type.in_([r.value for r in resource_types])) + if name: + statement = statement.where(GroupResource.third_id.like(f'%{name}%')) + if page_num and page_size: + statement = statement.offset(page_size * (page_num - 1)).limit(page_size) + return session.exec(statement).all() + @classmethod def get_resource_group(cls, resource_type: ResourceTypeEnum, third_id: str) -> list[GroupResource]: """ diff --git a/src/backend/bisheng/database/models/knowledge.py b/src/backend/bisheng/database/models/knowledge.py index 52e340505..65ee41907 100644 --- a/src/backend/bisheng/database/models/knowledge.py +++ b/src/backend/bisheng/database/models/knowledge.py @@ -2,6 +2,8 @@ from datetime import datetime from typing import Any, List, Optional, Union from sqlmodel.sql.expression import Select, SelectOfScalar +from enum import Enum +from typing import Any, List, Optional, Tuple from bisheng.database.base import session_getter from bisheng.database.models.base import SQLModelSerializable @@ -14,9 +16,16 @@ from sqlalchemy import Column, DateTime, func, text, delete, update from sqlmodel import Field, or_, select +class KnowledgeTypeEnum(Enum): + QA = 1 + NORMAL = 0 + + class KnowledgeBase(SQLModelSerializable): user_id: Optional[int] = Field(index=True) name: str = Field(index=True, min_length=1, max_length=30, description="知识库名, 最少一个字符，最多30个字符") + name: str = Field(index=True) + type: str = Field(index=False, default=0, description='0 为普通知识库，1 为QA知识库') description: Optional[str] = Field(index=True) model: Optional[str] = Field(index=False) collection_name: Optional[str] = Field(index=False) @@ -81,7 +90,7 @@ class KnowledgeDao(KnowledgeBase): return session.get(Knowledge, knowledge_id) @classmethod - def get_list_by_ids(cls, ids: List[int]): + def get_list_by_ids(cls, ids: List[int]) -> List[Knowledge]: with session_getter() as session: return session.exec(select(Knowledge).where(Knowledge.id.in_(ids))).all() diff --git a/src/backend/bisheng/database/models/knowledge_file.py b/src/backend/bisheng/database/models/knowledge_file.py index ac535173e..560e8a153 100644 --- a/src/backend/bisheng/database/models/knowledge_file.py +++ b/src/backend/bisheng/database/models/knowledge_file.py @@ -1,10 +1,13 @@ +import json from datetime import datetime from enum import Enum from typing import List, Optional from bisheng.database.base import session_getter from bisheng.database.models.base import SQLModelSerializable -from sqlalchemy import Column, DateTime, String, text +# if TYPE_CHECKING: +from pydantic import validator +from sqlalchemy import JSON, Column, DateTime, String, text from sqlmodel import Field, delete, func, select @@ -40,10 +43,54 @@ class KnowledgeFileBase(SQLModelSerializable): onupdate=text('CURRENT_TIMESTAMP'))) +class QAKnowledgeBase(SQLModelSerializable): + user_id: Optional[int] = Field(index=True) + knowledge_id: int = Field(index=True) + questions: List[str] = Field(index=False) + answers: str = Field(index=False) + source: Optional[int] = Field(index=False, description='0: 未知 1: 手动；2: 审计, 3: api') + status: Optional[int] = Field(index=False, description='1: 解析中；2: 解析成功；3: 解析失败') + extra_meta: Optional[str] = Field(index=False) + remark: Optional[str] = Field(sa_column=Column(String(length=512))) + create_time: Optional[datetime] = Field( + sa_column=Column(DateTime, nullable=False, server_default=text('CURRENT_TIMESTAMP'))) + update_time: Optional[datetime] = Field( + sa_column=Column(DateTime, + nullable=False, + server_default=text('CURRENT_TIMESTAMP'), + onupdate=text('CURRENT_TIMESTAMP'))) + + @validator('questions') + def validate_json(v): + # dict_keys(['description', 'name', 'id', 'data']) + if not v: + return v + if not isinstance(v, List): + raise ValueError('question must be a valid JSON') + + return v + + @validator('answers') + def validate_answer(v): + # dict_keys(['description', 'name', 'id', 'data']) + if not v: + return v + if isinstance(v, List): + return json.dumps(v, ensure_ascii=False) + + return v + + class KnowledgeFile(KnowledgeFileBase, table=True): id: Optional[int] = Field(default=None, primary_key=True) +class QAKnowledge(QAKnowledgeBase, table=True): + id: Optional[int] = Field(default=None, primary_key=True) + questions: Optional[List[str]] = Field(default=None, sa_column=Column(JSON)) + answers: Optional[str] = Field(default=None, sa_column=Column(String(length=2048))) + + class KnowledgeFileRead(KnowledgeFileBase): id: int @@ -52,6 +99,12 @@ class KnowledgeFileCreate(KnowledgeFileBase): pass +class QAKnowledgeUpsert(QAKnowledgeBase): + """支持修改""" + id: Optional[int] + answers: Optional[List[str]] + + class KnowledgeFileDao(KnowledgeFileBase): @classmethod @@ -142,3 +195,74 @@ class KnowledgeFileDao(KnowledgeFileBase): statement = statement.where(KnowledgeFile.status == status) with session_getter() as session: return session.scalar(statement) + +class QAKnoweldgeDao(QAKnowledgeBase): + + @classmethod + def get_qa_knowledge_by_knowledge_id(cls, knowledge_id: int): + with session_getter() as session: + return session.exec( + select(QAKnowledge).where(QAKnowledge.knowledge_id == knowledge_id)).all() + + @classmethod + def get_qa_knowledge_by_knowledge_ids(cls, knowledge_ids: List[int]) -> List[QAKnowledge]: + with session_getter() as session: + return session.exec( + select(QAKnowledge).where(QAKnowledge.knowledge_id.in_(knowledge_ids))).all() + + @classmethod + def get_qa_knowledge_by_primary_id(cls, qa_id: int) -> QAKnowledge: + with session_getter() as session: + return session.exec(select(QAKnowledge).where(QAKnowledge.id == qa_id)).first() + + @classmethod + def update(cls, qa_knowledge: QAKnowledge): + if qa_knowledge.id is None: + raise ValueError('id不能为空') + with session_getter() as session: + session.add(QAKnowledge.validate(qa_knowledge)) + session.commit() + session.refresh(qa_knowledge) + return qa_knowledge + + @classmethod + def delete_batch(cls, qa_ids: List[int]) -> bool: + with session_getter() as session: + session.exec(delete(QAKnowledge).where(QAKnowledge.id.in_(qa_ids))) + session.commit() + return True + + @classmethod + def select_list(cls, ids: List[int]) -> List[QAKnowledge]: + with session_getter() as session: + QAKnowledges = session.exec(select(QAKnowledge).where(QAKnowledge.id.in_(ids))).all() + if not QAKnowledges: + raise ValueError('知识库不存在') + return QAKnowledges + + @classmethod + def insert_qa(cls, qa_knowledge: QAKnowledgeUpsert): + with session_getter() as session: + qa = QAKnowledge.validate(qa_knowledge) + session.add(qa) + session.commit() + session.refresh(qa) + return qa + + @classmethod + def total_count(cls, sql): + with session_getter() as session: + return session.scalar(sql) + + @classmethod + def query_by_condition(cls, sql): + with session_getter() as session: + return session.exec(sql).all() + + @classmethod + def query_by_condition_v1(cls, source: List[int], create_start: str, create_end: str): + with session_getter() as session: + sql = select(QAKnowledge).where(QAKnowledge.source.in_(source)).where( + QAKnowledge.create_time.between(create_start, create_end)) + + return session.exec(sql).all() diff --git a/src/backend/bisheng/database/models/message.py b/src/backend/bisheng/database/models/message.py index a072ee09d..fd25a9643 100644 --- a/src/backend/bisheng/database/models/message.py +++ b/src/backend/bisheng/database/models/message.py @@ -1,12 +1,12 @@ from datetime import datetime -from typing import Dict, Optional, List +from typing import Dict, List, Optional, Tuple from uuid import UUID from bisheng.database.base import session_getter from bisheng.database.models.base import SQLModelSerializable from loguru import logger from pydantic import BaseModel -from sqlalchemy import JSON, Column, DateTime, String, Text, func, text, update +from sqlalchemy import JSON, Column, DateTime, String, Text, case, func, text, update from sqlmodel import Field, delete, select @@ -22,6 +22,7 @@ class MessageBase(SQLModelSerializable): user_id: Optional[str] = Field(index=True, description='用户id') liked: Optional[int] = Field(index=False, default=0, description='用户是否喜欢 0未评价/1 喜欢/2 不喜欢') solved: Optional[int] = Field(index=False, default=0, description='用户是否喜欢 0未评价/1 解决/2 未解决') + copied: Optional[int] = Field(index=False, default=0, description='用户是否复制 0：未复制 1：已复制') sender: Optional[str] = Field(index=False, default='', description='autogen 的发送方') receiver: Optional[Dict] = Field(index=False, default=None, description='autogen 的发送方') intermediate_steps: Optional[str] = Field(sa_column=Column(Text), description='过程日志') @@ -74,6 +75,48 @@ class MessageDao(MessageBase): with session_getter() as session: return session.scalar(base_condition) + @classmethod + def app_list_group_by_chat_id( + cls, + page_size: int, + page_num: int, + flow_ids: Optional[list[str]], + user_ids: Optional[list[int]], + ) -> Tuple[List[Dict], int]: + with session_getter() as session: + count_stat = select(func.count(func.distinct(ChatMessage.chat_id))) + sql = select(ChatMessage.chat_id, ChatMessage.user_id, ChatMessage.flow_id, + func.max(ChatMessage.create_time).label('create_time'), + func.sum(case((ChatMessage.liked == 1, 1), else_=0)), + func.sum(case((ChatMessage.liked == 2, 1), else_=0)), + func.sum(case((ChatMessage.copied == 1, 1), else_=0)), ) + + if flow_ids: + count_stat = count_stat.where(ChatMessage.flow_id.in_(flow_ids)) + sql = sql.where(ChatMessage.flow_id.in_(flow_ids)) + if user_ids: + count_stat = count_stat.where(ChatMessage.user_id.in_(user_ids)) + sql = sql.where(ChatMessage.user_id.in_(user_ids)) + sql = sql.group_by(ChatMessage.chat_id, ChatMessage.user_id, + ChatMessage.flow_id).order_by( + func.max(ChatMessage.create_time).desc()).offset( + page_size * (page_num - 1)).limit(page_size) + + res_list = session.exec(sql).all() + total_count = session.scalar(count_stat) + + dict_res = [{ + 'chat_id': chat_id, + 'user_id': user_id, + 'flow_id': flow_id, + 'like_count': like_num, + 'dislike_count': dislike_num, + 'copied_count': copied_num, + 'create_time': create_time + } for chat_id, user_id, flow_id, create_time, like_num, dislike_num, copied_num in res_list] + logger.info(res_list) + return dict_res, total_count + class ChatMessageDao(MessageBase): @@ -92,7 +135,8 @@ class ChatMessageDao(MessageBase): """ 获取每个会话最近的一次消息内容 """ - statement = select(ChatMessage.chat_id, func.max(ChatMessage.id)).where(ChatMessage.chat_id.in_(chat_ids)) + statement = select(ChatMessage.chat_id, + func.max(ChatMessage.id)).where(ChatMessage.chat_id.in_(chat_ids)) if category: statement = statement.where(ChatMessage.category == category) statement = statement.group_by(ChatMessage.chat_id) @@ -163,8 +207,7 @@ class ChatMessageDao(MessageBase): @classmethod def update_message(cls, message_id: int, user_id: int, message: str): with session_getter() as session: - statement = update(ChatMessage).where( - ChatMessage.id == message_id).where( + statement = update(ChatMessage).where(ChatMessage.id == message_id).where( ChatMessage.user_id == user_id).values(message=message) session.exec(statement) session.commit() @@ -176,3 +219,10 @@ class ChatMessageDao(MessageBase): session.commit() session.refresh(message) return message + + @classmethod + def update_message_copied(cls, message_id: int, copied: int): + with session_getter() as session: + statement = update(ChatMessage).where(ChatMessage.id == message_id).values(copied=copied) + session.exec(statement) + session.commit() diff --git a/src/backend/bisheng/database/models/preset_train.py b/src/backend/bisheng/database/models/preset_train.py index cdf49dea6..ed681ffa7 100644 --- a/src/backend/bisheng/database/models/preset_train.py +++ b/src/backend/bisheng/database/models/preset_train.py @@ -1,9 +1,10 @@ from datetime import datetime -from typing import List, Optional +from typing import List, Optional, Tuple from uuid import UUID, uuid4 from bisheng.database.base import session_getter from bisheng.database.models.base import SQLModelSerializable +from sqlalchemy import func from sqlmodel import Column, DateTime, Field, select, text @@ -14,10 +15,13 @@ class PresetTrainBase(SQLModelSerializable): name: str = Field(default='', index=True, description='上传的文件名字') user_id: str = Field(default='', index=True, description='创建人ID') user_name: str = Field(default='', index=True, description='创建人姓名') + type: int = Field(default=0, index=True, description='0 文件 1 QA') create_time: Optional[datetime] = Field(sa_column=Column( DateTime, nullable=False, index=True, server_default=text('CURRENT_TIMESTAMP'))) - update_time: Optional[datetime] = Field(sa_column=Column( - DateTime, nullable=False, server_default=text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'))) + update_time: Optional[datetime] = Field( + sa_column=Column(DateTime, + nullable=False, + server_default=text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'))) class PresetTrain(PresetTrainBase, table=True): @@ -54,3 +58,20 @@ class PresetTrainDao(PresetTrainBase): with session_getter() as session: statement = select(PresetTrain) return session.exec(statement).all() + + @classmethod + def search_name(cls, + keyword: str = None, + page_size: int = None, + page_num: int = None) -> Tuple[List[PresetTrain], int]: + with session_getter() as session: + statement = select(PresetTrain) + count = select(func.count(PresetTrain.id)) + if keyword: + statement = statement.where(PresetTrain.name.like('%{}%'.format(keyword))) + count = count.where(PresetTrain.name.like('%{}%'.format(keyword))) + if page_num and page_size: + statement = statement.offset((page_num - 1) * page_size).limit(page_size) + statement = statement.order_by(PresetTrain.create_time.desc()) + + return session.exec(statement).all(), session.scalar(count) diff --git a/src/backend/bisheng/database/models/user.py b/src/backend/bisheng/database/models/user.py index 30d6a6bbc..20aaae3ef 100644 --- a/src/backend/bisheng/database/models/user.py +++ b/src/backend/bisheng/database/models/user.py @@ -1,15 +1,14 @@ from datetime import datetime from typing import List, Optional -from pydantic import validator -from sqlalchemy import Column, DateTime, text, func -from sqlmodel import Field, select - from bisheng.database.base import session_getter from bisheng.database.models.base import SQLModelSerializable -from bisheng.database.models.role import DefaultRole, AdminRole +from bisheng.database.models.role import AdminRole, DefaultRole from bisheng.database.models.user_group import UserGroup from bisheng.database.models.user_role import UserRole +from pydantic import validator +from sqlalchemy import Column, DateTime, func, text +from sqlmodel import Field, select class UserBase(SQLModelSerializable): @@ -38,10 +37,9 @@ class UserBase(SQLModelSerializable): class User(UserBase, table=True): user_id: Optional[int] = Field(default=None, primary_key=True) password: str = Field(index=False) - password_update_time: Optional[datetime] = Field(sa_column=Column(DateTime, - nullable=False, - server_default=text('CURRENT_TIMESTAMP')), - description="密码最近的修改时间") + password_update_time: Optional[datetime] = Field(sa_column=Column( + DateTime, nullable=False, server_default=text('CURRENT_TIMESTAMP')), + description='密码最近的修改时间') class UserRead(UserBase): @@ -65,7 +63,7 @@ class UserLogin(UserBase): class UserCreate(UserBase): - password: Optional[str] = Field(default="") + password: Optional[str] = Field(default='') captcha_key: Optional[str] captcha: Optional[str] @@ -104,7 +102,11 @@ class UserDao(UserBase): return user @classmethod - def filter_users(cls, user_ids: List[int], keyword: str = None, page: int = 0, limit: int = 0) -> (List[User], int): + def filter_users(cls, + user_ids: List[int], + keyword: str = None, + page: int = 0, + limit: int = 0) -> (List[User], int): statement = select(User) count_statement = select(func.count(User.user_id)) if user_ids: @@ -125,6 +127,12 @@ class UserDao(UserBase): statement = select(User).where(User.user_name == user_name) return session.exec(statement).first() + @classmethod + def search_user_by_name(cls, user_name: str) -> List[User] | None: + with session_getter() as session: + statement = select(User).where(User.user_name.like('%{}%'.format(user_name))) + return session.exec(statement).all() + @classmethod def create_user(cls, db_user: User) -> User: with session_getter() as session: @@ -164,7 +172,8 @@ class UserDao(UserBase): return user @classmethod - def add_user_with_groups_and_roles(cls, user: User, group_ids: List[int], role_ids: List[int]) -> User: + def add_user_with_groups_and_roles(cls, user: User, group_ids: List[int], + role_ids: List[int]) -> User: with session_getter() as session: session.add(user) session.flush() diff --git a/src/backend/bisheng/graph/utils.py b/src/backend/bisheng/graph/utils.py index eb2a24e09..5eb7920a7 100644 --- a/src/backend/bisheng/graph/utils.py +++ b/src/backend/bisheng/graph/utils.py @@ -243,7 +243,8 @@ def update_source_handle(new_edge, g_nodes, g_edges): dict: The updated edge with the new source handle. """ last_node = copy.deepcopy(find_last_node(g_nodes, g_edges)) - new_edge['sourceHandle'] = new_edge['sourceHandle'].replace(new_edge['source'], last_node['id']) + new_edge['sourceHandle'] = new_edge['sourceHandle'].replace(new_edge['source'], + last_node['id']) new_edge['source'] = last_node['id'] return new_edge @@ -273,3 +274,40 @@ def get_updated_edges(base_flow, g_nodes, g_edges, group_node_id): if edge['target'] == group_node_id or edge['source'] == group_node_id: updated_edges.append(new_edge) return updated_edges + + +def find_next_node(graph_data: Dict, node_id: str) -> List[Dict]: + """ + Finds the next node in the graph data based on the given node id. + """ + nodes = graph_data.get('nodes', []) + edges = graph_data.get('edges', []) + + edges_ = [e['target'] for e in edges if e['source'] == node_id] + return [n for n in nodes if n['id'] in edges_] + + +def cut_graph_bynode(graph_data: Dict, node_id: str) -> List[Dict]: + """ + 通过node_id 找到和node相关的所有依赖节点。 + """ + nodes = graph_data.get('nodes', []) + edges = graph_data.get('edges', []) + + nodes_new_list = [] + edges_new_list = [] + iflast = True + for e in edges: + if e['target'] == node_id: + iflast = False + node_list, edge_list = cut_graph_bynode(graph_data, e['source']) + nodes_new_list.extend(node_list) + edges_new_list.extend(edge_list) + nodes_new_list.append([n for n in nodes if n['id'] == node_id][0]) + edges_new_list.append(e) + if iflast: + for node in nodes: + if node['id'] == node_id: + return [node], [] + + return nodes_new_list, edges_new_list diff --git a/src/backend/bisheng/interface/initialize/loading.py b/src/backend/bisheng/interface/initialize/loading.py index 8856355d0..020e09434 100644 --- a/src/backend/bisheng/interface/initialize/loading.py +++ b/src/backend/bisheng/interface/initialize/loading.py @@ -25,6 +25,7 @@ from bisheng.interface.wrappers.base import wrapper_creator from bisheng.settings import settings from bisheng.utils import validate from bisheng.utils.constants import NODE_ID_DICT, PRESET_QUESTION +from bisheng.utils.embedding import decide_embeddings from bisheng_langchain.vectorstores import VectorStoreFilterRetriever from langchain.agents import agent as agent_module from langchain.agents.agent import AgentExecutor @@ -39,8 +40,6 @@ from loguru import logger from pydantic import SecretStr, ValidationError, create_model from pydantic.fields import FieldInfo -from bisheng.utils.embedding import decide_embeddings - if TYPE_CHECKING: from bisheng import CustomComponent @@ -211,6 +210,13 @@ def instantiate_input_output(node_type, class_object, params, id_dict): return [file_path, file_name2 if file_name2 else file_path[1]] else: return '' + if 'file_path' in params: + file_path = params['file_path'] + if not file_path: + return '' + if isinstance(file_path, list): + params['file_path'] = file_path[0] + return class_object(**params).text() @@ -474,33 +480,35 @@ def instantiate_vectorstore(node_type: str, class_object: Type[VectorStore], par # 过滤掉用户没有权限的知识库 # TODO zgq 后续统一技能执行流程后将和业务有关的逻辑都迁移到初始化技能对象之前 - if node_type == "MilvusWithPermissionCheck" or node_type == "ElasticsearchWithPermissionCheck": - col_name = "collection_name" - if node_type == "ElasticsearchWithPermissionCheck": - col_name = "index_name" + if node_type == 'MilvusWithPermissionCheck' or node_type == 'ElasticsearchWithPermissionCheck': + col_name = 'collection_name' + if node_type == 'ElasticsearchWithPermissionCheck': + col_name = 'index_name' # 获取执行用户 有权限查看的知识库列表 - knowledge_ids = [one["key"] for one in params[col_name]] + knowledge_ids = [one['key'] for one in params[col_name]] knowledge_list = KnowledgeDao.judge_knowledge_permission(user_name, knowledge_ids) - logger.debug(f"{node_type} after filter, get knowledge_list: {knowledge_list}") + logger.debug(f'{node_type} after filter, get knowledge_list: {knowledge_list}') if not knowledge_list: - logger.warning(f"{node_type}: after filter, get zero knowledge") + logger.warning(f'{node_type}: after filter, get zero knowledge') # 没有任何知识库的话，提供假的embedding和空的collection_name - if node_type == "MilvusWithPermissionCheck": + if node_type == 'MilvusWithPermissionCheck': params[col_name] = [] - params["partition_keys"] = [] + params['partition_keys'] = [] for knowledge in knowledge_list: params[col_name].append(knowledge.collection_name) - if knowledge.collection_name.startswith("partition"): - params["partition_keys"].append(knowledge.id) + if knowledge.collection_name.startswith('partition'): + params['partition_keys'].append(knowledge.id) else: - params["partition_keys"].append(None) - params["embedding"] = decide_embeddings(knowledge_list[0].model) if knowledge_list else FakeEmbedding() + params['partition_keys'].append(None) + params['embedding'] = decide_embeddings( + knowledge_list[0].model) if knowledge_list else FakeEmbedding() else: - params[col_name] = [knowledge.index_name or knowledge.collection_name - for knowledge in knowledge_list] + params[col_name] = [ + knowledge.index_name or knowledge.collection_name for knowledge in knowledge_list + ] if initializer := vecstore_initializer.get(class_object.__name__): vecstore = initializer(class_object, params, search_kwargs) @@ -565,8 +573,8 @@ def instantiate_documentloader(class_object: Type[BaseLoader], params: Dict): def instantiate_textsplitter( - class_object, - params: Dict, + class_object, + params: Dict, ): try: documents = params.pop('documents') @@ -577,7 +585,7 @@ def instantiate_textsplitter( 'Try changing the chunk_size of the Text Splitter.') from exc if ('separator_type' in params - and params['separator_type'] == 'Text') or 'separator_type' not in params: + and params['separator_type'] == 'Text') or 'separator_type' not in params: params.pop('separator_type', None) # separators might come in as an escaped string like \\n # so we need to convert it to a string @@ -608,7 +616,7 @@ def replace_zero_shot_prompt_with_prompt_template(nodes): # Build Prompt Template tools = [ tool for tool in nodes if tool['type'] != 'chatOutputNode' - and 'Tool' in tool['data']['node']['base_classes'] + and 'Tool' in tool['data']['node']['base_classes'] ] node['data'] = build_prompt_template(prompt=node['data'], tools=tools) break diff --git a/src/backend/bisheng/interface/vector_store/custom.py b/src/backend/bisheng/interface/vector_store/custom.py index 78deac82a..63861cfe0 100644 --- a/src/backend/bisheng/interface/vector_store/custom.py +++ b/src/backend/bisheng/interface/vector_store/custom.py @@ -1,20 +1,19 @@ from abc import ABC -from typing import Optional, Any, List, Tuple, Dict, TYPE_CHECKING, Iterable, Callable +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Tuple import jieba +from bisheng_langchain.vectorstores.elastic_keywords_search import DEFAULT_PROMPT +from bisheng_langchain.vectorstores.milvus import DEFAULT_MILVUS_CONNECTION from langchain.chains.llm import LLMChain +from langchain.docstore.document import Document +from langchain.embeddings.base import Embeddings +from langchain.utils import get_from_dict_or_env +from langchain.vectorstores.base import VectorStore +from langchain_community.vectorstores.milvus import Milvus as MilvusLangchain from langchain_core.language_models import BaseLLM from langchain_core.prompts import PromptTemplate -from langchain.utils import get_from_dict_or_env -from langchain.embeddings.base import Embeddings -from langchain_community.vectorstores.milvus import Milvus as MilvusLangchain, Milvus -from langchain.docstore.document import Document -from langchain.vectorstores.base import VectorStore from loguru import logger -from bisheng_langchain.vectorstores.milvus import DEFAULT_MILVUS_CONNECTION -from bisheng_langchain.vectorstores.elastic_keywords_search import DEFAULT_PROMPT - if TYPE_CHECKING: from elasticsearch import Elasticsearch # noqa: F401 @@ -49,16 +48,65 @@ class MilvusWithPermissionCheck(MilvusLangchain): # Default search params when one is not provided. self.default_search_params = { - 'IVF_FLAT': {'metric_type': 'L2', 'params': {'nprobe': 10}}, - 'IVF_SQ8': {'metric_type': 'L2', 'params': {'nprobe': 10}}, - 'IVF_PQ': {'metric_type': 'L2', 'params': {'nprobe': 10}}, - 'HNSW': {'metric_type': 'L2', 'params': {'ef': 100}}, - 'RHNSW_FLAT': {'metric_type': 'L2', 'params': {'ef': 10}}, - 'RHNSW_SQ': {'metric_type': 'L2', 'params': {'ef': 10}}, - 'RHNSW_PQ': {'metric_type': 'L2', 'params': {'ef': 10}}, - 'IVF_HNSW': {'metric_type': 'L2', 'params': {'nprobe': 10, 'ef': 10}}, - 'ANNOY': {'metric_type': 'L2', 'params': {'search_k': 10}}, - 'AUTOINDEX': {'metric_type': 'L2', 'params': {}}, + 'IVF_FLAT': { + 'metric_type': 'L2', + 'params': { + 'nprobe': 10 + } + }, + 'IVF_SQ8': { + 'metric_type': 'L2', + 'params': { + 'nprobe': 10 + } + }, + 'IVF_PQ': { + 'metric_type': 'L2', + 'params': { + 'nprobe': 10 + } + }, + 'HNSW': { + 'metric_type': 'L2', + 'params': { + 'ef': 100 + } + }, + 'RHNSW_FLAT': { + 'metric_type': 'L2', + 'params': { + 'ef': 10 + } + }, + 'RHNSW_SQ': { + 'metric_type': 'L2', + 'params': { + 'ef': 10 + } + }, + 'RHNSW_PQ': { + 'metric_type': 'L2', + 'params': { + 'ef': 10 + } + }, + 'IVF_HNSW': { + 'metric_type': 'L2', + 'params': { + 'nprobe': 10, + 'ef': 10 + } + }, + 'ANNOY': { + 'metric_type': 'L2', + 'params': { + 'search_k': 10 + } + }, + 'AUTOINDEX': { + 'metric_type': 'L2', + 'params': {} + }, } self.embedding_func = embedding_function @@ -106,7 +154,7 @@ class MilvusWithPermissionCheck(MilvusLangchain): raise e if not self.col: - logger.warning("No collection found, please confirm user have knowledge access") + logger.warning('No collection found, please confirm user have knowledge access') # raise ValueError("No collection found, please confirm collection name correctly.") # Initialize the vector store self._init() @@ -115,13 +163,14 @@ class MilvusWithPermissionCheck(MilvusLangchain): from pymilvus import connections connections.remove_connection(using) - def _init(self, - embeddings: Optional[list] = None, - metadatas: Optional[list[dict]] = None, - partition_names: Optional[list] = None, - replica_number: int = 1, - timeout: Optional[float] = None, - ) -> None: + def _init( + self, + embeddings: Optional[list] = None, + metadatas: Optional[list[dict]] = None, + partition_names: Optional[list] = None, + replica_number: int = 1, + timeout: Optional[float] = None, + ) -> None: self._extract_fields(col_index=0) self._create_search_params() self._load() @@ -175,18 +224,18 @@ class MilvusWithPermissionCheck(MilvusLangchain): @classmethod def from_texts( - cls, - texts: List[str], - embedding: Embeddings, - metadatas: Optional[List[dict]] = None, - collection_name: list[str] = None, - connection_args: dict[str, Any] = DEFAULT_MILVUS_CONNECTION, - consistency_level: str = "Session", - index_params: Optional[dict] = None, - search_params: Optional[dict] = None, - drop_old: bool = False, - no_embedding: bool = False, - **kwargs: Any, + cls, + texts: List[str], + embedding: Embeddings, + metadatas: Optional[List[dict]] = None, + collection_name: list[str] = None, + connection_args: dict[str, Any] = DEFAULT_MILVUS_CONNECTION, + consistency_level: str = 'Session', + index_params: Optional[dict] = None, + search_params: Optional[dict] = None, + drop_old: bool = False, + no_embedding: bool = False, + **kwargs: Any, ): """ no insert data into milvus, only search from milvus @@ -204,13 +253,13 @@ class MilvusWithPermissionCheck(MilvusLangchain): return vector_db def similarity_search( - self, - query: str, - k: int = 4, - param: Optional[dict] = None, - expr: Optional[str] = None, - timeout: Optional[int] = None, - **kwargs: Any, + self, + query: str, + k: int = 4, + param: Optional[dict] = None, + expr: Optional[str] = None, + timeout: Optional[int] = None, + **kwargs: Any, ) -> List[Document]: """Perform a similarity search against the query string. @@ -236,13 +285,13 @@ class MilvusWithPermissionCheck(MilvusLangchain): return [doc for doc, _ in res] def similarity_search_by_vector( - self, - embedding: List[float], - k: int = 4, - param: Optional[dict] = None, - expr: Optional[str] = None, - timeout: Optional[int] = None, - **kwargs: Any, + self, + embedding: List[float], + k: int = 4, + param: Optional[dict] = None, + expr: Optional[str] = None, + timeout: Optional[int] = None, + **kwargs: Any, ) -> List[Document]: """Perform a similarity search against the query string. @@ -268,13 +317,13 @@ class MilvusWithPermissionCheck(MilvusLangchain): return [doc for doc, _ in res] def similarity_search_with_score( - self, - query: str, - k: int = 4, - param: Optional[dict] = None, - expr: Optional[str] = None, - timeout: Optional[int] = None, - **kwargs: Any, + self, + query: str, + k: int = 4, + param: Optional[dict] = None, + expr: Optional[str] = None, + timeout: Optional[int] = None, + **kwargs: Any, ) -> List[Tuple[Document, float]]: """Perform a search on a query string and return results with score. @@ -314,13 +363,13 @@ class MilvusWithPermissionCheck(MilvusLangchain): return res def similarity_search_with_score_by_vector( - self, - embedding: List[float], - k: int = 4, - param: Optional[dict] = None, - expr: Optional[str] = None, - timeout: Optional[int] = None, - **kwargs: Any, + self, + embedding: List[float], + k: int = 4, + param: Optional[dict] = None, + expr: Optional[str] = None, + timeout: Optional[int] = None, + **kwargs: Any, ) -> List[Tuple[Document, float]]: """Perform a search on a query string and return results with score. @@ -355,7 +404,7 @@ class MilvusWithPermissionCheck(MilvusLangchain): output_fields = self.fields[:] output_fields.remove(self._vector_field) - finally_k = kwargs.pop("k", k) + finally_k = kwargs.pop('k', k) ret = [] @@ -408,13 +457,13 @@ class ElasticsearchWithPermissionCheck(VectorStore, ABC): """ def __init__( - self, - elasticsearch_url: str, - index_name: List[str], - drop_old: Optional[bool] = False, - *, - ssl_verify: Optional[Dict[str, Any]] = None, - llm_chain: Optional[LLMChain] = None, + self, + elasticsearch_url: str, + index_name: List[str], + drop_old: Optional[bool] = False, + *, + ssl_verify: Optional[Dict[str, Any]] = None, + llm_chain: Optional[LLMChain] = None, ): """Initialize with necessary components.""" try: @@ -477,10 +526,11 @@ class ElasticsearchWithPermissionCheck(VectorStore, ABC): keywords = eval(keywords_str) if not isinstance(keywords, list): raise ValueError('Keywords extracted by llm is not list.') - except Exception as e: + except Exception: keywords = jieba.analyse.extract_tags(query, topK=10, withWeight=False) else: keywords = jieba.analyse.extract_tags(query, topK=10, withWeight=False) + keywords = keywords or [query] logger.debug(f'finally search keywords: {keywords}') match_query = {'bool': {must_or_should: []}} for key in keywords: @@ -491,13 +541,12 @@ class ElasticsearchWithPermissionCheck(VectorStore, ABC): response = self.client_search(self.client, one_index_name, match_query, size=k) hits = [hit for hit in response['hits']['hits']] for hit in hits: - ret.append(( - Document(page_content=hit['_source']['text'], metadata=hit['_source']['metadata']), - hit['_score'] - )) - logger.debug(f'ElasticsearchWithPermissionCheck Search {one_index_name} results: {hits}') + ret.append((Document(page_content=hit['_source']['text'], + metadata=hit['_source']['metadata']), hit['_score'])) + logger.debug( + f'ElasticsearchWithPermissionCheck Search {one_index_name} results: {hits}') logger.debug(f'ElasticsearchWithPermissionCheck Search all results: {len(ret)}') - finally_k = kwargs.pop("finally_k", k) + finally_k = kwargs.pop('finally_k', k) ret.sort(key=lambda x: x[1], reverse=True) ret = ret[:finally_k] logger.debug(f'ElasticsearchWithPermissionCheck Search finally results: {len(ret)}') @@ -515,17 +564,17 @@ class ElasticsearchWithPermissionCheck(VectorStore, ABC): @classmethod def from_texts( - cls, - texts: List[str], - embedding: Embeddings, - metadatas: Optional[List[dict]] = None, - ids: Optional[List[str]] = None, - index_name: Optional[List[str]] = None, - refresh_indices: bool = True, - llm: Optional[BaseLLM] = None, - prompt: Optional[PromptTemplate] = DEFAULT_PROMPT, - drop_old: Optional[bool] = False, - **kwargs: Any, + cls, + texts: List[str], + embedding: Embeddings, + metadatas: Optional[List[dict]] = None, + ids: Optional[List[str]] = None, + index_name: Optional[List[str]] = None, + refresh_indices: bool = True, + llm: Optional[BaseLLM] = None, + prompt: Optional[PromptTemplate] = DEFAULT_PROMPT, + drop_old: Optional[bool] = False, + **kwargs: Any, ): """Construct ElasticKeywordsSearch wrapper from raw documents. diff --git a/src/backend/bisheng/main.py b/src/backend/bisheng/main.py index e20595c4c..9baa756b9 100644 --- a/src/backend/bisheng/main.py +++ b/src/backend/bisheng/main.py @@ -21,12 +21,15 @@ from fastapi_jwt_auth.exceptions import AuthJWTException from loguru import logger -def handle_http_exception(req: Request, exc: HTTPException) -> ORJSONResponse: - msg = { - 'status_code': exc.status_code, - 'status_message': exc.detail['error'] if isinstance(exc.detail, dict) else exc.detail - } - logger.error(f'{req.method} {req.url} {exc.status_code} {exc.detail}') +def handle_http_exception(req: Request, exc: Exception) -> ORJSONResponse: + if isinstance(exc, HTTPException): + msg = { + 'status_code': exc.status_code, + 'status_message': exc.detail['error'] if isinstance(exc.detail, dict) else exc.detail + } + else: + msg = {'status_code': 500, 'status_message': str(exc)} + logger.error(f'{req.method} {req.url} {str(exc)}') return ORJSONResponse(content=msg) @@ -38,7 +41,8 @@ def handle_request_validation_error(req: Request, exc: RequestValidationError) - _EXCEPTION_HANDLERS = { HTTPException: handle_http_exception, - RequestValidationError: handle_request_validation_error + RequestValidationError: handle_request_validation_error, + Exception: handle_http_exception } diff --git a/src/backend/bisheng/template/frontend_node/input_output.py b/src/backend/bisheng/template/frontend_node/input_output.py index 896dce857..7477007b4 100644 --- a/src/backend/bisheng/template/frontend_node/input_output.py +++ b/src/backend/bisheng/template/frontend_node/input_output.py @@ -5,10 +5,30 @@ from bisheng.template.frontend_node.base import FrontendNode from bisheng.template.template.base import Template +def build_file_field(suffixes: list, + fileTypes: list, + name: str = 'file_path', + fieldType='fileNode') -> TemplateField: + """Build a template field for a document loader.""" + return TemplateField( + field_type=fieldType, + required=True, + show=True, + name=name, + value='', + suffixes=suffixes, + fileTypes=fileTypes, + ) + + class InputOutputNode(FrontendNode): name: str = 'InputOutputNode' base_classes: list[str] = ['input', 'output'] + def add_extra_base_classes(self) -> None: + if self.name in {'AudioInputNode', 'FileInputNode'}: + self.base_classes.append('input') + def add_extra_fields(self) -> None: pass @@ -36,6 +56,25 @@ class InputOutputNode(FrontendNode): field.show = True field.field_type = 'variable' field.required = True + if name == 'AudioInputNode': + if field.name == 'file_path': + field.show = True + field.field_type = 'fileNode' + field.required = True + field.suffixes = ['.mp3'] + field.fileTypes = ['mp3'] + elif field.name == 'openai_proxy': + field.show = True + + if name == 'FileInputNode': + if field.name == 'file_path': + field.show = True + field.field_type = 'fileNode' + field.required = True + field.suffixes = ['.jpg', '.png', '.jpeg'] + field.fileTypes = ['jpg', 'png', 'jpeg'] + elif field.name == 'openai_proxy': + field.show = True class InputNode(FrontendNode): @@ -72,14 +111,12 @@ class InputFileNode(FrontendNode): name='file_path', value='', ), - TemplateField( - field_type='str', - show=True, - name='file_type', - placeholder='提示上传文件类型', - display_name='Name', - info='Tips for which file should upload' - ), + TemplateField(field_type='str', + show=True, + name='file_type', + placeholder='提示上传文件类型', + display_name='Name', + info='Tips for which file should upload'), ], ) description: str = """输入节点，用来自动对接输入""" diff --git a/src/backend/bisheng/utils/constants.py b/src/backend/bisheng/utils/constants.py index a6fca512f..1187c1320 100644 --- a/src/backend/bisheng/utils/constants.py +++ b/src/backend/bisheng/utils/constants.py @@ -8,15 +8,8 @@ OPENAI_MODELS = [ 'text-ada-001', ] CHAT_OPENAI_MODELS = [ - 'gpt-3.5-turbo-0613', - 'gpt-3.5-turbo', - 'gpt-3.5-turbo-16k-0613', - 'gpt-3.5-turbo-16k', - 'gpt-4-0613', - 'gpt-4-32k-0613', - 'gpt-4', - 'gpt-4-32k', - 'gpt-4-1106-preview', + 'gpt-3.5-turbo-0613', 'gpt-3.5-turbo', 'gpt-3.5-turbo-16k-0613', 'gpt-3.5-turbo-16k', + 'gpt-4-0613', 'gpt-4-32k-0613', 'gpt-4', 'gpt-4-32k', 'gpt-4-1106-preview', 'gpt-4o' ] ANTHROPIC_MODELS = [ diff --git a/src/backend/test/milvus_trans.py b/src/backend/test/milvus_trans.py index 37b12036f..88e8ec1de 100644 --- a/src/backend/test/milvus_trans.py +++ b/src/backend/test/milvus_trans.py @@ -154,18 +154,19 @@ import requests def milvus_clean(): - milvus_cli = MilvusClient(uri='http://192.168.106.116:19530') + milvus_cli = MilvusClient(uri='http://192.168.106.109:19530') collection = milvus_cli.list_collections() for col in collection: - if col.startswith('rag'): + if col.startswith('tmp'): print(col) - collection_col = Collection(col, using=milvus_cli._using) - # milvus_cli.drop_collection(col) - try: - collection_col.release(timeout=1) - except Exception: - continue + # collection_col = Collection(col, using=milvus_cli._using) + milvus_cli.drop_collection(col) + + # try: + # collection_col.release(timeout=1) + # except Exception: + # continue def elastic_clean(): @@ -184,6 +185,6 @@ def elastic_clean(): print(x) -# milvus_clean() -elastic_clean() +milvus_clean() +# elastic_clean() # milvus_trans() diff --git a/src/bisheng-langchain/bisheng_langchain/chains/qa_generation/base_v2.py b/src/bisheng-langchain/bisheng_langchain/chains/qa_generation/base_v2.py index 1458eb55a..ce678e61a 100644 --- a/src/bisheng-langchain/bisheng_langchain/chains/qa_generation/base_v2.py +++ b/src/bisheng-langchain/bisheng_langchain/chains/qa_generation/base_v2.py @@ -134,8 +134,6 @@ class TrainsetGenerator: chunk_size: int = 1024, seed: int = 42, prompt: Optional[ChatPromptTemplate] = SEED_QUESTION_CHAT_PROMPT, - filter_lowquality_context: bool = False, - filter_lowquality_question: bool = False, answer_prompt: Optional[HumanMessagePromptTemplate] = ANSWER_FORMULATE, ) -> None: self.generator_llm = generator_llm @@ -154,8 +152,6 @@ class TrainsetGenerator: self.threshold = 5.0 self.rng = default_rng(seed) self.prompt = prompt - self.filter_lowquality_context = filter_lowquality_context - self.filter_lowquality_question = filter_lowquality_question if answer_prompt is None: answer_prompt = ANSWER_FORMULATE self.answer_prompt = answer_prompt @@ -167,8 +163,6 @@ class TrainsetGenerator: chunk_size: int = 512, trainset_distribution: dict = DEFAULT_TRAIN_DISTRIBUTION, prompt: Optional[ChatPromptTemplate] = SEED_QUESTION_CHAT_PROMPT, - filter_lowquality_context: bool = False, - filter_lowquality_question: bool = False, answer_prompt: Optional[PromptTemplate] = ANSWER_FORMULATE, ): generator_llm = llm @@ -179,8 +173,6 @@ class TrainsetGenerator: chunk_size=chunk_size, trainset_distribution=trainset_distribution, prompt=prompt, - filter_lowquality_context=filter_lowquality_context, - filter_lowquality_question=filter_lowquality_question, answer_prompt=answer_prompt, ) @@ -324,17 +316,14 @@ class TrainsetGenerator: ) text_chunk = " ".join([node.get_content() for node in nodes]) - if self.filter_lowquality_context: - score = self._filter_context(text_chunk) - if not score: - continue + score = self._filter_context(text_chunk) + if not score: + continue seed_question = self._seed_question(text_chunk) question = seed_question - if self.filter_lowquality_question: - is_valid_question = self._filter_question(question) - else: - is_valid_question = True + # is_valid_question = self._filter_question(question) + is_valid_question = True if is_valid_question: context = [text_chunk] * len(question.split("\n")) is_conv = len(context) > 1 @@ -372,8 +361,6 @@ class QAGenerationChainV2(Chain): llm: BaseLanguageModel, k: Optional[int] = None, chunk_size: int = 512, - filter_lowquality_context: bool = False, - filter_lowquality_question: bool = False, question_prompt: Optional[ChatPromptTemplate] = SEED_QUESTION_CHAT_PROMPT, answer_prompt: Optional[HumanMessagePromptTemplate] = ANSWER_FORMULATE, **kwargs: Any, @@ -390,14 +377,8 @@ class QAGenerationChainV2(Chain): Returns: a QAGenerationChain class """ - generator = TrainsetGenerator.from_default( - llm, - chunk_size=chunk_size, - prompt=question_prompt, - answer_prompt=answer_prompt, - filter_lowquality_context=filter_lowquality_context, - filter_lowquality_question=filter_lowquality_question - ) + generator = TrainsetGenerator.from_default(llm, chunk_size=chunk_size, prompt=question_prompt, + answer_prompt=answer_prompt) return cls(documents=documents, generator=generator, k=k, **kwargs) @property @@ -424,14 +405,14 @@ class QAGenerationChainV2(Chain): dataset = self.generator.generate(documents=self.documents, train_size=self.k) df = dataset.to_pandas() qa_pairs = df.to_dict("records") - qa = [] + qa = '' for pair in qa_pairs: - qa.append({ - "question": pair["question"], - "answer": pair["ground_truth"][0], - "context": pair["ground_truth_context"][0], - }) - qa = f'```json\n{json.dumps(qa, ensure_ascii=False, indent=4)}\n```' + qa += json.dumps( + { + "question": pair["question"], + "answer": pair["ground_truth"][0], + "context": pair["ground_truth_context"][0], + }, ensure_ascii=False) return {self.output_key: qa} async def _acall( diff --git a/src/bisheng-langchain/bisheng_langchain/rag/init_retrievers/keyword_retriever.py b/src/bisheng-langchain/bisheng_langchain/rag/init_retrievers/keyword_retriever.py index 4f26bc499..394fdc45d 100644 --- a/src/bisheng-langchain/bisheng_langchain/rag/init_retrievers/keyword_retriever.py +++ b/src/bisheng-langchain/bisheng_langchain/rag/init_retrievers/keyword_retriever.py @@ -16,7 +16,7 @@ from langchain.text_splitter import TextSplitter class KeywordRetriever(BaseRetriever): - keyword_store: VectorStore + keyword_store: ElasticKeywordsSearch text_splitter: TextSplitter search_type: str = 'similarity' search_kwargs: dict = Field(default_factory=dict) diff --git a/src/bisheng-langchain/bisheng_langchain/vectorstores/elastic_keywords_search.py b/src/bisheng-langchain/bisheng_langchain/vectorstores/elastic_keywords_search.py index 30462eade..62ec52c59 100644 --- a/src/bisheng-langchain/bisheng_langchain/vectorstores/elastic_keywords_search.py +++ b/src/bisheng-langchain/bisheng_langchain/vectorstores/elastic_keywords_search.py @@ -13,6 +13,7 @@ from langchain.llms.base import BaseLLM from langchain.prompts.prompt import PromptTemplate from langchain.utils import get_from_dict_or_env from langchain.vectorstores.base import VectorStore +from loguru import logger if TYPE_CHECKING: from elasticsearch import Elasticsearch # noqa: F401 @@ -326,6 +327,49 @@ class ElasticKeywordsSearch(VectorStore, ABC): response = client.search(index=index_name, body={'query': script_query, 'size': size}) return response - def delete(self, **kwargs: Any) -> None: + def delete_index(self, **kwargs: Any) -> None: # TODO: Check if this can be done in bulk self.client.indices.delete(index=self.index_name) + + def delete( + self, + ids: Optional[List[str]] = None, + refresh_indices: Optional[bool] = True, + **kwargs: Any, + ) -> Optional[bool]: + """Delete documents from the Elasticsearch index. + + Args: + ids: List of ids of documents to delete. + refresh_indices: Whether to refresh the index + after deleting documents. Defaults to True. + """ + try: + from elasticsearch.helpers import BulkIndexError, bulk + except ImportError: + raise ImportError('Could not import elasticsearch python package. ' + 'Please install it with `pip install elasticsearch`.') + + body = [] + + if ids is None: + raise ValueError('ids must be provided.') + + for _id in ids: + body.append({'_op_type': 'delete', '_index': self.index_name, '_id': _id}) + + if len(body) > 0: + try: + bulk(self.client, body, refresh=refresh_indices, ignore_status=404) + logger.debug(f'Deleted {len(body)} texts from index') + + return True + except BulkIndexError as e: + logger.error(f'Error deleting texts: {e}') + firstError = e.errors[0].get('index', {}).get('error', {}) + logger.error(f"First error reason: {firstError.get('reason')}") + raise e + + else: + logger.debug('No texts to delete from index') + return False diff --git a/src/bisheng-langchain/tests/chains/test_qa_generation.py b/src/bisheng-langchain/tests/chains/test_qa_generation.py index c6ec3518d..9f84a68bb 100644 --- a/src/bisheng-langchain/tests/chains/test_qa_generation.py +++ b/src/bisheng-langchain/tests/chains/test_qa_generation.py @@ -32,18 +32,8 @@ def generator(): unstructured_api_url='https://bisheng.dataelem.com/api/v1/etl4llm/predict', ) documents = loader.load() - k = 5 - chunk_size = 512 - filter_lowquality_context = False - filter_lowquality_question = False - # qa_generator = QAGenerationChain.from_llm(documents, llm, k=k, chunk_size=chunk_size, filter_lowquality_context=filter_lowquality_context, filter_lowquality_question=filter_lowquality_question) - qa_generator = QAGenerationChainV2.from_llm(documents, - llm, - k=k, - chunk_size=chunk_size, - filter_lowquality_context=filter_lowquality_context, - filter_lowquality_question=filter_lowquality_question - ) + # qa_generator = QAGenerationChain.from_llm(documents, llm, k=5) + qa_generator = QAGenerationChainV2.from_llm(documents, llm) inputs = {'begin': '开始'} response = qa_generator(inputs) question_answers = response['questions']