From 8b4393188655fcdef998d08d0cb2b1428f614ec0 Mon Sep 17 00:00:00 2001 From: wizardchen Date: Wed, 13 Aug 2025 21:56:20 +0800 Subject: [PATCH] feat: support minio storage --- .env.example | 16 +- Makefile | 6 + docker-compose.yml | 50 +- docker/Dockerfile.docreader | 4 +- frontend/src/api/initialization/index.ts | 59 +- frontend/src/views/chat/components/botmsg.vue | 2 +- .../initialization/InitializationConfig.vue | 1539 +++++++++++++---- internal/application/service/knowledge.go | 51 +- internal/application/service/knowledgebase.go | 2 +- internal/handler/initialization.go | 408 +++-- internal/router/router.go | 1 + internal/types/knowledgebase.go | 12 +- scripts/start_all.sh | 169 +- services/docreader/requirements.txt | 1 + services/docreader/scripts/generate_proto.sh | 0 services/docreader/src/parser/__init__.py | 3 +- services/docreader/src/parser/base_parser.py | 213 +-- services/docreader/src/parser/config.py | 21 + services/docreader/src/parser/docx_parser.py | 897 +++++----- services/docreader/src/parser/image_parser.py | 38 +- .../docreader/src/parser/markdown_parser.py | 4 +- services/docreader/src/parser/parser.py | 22 +- services/docreader/src/parser/pdf_parser.py | 4 +- services/docreader/src/parser/storage.py | 363 ++++ services/docreader/src/parser/text_parser.py | 3 +- services/docreader/src/parser/web_parser.py | 4 +- services/docreader/src/proto/docreader.pb.go | 215 ++- services/docreader/src/proto/docreader.proto | 26 +- services/docreader/src/server/server.py | 96 +- 29 files changed, 2735 insertions(+), 1494 deletions(-) mode change 100644 => 100755 services/docreader/scripts/generate_proto.sh create mode 100644 services/docreader/src/parser/config.py create mode 100644 services/docreader/src/parser/storage.py diff --git a/.env.example b/.env.example index 6879932d9..a321117ae 100644 --- a/.env.example +++ b/.env.example @@ -7,8 +7,7 @@ # 可选值: debug(开发模式，有详细日志), release(生产模式) GIN_MODE=debug -# ollama -# Ollama 服务的基准 URL，用于连接本地运行的 Ollama LLM 服务 +# Ollama 服务的基准 URL，用于连接本地/其他服务器上运行的 Ollama 服务 OLLAMA_BASE_URL=http://host.docker.internal:11435 # 存储配置 @@ -25,9 +24,6 @@ STORAGE_TYPE=local STREAM_MANAGER_TYPE=redis # 主数据库配置 -# 数据库地址，可以是主机名或IP地址 -DB_HOST=postgres - # 数据库端口，默认为5432 DB_PORT=5432 @@ -41,9 +37,6 @@ DB_PASSWORD=postgres123!@# DB_NAME=WeKnora # 如果使用 redis 作为流处理后端，需要配置以下参数 -# Redis地址，可以是主机名或IP地址 -REDIS_HOST=redis - # Redis端口，默认为6379 REDIS_PORT=6379 @@ -64,6 +57,10 @@ TENANT_AES_KEY=weknorarag-api-key-secret-secret # 是否开启知识图谱构建和检索（构建阶段需调用大模型，耗时较长） ENABLE_GRAPH_RAG=false +MINIO_PORT=9000 + +MINIO_CONSOLE_PORT=9001 + # 如果使用ElasticSearch作为向量存储，需要配置以下参数 # ElasticSearch地址，例如 http://localhost:9200 # ELASTICSEARCH_ADDR=your_elasticsearch_addr @@ -78,9 +75,6 @@ ENABLE_GRAPH_RAG=false # ELASTICSEARCH_INDEX=WeKnora # 如果使用MinIO作为文件存储，需要配置以下参数 -# MinIO服务器地址，例如 minio:9000 -# MINIO_ENDPOINT=minio:9000 - # MinIO访问密钥 # MINIO_ACCESS_KEY_ID=your_minio_access_key diff --git a/Makefile b/Makefile index 3ddc72463..609f8d678 100644 --- a/Makefile +++ b/Makefile @@ -90,5 +90,11 @@ clean-db: @if [ $$(docker volume ls -q -f name=weknora_postgres-data) ]; then \ docker volume rm weknora_postgres-data; \ fi + @if [ $$(docker volume ls -q -f name=weknora_minio_data) ]; then \ + docker volume rm weknora_minio_data; \ + fi + @if [ $$(docker volume ls -q -f name=weknora_redis_data) ]; then \ + docker volume rm weknora_redis_data; \ + fi diff --git a/docker-compose.yml b/docker-compose.yml index 420582bc8..8edce0d61 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3.8" - services: app: build: @@ -13,9 +11,9 @@ services: - ./config:/app/config environment: - GIN_MODE=${GIN_MODE} - - DB_DRIVER=${DB_DRIVER} - - DB_HOST=${DB_HOST} - - DB_PORT=${DB_PORT} + - DB_DRIVER=postgres + - DB_HOST=postgres + - DB_PORT=5432 - DB_USER=${DB_USER} - DB_PASSWORD=${DB_PASSWORD} - DB_NAME=${DB_NAME} @@ -34,13 +32,13 @@ services: - DOCREADER_ADDR=docreader:50051 - STORAGE_TYPE=${STORAGE_TYPE} - LOCAL_STORAGE_BASE_DIR=${LOCAL_STORAGE_BASE_DIR} - - MINIO_ENDPOINT=${MINIO_ENDPOINT} - - MINIO_ACCESS_KEY_ID=${MINIO_ACCESS_KEY_ID} - - MINIO_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY} + - MINIO_ENDPOINT=minio:9000 + - MINIO_ACCESS_KEY_ID=${MINIO_ACCESS_KEY_ID:-minioadmin} + - MINIO_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minioadmin} - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME} - - OLLAMA_BASE_URL=${OLLAMA_BASE_URL} + - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-http://host.docker.internal:11434} - STREAM_MANAGER_TYPE=${STREAM_MANAGER_TYPE} - - REDIS_ADDR=${REDIS_HOST}:${REDIS_PORT} + - REDIS_ADDR=redis:6379 - REDIS_PASSWORD=${REDIS_PASSWORD} - REDIS_DB=${REDIS_DB} - REDIS_PREFIX=${REDIS_PREFIX} @@ -57,19 +55,39 @@ services: - INIT_RERANK_MODEL_NAME=${INIT_RERANK_MODEL_NAME} - INIT_RERANK_MODEL_BASE_URL=${INIT_RERANK_MODEL_BASE_URL} - INIT_RERANK_MODEL_API_KEY=${INIT_RERANK_MODEL_API_KEY} - - INIT_TEST_TENANT_ID=${INIT_TEST_TENANT_ID} - - INIT_TEST_KNOWLEDGE_BASE_ID=${INIT_TEST_KNOWLEDGE_BASE_ID} depends_on: redis: condition: service_started postgres: condition: service_healthy + minio: + condition: service_started networks: - WeKnora-network restart: unless-stopped extra_hosts: - "host.docker.internal:host-gateway" + minio: + image: minio/minio:latest + container_name: WeKnora-minio + ports: + - "${MINIO_PORT:-9000}:9000" + - "${MINIO_CONSOLE_PORT:-9001}:9001" + environment: + - MINIO_ROOT_USER=${MINIO_ACCESS_KEY_ID:-minioadmin} + - MINIO_ROOT_PASSWORD=${MINIO_SECRET_ACCESS_KEY:-minioadmin} + command: server --console-address ":9001" /data + volumes: + - minio_data:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + networks: + - WeKnora-network + frontend: build: context: ./frontend @@ -103,6 +121,13 @@ services: - VLM_MODEL_BASE_URL=${VLM_MODEL_BASE_URL} - VLM_MODEL_NAME=${VLM_MODEL_NAME} - VLM_MODEL_API_KEY=${VLM_MODEL_API_KEY} + - STORAGE_TYPE=${STORAGE_TYPE} + - MINIO_PUBLIC_ENDPOINT=http://localhost:${MINIO_PORT:-9000} + - MINIO_ENDPOINT=minio:9000 + - MINIO_ACCESS_KEY_ID=${MINIO_ACCESS_KEY_ID:-minioadmin} + - MINIO_SECRET_ACCESS_KEY=${MINIO_SECRET_ACCESS_KEY:-minioadmin} + - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME} + - MINIO_USE_SSL=${MINIO_USE_SSL} networks: - WeKnora-network restart: unless-stopped @@ -175,3 +200,4 @@ volumes: data-files: jaeger_data: redis_data: + minio_data: diff --git a/docker/Dockerfile.docreader b/docker/Dockerfile.docreader index cbe79e898..5330b3662 100644 --- a/docker/Dockerfile.docreader +++ b/docker/Dockerfile.docreader @@ -99,8 +99,8 @@ RUN mkdir -p /tmp/libreoffice && \ cd LibreOffice_25.2.4*_Linux_x86-64_deb/DEBS && \ dpkg -i *.deb; \ elif [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then \ - wget -q https://mirrors.tuna.tsinghua.edu.cn/libreoffice/libreoffice/testing/25.8.0/deb/aarch64/LibreOffice_25.8.0.2_Linux_aarch64_deb.tar.gz && \ - tar -xzf LibreOffice_25.8.0.2_Linux_aarch64_deb.tar.gz && \ + wget -q https://mirrors.aliyun.com/libreoffice/testing/25.8.0/deb/aarch64/LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \ + tar -xzf LibreOffice_25.8.0.3_Linux_aarch64_deb.tar.gz && \ cd LibreOffice_25.8.0*_Linux_aarch64_deb/DEBS && \ dpkg -i *.deb; \ else \ diff --git a/frontend/src/api/initialization/index.ts b/frontend/src/api/initialization/index.ts index 875179be3..ee0d7a5ac 100644 --- a/frontend/src/api/initialization/index.ts +++ b/frontend/src/api/initialization/index.ts @@ -22,6 +22,7 @@ export interface InitializationConfig { }; multimodal: { enabled: boolean; + storageType: 'cos' | 'minio'; vlm?: { modelName: string; baseUrl: string; @@ -36,12 +37,18 @@ export interface InitializationConfig { appId: string; pathPrefix?: string; }; + minio?: { + bucketName: string; + pathPrefix?: string; + }; }; documentSplitting: { chunkSize: number; chunkOverlap: number; separators: string[]; }; + // Frontend-only hint for storage selection UI + storageType?: 'cos' | 'minio'; } // 下载任务状态类型 @@ -88,7 +95,7 @@ export function initializeSystem(config: InitializationConfig): Promise { } // 检查Ollama服务状态 -export function checkOllamaStatus(): Promise<{ available: boolean; version?: string; error?: string }> { +export function checkOllamaStatus(): Promise<{ available: boolean; version?: string; error?: string; baseUrl?: string }> { return new Promise((resolve, reject) => { get('/api/v1/initialization/ollama/status') .then((response: any) => { @@ -101,6 +108,20 @@ export function checkOllamaStatus(): Promise<{ available: boolean; version?: str }); } +// 列出已安装的 Ollama 模型 +export function listOllamaModels(): Promise { + return new Promise((resolve, reject) => { + get('/api/v1/initialization/ollama/models') + .then((response: any) => { + resolve((response.data && response.data.models) || []); + }) + .catch((error: any) => { + console.error('获取 Ollama 模型列表失败:', error); + resolve([]); + }); + }); +} + // 检查Ollama模型状态 export function checkOllamaModels(models: string[]): Promise<{ models: Record }> { return new Promise((resolve, reject) => { @@ -218,12 +239,17 @@ export function testMultimodalFunction(testData: { vlm_base_url: string; vlm_api_key?: string; vlm_interface_type?: string; - cos_secret_id: string; - cos_secret_key: string; - cos_region: string; - cos_bucket_name: string; - cos_app_id: string; + storage_type?: 'cos'|'minio'; + // COS optional fields (required only when storage_type === 'cos') + cos_secret_id?: string; + cos_secret_key?: string; + cos_region?: string; + cos_bucket_name?: string; + cos_app_id?: string; cos_path_prefix?: string; + // MinIO optional fields + minio_bucket_name?: string; + minio_path_prefix?: string; chunk_size: number; chunk_overlap: number; separators: string[]; @@ -245,14 +271,21 @@ export function testMultimodalFunction(testData: { if (testData.vlm_interface_type) { formData.append('vlm_interface_type', testData.vlm_interface_type); } - formData.append('cos_secret_id', testData.cos_secret_id); - formData.append('cos_secret_key', testData.cos_secret_key); - formData.append('cos_region', testData.cos_region); - formData.append('cos_bucket_name', testData.cos_bucket_name); - formData.append('cos_app_id', testData.cos_app_id); - if (testData.cos_path_prefix) { - formData.append('cos_path_prefix', testData.cos_path_prefix); + if (testData.storage_type) { + formData.append('storage_type', testData.storage_type); } + // Append COS fields only when storage_type is COS + if (testData.storage_type === 'cos') { + if (testData.cos_secret_id) formData.append('cos_secret_id', testData.cos_secret_id); + if (testData.cos_secret_key) formData.append('cos_secret_key', testData.cos_secret_key); + if (testData.cos_region) formData.append('cos_region', testData.cos_region); + if (testData.cos_bucket_name) formData.append('cos_bucket_name', testData.cos_bucket_name); + if (testData.cos_app_id) formData.append('cos_app_id', testData.cos_app_id); + if (testData.cos_path_prefix) formData.append('cos_path_prefix', testData.cos_path_prefix); + } + // MinIO fields + if (testData.minio_bucket_name) formData.append('minio_bucket_name', testData.minio_bucket_name); + if (testData.minio_path_prefix) formData.append('minio_path_prefix', testData.minio_path_prefix); formData.append('chunk_size', testData.chunk_size.toString()); formData.append('chunk_overlap', testData.chunk_overlap.toString()); formData.append('separators', JSON.stringify(testData.separators)); diff --git a/frontend/src/views/chat/components/botmsg.vue b/frontend/src/views/chat/components/botmsg.vue index 54badc88f..e2b71750f 100644 --- a/frontend/src/views/chat/components/botmsg.vue +++ b/frontend/src/views/chat/components/botmsg.vue @@ -158,7 +158,7 @@ function splitMarkdownByImages(markdown) { function isLink(str) { const trimmedStr = str.trim(); // 正则表达式匹配常见链接格式 - const urlPattern = /^(https?:\/\/|ftp:\/\/|www\.)[\w-]+(\.[\w-]+)+([\w.,@?^=%&:/~+#-]*[\w@?^=%&/~+#-])?/i; + const urlPattern = /^(https?:\/\/|ftp:\/\/|www\.)(?:(?:[\w-]+(?:\.[\w-]+)*)|(?:\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?:\[[a-fA-F0-9:]+\]))(?::\d{1,5})?(?:[\/\w.,@?^=%&:~+#-]*[\w@?^=%&\/~+#-])?/i; return urlPattern.test(trimmedStr); } diff --git a/frontend/src/views/initialization/InitializationConfig.vue b/frontend/src/views/initialization/InitializationConfig.vue index 4a2a71ee5..097021cfb 100644 --- a/frontend/src/views/initialization/InitializationConfig.vue +++ b/frontend/src/views/initialization/InitializationConfig.vue @@ -1,13 +1,57 @@